diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index 15caa87ad4..59144589fc 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -349,7 +349,8 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded( used_encoder.switched_from_at = now; frame_in_flight.OnFrameEncoded( now, encoded_image._frameType, DataSize::Bytes(encoded_image.size()), - stats.target_encode_bitrate, stats.qp, used_encoder); + stats.target_encode_bitrate, encoded_image.SpatialIndex().value_or(0), + stats.qp, used_encoder); if (options_.report_infra_metrics) { analyzer_stats_.on_frame_encoded_processing_time_ms.AddSample( @@ -1136,9 +1137,15 @@ void DefaultVideoQualityAnalyzer::ReportResults( "target_encode_bitrate", test_case_name, stats.target_encode_bitrate / 1000, Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, metric_metadata); - metrics_logger_->LogMetric("qp", test_case_name, stats.qp, Unit::kUnitless, - ImprovementDirection::kSmallerIsBetter, - metric_metadata); + for (const auto& [spatial_layer, qp] : stats.spatial_layers_qp) { + std::map qp_metadata = metric_metadata; + qp_metadata[MetricMetadataKey::kSpatialLayerMetadataKey] = + std::to_string(spatial_layer); + metrics_logger_->LogMetric("qp_sl" + std::to_string(spatial_layer), + test_case_name, qp, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, + std::move(qp_metadata)); + } metrics_logger_->LogSingleValueMetric( "actual_encode_bitrate", test_case_name, static_cast(stats.total_encoded_images_payload) / diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc index 24f829e089..df34dadaf0 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc @@ -88,13 +88,14 @@ void FrameInFlight::OnFrameEncoded(webrtc::Timestamp time, VideoFrameType frame_type, DataSize encoded_image_size, uint32_t target_encode_bitrate, + int spatial_layer, int qp, StreamCodecInfo used_encoder) { encoded_time_ = time; frame_type_ = frame_type; encoded_image_size_ = encoded_image_size; target_encode_bitrate_ += target_encode_bitrate; - qp_values_.AddSample(SamplesStatsCounter::StatsSample{ + spatial_layers_qp_[spatial_layer].AddSample(SamplesStatsCounter::StatsSample{ .value = static_cast(qp), .time = time}); // Update used encoder info. If simulcast/SVC is used, this method can // be called multiple times, in such case we should preserve the value @@ -185,7 +186,7 @@ FrameStats FrameInFlight::GetStatsForPeer(size_t peer) const { stats.encoded_frame_type = frame_type_; stats.encoded_image_size = encoded_image_size_; stats.used_encoder = used_encoder_; - stats.qp_values = qp_values_; + stats.spatial_layers_qp = spatial_layers_qp_; absl::optional receiver_stats = MaybeGetValue(receiver_stats_, peer); diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h index 7ee910effe..52a526d09b 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h @@ -88,6 +88,7 @@ class FrameInFlight { VideoFrameType frame_type, DataSize encoded_image_size, uint32_t target_encode_bitrate, + int spatial_layer, int qp, StreamCodecInfo used_encoder); @@ -154,7 +155,9 @@ class FrameInFlight { VideoFrameType frame_type_ = VideoFrameType::kEmptyFrame; DataSize encoded_image_size_ = DataSize::Bytes(0); uint32_t target_encode_bitrate_ = 0; - SamplesStatsCounter qp_values_; + // Sender side qp values per spatial layer. In case when spatial layer is not + // set for `webrtc::EncodedImage`, 0 is used as default. + std::map spatial_layers_qp_; // Can be not set if frame was dropped by encoder. absl::optional used_encoder_ = absl::nullopt; // Map from the receiver peer's index to frame stats for that peer. diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc index fe25e97ede..cbc0b7e8f3 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc @@ -468,10 +468,12 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( frame_stats.encoded_image_size.bytes(); stats->target_encode_bitrate.AddSample(StatsSample( frame_stats.target_encode_bitrate, frame_stats.encoded_time, metadata)); - for (SamplesStatsCounter::StatsSample qp : - frame_stats.qp_values.GetTimedSamples()) { - qp.metadata = metadata; - stats->qp.AddSample(std::move(qp)); + for (const auto& [spatial_layer, qp_values] : + frame_stats.spatial_layers_qp) { + for (SamplesStatsCounter::StatsSample qp : qp_values.GetTimedSamples()) { + qp.metadata = metadata; + stats->spatial_layers_qp[spatial_layer].AddSample(std::move(qp)); + } } // Stats sliced on encoded frame type. diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc index 9817e7c456..8d3cd47ed6 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc @@ -32,6 +32,7 @@ using ::testing::Each; using ::testing::Eq; using ::testing::IsEmpty; using ::testing::Pair; +using ::testing::SizeIs; using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample; @@ -400,7 +401,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, ExpectEmpty(stats.time_between_freezes_ms); ExpectEmpty(stats.resolution_of_decoded_frame); ExpectEmpty(stats.target_encode_bitrate); - ExpectEmpty(stats.qp); + EXPECT_THAT(stats.spatial_layers_qp, IsEmpty()); ExpectEmpty(stats.recv_key_frame_size_bytes); ExpectEmpty(stats.recv_delta_frame_size_bytes); EXPECT_EQ(stats.total_encoded_images_payload, 0); @@ -460,7 +461,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, ExpectEmpty(stats.time_between_freezes_ms); ExpectEmpty(stats.resolution_of_decoded_frame); ExpectEmpty(stats.target_encode_bitrate); - ExpectEmpty(stats.qp); + EXPECT_THAT(stats.spatial_layers_qp, IsEmpty()); ExpectEmpty(stats.recv_key_frame_size_bytes); ExpectEmpty(stats.recv_delta_frame_size_bytes); EXPECT_EQ(stats.total_encoded_images_payload, 0); @@ -501,8 +502,10 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.encoded_image_size = DataSize::Bytes(1000); frame_stats.target_encode_bitrate = 2000; - frame_stats.qp_values = StatsCounter( - /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}}); + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; comparator.Start(/*max_threads_count=*/1); comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, @@ -531,7 +534,9 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, ExpectEmpty(stats.resolution_of_decoded_frame); ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, /*value=*/2000.0); - ExpectSizeAndAllElementsAre(stats.qp, /*size=*/2, /*value=*/5.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); ExpectEmpty(stats.recv_key_frame_size_bytes); ExpectEmpty(stats.recv_delta_frame_size_bytes); EXPECT_EQ(stats.total_encoded_images_payload, 1000); @@ -573,8 +578,10 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameDelta; frame_stats.encoded_image_size = DataSize::Bytes(1000); frame_stats.target_encode_bitrate = 2000; - frame_stats.qp_values = StatsCounter( - /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}}); + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; comparator.Start(/*max_threads_count=*/1); comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, @@ -603,7 +610,9 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, ExpectEmpty(stats.resolution_of_decoded_frame); ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, /*value=*/2000.0); - ExpectSizeAndAllElementsAre(stats.qp, /*size=*/2, /*value=*/5.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); ExpectEmpty(stats.recv_key_frame_size_bytes); ExpectEmpty(stats.recv_delta_frame_size_bytes); EXPECT_EQ(stats.total_encoded_images_payload, 1000); @@ -645,8 +654,10 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.encoded_image_size = DataSize::Bytes(1000); frame_stats.target_encode_bitrate = 2000; - frame_stats.qp_values = StatsCounter( - /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}}); + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; // Frame pre decoded frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.pre_decoded_image_size = DataSize::Bytes(500); @@ -681,7 +692,9 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, ExpectEmpty(stats.resolution_of_decoded_frame); ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, /*value=*/2000.0); - ExpectSizeAndAllElementsAre(stats.qp, /*size=*/2, /*value=*/5.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, /*value=*/500.0); ExpectEmpty(stats.recv_delta_frame_size_bytes); @@ -724,8 +737,10 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.encoded_image_size = DataSize::Bytes(1000); frame_stats.target_encode_bitrate = 2000; - frame_stats.qp_values = StatsCounter( - /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}}); + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; // Frame pre decoded frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.pre_decoded_image_size = DataSize::Bytes(500); @@ -767,7 +782,9 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, EXPECT_GE(GetFirstOrDie(stats.resolution_of_decoded_frame), 200 * 100.0); ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, /*value=*/2000.0); - ExpectSizeAndAllElementsAre(stats.qp, /*size=*/2, /*value=*/5.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, /*value=*/500.0); ExpectEmpty(stats.recv_delta_frame_size_bytes); @@ -811,8 +828,10 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.encoded_image_size = DataSize::Bytes(1000); frame_stats.target_encode_bitrate = 2000; - frame_stats.qp_values = StatsCounter( - /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}}); + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; // Frame pre decoded frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.pre_decoded_image_size = DataSize::Bytes(500); @@ -851,7 +870,9 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, ExpectEmpty(stats.resolution_of_decoded_frame); ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, /*value=*/2000.0); - ExpectSizeAndAllElementsAre(stats.qp, /*size=*/2, /*value=*/5.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, /*value=*/500.0); ExpectEmpty(stats.recv_delta_frame_size_bytes); @@ -915,7 +936,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, ExpectEmpty(stats.time_between_freezes_ms); ExpectEmpty(stats.resolution_of_decoded_frame); ExpectEmpty(stats.target_encode_bitrate); - ExpectEmpty(stats.qp); + EXPECT_THAT(stats.spatial_layers_qp, IsEmpty()); ExpectEmpty(stats.recv_key_frame_size_bytes); ExpectEmpty(stats.recv_delta_frame_size_bytes); EXPECT_EQ(stats.total_encoded_images_payload, 0); @@ -975,7 +996,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, ExpectEmpty(stats.time_between_freezes_ms); ExpectEmpty(stats.resolution_of_decoded_frame); ExpectEmpty(stats.target_encode_bitrate); - ExpectEmpty(stats.qp); + EXPECT_THAT(stats.spatial_layers_qp, IsEmpty()); ExpectEmpty(stats.recv_key_frame_size_bytes); ExpectEmpty(stats.recv_delta_frame_size_bytes); EXPECT_EQ(stats.total_encoded_images_payload, 0); @@ -1016,8 +1037,10 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.encoded_image_size = DataSize::Bytes(1000); frame_stats.target_encode_bitrate = 2000; - frame_stats.qp_values = StatsCounter( - /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}}); + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; comparator.Start(/*max_threads_count=*/1); comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, @@ -1046,7 +1069,9 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, ExpectEmpty(stats.resolution_of_decoded_frame); ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, /*value=*/2000.0); - ExpectSizeAndAllElementsAre(stats.qp, /*size=*/2, /*value=*/5.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); ExpectEmpty(stats.recv_key_frame_size_bytes); ExpectEmpty(stats.recv_delta_frame_size_bytes); EXPECT_EQ(stats.total_encoded_images_payload, 1000); @@ -1088,8 +1113,10 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameDelta; frame_stats.encoded_image_size = DataSize::Bytes(1000); frame_stats.target_encode_bitrate = 2000; - frame_stats.qp_values = StatsCounter( - /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}}); + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; comparator.Start(/*max_threads_count=*/1); comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, @@ -1118,7 +1145,9 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, ExpectEmpty(stats.resolution_of_decoded_frame); ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, /*value=*/2000.0); - ExpectSizeAndAllElementsAre(stats.qp, /*size=*/2, /*value=*/5.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); ExpectEmpty(stats.recv_key_frame_size_bytes); ExpectEmpty(stats.recv_delta_frame_size_bytes); EXPECT_EQ(stats.total_encoded_images_payload, 1000); @@ -1237,8 +1266,10 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.encoded_image_size = DataSize::Bytes(1000); frame_stats.target_encode_bitrate = 2000; - frame_stats.qp_values = StatsCounter( - /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}}); + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; // Frame pre decoded frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.pre_decoded_image_size = DataSize::Bytes(500); @@ -1278,7 +1309,9 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, ExpectEmpty(stats.resolution_of_decoded_frame); ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, /*value=*/2000.0); - ExpectSizeAndAllElementsAre(stats.qp, /*size=*/2, /*value=*/5.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); ExpectEmpty(stats.recv_key_frame_size_bytes); ExpectEmpty(stats.recv_delta_frame_size_bytes); EXPECT_EQ(stats.total_encoded_images_payload, 1000); @@ -1321,8 +1354,10 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.encoded_image_size = DataSize::Bytes(1000); frame_stats.target_encode_bitrate = 2000; - frame_stats.qp_values = StatsCounter( - /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}}); + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; // Frame pre decoded frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.pre_decoded_image_size = DataSize::Bytes(500); @@ -1361,7 +1396,9 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, ExpectEmpty(stats.resolution_of_decoded_frame); ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, /*value=*/2000.0); - ExpectSizeAndAllElementsAre(stats.qp, /*size=*/2, /*value=*/5.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, /*value=*/500.0); ExpectEmpty(stats.recv_delta_frame_size_bytes); @@ -1409,8 +1446,10 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.encoded_image_size = DataSize::Bytes(1000); frame_stats.target_encode_bitrate = 2000; - frame_stats.qp_values = StatsCounter( - /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}}); + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; // Frame pre decoded frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.pre_decoded_image_size = DataSize::Bytes(500); @@ -1453,7 +1492,9 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, EXPECT_GE(GetFirstOrDie(stats.resolution_of_decoded_frame), 200 * 100.0); ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, /*value=*/2000.0); - ExpectSizeAndAllElementsAre(stats.qp, /*size=*/2, /*value=*/5.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, /*value=*/500.0); ExpectEmpty(stats.recv_delta_frame_size_bytes); @@ -1498,8 +1539,10 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, AllStatsHaveMetadataSet) { frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.encoded_image_size = DataSize::Bytes(1000); frame_stats.target_encode_bitrate = 2000; - frame_stats.qp_values = StatsCounter( - /*samples=*/{{5, Timestamp::Seconds(1)}, {5, Timestamp::Seconds(2)}}); + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; // Frame pre decoded frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; frame_stats.pre_decoded_image_size = DataSize::Bytes(500); @@ -1536,7 +1579,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, AllStatsHaveMetadataSet) { AssertFirstMetadataHasField(stats.resolution_of_decoded_frame, "frame_id", "1"); AssertFirstMetadataHasField(stats.target_encode_bitrate, "frame_id", "1"); - AssertFirstMetadataHasField(stats.qp, "frame_id", "1"); + AssertFirstMetadataHasField(stats.spatial_layers_qp[0], "frame_id", "1"); AssertFirstMetadataHasField(stats.recv_key_frame_size_bytes, "frame_id", "1"); ExpectEmpty(stats.recv_delta_frame_size_bytes); diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h index 3e65e2b888..10f1314f46 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h @@ -64,10 +64,9 @@ struct FrameStats { VideoFrameType pre_decoded_frame_type = VideoFrameType::kEmptyFrame; DataSize pre_decoded_image_size = DataSize::Bytes(0); uint32_t target_encode_bitrate = 0; - // There can be multiple qp values for single video frame when simulcast - // or SVC is used. In such case multiple EncodedImage's are created by encoder - // and each of it will have its own qp value. - SamplesStatsCounter qp_values; + // Sender side qp values per spatial layer. In case when spatial layer is not + // set for `webrtc::EncodedImage`, 0 is used as default. + std::map spatial_layers_qp; absl::optional decoded_frame_width = absl::nullopt; absl::optional decoded_frame_height = absl::nullopt; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc index 106daac584..f5029ac956 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc @@ -279,7 +279,7 @@ TEST(DefaultVideoQualityAnalyzerMetricNamesTest, MetricNamesForP2PAreCorrect) { .improvement_direction = ImprovementDirection::kNeitherIsBetter}, MetricValidationInfo{ .test_case = "test_case/alice_video", - .name = "qp", + .name = "qp_sl0", .unit = Unit::kUnitless, .improvement_direction = ImprovementDirection::kSmallerIsBetter}, MetricValidationInfo{ @@ -453,7 +453,7 @@ TEST(DefaultVideoQualityAnalyzerMetricNamesTest, .improvement_direction = ImprovementDirection::kNeitherIsBetter}, MetricValidationInfo{ .test_case = "test_case/alice_video_alice_bob", - .name = "qp", + .name = "qp_sl0", .unit = Unit::kUnitless, .improvement_direction = ImprovementDirection::kSmallerIsBetter}, MetricValidationInfo{ @@ -595,7 +595,7 @@ TEST(DefaultVideoQualityAnalyzerMetricNamesTest, .improvement_direction = ImprovementDirection::kNeitherIsBetter}, MetricValidationInfo{ .test_case = "test_case/alice_video_alice_charlie", - .name = "qp", + .name = "qp_sl0", .unit = Unit::kUnitless, .improvement_direction = ImprovementDirection::kSmallerIsBetter}, MetricValidationInfo{ diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h index a71dad71c1..175f777b68 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h @@ -145,7 +145,9 @@ struct StreamStats { SamplesStatsCounter time_between_freezes_ms; SamplesStatsCounter resolution_of_decoded_frame; SamplesStatsCounter target_encode_bitrate; - SamplesStatsCounter qp; + // Sender side qp values per spatial layer. In case when spatial layer is not + // set for `webrtc::EncodedImage`, 0 is used as default. + std::map spatial_layers_qp; int64_t total_encoded_images_payload = 0; // Counters on which phase how many frames were dropped. diff --git a/test/pc/e2e/metric_metadata_keys.h b/test/pc/e2e/metric_metadata_keys.h index 2fee0cbcb0..fbcd3b90fe 100644 --- a/test/pc/e2e/metric_metadata_keys.h +++ b/test/pc/e2e/metric_metadata_keys.h @@ -37,6 +37,8 @@ class MetricMetadataKey { // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. static constexpr char kExperimentalTestNameMetadataKey[] = "experimental_test_name"; + // Represents index of a video spatial layer to which metric belongs. + static constexpr char kSpatialLayerMetadataKey[] = "spatial_layer"; private: MetricMetadataKey() = default; diff --git a/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc b/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc index d5f46f3ccc..8a47e108e0 100644 --- a/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc +++ b/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc @@ -534,7 +534,7 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest, "test_case"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", - .name = "qp", + .name = "qp_sl0", .unit = Unit::kUnitless, .improvement_direction = ImprovementDirection::kSmallerIsBetter, .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, @@ -542,6 +542,7 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest, "alice_video"}, {MetricMetadataKey::kSenderMetadataKey, "alice"}, {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kSpatialLayerMetadataKey, "0"}, {MetricMetadataKey::kExperimentalTestNameMetadataKey, "test_case"}}}, MetricValidationInfo{ @@ -798,7 +799,7 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest, "test_case"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", - .name = "qp", + .name = "qp_sl0", .unit = Unit::kUnitless, .improvement_direction = ImprovementDirection::kSmallerIsBetter, .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, @@ -806,6 +807,7 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest, "bob_video"}, {MetricMetadataKey::kSenderMetadataKey, "bob"}, {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kSpatialLayerMetadataKey, "0"}, {MetricMetadataKey::kExperimentalTestNameMetadataKey, "test_case"}}}, MetricValidationInfo{