From 6566bd548483c0fcb5754afb365f9a22fa45a474 Mon Sep 17 00:00:00 2001 From: Rasmus Brandt Date: Wed, 4 May 2022 13:54:10 +0200 Subject: [PATCH] Add some more frame stats to PCLF MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Metrics for calculating sent/recv key frame rate: num_encoded_frames -- number of encoded frames num_decoded_frames -- number of decoded frames num_send_key_frames -- number of sent keyframes num_recv_key_frames -- number of received keyframes Metrics for frame size (in bytes) distribution: recv_key_frame_size_bytes -- key frame size recv_delta_frame_size_bytes -- delta frame size The stats are only reported if the user so requests. This is because the stats may not be useful for regression tracking, but mainly for adhoc analysis. Tested: ``` ~/s/w/src (frame_stats↑1|✚1) $ autoninja -C out/Debug && out/Debug/test_support_unittests --gtest_filter="PeerConnectionE2EQualityTestSmokeTest.Smoke" 2>/dev/null Note: Google Test filter = PeerConnectionE2EQualityTestSmokeTest.Smoke [==========] Running 1 test from 1 test suite. [----------] Global test environment set-up. [----------] 1 test from PeerConnectionE2EQualityTestSmokeTest [ RUN ] PeerConnectionE2EQualityTestSmokeTest.Smoke ... RESULT num_encoded_frames: Smoke/alice-video= 31 count_biggerIsBetter RESULT num_decoded_frames: Smoke/alice-video= 31 count_biggerIsBetter RESULT num_send_key_frames: Smoke/alice-video= 2 count_biggerIsBetter RESULT num_recv_key_frames: Smoke/alice-video= 2 count_biggerIsBetter RESULT recv_key_frame_size_bytes: Smoke/alice-video= {1847,126} count_biggerIsBetter RESULT recv_delta_frame_size_bytes: Smoke/alice-video= {1150.1034,393.66329} count_biggerIsBetter ... RESULT num_encoded_frames: Smoke/charlie-video= 31 count_biggerIsBetter RESULT num_decoded_frames: Smoke/charlie-video= 31 count_biggerIsBetter RESULT num_send_key_frames: Smoke/charlie-video= 2 count_biggerIsBetter RESULT num_recv_key_frames: Smoke/charlie-video= 2 count_biggerIsBetter RESULT recv_key_frame_size_bytes: Smoke/charlie-video= {1847,126} count_biggerIsBetter RESULT recv_delta_frame_size_bytes: Smoke/charlie-video= {1150.1034,393.66329} count_biggerIsBetter ... [----------] 1 test from PeerConnectionE2EQualityTestSmokeTest (2196 ms total) [----------] Global test environment tear-down [==========] 1 test from 1 test suite ran. (2196 ms total) [ PASSED ] 1 test. ``` Bug: webrtc:14019 Change-Id: I3d5ea0cf659a6299c6d94f5004cbf0c763b267e5 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/260924 Reviewed-by: Artem Titov Commit-Queue: Rasmus Brandt Cr-Commit-Position: refs/heads/main@{#36759} --- test/pc/e2e/BUILD.gn | 4 ++ .../video/default_video_quality_analyzer.cc | 42 +++++++++++++++++-- .../video/default_video_quality_analyzer.h | 17 ++++++-- ...ideo_quality_analyzer_frames_comparator.cc | 23 +++++++++- ...quality_analyzer_internal_shared_objects.h | 7 +++- ...lt_video_quality_analyzer_shared_objects.h | 13 ++++++ 6 files changed, 97 insertions(+), 9 deletions(-) diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 09629308c4..ab610a0364 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -662,10 +662,12 @@ if (!build_with_chromium) { "../../../api:array_view", "../../../api:video_quality_analyzer_api", "../../../api/numerics", + "../../../api/units:data_size", "../../../api/units:time_delta", "../../../api/units:timestamp", "../../../api/video:encoded_image", "../../../api/video:video_frame", + "../../../api/video:video_frame_type", "../../../api/video:video_rtp_headers", "../../../common_video", "../../../rtc_base:checks", @@ -706,8 +708,10 @@ if (!build_with_chromium) { "../../../api:array_view", "../../../api:scoped_refptr", "../../../api/numerics:numerics", + "../../../api/units:data_size", "../../../api/units:timestamp", "../../../api/video:video_frame", + "../../../api/video:video_frame_type", "../../../common_video", "../../../rtc_base:checks", "../../../rtc_base:platform_thread", diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index ecc92f28b9..abb9290105 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -317,7 +317,8 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded( used_encoder.last_frame_id = frame_id; used_encoder.switched_on_at = now; used_encoder.switched_from_at = now; - it->second.OnFrameEncoded(now, encoded_image.size(), + it->second.OnFrameEncoded(now, encoded_image._frameType, + DataSize::Bytes(encoded_image.size()), stats.target_encode_bitrate, used_encoder); } @@ -363,7 +364,9 @@ void DefaultVideoQualityAnalyzer::OnFramePreDecode( ->receive_time(); it->second.OnFramePreDecode(peer_index, /*received_time=*/last_receive_time, - /*decode_start_time=*/Now()); + /*decode_start_time=*/Now(), + input_image._frameType, + DataSize::Bytes(input_image.size())); } void DefaultVideoQualityAnalyzer::OnFrameDecoded( @@ -867,6 +870,28 @@ void DefaultVideoQualityAnalyzer::ReportResults( static_cast(stats.total_encoded_images_payload) / static_cast(test_duration.us()) * kMicrosPerSecond, "bytesPerSecond", /*important=*/false, ImproveDirection::kNone); + + if (options_.report_detailed_frame_stats) { + test::PrintResult("num_encoded_frames", "", test_case_name, + frame_counters.encoded, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + test::PrintResult("num_decoded_frames", "", test_case_name, + frame_counters.decoded, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + test::PrintResult("num_send_key_frames", "", test_case_name, + stats.num_send_key_frames, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + test::PrintResult("num_recv_key_frames", "", test_case_name, + stats.num_recv_key_frames, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + + ReportResult("recv_key_frame_size_bytes", test_case_name, + stats.recv_key_frame_size_bytes, "count", + ImproveDirection::kBiggerIsBetter); + ReportResult("recv_delta_frame_size_bytes", test_case_name, + stats.recv_delta_frame_size_bytes, "count", + ImproveDirection::kBiggerIsBetter); + } } void DefaultVideoQualityAnalyzer::ReportResult( @@ -1046,10 +1071,12 @@ bool DefaultVideoQualityAnalyzer::FrameInFlight::HaveAllPeersReceived() const { void DefaultVideoQualityAnalyzer::FrameInFlight::OnFrameEncoded( webrtc::Timestamp time, - int64_t encoded_image_size, + VideoFrameType frame_type, + DataSize encoded_image_size, uint32_t target_encode_bitrate, StreamCodecInfo used_encoder) { encoded_time_ = time; + frame_type_ = frame_type; encoded_image_size_ = encoded_image_size; target_encode_bitrate_ += target_encode_bitrate; // Update used encoder info. If simulcast/SVC is used, this method can @@ -1070,9 +1097,13 @@ void DefaultVideoQualityAnalyzer::FrameInFlight::OnFrameEncoded( void DefaultVideoQualityAnalyzer::FrameInFlight::OnFramePreDecode( size_t peer, webrtc::Timestamp received_time, - webrtc::Timestamp decode_start_time) { + webrtc::Timestamp decode_start_time, + VideoFrameType frame_type, + DataSize encoded_image_size) { receiver_stats_[peer].received_time = received_time; receiver_stats_[peer].decode_start_time = decode_start_time; + receiver_stats_[peer].frame_type = frame_type; + receiver_stats_[peer].encoded_image_size = encoded_image_size; } bool DefaultVideoQualityAnalyzer::FrameInFlight::HasReceivedTime( @@ -1134,6 +1165,7 @@ FrameStats DefaultVideoQualityAnalyzer::FrameInFlight::GetStatsForPeer( stats.pre_encode_time = pre_encode_time_; stats.encoded_time = encoded_time_; stats.target_encode_bitrate = target_encode_bitrate_; + stats.encoded_frame_type = frame_type_; stats.encoded_image_size = encoded_image_size_; stats.used_encoder = used_encoder_; @@ -1148,6 +1180,8 @@ FrameStats DefaultVideoQualityAnalyzer::FrameInFlight::GetStatsForPeer( stats.rendered_frame_width = receiver_stats->rendered_frame_width; stats.rendered_frame_height = receiver_stats->rendered_frame_height; stats.used_decoder = receiver_stats->used_decoder; + stats.pre_decoded_frame_type = receiver_stats->frame_type; + stats.pre_decoded_image_size = receiver_stats->encoded_image_size; } return stats; } diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index bbda276767..ee5d499120 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -22,9 +22,11 @@ #include "api/array_view.h" #include "api/numerics/samples_stats_counter.h" #include "api/test/video_quality_analyzer_interface.h" +#include "api/units/data_size.h" #include "api/units/timestamp.h" #include "api/video/encoded_image.h" #include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" #include "rtc_base/event.h" #include "rtc_base/platform_thread.h" #include "rtc_base/synchronization/mutex.h" @@ -179,6 +181,10 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { Timestamp rendered_time = Timestamp::MinusInfinity(); Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); + // Type and encoded size of received frame. + VideoFrameType frame_type = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size = DataSize::Bytes(0); + absl::optional rendered_frame_width = absl::nullopt; absl::optional rendered_frame_height = absl::nullopt; @@ -217,7 +223,8 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { void SetPreEncodeTime(webrtc::Timestamp time) { pre_encode_time_ = time; } void OnFrameEncoded(webrtc::Timestamp time, - int64_t encoded_image_size, + VideoFrameType frame_type, + DataSize encoded_image_size, uint32_t target_encode_bitrate, StreamCodecInfo used_encoder); @@ -225,7 +232,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { void OnFramePreDecode(size_t peer, webrtc::Timestamp received_time, - webrtc::Timestamp decode_start_time); + webrtc::Timestamp decode_start_time, + VideoFrameType frame_type, + DataSize encoded_image_size); bool HasReceivedTime(size_t peer) const; @@ -267,7 +276,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { Timestamp captured_time_; Timestamp pre_encode_time_ = Timestamp::MinusInfinity(); Timestamp encoded_time_ = Timestamp::MinusInfinity(); - int64_t encoded_image_size_ = 0; + // Type and encoded size of sent frame. + VideoFrameType frame_type_ = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size_ = DataSize::Bytes(0); uint32_t target_encode_bitrate_ = 0; // Can be not set if frame was dropped by encoder. absl::optional used_encoder_ = absl::nullopt; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc index 904671001a..cbe1a709f7 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc @@ -18,6 +18,7 @@ #include "api/array_view.h" #include "api/scoped_refptr.h" #include "api/video/i420_buffer.h" +#include "api/video/video_frame_type.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "rtc_base/checks.h" #include "rtc_base/platform_thread.h" @@ -425,9 +426,15 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( (frame_stats.encoded_time - frame_stats.pre_encode_time).ms(), frame_stats.encoded_time)); stats->encode_frame_rate.AddEvent(frame_stats.encoded_time); - stats->total_encoded_images_payload += frame_stats.encoded_image_size; + stats->total_encoded_images_payload += + frame_stats.encoded_image_size.bytes(); stats->target_encode_bitrate.AddSample(StatsSample( frame_stats.target_encode_bitrate, frame_stats.encoded_time)); + + // Stats sliced on encoded frame type. + if (frame_stats.encoded_frame_type == VideoFrameType::kVideoFrameKey) { + ++stats->num_send_key_frames; + } } // Next stats can be calculated only if frame was received on remote side. if (comparison.type != FrameComparisonType::kDroppedFrame) { @@ -447,6 +454,20 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( stats->transport_time_ms.AddSample(StatsSample( (frame_stats.decode_start_time - frame_stats.encoded_time).ms(), frame_stats.decode_start_time)); + + // Stats sliced on decoded frame type. + if (frame_stats.pre_decoded_frame_type == + VideoFrameType::kVideoFrameKey) { + ++stats->num_recv_key_frames; + stats->recv_key_frame_size_bytes.AddSample( + StatsSample(frame_stats.pre_decoded_image_size.bytes(), + frame_stats.decode_start_time)); + } else if (frame_stats.pre_decoded_frame_type == + VideoFrameType::kVideoFrameDelta) { + stats->recv_delta_frame_size_bytes.AddSample( + StatsSample(frame_stats.pre_decoded_image_size.bytes(), + frame_stats.decode_start_time)); + } } if (frame_stats.decode_end_time.IsFinite()) { stats->decode_time_ms.AddSample(StatsSample( diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h index d13eb21c3e..04f653c02b 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h @@ -16,8 +16,10 @@ #include #include "absl/types/optional.h" +#include "api/units/data_size.h" #include "api/units/timestamp.h" #include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" namespace webrtc { @@ -53,7 +55,10 @@ struct FrameStats { Timestamp rendered_time = Timestamp::MinusInfinity(); Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); - int64_t encoded_image_size = 0; + VideoFrameType encoded_frame_type = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size = DataSize::Bytes(0); + VideoFrameType pre_decoded_frame_type = VideoFrameType::kEmptyFrame; + DataSize pre_decoded_image_size = DataSize::Bytes(0); uint32_t target_encode_bitrate = 0; absl::optional rendered_frame_width = absl::nullopt; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h index 3b52a80d35..8f0afd36f7 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h @@ -96,8 +96,10 @@ struct StreamStats { // The time when the first frame of this stream was captured. Timestamp stream_started_time; + // Spatial quality metrics. SamplesStatsCounter psnr; SamplesStatsCounter ssim; + // Time from frame encoded (time point on exit from encoder) to the // encoded image received in decoder (time point on entrance to decoder). SamplesStatsCounter transport_time_ms; @@ -129,6 +131,14 @@ struct StreamStats { // Counters on which phase how many frames were dropped. std::map dropped_by_phase; + // Frame count metrics. + int64_t num_send_key_frames = 0; + int64_t num_recv_key_frames = 0; + + // Encoded frame size (in bytes) metrics. + SamplesStatsCounter recv_key_frame_size_bytes; + SamplesStatsCounter recv_delta_frame_size_bytes; + // Vector of encoders used for this stream by sending client. std::vector encoders; // Vectors of decoders used for this stream by receiving client. @@ -214,6 +224,9 @@ struct DefaultVideoQualityAnalyzerOptions { bool compute_ssim = true; // If true, weights the luma plane more than the chroma planes in the PSNR. bool use_weighted_psnr = false; + // Tells DefaultVideoQualityAnalyzer if detailed frame stats should be + // reported. + bool report_detailed_frame_stats = false; // If true DefaultVideoQualityAnalyzer will try to adjust frames before // computing PSNR and SSIM for them. In some cases picture may be shifted by // a few pixels after the encode/decode step. Those difference is invisible