From d49d49ad89e67d1a3c63fbc638af445af5648875 Mon Sep 17 00:00:00 2001 From: Sergey Silkin Date: Wed, 7 Dec 2022 15:29:09 +0100 Subject: [PATCH] Report total and squared inter frame delays measured in OnRenderedFrame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After https://webrtc-review.googlesource.com/c/src/+/160042 we ended up with two sets of metrics representing total and total squared inter frame delays: old is measured in OnDecodedFrame and new in OnRenderedFrame. Reporting of old metrics was unshipped in https://webrtc-review.googlesource.com/c/src/+/278100. The metrics are used for calculation of harmonic frame rate and are desired to be measured as close as possible to rendering. This CL removes calculation of inter frame delay metrics from OnDecodedFrame and reports the metrics calculated in OnRenderedFrame to the stats. Bug: webrtc:11108, b/261512902 Change-Id: Ia21b321aab3a1ac0b6136dc0df7d95f2f0fd24c6 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/286842 Commit-Queue: Sergey Silkin Reviewed-by: Erik Språng Reviewed-by: Harald Alvestrand Reviewed-by: Henrik Boström Cr-Commit-Position: refs/heads/main@{#38909} --- call/video_receive_stream.h | 2 - media/base/media_channel.h | 2 - video/g3doc/stats.md | 6 +- video/receive_statistics_proxy2.cc | 11 ++-- video/receive_statistics_proxy2_unittest.cc | 72 +++------------------ video/video_analyzer.cc | 37 +++++------ video/video_analyzer.h | 4 +- 7 files changed, 35 insertions(+), 99 deletions(-) diff --git a/call/video_receive_stream.h b/call/video_receive_stream.h index 2e2742a814..0fa257e5ba 100644 --- a/call/video_receive_stream.h +++ b/call/video_receive_stream.h @@ -132,8 +132,6 @@ class VideoReceiveStreamInterface : public MediaReceiveStreamInterface { uint32_t pause_count = 0; uint32_t total_freezes_duration_ms = 0; uint32_t total_pauses_duration_ms = 0; - uint32_t total_frames_duration_ms = 0; - double sum_squared_frame_durations = 0.0; VideoContentType content_type = VideoContentType::UNSPECIFIED; diff --git a/media/base/media_channel.h b/media/base/media_channel.h index ee9a4724b0..233f1185dc 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -628,8 +628,6 @@ struct VideoReceiverInfo : public MediaReceiverInfo { uint32_t pause_count = 0; uint32_t total_freezes_duration_ms = 0; uint32_t total_pauses_duration_ms = 0; - uint32_t total_frames_duration_ms = 0; - double sum_squared_frame_durations = 0.0; uint32_t jitter_ms = 0; webrtc::VideoContentType content_type = webrtc::VideoContentType::UNSPECIFIED; diff --git a/video/g3doc/stats.md b/video/g3doc/stats.md index 669651e262..0a423e12ff 100644 --- a/video/g3doc/stats.md +++ b/video/g3doc/stats.md @@ -148,8 +148,6 @@ Updated after a frame has been decoded, `VCMDecodedFrameCallback::Decoded`. * `qp_sum` - sum of quantizer values of decoded frames [[rtcinboundrtpstreamstats-qpsum]]. * `content_type` - content type (UNSPECIFIED/SCREENSHARE). * `interframe_delay_max_ms` - max inter-frame delay within a time window between decoded frames. -* `total_inter_frame_delay` - sum of inter-frame delay in seconds between decoded frames [[rtcinboundrtpstreamstats-totalinterframedelay]]. -* `total_squared_inter_frame_delay` - sum of squared inter-frame delays in seconds between decoded frames [[rtcinboundrtpstreamstats-totalsquaredinterframedelay]]. Updated before a frame is sent to the renderer, `VideoReceiveStream2::OnFrame`. * `frames_rendered` - total number of rendered frames. @@ -162,8 +160,8 @@ Updated before a frame is sent to the renderer, `VideoReceiveStream2::OnFrame`. * `pause_count` - total number of detected pauses. * `total_freezes_duration_ms` - total duration of freezes in ms. * `total_pauses_duration_ms` - total duration of pauses in ms. -* `total_frames_duration_ms` - time in ms between the last rendered frame and the first rendered frame. -* `sum_squared_frame_durations` - sum of squared inter-frame delays in seconds between rendered frames. +* `total_inter_frame_delay` - sum of inter-frame delay in seconds between rendered frames [[rtcinboundrtpstreamstats-totalinterframedelay]]. +* `total_squared_inter_frame_delay` - sum of squared inter-frame delays in seconds between rendered frames [[rtcinboundrtpstreamstats-totalsquaredinterframedelay]]. `ReceiveStatisticsImpl::OnRtpPacket` is updated for received RTP packets. From `ReceiveStatistics`: * `total_bitrate_bps` - incoming bitrate in bps. diff --git a/video/receive_statistics_proxy2.cc b/video/receive_statistics_proxy2.cc index 297f5d3de9..cb34f98023 100644 --- a/video/receive_statistics_proxy2.cc +++ b/video/receive_statistics_proxy2.cc @@ -624,10 +624,11 @@ VideoReceiveStreamInterface::Stats ReceiveStatisticsProxy::GetStats() const { video_quality_observer_->TotalFreezesDurationMs(); stats_.total_pauses_duration_ms = video_quality_observer_->TotalPausesDurationMs(); - stats_.total_frames_duration_ms = - video_quality_observer_->TotalFramesDurationMs(); - stats_.sum_squared_frame_durations = + stats_.total_inter_frame_delay = + video_quality_observer_->TotalFramesDurationMs() / 1000.0; + stats_.total_squared_inter_frame_delay = video_quality_observer_->SumSquaredFrameDurationsSec(); + stats_.content_type = last_content_type_; stats_.timing_frame_info = timing_frame_info_counter_.Max(now_ms); stats_.jitter_buffer_delay_seconds = @@ -839,10 +840,6 @@ void ReceiveStatisticsProxy::OnDecodedFrame( int64_t interframe_delay_ms = frame_meta.decode_timestamp.ms() - *last_decoded_frame_time_ms_; RTC_DCHECK_GE(interframe_delay_ms, 0); - double interframe_delay = interframe_delay_ms / 1000.0; - stats_.total_inter_frame_delay += interframe_delay; - stats_.total_squared_inter_frame_delay += - interframe_delay * interframe_delay; interframe_delay_max_moving_.Add(interframe_delay_ms, frame_meta.decode_timestamp.ms()); content_specific_stats->interframe_delay_counter.Add(interframe_delay_ms); diff --git a/video/receive_statistics_proxy2_unittest.cc b/video/receive_statistics_proxy2_unittest.cc index f0869c4341..0c628f7b83 100644 --- a/video/receive_statistics_proxy2_unittest.cc +++ b/video/receive_statistics_proxy2_unittest.cc @@ -328,62 +328,6 @@ TEST_F(ReceiveStatisticsProxy2Test, ReportsContentType) { videocontenttypehelpers::ToString(FlushAndGetStats().content_type)); } -TEST_F(ReceiveStatisticsProxy2Test, ReportsMaxTotalInterFrameDelay) { - webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); - const TimeDelta kInterFrameDelay1 = TimeDelta::Millis(100); - const TimeDelta kInterFrameDelay2 = TimeDelta::Millis(200); - const TimeDelta kInterFrameDelay3 = TimeDelta::Millis(300); - double expected_total_inter_frame_delay = 0; - double expected_total_squared_inter_frame_delay = 0; - EXPECT_EQ(expected_total_inter_frame_delay, - statistics_proxy_->GetStats().total_inter_frame_delay); - EXPECT_EQ(expected_total_squared_inter_frame_delay, - statistics_proxy_->GetStats().total_squared_inter_frame_delay); - - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); - EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay, - FlushAndGetStats().total_inter_frame_delay); - EXPECT_DOUBLE_EQ(expected_total_squared_inter_frame_delay, - FlushAndGetStats().total_squared_inter_frame_delay); - - time_controller_.AdvanceTime(kInterFrameDelay1); - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); - expected_total_inter_frame_delay += kInterFrameDelay1.seconds(); - expected_total_squared_inter_frame_delay += - pow(kInterFrameDelay1.seconds(), 2.0); - EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay, - FlushAndGetStats().total_inter_frame_delay); - EXPECT_DOUBLE_EQ( - expected_total_squared_inter_frame_delay, - statistics_proxy_->GetStats().total_squared_inter_frame_delay); - - time_controller_.AdvanceTime(kInterFrameDelay2); - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); - expected_total_inter_frame_delay += kInterFrameDelay2.seconds(); - expected_total_squared_inter_frame_delay += - pow(kInterFrameDelay2.seconds(), 2.0); - EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay, - FlushAndGetStats().total_inter_frame_delay); - EXPECT_DOUBLE_EQ( - expected_total_squared_inter_frame_delay, - statistics_proxy_->GetStats().total_squared_inter_frame_delay); - - time_controller_.AdvanceTime(kInterFrameDelay3); - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); - expected_total_inter_frame_delay += kInterFrameDelay3.seconds(); - expected_total_squared_inter_frame_delay += - pow(kInterFrameDelay3.seconds(), 2.0); - EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay, - FlushAndGetStats().total_inter_frame_delay); - EXPECT_DOUBLE_EQ( - expected_total_squared_inter_frame_delay, - statistics_proxy_->GetStats().total_squared_inter_frame_delay); -} - TEST_F(ReceiveStatisticsProxy2Test, ReportsMaxInterframeDelay) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); const TimeDelta kInterframeDelay1 = TimeDelta::Millis(100); @@ -503,9 +447,9 @@ TEST_F(ReceiveStatisticsProxy2Test, PauseBeforeFirstAndAfterLastFrameIgnored) { EXPECT_EQ(0u, stats.total_pauses_duration_ms); } -TEST_F(ReceiveStatisticsProxy2Test, ReportsFramesDuration) { +TEST_F(ReceiveStatisticsProxy2Test, ReportsTotalInterFrameDelay) { VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats(); - ASSERT_EQ(0u, stats.total_frames_duration_ms); + ASSERT_EQ(0.0, stats.total_inter_frame_delay); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); @@ -519,12 +463,12 @@ TEST_F(ReceiveStatisticsProxy2Test, ReportsFramesDuration) { } stats = statistics_proxy_->GetStats(); - EXPECT_EQ(10 * 30u, stats.total_frames_duration_ms); + EXPECT_EQ(10 * 30 / 1000.0, stats.total_inter_frame_delay); } -TEST_F(ReceiveStatisticsProxy2Test, ReportsSumSquaredFrameDurations) { +TEST_F(ReceiveStatisticsProxy2Test, ReportsTotalSquaredInterFrameDelay) { VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats(); - ASSERT_EQ(0u, stats.sum_squared_frame_durations); + ASSERT_EQ(0.0, stats.total_squared_inter_frame_delay); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); for (int i = 0; i <= 10; ++i) { @@ -533,10 +477,10 @@ TEST_F(ReceiveStatisticsProxy2Test, ReportsSumSquaredFrameDurations) { } stats = statistics_proxy_->GetStats(); - const double kExpectedSumSquaredFrameDurationsSecs = + const double kExpectedTotalSquaredInterFrameDelaySecs = 10 * (30 / 1000.0 * 30 / 1000.0); - EXPECT_EQ(kExpectedSumSquaredFrameDurationsSecs, - stats.sum_squared_frame_durations); + EXPECT_EQ(kExpectedTotalSquaredInterFrameDelaySecs, + stats.total_squared_inter_frame_delay); } TEST_F(ReceiveStatisticsProxy2Test, OnDecodedFrameWithoutQpQpSumWontExist) { diff --git a/video/video_analyzer.cc b/video/video_analyzer.cc index 3077a77b2c..bc350c3ff3 100644 --- a/video/video_analyzer.cc +++ b/video/video_analyzer.cc @@ -99,8 +99,8 @@ VideoAnalyzer::VideoAnalyzer(test::LayerFilteringTransport* transport, mean_decode_time_ms_(0.0), freeze_count_(0), total_freezes_duration_ms_(0), - total_frames_duration_ms_(0), - sum_squared_frame_durations_(0), + total_inter_frame_delay_(0), + total_squared_inter_frame_delay_(0), decode_frame_rate_(0), render_frame_rate_(0), last_fec_bytes_(0), @@ -502,6 +502,14 @@ void VideoAnalyzer::PollStats() { if (receive_stream_ != nullptr) { VideoReceiveStreamInterface::Stats receive_stats = receive_stream_->GetStats(); + + // Freeze metrics. + freeze_count_ = receive_stats.freeze_count; + total_freezes_duration_ms_ = receive_stats.total_freezes_duration_ms; + total_inter_frame_delay_ = receive_stats.total_inter_frame_delay; + total_squared_inter_frame_delay_ = + receive_stats.total_squared_inter_frame_delay; + // `total_decode_time_ms` gives a good estimate of the mean decode time, // `decode_ms` is used to keep track of the standard deviation. if (receive_stats.frames_decoded > 0) @@ -518,20 +526,12 @@ void VideoAnalyzer::PollStats() { // `frames_decoded` and `frames_rendered` are used because they are more // accurate than `decode_frame_rate` and `render_frame_rate`. // The latter two are calculated on a momentary basis. - const double total_frames_duration_sec_double = - static_cast(receive_stats.total_frames_duration_ms) / 1000.0; - if (total_frames_duration_sec_double > 0) { - decode_frame_rate_ = static_cast(receive_stats.frames_decoded) / - total_frames_duration_sec_double; - render_frame_rate_ = static_cast(receive_stats.frames_rendered) / - total_frames_duration_sec_double; + if (total_inter_frame_delay_ > 0) { + decode_frame_rate_ = + receive_stats.frames_decoded / total_inter_frame_delay_; + render_frame_rate_ = + receive_stats.frames_rendered / total_inter_frame_delay_; } - - // Freeze metrics. - freeze_count_ = receive_stats.freeze_count; - total_freezes_duration_ms_ = receive_stats.total_freezes_duration_ms; - total_frames_duration_ms_ = receive_stats.total_frames_duration_ms; - sum_squared_frame_durations_ = receive_stats.sum_squared_frame_durations; } if (audio_receive_stream_ != nullptr) { @@ -673,7 +673,7 @@ void VideoAnalyzer::PrintResults() { const double total_freezes_duration_ms_double = static_cast(total_freezes_duration_ms_); const double total_frames_duration_ms_double = - static_cast(total_frames_duration_ms_); + total_inter_frame_delay_ / rtc::kNumMillisecsPerSec; if (total_frames_duration_ms_double > 0) { GetGlobalMetricsLogger()->LogSingleValueMetric( @@ -701,10 +701,11 @@ void VideoAnalyzer::PrintResults() { : 0, Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); - if (1000 * sum_squared_frame_durations_ > 0) { + if (total_squared_inter_frame_delay_ > 0) { GetGlobalMetricsLogger()->LogSingleValueMetric( "harmonic_frame_rate_fps", test_label_, - total_frames_duration_ms_double / (1000 * sum_squared_frame_durations_), + total_frames_duration_ms_double / + (1000 * total_squared_inter_frame_delay_), Unit::kHertz, ImprovementDirection::kBiggerIsBetter); } diff --git a/video/video_analyzer.h b/video/video_analyzer.h index 2cee5e1b92..9186d78597 100644 --- a/video/video_analyzer.h +++ b/video/video_analyzer.h @@ -263,8 +263,8 @@ class VideoAnalyzer : public PacketReceiver, SamplesStatsCounter time_between_freezes_ RTC_GUARDED_BY(comparison_lock_); uint32_t freeze_count_ RTC_GUARDED_BY(comparison_lock_); uint32_t total_freezes_duration_ms_ RTC_GUARDED_BY(comparison_lock_); - uint32_t total_frames_duration_ms_ RTC_GUARDED_BY(comparison_lock_); - double sum_squared_frame_durations_ RTC_GUARDED_BY(comparison_lock_); + double total_inter_frame_delay_ RTC_GUARDED_BY(comparison_lock_); + double total_squared_inter_frame_delay_ RTC_GUARDED_BY(comparison_lock_); double decode_frame_rate_ RTC_GUARDED_BY(comparison_lock_); double render_frame_rate_ RTC_GUARDED_BY(comparison_lock_);