From cdc959fb42406e2f2a3ac4cc7604430c0cf1db75 Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Wed, 10 Oct 2018 13:15:09 +0200 Subject: [PATCH] Compute video freeze metrics on rendered frames instead of on decoded Bug: webrtc:9828 Change-Id: I1390c736785759a2d8712e71398db4f5069ebcd4 Reviewed-on: https://webrtc-review.googlesource.com/c/105100 Reviewed-by: Stefan Holmer Commit-Queue: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#25116} --- video/receive_statistics_proxy.cc | 3 + video/receive_statistics_proxy_unittest.cc | 8 +++ video/video_quality_observer.cc | 72 ++++++++++++++-------- video/video_quality_observer.h | 6 +- 4 files changed, 63 insertions(+), 26 deletions(-) diff --git a/video/receive_statistics_proxy.cc b/video/receive_statistics_proxy.cc index ccd2c20f65..bffae167fc 100644 --- a/video/receive_statistics_proxy.cc +++ b/video/receive_statistics_proxy.cc @@ -759,6 +759,9 @@ void ReceiveStatisticsProxy::OnRenderedFrame(const VideoFrame& frame) { RTC_DCHECK_GT(height, 0); int64_t now_ms = clock_->TimeInMilliseconds(); rtc::CritScope lock(&crit_); + + video_quality_observer_->OnRenderedFrame(now_ms); + ContentSpecificStats* content_specific_stats = &content_specific_stats_[last_content_type_]; renders_fps_estimator_.Update(1, now_ms); diff --git a/video/receive_statistics_proxy_unittest.cc b/video/receive_statistics_proxy_unittest.cc index b8a96b2f5e..b74e4601aa 100644 --- a/video/receive_statistics_proxy_unittest.cc +++ b/video/receive_statistics_proxy_unittest.cc @@ -1057,15 +1057,19 @@ TEST_P(ReceiveStatisticsProxyTest, FreezesAreReported) { const int kFreezeDelayMs = 200; const int kCallDurationMs = kMinRequiredSamples * kInterFrameDelayMs + kFreezeDelayMs; + webrtc::VideoFrame frame(webrtc::I420Buffer::Create(1, 1), 0, 0, + webrtc::kVideoRotation_0); for (int i = 0; i < kMinRequiredSamples; ++i) { statistics_proxy_->OnDecodedFrame(absl::nullopt, kWidth, kHeight, content_type); + statistics_proxy_->OnRenderedFrame(frame); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } // Add extra freeze. fake_clock_.AdvanceTimeMilliseconds(kFreezeDelayMs); statistics_proxy_->OnDecodedFrame(absl::nullopt, kWidth, kHeight, content_type); + statistics_proxy_->OnRenderedFrame(frame); statistics_proxy_.reset(); const int kExpectedTimeBetweenFreezes = @@ -1095,9 +1099,12 @@ TEST_P(ReceiveStatisticsProxyTest, PausesAreIgnored) { const VideoContentType content_type = GetParam(); const int kInterFrameDelayMs = 33; const int kPauseDurationMs = 10000; + webrtc::VideoFrame frame(webrtc::I420Buffer::Create(1, 1), 0, 0, + webrtc::kVideoRotation_0); for (int i = 0; i <= kMinRequiredSamples; ++i) { statistics_proxy_->OnDecodedFrame(absl::nullopt, kWidth, kHeight, content_type); + statistics_proxy_->OnRenderedFrame(frame); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } // Add a pause. @@ -1108,6 +1115,7 @@ TEST_P(ReceiveStatisticsProxyTest, PausesAreIgnored) { for (int i = 0; i <= kMinRequiredSamples * 3; ++i) { statistics_proxy_->OnDecodedFrame(absl::nullopt, kWidth, kHeight, content_type); + statistics_proxy_->OnRenderedFrame(frame); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } diff --git a/video/video_quality_observer.cc b/video/video_quality_observer.cc index b05c3535e0..c4a574abfe 100644 --- a/video/video_quality_observer.cc +++ b/video/video_quality_observer.cc @@ -34,7 +34,9 @@ const int kBlockyQpThresholdVp9 = 60; // TODO(ilnik): tune this value. VideoQualityObserver::VideoQualityObserver(VideoContentType content_type) : last_frame_decoded_ms_(-1), + last_frame_rendered_ms_(-1), num_frames_decoded_(0), + num_frames_rendered_(0), first_frame_decoded_ms_(-1), last_frame_pixels_(0), last_frame_qp_(0), @@ -119,6 +121,46 @@ void VideoQualityObserver::UpdateHistograms() { RTC_LOG(LS_INFO) << log_stream.str(); } +void VideoQualityObserver::OnRenderedFrame(int64_t now_ms) { + if (num_frames_rendered_ == 0) { + last_unfreeze_time_ = now_ms; + } + + ++num_frames_rendered_; + + if (!is_paused_ && num_frames_rendered_ > 1) { + // Process inter-frame delay. + int64_t interframe_delay_ms = now_ms - last_frame_rendered_ms_; + render_interframe_delays_.Add(interframe_delay_ms); + absl::optional avg_interframe_delay = + render_interframe_delays_.Avg(kMinFrameSamplesToDetectFreeze); + // Check if it was a freeze. + if (avg_interframe_delay && + interframe_delay_ms >= + std::max(3 * *avg_interframe_delay, + *avg_interframe_delay + kMinIncreaseForFreezeMs)) { + freezes_durations_.Add(interframe_delay_ms); + smooth_playback_durations_.Add(last_frame_rendered_ms_ - + last_unfreeze_time_); + last_unfreeze_time_ = now_ms; + } + } + + if (is_paused_) { + // If the stream was paused since the previous frame, do not count the + // pause toward smooth playback. Explicitly count the part before it and + // start the new smooth playback interval from this frame. + is_paused_ = false; + if (last_frame_rendered_ms_ > last_unfreeze_time_) { + smooth_playback_durations_.Add(last_frame_rendered_ms_ - + last_unfreeze_time_); + } + last_unfreeze_time_ = now_ms; + } + + last_frame_rendered_ms_ = now_ms; +} + void VideoQualityObserver::OnDecodedFrame(absl::optional qp, int width, int height, @@ -126,7 +168,6 @@ void VideoQualityObserver::OnDecodedFrame(absl::optional qp, VideoCodecType codec) { if (num_frames_decoded_ == 0) { first_frame_decoded_ms_ = now_ms; - last_unfreeze_time_ = now_ms; } ++num_frames_decoded_; @@ -134,21 +175,14 @@ void VideoQualityObserver::OnDecodedFrame(absl::optional qp, if (!is_paused_ && num_frames_decoded_ > 1) { // Process inter-frame delay. int64_t interframe_delay_ms = now_ms - last_frame_decoded_ms_; - interframe_delays_.Add(interframe_delay_ms); + decode_interframe_delays_.Add(interframe_delay_ms); absl::optional avg_interframe_delay = - interframe_delays_.Avg(kMinFrameSamplesToDetectFreeze); - // Check if it was a freeze. - if (avg_interframe_delay && - interframe_delay_ms >= + decode_interframe_delays_.Avg(kMinFrameSamplesToDetectFreeze); + // Count spatial metrics if there were no freeze. + if (!avg_interframe_delay || + interframe_delay_ms < std::max(3 * *avg_interframe_delay, *avg_interframe_delay + kMinIncreaseForFreezeMs)) { - freezes_durations_.Add(interframe_delay_ms); - smooth_playback_durations_.Add(last_frame_decoded_ms_ - - last_unfreeze_time_); - last_unfreeze_time_ = now_ms; - } else { - // Only count inter-frame delay as playback time if there - // was no freeze. time_in_resolution_ms_[current_resolution_] += interframe_delay_ms; absl::optional qp_blocky_threshold; // TODO(ilnik): add other codec types when we have QP for them. @@ -168,18 +202,6 @@ void VideoQualityObserver::OnDecodedFrame(absl::optional qp, } } - if (is_paused_) { - // If the stream was paused since the previous frame, do not count the - // pause toward smooth playback. Explicitly count the part before it and - // start the new smooth playback interval from this frame. - is_paused_ = false; - if (last_frame_decoded_ms_ > last_unfreeze_time_) { - smooth_playback_durations_.Add(last_frame_decoded_ms_ - - last_unfreeze_time_); - } - last_unfreeze_time_ = now_ms; - } - int64_t pixels = width * height; if (pixels >= kPixelsInHighResolution) { current_resolution_ = Resolution::High; diff --git a/video/video_quality_observer.h b/video/video_quality_observer.h index 8ba4af3984..a73de92c91 100644 --- a/video/video_quality_observer.h +++ b/video/video_quality_observer.h @@ -35,6 +35,7 @@ class VideoQualityObserver { int height, int64_t now_ms, VideoCodecType codec); + void OnRenderedFrame(int64_t now_ms); void OnStreamInactive(); @@ -48,13 +49,16 @@ class VideoQualityObserver { }; int64_t last_frame_decoded_ms_; + int64_t last_frame_rendered_ms_; int64_t num_frames_decoded_; + int64_t num_frames_rendered_; int64_t first_frame_decoded_ms_; int64_t last_frame_pixels_; uint8_t last_frame_qp_; // Decoded timestamp of the last delayed frame. int64_t last_unfreeze_time_; - rtc::SampleCounter interframe_delays_; + rtc::SampleCounter render_interframe_delays_; + rtc::SampleCounter decode_interframe_delays_; // An inter-frame delay is counted as a freeze if it's significantly longer // than average inter-frame delay. rtc::SampleCounter freezes_durations_;