diff --git a/api/stats/rtcstats_objects.h b/api/stats/rtcstats_objects.h index b27bf73df6..8abf224e09 100644 --- a/api/stats/rtcstats_objects.h +++ b/api/stats/rtcstats_objects.h @@ -314,6 +314,14 @@ class RTC_EXPORT RTCMediaStreamTrackStats final : public RTCStats { // TODO(kuddai): Add description to standard. crbug.com/webrtc/10042 RTCNonStandardStatsMember jitter_buffer_flushes; RTCNonStandardStatsMember delayed_packet_outage_samples; + // Non-standard video-only members. + // https://henbos.github.io/webrtc-provisional-stats/#RTCVideoReceiverStats-dict* + RTCNonStandardStatsMember freeze_count; + RTCNonStandardStatsMember pause_count; + RTCNonStandardStatsMember total_freezes_duration; + RTCNonStandardStatsMember total_pauses_duration; + RTCNonStandardStatsMember total_frames_duration; + RTCNonStandardStatsMember sum_squared_frame_durations; }; // https://w3c.github.io/webrtc-stats/#pcstats-dict* diff --git a/call/video_receive_stream.h b/call/video_receive_stream.h index 3a5591782f..ecff63d62b 100644 --- a/call/video_receive_stream.h +++ b/call/video_receive_stream.h @@ -90,6 +90,13 @@ class VideoReceiveStream { int width = 0; int height = 0; + uint32_t freeze_count = 0; + uint32_t pause_count = 0; + uint32_t total_freezes_duration_ms = 0; + uint32_t total_pauses_duration_ms = 0; + uint32_t total_frames_duration_ms = 0; + double sum_squared_frame_durations = 0.0; + VideoContentType content_type = VideoContentType::UNSPECIFIED; int sync_offset_ms = std::numeric_limits::max(); diff --git a/media/base/media_channel.h b/media/base/media_channel.h index a55c191696..d20de7571f 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -533,6 +533,12 @@ struct VideoReceiverInfo : public MediaReceiverInfo { uint32_t frames_rendered = 0; absl::optional qp_sum; int64_t interframe_delay_max_ms = -1; + uint32_t freeze_count = 0; + uint32_t pause_count = 0; + uint32_t total_freezes_duration_ms = 0; + uint32_t total_pauses_duration_ms = 0; + uint32_t total_frames_duration_ms = 0; + double sum_squared_frame_durations = 0.0; webrtc::VideoContentType content_type = webrtc::VideoContentType::UNSPECIFIED; diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 724fe8f523..503e46541d 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -2515,6 +2515,12 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::GetVideoReceiverInfo( info.first_frame_received_to_decoded_ms = stats.first_frame_received_to_decoded_ms; info.interframe_delay_max_ms = stats.interframe_delay_max_ms; + info.freeze_count = stats.freeze_count; + info.pause_count = stats.pause_count; + info.total_freezes_duration_ms = stats.total_freezes_duration_ms; + info.total_pauses_duration_ms = stats.total_pauses_duration_ms; + info.total_frames_duration_ms = stats.total_frames_duration_ms; + info.sum_squared_frame_durations = stats.sum_squared_frame_durations; info.content_type = stats.content_type; diff --git a/pc/rtc_stats_collector.cc b/pc/rtc_stats_collector.cc index e66a504b45..a25a785e21 100644 --- a/pc/rtc_stats_collector.cc +++ b/pc/rtc_stats_collector.cc @@ -523,6 +523,20 @@ ProduceMediaStreamTrackStatsFromVideoReceiverInfo( video_receiver_info.frames_rendered); video_track_stats->frames_dropped = video_receiver_info.frames_received - video_receiver_info.frames_rendered; + video_track_stats->freeze_count = video_receiver_info.freeze_count; + video_track_stats->pause_count = video_receiver_info.pause_count; + video_track_stats->total_freezes_duration = + static_cast(video_receiver_info.total_freezes_duration_ms) / + rtc::kNumMillisecsPerSec; + video_track_stats->total_pauses_duration = + static_cast(video_receiver_info.total_pauses_duration_ms) / + rtc::kNumMillisecsPerSec; + video_track_stats->total_frames_duration = + static_cast(video_receiver_info.total_frames_duration_ms) / + rtc::kNumMillisecsPerSec; + video_track_stats->sum_squared_frame_durations = + video_receiver_info.sum_squared_frame_durations; + return video_track_stats; } diff --git a/pc/rtc_stats_collector_unittest.cc b/pc/rtc_stats_collector_unittest.cc index 468e2c4bca..74799d1be2 100644 --- a/pc/rtc_stats_collector_unittest.cc +++ b/pc/rtc_stats_collector_unittest.cc @@ -1552,6 +1552,12 @@ TEST_F(RTCStatsCollectorTest, video_receiver_info_ssrc3.frames_received = 1000; video_receiver_info_ssrc3.frames_decoded = 995; video_receiver_info_ssrc3.frames_rendered = 990; + video_receiver_info_ssrc3.freeze_count = 3; + video_receiver_info_ssrc3.pause_count = 2; + video_receiver_info_ssrc3.total_freezes_duration_ms = 1000; + video_receiver_info_ssrc3.total_pauses_duration_ms = 10000; + video_receiver_info_ssrc3.total_frames_duration_ms = 15000; + video_receiver_info_ssrc3.sum_squared_frame_durations = 1.5; stats_->CreateMockRtpSendersReceiversAndChannels( {}, {}, {}, @@ -1591,6 +1597,13 @@ TEST_F(RTCStatsCollectorTest, expected_remote_video_track_ssrc3.frames_received = 1000; expected_remote_video_track_ssrc3.frames_decoded = 995; expected_remote_video_track_ssrc3.frames_dropped = 1000 - 990; + expected_remote_video_track_ssrc3.freeze_count = 3; + expected_remote_video_track_ssrc3.pause_count = 2; + expected_remote_video_track_ssrc3.total_freezes_duration = 1; + expected_remote_video_track_ssrc3.total_pauses_duration = 10; + expected_remote_video_track_ssrc3.total_frames_duration = 15; + expected_remote_video_track_ssrc3.sum_squared_frame_durations = 1.5; + ASSERT_TRUE(report->Get(expected_remote_video_track_ssrc3.id())); EXPECT_EQ(expected_remote_video_track_ssrc3, report->Get(expected_remote_video_track_ssrc3.id()) diff --git a/pc/rtc_stats_integrationtest.cc b/pc/rtc_stats_integrationtest.cc index afe7649562..f0fc2f3bdf 100644 --- a/pc/rtc_stats_integrationtest.cc +++ b/pc/rtc_stats_integrationtest.cc @@ -560,6 +560,18 @@ class RTCStatsReportVerifier { media_stream_track.frames_decoded); verifier.TestMemberIsNonNegative( media_stream_track.frames_dropped); + verifier.TestMemberIsNonNegative( + media_stream_track.freeze_count); + verifier.TestMemberIsNonNegative( + media_stream_track.pause_count); + verifier.TestMemberIsNonNegative( + media_stream_track.total_freezes_duration); + verifier.TestMemberIsNonNegative( + media_stream_track.total_pauses_duration); + verifier.TestMemberIsNonNegative( + media_stream_track.total_frames_duration); + verifier.TestMemberIsNonNegative( + media_stream_track.sum_squared_frame_durations); } else { verifier.TestMemberIsNonNegative( media_stream_track.frames_sent); @@ -568,6 +580,16 @@ class RTCStatsReportVerifier { verifier.TestMemberIsUndefined(media_stream_track.frames_received); verifier.TestMemberIsUndefined(media_stream_track.frames_decoded); verifier.TestMemberIsUndefined(media_stream_track.frames_dropped); + verifier.TestMemberIsUndefined(media_stream_track.freeze_count); + verifier.TestMemberIsUndefined(media_stream_track.pause_count); + verifier.TestMemberIsUndefined( + media_stream_track.total_freezes_duration); + verifier.TestMemberIsUndefined( + media_stream_track.total_pauses_duration); + verifier.TestMemberIsUndefined( + media_stream_track.total_frames_duration); + verifier.TestMemberIsUndefined( + media_stream_track.sum_squared_frame_durations); } verifier.TestMemberIsUndefined(media_stream_track.frames_corrupted); verifier.TestMemberIsUndefined(media_stream_track.partial_frames_lost); @@ -593,6 +615,13 @@ class RTCStatsReportVerifier { verifier.TestMemberIsUndefined(media_stream_track.frames_corrupted); verifier.TestMemberIsUndefined(media_stream_track.partial_frames_lost); verifier.TestMemberIsUndefined(media_stream_track.full_frames_lost); + verifier.TestMemberIsUndefined(media_stream_track.freeze_count); + verifier.TestMemberIsUndefined(media_stream_track.pause_count); + verifier.TestMemberIsUndefined(media_stream_track.total_freezes_duration); + verifier.TestMemberIsUndefined(media_stream_track.total_pauses_duration); + verifier.TestMemberIsUndefined(media_stream_track.total_frames_duration); + verifier.TestMemberIsUndefined( + media_stream_track.sum_squared_frame_durations); // Audio-only members verifier.TestMemberIsNonNegative(media_stream_track.audio_level); verifier.TestMemberIsNonNegative( diff --git a/rtc_base/numerics/sample_counter.cc b/rtc_base/numerics/sample_counter.cc index bab47f165a..7f76b743d2 100644 --- a/rtc_base/numerics/sample_counter.cc +++ b/rtc_base/numerics/sample_counter.cc @@ -57,6 +57,13 @@ absl::optional SampleCounter::Max() const { return max_; } +absl::optional SampleCounter::Sum(int64_t min_required_samples) const { + RTC_DCHECK_GT(min_required_samples, 0); + if (num_samples_ < min_required_samples) + return absl::nullopt; + return sum_; +} + int64_t SampleCounter::NumSamples() const { return num_samples_; } diff --git a/rtc_base/numerics/sample_counter.h b/rtc_base/numerics/sample_counter.h index 18bd36b8f3..93d39c3c8d 100644 --- a/rtc_base/numerics/sample_counter.h +++ b/rtc_base/numerics/sample_counter.h @@ -26,6 +26,7 @@ class SampleCounter { void Add(int sample); absl::optional Avg(int64_t min_required_samples) const; absl::optional Max() const; + absl::optional Sum(int64_t min_required_samples) const; int64_t NumSamples() const; void Reset(); // Adds all the samples from the |other| SampleCounter as if they were all diff --git a/rtc_base/numerics/sample_counter_unittest.cc b/rtc_base/numerics/sample_counter_unittest.cc index 57812fcb4e..e87c8094c3 100644 --- a/rtc_base/numerics/sample_counter_unittest.cc +++ b/rtc_base/numerics/sample_counter_unittest.cc @@ -33,6 +33,7 @@ TEST(SampleCounterTest, NotEnoughSamples) { counter.Add(value); } EXPECT_THAT(counter.Avg(kMinSamples), Eq(absl::nullopt)); + EXPECT_THAT(counter.Sum(kMinSamples), Eq(absl::nullopt)); EXPECT_THAT(counter.Max(), Eq(5)); } @@ -43,6 +44,7 @@ TEST(SampleCounterTest, EnoughSamples) { counter.Add(value); } EXPECT_THAT(counter.Avg(kMinSamples), Eq(3)); + EXPECT_THAT(counter.Sum(kMinSamples), Eq(15)); EXPECT_THAT(counter.Max(), Eq(5)); } diff --git a/stats/rtcstats_objects.cc b/stats/rtcstats_objects.cc index 2451e12169..473a3f3334 100644 --- a/stats/rtcstats_objects.cc +++ b/stats/rtcstats_objects.cc @@ -381,7 +381,13 @@ WEBRTC_RTCSTATS_IMPL(RTCMediaStreamTrackStats, RTCStats, "track", &concealed_samples, &concealment_events, &jitter_buffer_flushes, - &delayed_packet_outage_samples); + &delayed_packet_outage_samples, + &freeze_count, + &pause_count, + &total_freezes_duration, + &total_pauses_duration, + &total_frames_duration, + &sum_squared_frame_durations); // clang-format on RTCMediaStreamTrackStats::RTCMediaStreamTrackStats(const std::string& id, @@ -420,7 +426,13 @@ RTCMediaStreamTrackStats::RTCMediaStreamTrackStats(std::string&& id, concealed_samples("concealedSamples"), concealment_events("concealmentEvents"), jitter_buffer_flushes("jitterBufferFlushes"), - delayed_packet_outage_samples("delayedPacketOutageSamples") { + delayed_packet_outage_samples("delayedPacketOutageSamples"), + freeze_count("freezeCount"), + pause_count("pauseCount"), + total_freezes_duration("totalFreezesDuration"), + total_pauses_duration("totalPausesDuration"), + total_frames_duration("totalFramesDuration"), + sum_squared_frame_durations("sumOfSquaredFramesDuration") { RTC_DCHECK(kind == RTCMediaStreamTrackKind::kAudio || kind == RTCMediaStreamTrackKind::kVideo); } @@ -455,7 +467,13 @@ RTCMediaStreamTrackStats::RTCMediaStreamTrackStats( concealed_samples(other.concealed_samples), concealment_events(other.concealment_events), jitter_buffer_flushes(other.jitter_buffer_flushes), - delayed_packet_outage_samples(other.delayed_packet_outage_samples) {} + delayed_packet_outage_samples(other.delayed_packet_outage_samples), + freeze_count(other.freeze_count), + pause_count(other.pause_count), + total_freezes_duration(other.total_freezes_duration), + total_pauses_duration(other.total_pauses_duration), + total_frames_duration(other.total_frames_duration), + sum_squared_frame_durations(other.sum_squared_frame_durations) {} RTCMediaStreamTrackStats::~RTCMediaStreamTrackStats() {} diff --git a/video/receive_statistics_proxy.cc b/video/receive_statistics_proxy.cc index 28e2becba2..b3f0710695 100644 --- a/video/receive_statistics_proxy.cc +++ b/video/receive_statistics_proxy.cc @@ -582,8 +582,18 @@ VideoReceiveStream::Stats ReceiveStatisticsProxy::GetStats() const { static_cast(total_byte_tracker_.ComputeRate() * 8); stats_.interframe_delay_max_ms = interframe_delay_max_moving_.Max(now_ms).value_or(-1); - stats_.timing_frame_info = timing_frame_info_counter_.Max(now_ms); + stats_.freeze_count = video_quality_observer_->NumFreezes(); + stats_.pause_count = video_quality_observer_->NumPauses(); + stats_.total_freezes_duration_ms = + video_quality_observer_->TotalFreezesDurationMs(); + stats_.total_pauses_duration_ms = + video_quality_observer_->TotalPausesDurationMs(); + stats_.total_frames_duration_ms = + video_quality_observer_->TotalFramesDurationMs(); + stats_.sum_squared_frame_durations = + video_quality_observer_->SumSquaredFrameDurationsSec(); stats_.content_type = last_content_type_; + stats_.timing_frame_info = timing_frame_info_counter_.Max(now_ms); return stats_; } diff --git a/video/receive_statistics_proxy_unittest.cc b/video/receive_statistics_proxy_unittest.cc index ac75aad42e..3d870b4376 100644 --- a/video/receive_statistics_proxy_unittest.cc +++ b/video/receive_statistics_proxy_unittest.cc @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include "api/scoped_refptr.h" #include "api/video/i420_buffer.h" @@ -30,12 +32,10 @@ const uint32_t kRemoteSsrc = 456; const int kMinRequiredSamples = 200; const int kWidth = 1280; const int kHeight = 720; - } // namespace // TODO(sakal): ReceiveStatisticsProxy is lacking unittesting. -class ReceiveStatisticsProxyTest - : public ::testing::TestWithParam { +class ReceiveStatisticsProxyTest : public ::testing::Test { public: ReceiveStatisticsProxyTest() : fake_clock_(1234), config_(GetTestConfig()) {} virtual ~ReceiveStatisticsProxyTest() {} @@ -167,13 +167,14 @@ TEST_F(ReceiveStatisticsProxyTest, ReportsContentType) { const std::string kScreenshareString("screen"); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); EXPECT_EQ(kRealtimeString, videocontenttypehelpers::ToString( - statistics_proxy_->GetStats().content_type)); + statistics_proxy_->GetStats().content_type)); statistics_proxy_->OnDecodedFrame(frame, 3u, VideoContentType::SCREENSHARE); - EXPECT_EQ(kScreenshareString, videocontenttypehelpers::ToString( - statistics_proxy_->GetStats().content_type)); + EXPECT_EQ(kScreenshareString, + videocontenttypehelpers::ToString( + statistics_proxy_->GetStats().content_type)); statistics_proxy_->OnDecodedFrame(frame, 3u, VideoContentType::UNSPECIFIED); EXPECT_EQ(kRealtimeString, videocontenttypehelpers::ToString( - statistics_proxy_->GetStats().content_type)); + statistics_proxy_->GetStats().content_type)); } TEST_F(ReceiveStatisticsProxyTest, ReportsMaxInterframeDelay) { @@ -237,6 +238,107 @@ TEST_F(ReceiveStatisticsProxyTest, ReportInterframeDelayInWindow) { statistics_proxy_->GetStats().interframe_delay_max_ms); } +TEST_F(ReceiveStatisticsProxyTest, ReportsFreezeMetrics) { + const int64_t kFreezeDurationMs = 1000; + + VideoReceiveStream::Stats stats = statistics_proxy_->GetStats(); + EXPECT_EQ(0u, stats.freeze_count); + EXPECT_FALSE(stats.total_freezes_duration_ms); + + webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); + for (size_t i = 0; i < VideoQualityObserver::kMinFrameSamplesToDetectFreeze; + ++i) { + fake_clock_.AdvanceTimeMilliseconds(30); + statistics_proxy_->OnRenderedFrame(frame); + } + + // Freeze. + fake_clock_.AdvanceTimeMilliseconds(kFreezeDurationMs); + statistics_proxy_->OnRenderedFrame(frame); + + stats = statistics_proxy_->GetStats(); + EXPECT_EQ(1u, stats.freeze_count); + EXPECT_EQ(kFreezeDurationMs, stats.total_freezes_duration_ms); +} + +TEST_F(ReceiveStatisticsProxyTest, ReportsPauseMetrics) { + VideoReceiveStream::Stats stats = statistics_proxy_->GetStats(); + ASSERT_EQ(0u, stats.pause_count); + ASSERT_EQ(0u, stats.total_pauses_duration_ms); + + webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); + statistics_proxy_->OnRenderedFrame(frame); + + // Pause. + fake_clock_.AdvanceTimeMilliseconds(5432); + statistics_proxy_->OnStreamInactive(); + statistics_proxy_->OnRenderedFrame(frame); + + stats = statistics_proxy_->GetStats(); + EXPECT_EQ(1u, stats.pause_count); + EXPECT_EQ(5432u, stats.total_pauses_duration_ms); +} + +TEST_F(ReceiveStatisticsProxyTest, PauseBeforeFirstAndAfterLastFrameIgnored) { + VideoReceiveStream::Stats stats = statistics_proxy_->GetStats(); + ASSERT_EQ(0u, stats.pause_count); + ASSERT_EQ(0u, stats.total_pauses_duration_ms); + + webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); + + // Pause -> Frame -> Pause + fake_clock_.AdvanceTimeMilliseconds(5000); + statistics_proxy_->OnStreamInactive(); + statistics_proxy_->OnRenderedFrame(frame); + + fake_clock_.AdvanceTimeMilliseconds(30); + statistics_proxy_->OnRenderedFrame(frame); + + fake_clock_.AdvanceTimeMilliseconds(5000); + statistics_proxy_->OnStreamInactive(); + + stats = statistics_proxy_->GetStats(); + EXPECT_EQ(0u, stats.pause_count); + EXPECT_EQ(0u, stats.total_pauses_duration_ms); +} + +TEST_F(ReceiveStatisticsProxyTest, ReportsFramesDuration) { + VideoReceiveStream::Stats stats = statistics_proxy_->GetStats(); + ASSERT_EQ(0u, stats.total_frames_duration_ms); + + webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); + + // Emulate delay before first frame is rendered. This is needed to ensure + // that frame duration only covers time since first frame is rendered and + // not the total time. + fake_clock_.AdvanceTimeMilliseconds(5432); + + for (int i = 0; i <= 10; ++i) { + fake_clock_.AdvanceTimeMilliseconds(30); + statistics_proxy_->OnRenderedFrame(frame); + } + + stats = statistics_proxy_->GetStats(); + EXPECT_EQ(10 * 30u, stats.total_frames_duration_ms); +} + +TEST_F(ReceiveStatisticsProxyTest, ReportsSumSquaredFrameDurations) { + VideoReceiveStream::Stats stats = statistics_proxy_->GetStats(); + ASSERT_EQ(0u, stats.sum_squared_frame_durations); + + webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); + for (int i = 0; i <= 10; ++i) { + fake_clock_.AdvanceTimeMilliseconds(30); + statistics_proxy_->OnRenderedFrame(frame); + } + + stats = statistics_proxy_->GetStats(); + const double kExpectedSumSquaredFrameDurationsSecs = + 10 * (30 / 1000.0 * 30 / 1000.0); + EXPECT_EQ(kExpectedSumSquaredFrameDurationsSecs, + stats.sum_squared_frame_durations); +} + TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameWithoutQpQpSumWontExist) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum); @@ -928,30 +1030,89 @@ TEST_F(ReceiveStatisticsProxyTest, RtcpHistogramsAreUpdated) { kNackPackets * 60 / metrics::kMinRunTimeInSeconds)); } -INSTANTIATE_TEST_SUITE_P(ContentTypes, - ReceiveStatisticsProxyTest, - ::testing::Values(VideoContentType::UNSPECIFIED, - VideoContentType::SCREENSHARE)); +class ReceiveStatisticsProxyTestWithFreezeDuration + : public ReceiveStatisticsProxyTest, + public testing::WithParamInterface< + std::tuple> { + protected: + const uint32_t frame_duration_ms_ = {std::get<0>(GetParam())}; + const uint32_t freeze_duration_ms_ = {std::get<1>(GetParam())}; + const uint32_t expected_freeze_count_ = {std::get<2>(GetParam())}; +}; -TEST_P(ReceiveStatisticsProxyTest, InterFrameDelaysAreReported) { - const VideoContentType content_type = GetParam(); +// It is a freeze if: +// frame_duration_ms >= max(3 * avg_frame_duration, avg_frame_duration + 150) +// where avg_frame_duration is average duration of last 30 frames including +// the current one. +// +// Condition 1: 3 * avg_frame_duration > avg_frame_duration + 150 +const auto kFreezeDetectionCond1Freeze = std::make_tuple(150, 483, 1); +const auto kFreezeDetectionCond1NotFreeze = std::make_tuple(150, 482, 0); +// Condition 2: 3 * avg_frame_duration < avg_frame_duration + 150 +const auto kFreezeDetectionCond2Freeze = std::make_tuple(30, 185, 1); +const auto kFreezeDetectionCond2NotFreeze = std::make_tuple(30, 184, 0); + +INSTANTIATE_TEST_CASE_P(_, + ReceiveStatisticsProxyTestWithFreezeDuration, + ::testing::Values(kFreezeDetectionCond1Freeze, + kFreezeDetectionCond1NotFreeze, + kFreezeDetectionCond2Freeze, + kFreezeDetectionCond2NotFreeze)); + +TEST_P(ReceiveStatisticsProxyTestWithFreezeDuration, FreezeDetection) { + VideoReceiveStream::Stats stats = statistics_proxy_->GetStats(); + EXPECT_EQ(0u, stats.freeze_count); + webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); + + // Add a very long frame. This is need to verify that average frame + // duration, which is supposed to be calculated as mean of durations of + // last 30 frames, is calculated correctly. + statistics_proxy_->OnRenderedFrame(frame); + fake_clock_.AdvanceTimeMilliseconds(2000); + + for (size_t i = 0; + i <= VideoQualityObserver::kAvgInterframeDelaysWindowSizeFrames; ++i) { + fake_clock_.AdvanceTimeMilliseconds(frame_duration_ms_); + statistics_proxy_->OnRenderedFrame(frame); + } + + fake_clock_.AdvanceTimeMilliseconds(freeze_duration_ms_); + statistics_proxy_->OnRenderedFrame(frame); + + stats = statistics_proxy_->GetStats(); + EXPECT_EQ(stats.freeze_count, expected_freeze_count_); +} + +class ReceiveStatisticsProxyTestWithContent + : public ReceiveStatisticsProxyTest, + public ::testing::WithParamInterface { + protected: + const webrtc::VideoContentType content_type_{GetParam()}; +}; + +INSTANTIATE_TEST_CASE_P(ContentTypes, + ReceiveStatisticsProxyTestWithContent, + ::testing::Values(VideoContentType::UNSPECIFIED, + VideoContentType::SCREENSHARE)); + +TEST_P(ReceiveStatisticsProxyTestWithContent, InterFrameDelaysAreReported) { const int kInterFrameDelayMs = 33; webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); for (int i = 0; i < kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } // One extra with double the interval. fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); statistics_proxy_.reset(); const int kExpectedInterFrame = (kInterFrameDelayMs * (kMinRequiredSamples - 1) + kInterFrameDelayMs * 2) / kMinRequiredSamples; - if (videocontenttypehelpers::IsScreenshare(content_type)) { + if (videocontenttypehelpers::IsScreenshare(content_type_)) { EXPECT_EQ( kExpectedInterFrame, metrics::MinSample("WebRTC.Video.Screenshare.InterframeDelayInMs")); @@ -966,28 +1127,28 @@ TEST_P(ReceiveStatisticsProxyTest, InterFrameDelaysAreReported) { } } -TEST_P(ReceiveStatisticsProxyTest, InterFrameDelaysPercentilesAreReported) { - const VideoContentType content_type = GetParam(); +TEST_P(ReceiveStatisticsProxyTestWithContent, + InterFrameDelaysPercentilesAreReported) { const int kInterFrameDelayMs = 33; const int kLastFivePercentsSamples = kMinRequiredSamples * 5 / 100; webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); for (int i = 0; i <= kMinRequiredSamples - kLastFivePercentsSamples; ++i) { fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); } // Last 5% of intervals are double in size. for (int i = 0; i < kLastFivePercentsSamples; ++i) { fake_clock_.AdvanceTimeMilliseconds(2 * kInterFrameDelayMs); - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); } // Final sample is outlier and 10 times as big. fake_clock_.AdvanceTimeMilliseconds(10 * kInterFrameDelayMs); - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); statistics_proxy_.reset(); const int kExpectedInterFrame = kInterFrameDelayMs * 2; - if (videocontenttypehelpers::IsScreenshare(content_type)) { + if (videocontenttypehelpers::IsScreenshare(content_type_)) { EXPECT_EQ(kExpectedInterFrame, metrics::MinSample( "WebRTC.Video.Screenshare.InterframeDelay95PercentileInMs")); @@ -998,13 +1159,13 @@ TEST_P(ReceiveStatisticsProxyTest, InterFrameDelaysPercentilesAreReported) { } } -TEST_P(ReceiveStatisticsProxyTest, MaxInterFrameDelayOnlyWithValidAverage) { - const VideoContentType content_type = GetParam(); +TEST_P(ReceiveStatisticsProxyTestWithContent, + MaxInterFrameDelayOnlyWithValidAverage) { const int kInterFrameDelayMs = 33; webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); for (int i = 0; i < kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } @@ -1019,13 +1180,12 @@ TEST_P(ReceiveStatisticsProxyTest, MaxInterFrameDelayOnlyWithValidAverage) { "WebRTC.Video.Screenshare.InterframeDelayMaxInMs")); } -TEST_P(ReceiveStatisticsProxyTest, MaxInterFrameDelayOnlyWithPause) { - const VideoContentType content_type = GetParam(); +TEST_P(ReceiveStatisticsProxyTestWithContent, MaxInterFrameDelayOnlyWithPause) { const int kInterFrameDelayMs = 33; webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); for (int i = 0; i <= kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } @@ -1036,12 +1196,12 @@ TEST_P(ReceiveStatisticsProxyTest, MaxInterFrameDelayOnlyWithPause) { // Insert two more frames. The interval during the pause should be disregarded // in the stats. - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); statistics_proxy_.reset(); - if (videocontenttypehelpers::IsScreenshare(content_type)) { + if (videocontenttypehelpers::IsScreenshare(content_type_)) { EXPECT_EQ( 1, metrics::NumSamples("WebRTC.Video.Screenshare.InterframeDelayInMs")); EXPECT_EQ(1, metrics::NumSamples( @@ -1062,8 +1222,7 @@ TEST_P(ReceiveStatisticsProxyTest, MaxInterFrameDelayOnlyWithPause) { } } -TEST_P(ReceiveStatisticsProxyTest, FreezesAreReported) { - const VideoContentType content_type = GetParam(); +TEST_P(ReceiveStatisticsProxyTestWithContent, FreezesAreReported) { const int kInterFrameDelayMs = 33; const int kFreezeDelayMs = 200; const int kCallDurationMs = @@ -1071,20 +1230,20 @@ TEST_P(ReceiveStatisticsProxyTest, FreezesAreReported) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); for (int i = 0; i < kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); statistics_proxy_->OnRenderedFrame(frame); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } // Add extra freeze. fake_clock_.AdvanceTimeMilliseconds(kFreezeDelayMs); - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); statistics_proxy_->OnRenderedFrame(frame); statistics_proxy_.reset(); const int kExpectedTimeBetweenFreezes = kInterFrameDelayMs * (kMinRequiredSamples - 1); const int kExpectedNumberFreezesPerMinute = 60 * 1000 / kCallDurationMs; - if (videocontenttypehelpers::IsScreenshare(content_type)) { + if (videocontenttypehelpers::IsScreenshare(content_type_)) { EXPECT_EQ( kFreezeDelayMs + kInterFrameDelayMs, metrics::MinSample("WebRTC.Video.Screenshare.MeanFreezeDurationMs")); @@ -1104,8 +1263,7 @@ TEST_P(ReceiveStatisticsProxyTest, FreezesAreReported) { } } -TEST_P(ReceiveStatisticsProxyTest, HarmonicFrameRateIsReported) { - const VideoContentType content_type = GetParam(); +TEST_P(ReceiveStatisticsProxyTestWithContent, HarmonicFrameRateIsReported) { const int kInterFrameDelayMs = 33; const int kFreezeDelayMs = 200; const int kCallDurationMs = @@ -1114,12 +1272,12 @@ TEST_P(ReceiveStatisticsProxyTest, HarmonicFrameRateIsReported) { for (int i = 0; i < kMinRequiredSamples; ++i) { fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); statistics_proxy_->OnRenderedFrame(frame); } // Add extra freeze. fake_clock_.AdvanceTimeMilliseconds(kFreezeDelayMs); - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); statistics_proxy_->OnRenderedFrame(frame); statistics_proxy_.reset(); @@ -1130,7 +1288,7 @@ TEST_P(ReceiveStatisticsProxyTest, HarmonicFrameRateIsReported) { kFreezeDelayMs / 1000.0 * kFreezeDelayMs / 1000.0; const int kExpectedHarmonicFrameRateFps = std::round(kCallDurationMs / (1000 * kSumSquaredInterframeDelaysSecs)); - if (videocontenttypehelpers::IsScreenshare(content_type)) { + if (videocontenttypehelpers::IsScreenshare(content_type_)) { EXPECT_EQ(kExpectedHarmonicFrameRateFps, metrics::MinSample("WebRTC.Video.Screenshare.HarmonicFrameRate")); } else { @@ -1139,14 +1297,13 @@ TEST_P(ReceiveStatisticsProxyTest, HarmonicFrameRateIsReported) { } } -TEST_P(ReceiveStatisticsProxyTest, PausesAreIgnored) { - const VideoContentType content_type = GetParam(); +TEST_P(ReceiveStatisticsProxyTestWithContent, PausesAreIgnored) { const int kInterFrameDelayMs = 33; const int kPauseDurationMs = 10000; webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); for (int i = 0; i <= kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); statistics_proxy_->OnRenderedFrame(frame); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } @@ -1156,7 +1313,7 @@ TEST_P(ReceiveStatisticsProxyTest, PausesAreIgnored) { // Second playback interval with triple the length. for (int i = 0; i <= kMinRequiredSamples * 3; ++i) { - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); statistics_proxy_->OnRenderedFrame(frame); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } @@ -1165,7 +1322,7 @@ TEST_P(ReceiveStatisticsProxyTest, PausesAreIgnored) { // Average of two playback intervals. const int kExpectedTimeBetweenFreezes = kInterFrameDelayMs * kMinRequiredSamples * 2; - if (videocontenttypehelpers::IsScreenshare(content_type)) { + if (videocontenttypehelpers::IsScreenshare(content_type_)) { EXPECT_EQ(-1, metrics::MinSample( "WebRTC.Video.Screenshare.MeanFreezeDurationMs")); EXPECT_EQ(kExpectedTimeBetweenFreezes, @@ -1178,26 +1335,25 @@ TEST_P(ReceiveStatisticsProxyTest, PausesAreIgnored) { } } -TEST_P(ReceiveStatisticsProxyTest, ManyPausesAtTheBeginning) { - const VideoContentType content_type = GetParam(); +TEST_P(ReceiveStatisticsProxyTestWithContent, ManyPausesAtTheBeginning) { const int kInterFrameDelayMs = 33; const int kPauseDurationMs = 10000; webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); for (int i = 0; i <= kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); statistics_proxy_->OnStreamInactive(); fake_clock_.AdvanceTimeMilliseconds(kPauseDurationMs); - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, content_type_); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } statistics_proxy_.reset(); // No freezes should be detected, as all long inter-frame delays were pauses. - if (videocontenttypehelpers::IsScreenshare(content_type)) { + if (videocontenttypehelpers::IsScreenshare(content_type_)) { EXPECT_EQ(-1, metrics::MinSample( "WebRTC.Video.Screenshare.MeanFreezeDurationMs")); } else { @@ -1205,21 +1361,20 @@ TEST_P(ReceiveStatisticsProxyTest, ManyPausesAtTheBeginning) { } } -TEST_P(ReceiveStatisticsProxyTest, TimeInHdReported) { - const VideoContentType content_type = GetParam(); +TEST_P(ReceiveStatisticsProxyTestWithContent, TimeInHdReported) { const int kInterFrameDelayMs = 20; webrtc::VideoFrame frame_hd = CreateFrame(1280, 720); webrtc::VideoFrame frame_sd = CreateFrame(640, 360); // HD frames. for (int i = 0; i < kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(frame_hd, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame_hd, absl::nullopt, content_type_); statistics_proxy_->OnRenderedFrame(frame_hd); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } // SD frames. for (int i = 0; i < 2 * kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(frame_sd, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame_sd, absl::nullopt, content_type_); statistics_proxy_->OnRenderedFrame(frame_sd); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } @@ -1228,7 +1383,7 @@ TEST_P(ReceiveStatisticsProxyTest, TimeInHdReported) { statistics_proxy_.reset(); const int kExpectedTimeInHdPercents = 33; - if (videocontenttypehelpers::IsScreenshare(content_type)) { + if (videocontenttypehelpers::IsScreenshare(content_type_)) { EXPECT_EQ( kExpectedTimeInHdPercents, metrics::MinSample("WebRTC.Video.Screenshare.TimeInHdPercentage")); @@ -1238,8 +1393,7 @@ TEST_P(ReceiveStatisticsProxyTest, TimeInHdReported) { } } -TEST_P(ReceiveStatisticsProxyTest, TimeInBlockyVideoReported) { - const VideoContentType content_type = GetParam(); +TEST_P(ReceiveStatisticsProxyTestWithContent, TimeInBlockyVideoReported) { const int kInterFrameDelayMs = 20; const int kHighQp = 80; const int kLowQp = 30; @@ -1247,23 +1401,23 @@ TEST_P(ReceiveStatisticsProxyTest, TimeInBlockyVideoReported) { // High quality frames. for (int i = 0; i < kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(frame, kLowQp, content_type); + statistics_proxy_->OnDecodedFrame(frame, kLowQp, content_type_); statistics_proxy_->OnRenderedFrame(frame); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } // Blocky frames. for (int i = 0; i < 2 * kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(frame, kHighQp, content_type); + statistics_proxy_->OnDecodedFrame(frame, kHighQp, content_type_); statistics_proxy_->OnRenderedFrame(frame); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } // Extra last frame. - statistics_proxy_->OnDecodedFrame(frame, kHighQp, content_type); + statistics_proxy_->OnDecodedFrame(frame, kHighQp, content_type_); statistics_proxy_->OnRenderedFrame(frame); statistics_proxy_.reset(); const int kExpectedTimeInHdPercents = 66; - if (videocontenttypehelpers::IsScreenshare(content_type)) { + if (videocontenttypehelpers::IsScreenshare(content_type_)) { EXPECT_EQ(kExpectedTimeInHdPercents, metrics::MinSample( "WebRTC.Video.Screenshare.TimeInBlockyVideoPercentage")); @@ -1273,8 +1427,7 @@ TEST_P(ReceiveStatisticsProxyTest, TimeInBlockyVideoReported) { } } -TEST_P(ReceiveStatisticsProxyTest, DownscalesReported) { - const VideoContentType content_type = GetParam(); +TEST_P(ReceiveStatisticsProxyTestWithContent, DownscalesReported) { const int kInterFrameDelayMs = 2000; // To ensure long enough call duration. webrtc::VideoFrame frame_hd = CreateFrame(1280, 720); @@ -1282,7 +1435,7 @@ TEST_P(ReceiveStatisticsProxyTest, DownscalesReported) { webrtc::VideoFrame frame_ld = CreateFrame(320, 180); // Call once to pass content type. - statistics_proxy_->OnDecodedFrame(frame_hd, absl::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(frame_hd, absl::nullopt, content_type_); statistics_proxy_->OnRenderedFrame(frame_hd); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); @@ -1297,7 +1450,7 @@ TEST_P(ReceiveStatisticsProxyTest, DownscalesReported) { statistics_proxy_.reset(); const int kExpectedDownscales = 30; // 2 per 4 seconds = 30 per minute. - if (videocontenttypehelpers::IsScreenshare(content_type)) { + if (videocontenttypehelpers::IsScreenshare(content_type_)) { EXPECT_EQ( kExpectedDownscales, metrics::MinSample( @@ -1309,9 +1462,10 @@ TEST_P(ReceiveStatisticsProxyTest, DownscalesReported) { } } -TEST_P(ReceiveStatisticsProxyTest, StatsAreSlicedOnSimulcastAndExperiment) { - VideoContentType content_type = GetParam(); +TEST_P(ReceiveStatisticsProxyTestWithContent, + StatsAreSlicedOnSimulcastAndExperiment) { const uint8_t experiment_id = 1; + webrtc::VideoContentType content_type = content_type_; videocontenttypehelpers::SetExperimentId(&content_type, experiment_id); const int kInterFrameDelayMs1 = 30; const int kInterFrameDelayMs2 = 50; diff --git a/video/video_quality_observer.cc b/video/video_quality_observer.cc index ebe5bb119c..2f1bb1b925 100644 --- a/video/video_quality_observer.cc +++ b/video/video_quality_observer.cc @@ -20,12 +20,13 @@ #include "system_wrappers/include/metrics.h" namespace webrtc { +const uint32_t VideoQualityObserver::kMinFrameSamplesToDetectFreeze = 5; +const uint32_t VideoQualityObserver::kMinIncreaseForFreezeMs = 150; +const uint32_t VideoQualityObserver::kAvgInterframeDelaysWindowSizeFrames = 30; namespace { -constexpr int kMinFrameSamplesToDetectFreeze = 5; constexpr int kMinVideoDurationMs = 3000; constexpr int kMinRequiredSamples = 1; -constexpr int kMinIncreaseForFreezeMs = 150; constexpr int kPixelsInHighResolution = 960 * 540; // CPU-adapted HD still counts. constexpr int kPixelsInMediumResolution = 640 * 360; @@ -41,7 +42,8 @@ VideoQualityObserver::VideoQualityObserver(VideoContentType content_type) first_frame_rendered_ms_(-1), last_frame_pixels_(0), is_last_frame_blocky_(false), - last_unfreeze_time_(0), + last_unfreeze_time_ms_(0), + render_interframe_delays_(kAvgInterframeDelaysWindowSizeFrames), sum_squared_interframe_delays_secs_(0.0), time_in_resolution_ms_(3, 0), current_resolution_(Resolution::Low), @@ -63,9 +65,9 @@ void VideoQualityObserver::UpdateHistograms() { char log_stream_buf[2 * 1024]; rtc::SimpleStringBuilder log_stream(log_stream_buf); - if (last_frame_rendered_ms_ > last_unfreeze_time_) { + if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) { smooth_playback_durations_.Add(last_frame_rendered_ms_ - - last_unfreeze_time_); + last_unfreeze_time_ms_); } std::string uma_prefix = videocontenttypehelpers::IsScreenshare(content_type_) @@ -135,32 +137,38 @@ void VideoQualityObserver::UpdateHistograms() { void VideoQualityObserver::OnRenderedFrame(const VideoFrame& frame, int64_t now_ms) { - if (num_frames_rendered_ == 0) { - first_frame_rendered_ms_ = last_unfreeze_time_ = now_ms; - } + RTC_DCHECK_LE(last_frame_rendered_ms_, now_ms); + RTC_DCHECK_LE(last_unfreeze_time_ms_, now_ms); - ++num_frames_rendered_; + if (num_frames_rendered_ == 0) { + first_frame_rendered_ms_ = last_unfreeze_time_ms_ = now_ms; + } auto blocky_frame_it = blocky_frames_.find(frame.timestamp()); - if (!is_paused_ && num_frames_rendered_ > 1) { + if (!is_paused_ && num_frames_rendered_ > 0) { // Process inter-frame delay. const int64_t interframe_delay_ms = now_ms - last_frame_rendered_ms_; - const float interframe_delays_secs = interframe_delay_ms / 1000.0; + const double interframe_delays_secs = interframe_delay_ms / 1000.0; sum_squared_interframe_delays_secs_ += interframe_delays_secs * interframe_delays_secs; - render_interframe_delays_.Add(interframe_delay_ms); - absl::optional avg_interframe_delay = - render_interframe_delays_.Avg(kMinFrameSamplesToDetectFreeze); - // Check if it was a freeze. - if (avg_interframe_delay && - interframe_delay_ms >= - std::max(3 * *avg_interframe_delay, - *avg_interframe_delay + kMinIncreaseForFreezeMs)) { + render_interframe_delays_.AddSample(interframe_delay_ms); + + bool was_freeze = false; + if (render_interframe_delays_.Size() >= kMinFrameSamplesToDetectFreeze) { + const absl::optional avg_interframe_delay = + render_interframe_delays_.GetAverageRoundedDown(); + RTC_DCHECK(avg_interframe_delay); + was_freeze = interframe_delay_ms >= + std::max(3 * *avg_interframe_delay, + *avg_interframe_delay + kMinIncreaseForFreezeMs); + } + + if (was_freeze) { freezes_durations_.Add(interframe_delay_ms); smooth_playback_durations_.Add(last_frame_rendered_ms_ - - last_unfreeze_time_); - last_unfreeze_time_ = now_ms; + last_unfreeze_time_ms_); + last_unfreeze_time_ms_ = now_ms; } else { // Count spatial metrics if there were no freeze. time_in_resolution_ms_[current_resolution_] += interframe_delay_ms; @@ -176,11 +184,15 @@ void VideoQualityObserver::OnRenderedFrame(const VideoFrame& frame, // pause toward smooth playback. Explicitly count the part before it and // start the new smooth playback interval from this frame. is_paused_ = false; - if (last_frame_rendered_ms_ > last_unfreeze_time_) { + if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) { smooth_playback_durations_.Add(last_frame_rendered_ms_ - - last_unfreeze_time_); + last_unfreeze_time_ms_); + } + last_unfreeze_time_ms_ = now_ms; + + if (num_frames_rendered_ > 0) { + pauses_durations_.Add(now_ms - last_frame_rendered_ms_); } - last_unfreeze_time_ = now_ms; } int64_t pixels = frame.width() * frame.height(); @@ -203,6 +215,8 @@ void VideoQualityObserver::OnRenderedFrame(const VideoFrame& frame, if (is_last_frame_blocky_) { blocky_frames_.erase(blocky_frames_.begin(), ++blocky_frame_it); } + + ++num_frames_rendered_; } void VideoQualityObserver::OnDecodedFrame(const VideoFrame& frame, @@ -241,4 +255,29 @@ void VideoQualityObserver::OnDecodedFrame(const VideoFrame& frame, void VideoQualityObserver::OnStreamInactive() { is_paused_ = true; } + +uint32_t VideoQualityObserver::NumFreezes() { + return freezes_durations_.NumSamples(); +} + +uint32_t VideoQualityObserver::NumPauses() { + return pauses_durations_.NumSamples(); +} + +uint32_t VideoQualityObserver::TotalFreezesDurationMs() { + return freezes_durations_.Sum(kMinRequiredSamples).value_or(0); +} + +uint32_t VideoQualityObserver::TotalPausesDurationMs() { + return pauses_durations_.Sum(kMinRequiredSamples).value_or(0); +} + +uint32_t VideoQualityObserver::TotalFramesDurationMs() { + return last_frame_rendered_ms_ - first_frame_rendered_ms_; +} + +double VideoQualityObserver::SumSquaredFrameDurationsSec() { + return sum_squared_interframe_delays_secs_; +} + } // namespace webrtc diff --git a/video/video_quality_observer.h b/video/video_quality_observer.h index 0c60d9a4c7..afa0156dfb 100644 --- a/video/video_quality_observer.h +++ b/video/video_quality_observer.h @@ -19,6 +19,7 @@ #include "api/video/video_codec_type.h" #include "api/video/video_content_type.h" #include "api/video/video_frame.h" +#include "rtc_base/numerics/moving_average.h" #include "rtc_base/numerics/sample_counter.h" namespace webrtc { @@ -40,6 +41,17 @@ class VideoQualityObserver { void OnStreamInactive(); + uint32_t NumFreezes(); + uint32_t NumPauses(); + uint32_t TotalFreezesDurationMs(); + uint32_t TotalPausesDurationMs(); + uint32_t TotalFramesDurationMs(); + double SumSquaredFrameDurationsSec(); + + static const uint32_t kMinFrameSamplesToDetectFreeze; + static const uint32_t kMinIncreaseForFreezeMs; + static const uint32_t kAvgInterframeDelaysWindowSizeFrames; + private: void UpdateHistograms(); @@ -55,12 +67,13 @@ class VideoQualityObserver { int64_t last_frame_pixels_; bool is_last_frame_blocky_; // Decoded timestamp of the last delayed frame. - int64_t last_unfreeze_time_; - rtc::SampleCounter render_interframe_delays_; + int64_t last_unfreeze_time_ms_; + rtc::MovingAverage render_interframe_delays_; double sum_squared_interframe_delays_secs_; // An inter-frame delay is counted as a freeze if it's significantly longer // than average inter-frame delay. rtc::SampleCounter freezes_durations_; + rtc::SampleCounter pauses_durations_; // Time between freezes. rtc::SampleCounter smooth_playback_durations_; // Counters for time spent in different resolutions. Time between each two