diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc index 46b8c95f61..4c0e381575 100644 --- a/modules/video_coding/generic_decoder.cc +++ b/modules/video_coding/generic_decoder.cc @@ -201,7 +201,8 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, decodedImage.set_timestamp_us( frame_info->render_time ? frame_info->render_time->us() : -1); _receiveCallback->FrameToRender(decodedImage, qp, decode_time, - frame_info->content_type); + frame_info->content_type, + frame_info->frame_type); } void VCMDecodedFrameCallback::OnDecoderInfoChanged( @@ -285,6 +286,7 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) { } else { frame_info.content_type = _last_keyframe_content_type; } + frame_info.frame_type = frame.FrameType(); _callback->Map(std::move(frame_info)); int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(), diff --git a/modules/video_coding/generic_decoder.h b/modules/video_coding/generic_decoder.h index 7dc6d34c01..5922b52b7c 100644 --- a/modules/video_coding/generic_decoder.h +++ b/modules/video_coding/generic_decoder.h @@ -46,6 +46,7 @@ struct FrameInfo { int64_t ntp_time_ms; RtpPacketInfos packet_infos; // ColorSpace is not stored here, as it might be modified by decoders. + VideoFrameType frame_type; }; class VCMDecodedFrameCallback : public DecodedImageCallback { diff --git a/modules/video_coding/generic_decoder_unittest.cc b/modules/video_coding/generic_decoder_unittest.cc index 68bc307e65..529f13fdb8 100644 --- a/modules/video_coding/generic_decoder_unittest.cc +++ b/modules/video_coding/generic_decoder_unittest.cc @@ -35,7 +35,8 @@ class ReceiveCallback : public VCMReceiveCallback { int32_t FrameToRender(VideoFrame& frame, absl::optional qp, TimeDelta decode_time, - VideoContentType content_type) override { + VideoContentType content_type, + VideoFrameType frame_type) override { frames_.push_back(frame); return 0; } diff --git a/modules/video_coding/include/video_coding_defines.h b/modules/video_coding/include/video_coding_defines.h index fe7d0fb383..d20af1b0be 100644 --- a/modules/video_coding/include/video_coding_defines.h +++ b/modules/video_coding/include/video_coding_defines.h @@ -53,7 +53,8 @@ class VCMReceiveCallback { virtual int32_t FrameToRender(VideoFrame& videoFrame, // NOLINT absl::optional qp, TimeDelta decode_time, - VideoContentType content_type) = 0; + VideoContentType content_type, + VideoFrameType frame_type) = 0; virtual void OnDroppedFrames(uint32_t frames_dropped); diff --git a/modules/video_coding/video_receiver2_unittest.cc b/modules/video_coding/video_receiver2_unittest.cc index 6edf1230d8..b524316d79 100644 --- a/modules/video_coding/video_receiver2_unittest.cc +++ b/modules/video_coding/video_receiver2_unittest.cc @@ -35,11 +35,14 @@ class MockVCMReceiveCallback : public VCMReceiveCallback { public: MockVCMReceiveCallback() = default; - MOCK_METHOD( - int32_t, - FrameToRender, - (VideoFrame&, absl::optional, TimeDelta, VideoContentType), - (override)); + MOCK_METHOD(int32_t, + FrameToRender, + (VideoFrame&, + absl::optional, + TimeDelta, + VideoContentType, + VideoFrameType), + (override)); MOCK_METHOD(void, OnIncomingPayloadType, (int), (override)); MOCK_METHOD(void, OnDecoderInfoChanged, diff --git a/modules/video_coding/video_receiver_unittest.cc b/modules/video_coding/video_receiver_unittest.cc index fe9674e521..8a706641a8 100644 --- a/modules/video_coding/video_receiver_unittest.cc +++ b/modules/video_coding/video_receiver_unittest.cc @@ -38,11 +38,14 @@ class MockVCMReceiveCallback : public VCMReceiveCallback { MockVCMReceiveCallback() {} virtual ~MockVCMReceiveCallback() {} - MOCK_METHOD( - int32_t, - FrameToRender, - (VideoFrame&, absl::optional, TimeDelta, VideoContentType), - (override)); + MOCK_METHOD(int32_t, + FrameToRender, + (VideoFrame&, + absl::optional, + TimeDelta, + VideoContentType, + VideoFrameType), + (override)); MOCK_METHOD(void, OnIncomingPayloadType, (int), (override)); MOCK_METHOD(void, OnDecoderInfoChanged, diff --git a/video/receive_statistics_proxy.cc b/video/receive_statistics_proxy.cc index 5bb30041e3..75512a2465 100644 --- a/video/receive_statistics_proxy.cc +++ b/video/receive_statistics_proxy.cc @@ -592,7 +592,8 @@ void ReceiveStatisticsProxy::OnCname(uint32_t ssrc, absl::string_view cname) { void ReceiveStatisticsProxy::OnDecodedFrame(const VideoFrame& frame, absl::optional qp, TimeDelta decode_time, - VideoContentType content_type) { + VideoContentType content_type, + VideoFrameType frame_type) { TimeDelta processing_delay = TimeDelta::Zero(); webrtc::Timestamp current_time = clock_->CurrentTime(); // TODO(bugs.webrtc.org/13984): some tests do not fill packet_infos(). @@ -615,11 +616,11 @@ void ReceiveStatisticsProxy::OnDecodedFrame(const VideoFrame& frame, // may be on. E.g. on iOS this gets called on // "com.apple.coremedia.decompressionsession.clientcallback" VideoFrameMetaData meta(frame, current_time); - worker_thread_->PostTask( - SafeTask(task_safety_.flag(), [meta, qp, decode_time, processing_delay, - assembly_time, content_type, this]() { + worker_thread_->PostTask(SafeTask( + task_safety_.flag(), [meta, qp, decode_time, processing_delay, + assembly_time, content_type, frame_type, this]() { OnDecodedFrame(meta, qp, decode_time, processing_delay, assembly_time, - content_type); + content_type, frame_type); })); } @@ -629,7 +630,8 @@ void ReceiveStatisticsProxy::OnDecodedFrame( TimeDelta decode_time, TimeDelta processing_delay, TimeDelta assembly_time, - VideoContentType content_type) { + VideoContentType content_type, + VideoFrameType frame_type) { RTC_DCHECK_RUN_ON(&main_thread_); const bool is_screenshare = @@ -651,6 +653,11 @@ void ReceiveStatisticsProxy::OnDecodedFrame( &content_specific_stats_[content_type]; ++stats_.frames_decoded; + if (frame_type == VideoFrameType::kVideoFrameKey) { + ++stats_.frame_counts.key_frames; + } else { + ++stats_.frame_counts.delta_frames; + } if (qp) { if (!stats_.qp_sum) { if (stats_.frames_decoded != 1) { @@ -760,12 +767,6 @@ void ReceiveStatisticsProxy::OnCompleteFrame(bool is_keyframe, VideoContentType content_type) { RTC_DCHECK_RUN_ON(&main_thread_); - if (is_keyframe) { - ++stats_.frame_counts.key_frames; - } else { - ++stats_.frame_counts.delta_frames; - } - // Content type extension is set only for keyframes and should be propagated // for all the following delta frames. Here we may receive frames out of order // and miscategorise some delta frames near the layer switch. diff --git a/video/receive_statistics_proxy.h b/video/receive_statistics_proxy.h index d8da3064fd..8e4941f961 100644 --- a/video/receive_statistics_proxy.h +++ b/video/receive_statistics_proxy.h @@ -58,7 +58,8 @@ class ReceiveStatisticsProxy : public VideoStreamBufferControllerStatsObserver, void OnDecodedFrame(const VideoFrame& frame, absl::optional qp, TimeDelta decode_time, - VideoContentType content_type); + VideoContentType content_type, + VideoFrameType frame_type); // Called asyncronously on the worker thread as a result of a call to the // above OnDecodedFrame method, which is called back on the thread where @@ -68,7 +69,8 @@ class ReceiveStatisticsProxy : public VideoStreamBufferControllerStatsObserver, TimeDelta decode_time, TimeDelta processing_delay, TimeDelta assembly_time, - VideoContentType content_type); + VideoContentType content_type, + VideoFrameType frame_type); void OnSyncOffsetUpdated(int64_t video_playout_ntp_ms, int64_t sync_offset_ms, diff --git a/video/receive_statistics_proxy_unittest.cc b/video/receive_statistics_proxy_unittest.cc index bc11efefd0..a30a7e4490 100644 --- a/video/receive_statistics_proxy_unittest.cc +++ b/video/receive_statistics_proxy_unittest.cc @@ -115,7 +115,8 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesFramesDecoded) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); for (uint32_t i = 1; i <= 3; ++i) { statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); EXPECT_EQ(i, FlushAndGetStats().frames_decoded); } } @@ -127,7 +128,8 @@ TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsReported) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); for (int i = 0; i < kRequiredSamples; ++i) { statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); time_controller_.AdvanceTime(1 / kFps); } FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); @@ -144,7 +146,8 @@ TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsNotReportedForTooFewSamples) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); for (int i = 0; i < kRequiredSamples - 1; ++i) { statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); time_controller_.AdvanceTime(1 / kFps); } FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); @@ -159,9 +162,9 @@ TEST_F(ReceiveStatisticsProxyTest, TimeDelta expected_total_decode_time = TimeDelta::Zero(); unsigned int expected_frames_decoded = 0; for (uint32_t i = 1; i <= 3; ++i) { - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, - TimeDelta::Millis(1), - VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame( + frame, absl::nullopt, TimeDelta::Millis(1), + VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey); expected_total_decode_time += TimeDelta::Millis(1); ++expected_frames_decoded; time_controller_.AdvanceTime(TimeDelta::Zero()); @@ -171,7 +174,8 @@ TEST_F(ReceiveStatisticsProxyTest, statistics_proxy_->GetStats().total_decode_time); } statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); ++expected_frames_decoded; expected_total_decode_time += TimeDelta::Millis(3); time_controller_.AdvanceTime(TimeDelta::Zero()); @@ -195,9 +199,9 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesProcessingDelay) { frame.set_packet_infos(RtpPacketInfos(packet_infos)); for (int i = 1; i <= 3; ++i) { time_controller_.AdvanceTime(kProcessingDelay); - statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, - TimeDelta::Millis(1), - VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame( + frame, absl::nullopt, TimeDelta::Millis(1), + VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey); expected_total_processing_delay += i * kProcessingDelay; ++expected_frames_decoded; time_controller_.AdvanceTime(TimeDelta::Zero()); @@ -208,7 +212,8 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesProcessingDelay) { } time_controller_.AdvanceTime(kProcessingDelay); statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); ++expected_frames_decoded; expected_total_processing_delay += 4 * kProcessingDelay; time_controller_.AdvanceTime(TimeDelta::Zero()); @@ -232,7 +237,8 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesAssemblyTime) { /*ssrc=*/{}, /*csrcs=*/{}, /*rtp_timestamp=*/{}, /*receive_time=*/Now())}; frame.set_packet_infos(RtpPacketInfos(single_packet_frame)); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Millis(1), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); ++expected_frames_decoded; time_controller_.AdvanceTime(TimeDelta::Zero()); EXPECT_EQ(expected_total_assembly_time, @@ -252,7 +258,8 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesAssemblyTime) { }; frame.set_packet_infos(RtpPacketInfos(ordered_frame)); statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); ++expected_frames_decoded; ++expected_frames_assembled_from_multiple_packets; expected_total_assembly_time += 2 * kAssemblyTime; @@ -276,7 +283,8 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesAssemblyTime) { }; frame.set_packet_infos(RtpPacketInfos(unordered_frame)); statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); ++expected_frames_decoded; ++expected_frames_assembled_from_multiple_packets; expected_total_assembly_time += 2 * kAssemblyTime; @@ -294,10 +302,12 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesQpSum) { EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); EXPECT_EQ(3u, FlushAndGetStats().qp_sum); statistics_proxy_->OnDecodedFrame(frame, 127u, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta); EXPECT_EQ(130u, FlushAndGetStats().qp_sum); } @@ -305,10 +315,12 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesTotalDecodeTime) { EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Millis(4), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); EXPECT_EQ(4u, FlushAndGetStats().total_decode_time.ms()); statistics_proxy_->OnDecodedFrame(frame, 127u, TimeDelta::Millis(7), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta); EXPECT_EQ(11u, FlushAndGetStats().total_decode_time.ms()); } @@ -319,11 +331,13 @@ TEST_F(ReceiveStatisticsProxyTest, ReportsContentType) { EXPECT_EQ(kRealtimeString, videocontenttypehelpers::ToString( statistics_proxy_->GetStats().content_type)); statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(), - VideoContentType::SCREENSHARE); + VideoContentType::SCREENSHARE, + VideoFrameType::kVideoFrameKey); EXPECT_EQ(kScreenshareString, videocontenttypehelpers::ToString(FlushAndGetStats().content_type)); statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta); EXPECT_EQ(kRealtimeString, videocontenttypehelpers::ToString(FlushAndGetStats().content_type)); } @@ -335,22 +349,26 @@ TEST_F(ReceiveStatisticsProxyTest, ReportsMaxInterframeDelay) { const TimeDelta kInterframeDelay3 = TimeDelta::Millis(100); EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); EXPECT_EQ(-1, FlushAndGetStats().interframe_delay_max_ms); time_controller_.AdvanceTime(kInterframeDelay1); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta); EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms); time_controller_.AdvanceTime(kInterframeDelay2); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta); EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms); time_controller_.AdvanceTime(kInterframeDelay3); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta); // kInterframeDelay3 is smaller than kInterframeDelay2. EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms); } @@ -362,23 +380,27 @@ TEST_F(ReceiveStatisticsProxyTest, ReportInterframeDelayInWindow) { const TimeDelta kInterframeDelay3 = TimeDelta::Millis(700); EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); EXPECT_EQ(-1, FlushAndGetStats().interframe_delay_max_ms); time_controller_.AdvanceTime(kInterframeDelay1); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta); EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms); time_controller_.AdvanceTime(kInterframeDelay2); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta); // Still first delay is the maximum EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms); time_controller_.AdvanceTime(kInterframeDelay3); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta); // Now the first sample is out of the window, so the second is the maximum. EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms); } @@ -487,7 +509,8 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameWithoutQpQpSumWontExist) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); EXPECT_EQ(absl::nullopt, FlushAndGetStats().qp_sum); } @@ -495,10 +518,12 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameWithoutQpResetsQpSum) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum); statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); EXPECT_EQ(3u, FlushAndGetStats().qp_sum); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta); EXPECT_EQ(absl::nullopt, FlushAndGetStats().qp_sum); } @@ -543,7 +568,7 @@ TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsOnCompleteFrame) { VideoContentType::UNSPECIFIED); VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats(); EXPECT_EQ(1, stats.network_frame_rate); - EXPECT_EQ(1, stats.frame_counts.key_frames); + EXPECT_EQ(0, stats.frame_counts.key_frames); EXPECT_EQ(0, stats.frame_counts.delta_frames); } @@ -628,16 +653,21 @@ TEST_F(ReceiveStatisticsProxyTest, TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsFrameCounts) { const int kKeyFrames = 3; const int kDeltaFrames = 22; + webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); for (int i = 0; i < kKeyFrames; i++) { - statistics_proxy_->OnCompleteFrame(true, 0, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); } for (int i = 0; i < kDeltaFrames; i++) { - statistics_proxy_->OnCompleteFrame(false, 0, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameDelta); } VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats(); - EXPECT_EQ(kKeyFrames, stats.frame_counts.key_frames); - EXPECT_EQ(kDeltaFrames, stats.frame_counts.delta_frames); + EXPECT_EQ(0, stats.frame_counts.key_frames); + EXPECT_EQ(0, stats.frame_counts.delta_frames); } TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsCName) { @@ -700,8 +730,15 @@ TEST_F(ReceiveStatisticsProxyTest, RespectsReportingIntervalForTimingFrames) { TEST_F(ReceiveStatisticsProxyTest, LifetimeHistogramIsUpdated) { const TimeDelta kLifetime = TimeDelta::Seconds(3); time_controller_.AdvanceTime(kLifetime); - // Need at least one frame to report stream lifetime. + // Need at least one decoded frame to report stream lifetime. + + webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnCompleteFrame(true, 1000, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame( + frame, absl::nullopt, TimeDelta::Millis(1000), + VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey); + FlushAndGetStats(); + statistics_proxy_->UpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); EXPECT_METRIC_EQ( @@ -841,10 +878,16 @@ TEST_F(ReceiveStatisticsProxyTest, KeyFrameHistogramNotUpdatedForTooFewSamples) { const bool kIsKeyFrame = false; const int kFrameSizeBytes = 1000; + webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); - for (int i = 0; i < kMinRequiredSamples - 1; ++i) + for (int i = 0; i < kMinRequiredSamples - 1; ++i) { statistics_proxy_->OnCompleteFrame(kIsKeyFrame, kFrameSizeBytes, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame( + frame, absl::nullopt, TimeDelta::Millis(1000), + VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta); + } + FlushAndGetStats(); EXPECT_EQ(0, statistics_proxy_->GetStats().frame_counts.key_frames); EXPECT_EQ(kMinRequiredSamples - 1, @@ -860,10 +903,16 @@ TEST_F(ReceiveStatisticsProxyTest, KeyFrameHistogramUpdatedForMinRequiredSamples) { const bool kIsKeyFrame = false; const int kFrameSizeBytes = 1000; + webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); - for (int i = 0; i < kMinRequiredSamples; ++i) + for (int i = 0; i < kMinRequiredSamples; ++i) { statistics_proxy_->OnCompleteFrame(kIsKeyFrame, kFrameSizeBytes, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame( + frame, absl::nullopt, TimeDelta::Millis(1000), + VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta); + } + FlushAndGetStats(); EXPECT_EQ(0, statistics_proxy_->GetStats().frame_counts.key_frames); EXPECT_EQ(kMinRequiredSamples, @@ -879,14 +928,23 @@ TEST_F(ReceiveStatisticsProxyTest, TEST_F(ReceiveStatisticsProxyTest, KeyFrameHistogramIsUpdated) { const int kFrameSizeBytes = 1000; + webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); - for (int i = 0; i < kMinRequiredSamples; ++i) + for (int i = 0; i < kMinRequiredSamples; ++i) { statistics_proxy_->OnCompleteFrame(true, kFrameSizeBytes, VideoContentType::UNSPECIFIED); - - for (int i = 0; i < kMinRequiredSamples; ++i) + statistics_proxy_->OnDecodedFrame( + frame, absl::nullopt, TimeDelta::Millis(1000), + VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey); + } + for (int i = 0; i < kMinRequiredSamples; ++i) { statistics_proxy_->OnCompleteFrame(false, kFrameSizeBytes, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame( + frame, absl::nullopt, TimeDelta::Millis(1000), + VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta); + } + FlushAndGetStats(); EXPECT_EQ(kMinRequiredSamples, statistics_proxy_->GetStats().frame_counts.key_frames); @@ -966,7 +1024,8 @@ TEST_F(ReceiveStatisticsProxyTest, DoesNotReportStaleFramerates) { frame.set_ntp_time_ms( time_controller_.GetClock()->CurrentNtpInMilliseconds()); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); statistics_proxy_->OnRenderedFrame(MetaData(frame)); time_controller_.AdvanceTime(1 / kDefaultFps); } @@ -1038,7 +1097,8 @@ TEST_F(ReceiveStatisticsProxyTest, ReceivedFrameHistogramsAreUpdated) { TEST_F(ReceiveStatisticsProxyTest, ZeroDelayReportedIfFrameNotDelayed) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); // Frame not delayed, delayed frames to render: 0%. statistics_proxy_->OnRenderedFrame( @@ -1060,7 +1120,8 @@ TEST_F(ReceiveStatisticsProxyTest, DelayedFrameHistogramsAreNotUpdatedIfMinRuntimeHasNotPassed) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); // Frame not delayed, delayed frames to render: 0%. statistics_proxy_->OnRenderedFrame( @@ -1081,7 +1142,8 @@ TEST_F(ReceiveStatisticsProxyTest, DelayedFramesHistogramsAreNotUpdatedIfNoRenderedFrames) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); // Min run time has passed. No rendered frames. time_controller_.AdvanceTime( @@ -1097,7 +1159,8 @@ TEST_F(ReceiveStatisticsProxyTest, TEST_F(ReceiveStatisticsProxyTest, DelayReportedIfFrameIsDelayed) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); // Frame delayed 1 ms, delayed frames to render: 100%. statistics_proxy_->OnRenderedFrame( @@ -1121,7 +1184,8 @@ TEST_F(ReceiveStatisticsProxyTest, DelayReportedIfFrameIsDelayed) { TEST_F(ReceiveStatisticsProxyTest, AverageDelayOfDelayedFramesIsReported) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - VideoContentType::UNSPECIFIED); + VideoContentType::UNSPECIFIED, + VideoFrameType::kVideoFrameKey); // Two frames delayed (6 ms, 10 ms), delayed frames to render: 50%. const int64_t kNowMs = Now().ms(); @@ -1274,13 +1338,15 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, InterFrameDelaysAreReported) { for (int i = 0; i < kMinRequiredSamples; ++i) { statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameKey); time_controller_.AdvanceTime(kInterFrameDelay); } // One extra with double the interval. time_controller_.AdvanceTime(kInterFrameDelay); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameDelta); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); const TimeDelta kExpectedInterFrame = @@ -1310,18 +1376,21 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, for (int i = 0; i <= kMinRequiredSamples - kLastFivePercentsSamples; ++i) { time_controller_.AdvanceTime(kInterFrameDelay); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameKey); } // Last 5% of intervals are double in size. for (int i = 0; i < kLastFivePercentsSamples; ++i) { time_controller_.AdvanceTime(2 * kInterFrameDelay); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameKey); } // Final sample is outlier and 10 times as big. time_controller_.AdvanceTime(10 * kInterFrameDelay); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameKey); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); const TimeDelta kExpectedInterFrame = kInterFrameDelay * 2; @@ -1344,7 +1413,8 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, for (int i = 0; i < kMinRequiredSamples; ++i) { statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameKey); time_controller_.AdvanceTime(kInterFrameDelay); } @@ -1367,7 +1437,8 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, MaxInterFrameDelayOnlyWithPause) { for (int i = 0; i <= kMinRequiredSamples; ++i) { statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameKey); time_controller_.AdvanceTime(kInterFrameDelay); } @@ -1378,10 +1449,12 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, MaxInterFrameDelayOnlyWithPause) { // Insert two more frames. The interval during the pause should be // disregarded in the stats. statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameKey); time_controller_.AdvanceTime(kInterFrameDelay); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameDelta); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); if (videocontenttypehelpers::IsScreenshare(content_type_)) { @@ -1416,18 +1489,18 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, FreezesAreReported) { for (int i = 0; i < kMinRequiredSamples; ++i) { VideoFrameMetaData meta = MetaData(frame); - statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(), - TimeDelta::Zero(), TimeDelta::Zero(), - content_type_); + statistics_proxy_->OnDecodedFrame( + meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(), + TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameKey); statistics_proxy_->OnRenderedFrame(meta); time_controller_.AdvanceTime(kInterFrameDelay); } // Add extra freeze. time_controller_.AdvanceTime(kFreezeDelay); VideoFrameMetaData meta = MetaData(frame); - statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(), - TimeDelta::Zero(), TimeDelta::Zero(), - content_type_); + statistics_proxy_->OnDecodedFrame( + meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(), + TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameDelta); statistics_proxy_->OnRenderedFrame(meta); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); @@ -1466,7 +1539,8 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, HarmonicFrameRateIsReported) { for (int i = 0; i < kMinRequiredSamples; ++i) { time_controller_.AdvanceTime(kFrameDuration); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameKey); statistics_proxy_->OnRenderedFrame(MetaData(frame)); } @@ -1474,14 +1548,16 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, HarmonicFrameRateIsReported) { // Add freeze. time_controller_.AdvanceTime(kFreezeDuration); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameDelta); statistics_proxy_->OnRenderedFrame(MetaData(frame)); // Add pause. time_controller_.AdvanceTime(kPauseDuration); statistics_proxy_->OnStreamInactive(); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameDelta); statistics_proxy_->OnRenderedFrame(MetaData(frame)); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); @@ -1511,9 +1587,9 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, PausesAreIgnored) { for (int i = 0; i <= kMinRequiredSamples; ++i) { VideoFrameMetaData meta = MetaData(frame); - statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(), - TimeDelta::Zero(), TimeDelta::Zero(), - content_type_); + statistics_proxy_->OnDecodedFrame( + meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(), + TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameKey); statistics_proxy_->OnRenderedFrame(meta); time_controller_.AdvanceTime(kInterFrameDelay); } @@ -1523,9 +1599,9 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, PausesAreIgnored) { // Second playback interval with triple the length. for (int i = 0; i <= kMinRequiredSamples * 3; ++i) { VideoFrameMetaData meta = MetaData(frame); - statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(), - TimeDelta::Zero(), TimeDelta::Zero(), - content_type_); + statistics_proxy_->OnDecodedFrame( + meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(), + TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameDelta); statistics_proxy_->OnRenderedFrame(meta); time_controller_.AdvanceTime(kInterFrameDelay); } @@ -1556,12 +1632,14 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, ManyPausesAtTheBeginning) { for (int i = 0; i <= kMinRequiredSamples; ++i) { statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameKey); time_controller_.AdvanceTime(kInterFrameDelay); statistics_proxy_->OnStreamInactive(); time_controller_.AdvanceTime(kPauseDuration); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameDelta); time_controller_.AdvanceTime(kInterFrameDelay); } @@ -1586,18 +1664,18 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, TimeInHdReported) { // HD frames. for (int i = 0; i < kMinRequiredSamples; ++i) { VideoFrameMetaData meta = MetaData(frame_hd); - statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(), - TimeDelta::Zero(), TimeDelta::Zero(), - content_type_); + statistics_proxy_->OnDecodedFrame( + meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(), + TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameKey); statistics_proxy_->OnRenderedFrame(meta); time_controller_.AdvanceTime(kInterFrameDelay); } // SD frames. for (int i = 0; i < 2 * kMinRequiredSamples; ++i) { VideoFrameMetaData meta = MetaData(frame_sd); - statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(), - TimeDelta::Zero(), TimeDelta::Zero(), - content_type_); + statistics_proxy_->OnDecodedFrame( + meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(), + TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameKey); statistics_proxy_->OnRenderedFrame(meta); time_controller_.AdvanceTime(kInterFrameDelay); } @@ -1626,24 +1704,25 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, TimeInBlockyVideoReported) { // High quality frames. for (int i = 0; i < kMinRequiredSamples; ++i) { VideoFrameMetaData meta = MetaData(frame); - statistics_proxy_->OnDecodedFrame(meta, kLowQp, TimeDelta::Zero(), - TimeDelta::Zero(), TimeDelta::Zero(), - content_type_); + statistics_proxy_->OnDecodedFrame( + meta, kLowQp, TimeDelta::Zero(), TimeDelta::Zero(), TimeDelta::Zero(), + content_type_, VideoFrameType::kVideoFrameKey); statistics_proxy_->OnRenderedFrame(meta); time_controller_.AdvanceTime(kInterFrameDelay); } // Blocky frames. for (int i = 0; i < 2 * kMinRequiredSamples; ++i) { VideoFrameMetaData meta = MetaData(frame); - statistics_proxy_->OnDecodedFrame(meta, kHighQp, TimeDelta::Zero(), - TimeDelta::Zero(), TimeDelta::Zero(), - content_type_); + statistics_proxy_->OnDecodedFrame( + meta, kHighQp, TimeDelta::Zero(), TimeDelta::Zero(), TimeDelta::Zero(), + content_type_, VideoFrameType::kVideoFrameKey); statistics_proxy_->OnRenderedFrame(meta); time_controller_.AdvanceTime(kInterFrameDelay); } // Extra last frame. statistics_proxy_->OnDecodedFrame(frame, kHighQp, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameKey); statistics_proxy_->OnRenderedFrame(MetaData(frame)); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); @@ -1670,7 +1749,8 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, DownscalesReported) { // Call once to pass content type. statistics_proxy_->OnDecodedFrame(frame_hd, absl::nullopt, TimeDelta::Zero(), - content_type_); + content_type_, + VideoFrameType::kVideoFrameKey); time_controller_.AdvanceTime(TimeDelta::Zero()); statistics_proxy_->OnRenderedFrame(MetaData(frame_hd)); @@ -1699,8 +1779,8 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, DecodeTimeReported) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); for (int i = 0; i < kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(frame, kLowQp, kDecodeTime, - content_type_); + statistics_proxy_->OnDecodedFrame(frame, kLowQp, kDecodeTime, content_type_, + VideoFrameType::kVideoFrameKey); time_controller_.AdvanceTime(kInterFrameDelay); } FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); diff --git a/video/video_stream_decoder2.cc b/video/video_stream_decoder2.cc index ba36d127b0..5640835c16 100644 --- a/video/video_stream_decoder2.cc +++ b/video/video_stream_decoder2.cc @@ -46,9 +46,10 @@ VideoStreamDecoder::~VideoStreamDecoder() { int32_t VideoStreamDecoder::FrameToRender(VideoFrame& video_frame, absl::optional qp, TimeDelta decode_time, - VideoContentType content_type) { + VideoContentType content_type, + VideoFrameType frame_type) { receive_stats_callback_->OnDecodedFrame(video_frame, qp, decode_time, - content_type); + content_type, frame_type); incoming_video_stream_->OnFrame(video_frame); return 0; } diff --git a/video/video_stream_decoder2.h b/video/video_stream_decoder2.h index 473d463186..19db810b7c 100644 --- a/video/video_stream_decoder2.h +++ b/video/video_stream_decoder2.h @@ -43,7 +43,8 @@ class VideoStreamDecoder : public VCMReceiveCallback { int32_t FrameToRender(VideoFrame& video_frame, absl::optional qp, TimeDelta decode_time, - VideoContentType content_type) override; + VideoContentType content_type, + VideoFrameType frame_type) override; void OnDroppedFrames(uint32_t frames_dropped) override; void OnIncomingPayloadType(int payload_type) override; void OnDecoderInfoChanged(