diff --git a/common_video/BUILD.gn b/common_video/BUILD.gn index c7a8df5b3e..4cff5700e4 100644 --- a/common_video/BUILD.gn +++ b/common_video/BUILD.gn @@ -32,6 +32,14 @@ rtc_library("frame_instrumentation_data") { sources = [ "frame_instrumentation_data.h" ] } +rtc_library("corruption_score_calculator") { + sources = [ "include/corruption_score_calculator.h" ] + deps = [ + ":frame_instrumentation_data", + "../api/video:video_frame", + ] +} + rtc_library("common_video") { visibility = [ "*" ] diff --git a/common_video/include/corruption_score_calculator.h b/common_video/include/corruption_score_calculator.h new file mode 100644 index 0000000000..77ea5335f0 --- /dev/null +++ b/common_video/include/corruption_score_calculator.h @@ -0,0 +1,34 @@ +/* + * Copyright 2024 The WebRTC project authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef COMMON_VIDEO_INCLUDE_CORRUPTION_SCORE_CALCULATOR_H_ +#define COMMON_VIDEO_INCLUDE_CORRUPTION_SCORE_CALCULATOR_H_ + +#include + +#include "api/video/video_frame.h" +#include "common_video/frame_instrumentation_data.h" + +namespace webrtc { + +// Allow classes to have their own implementations of how to calculate a score +// for automatic corruption detection. +class CorruptionScoreCalculator { + public: + virtual ~CorruptionScoreCalculator() = default; + + virtual std::optional CalculateCorruptionScore( + const VideoFrame& frame, + const FrameInstrumentationData& frame_instrumentation_data) = 0; +}; + +} // namespace webrtc + +#endif // COMMON_VIDEO_INCLUDE_CORRUPTION_SCORE_CALCULATOR_H_ diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index b79e0de6f4..ae832adb64 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -236,6 +236,8 @@ rtc_library("video_coding") { "../../api/video:video_rtp_headers", "../../api/video_codecs:video_codecs_api", "../../common_video", + "../../common_video:corruption_score_calculator", + "../../common_video:frame_instrumentation_data", "../../rtc_base:buffer", "../../rtc_base:byte_buffer", "../../rtc_base:checks", @@ -260,6 +262,7 @@ rtc_library("video_coding") { "../../system_wrappers", "../../system_wrappers:metrics", "../../video/config:encoder_config", + "../../video/corruption_detection:frame_instrumentation_evaluation", "../rtp_rtcp", "../rtp_rtcp:rtp_rtcp_format", "../rtp_rtcp:rtp_video_header", @@ -329,6 +332,7 @@ rtc_library("video_coding_legacy") { "../../api/video:video_rtp_headers", "../../api/video_codecs:video_codecs_api", "../../common_video", + "../../common_video:corruption_score_calculator", "../../modules/rtp_rtcp:rtp_video_header", "../../rtc_base:checks", "../../rtc_base:event_tracer", @@ -1196,6 +1200,7 @@ if (rtc_include_tests) { "../../api/video:builtin_video_bitrate_allocator_factory", "../../api/video:encoded_frame", "../../api/video:encoded_image", + "../../api/video:frame_buffer", "../../api/video:render_resolution", "../../api/video:video_adaptation", "../../api/video:video_bitrate_allocation", @@ -1207,6 +1212,8 @@ if (rtc_include_tests) { "../../api/video_codecs:video_codecs_api", "../../api/video_codecs:vp8_temporal_layers_factory", "../../common_video", + "../../common_video:corruption_score_calculator", + "../../common_video:frame_instrumentation_data", "../../common_video/generic_frame_descriptor", "../../common_video/test:utilities", "../../media:media_constants", diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc index 1568c29e7f..753d6d9a03 100644 --- a/modules/video_coding/generic_decoder.cc +++ b/modules/video_coding/generic_decoder.cc @@ -19,8 +19,11 @@ #include #include "absl/algorithm/container.h" +#include "absl/types/variant.h" #include "api/video/video_timing.h" #include "api/video_codecs/video_decoder.h" +#include "common_video/frame_instrumentation_data.h" +#include "common_video/include/corruption_score_calculator.h" #include "modules/include/module_common_types_public.h" #include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/checks.h" @@ -41,8 +44,11 @@ constexpr size_t kDecoderFrameMemoryLength = 10; VCMDecodedFrameCallback::VCMDecodedFrameCallback( VCMTiming* timing, Clock* clock, - const FieldTrialsView& field_trials) - : _clock(clock), _timing(timing) { + const FieldTrialsView& field_trials, + CorruptionScoreCalculator* corruption_score_calculator) + : _clock(clock), + _timing(timing), + corruption_score_calculator_(corruption_score_calculator) { ntp_offset_ = _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds(); } @@ -128,6 +134,18 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, return; } + // TODO: bugs.webrtc.org/358039777 - Use the score. + std::optional score; + if (corruption_score_calculator_ && + frame_info->frame_instrumentation_data.has_value()) { + if (const FrameInstrumentationData* data = + absl::get_if( + &*frame_info->frame_instrumentation_data)) { + score = corruption_score_calculator_->CalculateCorruptionScore( + decodedImage, *data); + } + } + decodedImage.set_ntp_time_ms(frame_info->ntp_time_ms); decodedImage.set_packet_infos(frame_info->packet_infos); decodedImage.set_rotation(frame_info->rotation); @@ -285,16 +303,22 @@ bool VCMGenericDecoder::Configure(const VideoDecoder::Settings& settings) { } int32_t VCMGenericDecoder::Decode(const EncodedFrame& frame, Timestamp now) { - return Decode(frame, now, frame.RenderTimeMs()); + return Decode(frame, now, frame.RenderTimeMs(), + frame.CodecSpecific()->frame_instrumentation_data); } int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) { - return Decode(frame, now, frame.RenderTimeMs()); + return Decode(frame, now, frame.RenderTimeMs(), + frame.CodecSpecific()->frame_instrumentation_data); } -int32_t VCMGenericDecoder::Decode(const EncodedImage& frame, - Timestamp now, - int64_t render_time_ms) { +int32_t VCMGenericDecoder::Decode( + const EncodedImage& frame, + Timestamp now, + int64_t render_time_ms, + const std::optional< + absl::variant>& + frame_instrumentation_data) { TRACE_EVENT("webrtc", "VCMGenericDecoder::Decode", perfetto::Flow::ProcessScoped(frame.RtpTimestamp())); FrameInfo frame_info; @@ -308,6 +332,7 @@ int32_t VCMGenericDecoder::Decode(const EncodedImage& frame, frame_info.timing = frame.video_timing(); frame_info.ntp_time_ms = frame.ntp_time_ms_; frame_info.packet_infos = frame.PacketInfos(); + frame_info.frame_instrumentation_data = frame_instrumentation_data; // Set correctly only for key frames. Thus, use latest key frame // content type. If the corresponding key frame was lost, decode will fail diff --git a/modules/video_coding/generic_decoder.h b/modules/video_coding/generic_decoder.h index 4859031590..5a4018da68 100644 --- a/modules/video_coding/generic_decoder.h +++ b/modules/video_coding/generic_decoder.h @@ -13,13 +13,17 @@ #include #include +#include #include #include +#include "absl/types/variant.h" #include "api/field_trials_view.h" #include "api/sequence_checker.h" #include "api/video/encoded_frame.h" #include "api/video_codecs/video_decoder.h" +#include "common_video/frame_instrumentation_data.h" +#include "common_video/include/corruption_score_calculator.h" #include "modules/video_coding/encoded_frame.h" #include "modules/video_coding/timing/timing.h" #include "rtc_base/synchronization/mutex.h" @@ -48,13 +52,18 @@ struct FrameInfo { RtpPacketInfos packet_infos; // ColorSpace is not stored here, as it might be modified by decoders. VideoFrameType frame_type; + std::optional< + absl::variant> + frame_instrumentation_data; }; class VCMDecodedFrameCallback : public DecodedImageCallback { public: - VCMDecodedFrameCallback(VCMTiming* timing, - Clock* clock, - const FieldTrialsView& field_trials); + VCMDecodedFrameCallback( + VCMTiming* timing, + Clock* clock, + const FieldTrialsView& field_trials, + CorruptionScoreCalculator* corruption_score_calculator); ~VCMDecodedFrameCallback() override; void SetUserReceiveCallback(VCMReceiveCallback* receiveCallback); VCMReceiveCallback* UserReceiveCallback(); @@ -86,6 +95,7 @@ class VCMDecodedFrameCallback : public DecodedImageCallback { Mutex lock_; std::deque frame_infos_ RTC_GUARDED_BY(lock_); int64_t ntp_offset_; + CorruptionScoreCalculator* const corruption_score_calculator_; }; class VCMGenericDecoder { @@ -120,7 +130,10 @@ class VCMGenericDecoder { private: int32_t Decode(const EncodedImage& frame, Timestamp now, - int64_t render_time_ms); + int64_t render_time_ms, + const std::optional>& + frame_instrumentation_data); VCMDecodedFrameCallback* _callback = nullptr; VideoDecoder* const decoder_; VideoContentType _last_keyframe_content_type; diff --git a/modules/video_coding/generic_decoder_unittest.cc b/modules/video_coding/generic_decoder_unittest.cc index fef8083a70..1700c931bb 100644 --- a/modules/video_coding/generic_decoder_unittest.cc +++ b/modules/video_coding/generic_decoder_unittest.cc @@ -13,15 +13,21 @@ #include #include #include +#include #include #include "api/array_view.h" #include "api/rtp_packet_infos.h" +#include "api/scoped_refptr.h" #include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/i420_buffer.h" #include "api/video/video_content_type.h" #include "api/video/video_frame.h" #include "api/video/video_frame_type.h" #include "api/video_codecs/video_decoder.h" +#include "common_video/frame_instrumentation_data.h" +#include "common_video/include/corruption_score_calculator.h" #include "common_video/test/utilities.h" #include "modules/video_coding/timing/timing.h" #include "system_wrappers/include/clock.h" @@ -34,6 +40,15 @@ namespace webrtc { namespace video_coding { +class MockCorruptionScoreCalculator : public CorruptionScoreCalculator { + public: + MOCK_METHOD(std::optional, + CalculateCorruptionScore, + (const VideoFrame& frame, + const FrameInstrumentationData& frame_instrumentation_data), + (override)); +}; + class ReceiveCallback : public VCMReceiveCallback { public: int32_t FrameToRender(VideoFrame& frame, @@ -81,7 +96,10 @@ class GenericDecoderTest : public ::testing::Test { clock_(time_controller_.GetClock()), timing_(time_controller_.GetClock(), field_trials_), decoder_(time_controller_.GetTaskQueueFactory()), - vcm_callback_(&timing_, time_controller_.GetClock(), field_trials_), + vcm_callback_(&timing_, + time_controller_.GetClock(), + field_trials_, + &corruption_score_calculator_), generic_decoder_(&decoder_) {} void SetUp() override { @@ -102,6 +120,7 @@ class GenericDecoderTest : public ::testing::Test { VCMDecodedFrameCallback vcm_callback_; VCMGenericDecoder generic_decoder_; ReceiveCallback user_callback_; + MockCorruptionScoreCalculator corruption_score_calculator_; }; TEST_F(GenericDecoderTest, PassesPacketInfos) { @@ -200,5 +219,24 @@ TEST_F(GenericDecoderTest, IsLowLatencyStreamActivatedByPlayoutDelay) { EXPECT_TRUE(decoded_frame->render_parameters().use_low_latency_rendering); } +TEST_F(GenericDecoderTest, CallCalculateCorruptionScoreInDecoded) { + EXPECT_CALL(corruption_score_calculator_, CalculateCorruptionScore); + + uint32_t rtp_timestamp = 1; + FrameInfo frame_info; + frame_info.frame_instrumentation_data = FrameInstrumentationData{}; + frame_info.rtp_timestamp = rtp_timestamp; + frame_info.decode_start = Timestamp::Zero(); + frame_info.content_type = VideoContentType::UNSPECIFIED; + frame_info.frame_type = VideoFrameType::kVideoFrameDelta; + VideoFrame video_frame = VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(5, 5)) + .set_rtp_timestamp(rtp_timestamp) + .build(); + vcm_callback_.Map(std::move(frame_info)); + + vcm_callback_.Decoded(video_frame); +} + } // namespace video_coding } // namespace webrtc diff --git a/modules/video_coding/video_receiver.cc b/modules/video_coding/video_receiver.cc index 0c54cb178a..75fc75dd9d 100644 --- a/modules/video_coding/video_receiver.cc +++ b/modules/video_coding/video_receiver.cc @@ -44,7 +44,10 @@ VideoReceiver::VideoReceiver(Clock* clock, : clock_(clock), _timing(timing), _receiver(_timing, clock_, field_trials), - _decodedFrameCallback(_timing, clock_, field_trials), + _decodedFrameCallback(_timing, + clock_, + field_trials, + /*corruption_score_calculator=*/nullptr), _frameTypeCallback(nullptr), _packetRequestCallback(nullptr), _scheduleKeyRequest(false), diff --git a/modules/video_coding/video_receiver2.cc b/modules/video_coding/video_receiver2.cc index 72f366dbe7..2b22b24959 100644 --- a/modules/video_coding/video_receiver2.cc +++ b/modules/video_coding/video_receiver2.cc @@ -19,6 +19,7 @@ #include "absl/algorithm/container.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_decoder.h" +#include "common_video/include/corruption_score_calculator.h" #include "modules/video_coding/decoder_database.h" #include "modules/video_coding/generic_decoder.h" #include "modules/video_coding/include/video_coding_defines.h" @@ -29,11 +30,16 @@ namespace webrtc { -VideoReceiver2::VideoReceiver2(Clock* clock, - VCMTiming* timing, - const FieldTrialsView& field_trials) +VideoReceiver2::VideoReceiver2( + Clock* clock, + VCMTiming* timing, + const FieldTrialsView& field_trials, + CorruptionScoreCalculator* corruption_score_calculator) : clock_(clock), - decoded_frame_callback_(timing, clock_, field_trials), + decoded_frame_callback_(timing, + clock_, + field_trials, + corruption_score_calculator), codec_database_() { decoder_sequence_checker_.Detach(); } diff --git a/modules/video_coding/video_receiver2.h b/modules/video_coding/video_receiver2.h index 0d3bdb464f..4dc0aa5a72 100644 --- a/modules/video_coding/video_receiver2.h +++ b/modules/video_coding/video_receiver2.h @@ -18,6 +18,7 @@ #include "api/sequence_checker.h" #include "api/video/encoded_frame.h" #include "api/video_codecs/video_decoder.h" +#include "common_video/include/corruption_score_calculator.h" #include "modules/video_coding/decoder_database.h" #include "modules/video_coding/generic_decoder.h" #include "modules/video_coding/timing/timing.h" @@ -35,7 +36,8 @@ class VideoReceiver2 { public: VideoReceiver2(Clock* clock, VCMTiming* timing, - const FieldTrialsView& field_trials); + const FieldTrialsView& field_trials, + CorruptionScoreCalculator* corruption_score_calculator); ~VideoReceiver2(); void RegisterReceiveCodec(uint8_t payload_type, diff --git a/modules/video_coding/video_receiver2_unittest.cc b/modules/video_coding/video_receiver2_unittest.cc index 8cdd8abab5..27c8bfb7b3 100644 --- a/modules/video_coding/video_receiver2_unittest.cc +++ b/modules/video_coding/video_receiver2_unittest.cc @@ -99,7 +99,8 @@ class VideoReceiver2Test : public ::testing::Test { SimulatedClock clock_{Timestamp::Millis(1337)}; VCMTiming timing_{&clock_, field_trials_}; NiceMock receive_callback_; - VideoReceiver2 receiver_{&clock_, &timing_, field_trials_}; + VideoReceiver2 receiver_{&clock_, &timing_, field_trials_, + /*corruption_score_calculator=*/nullptr}; }; TEST_F(VideoReceiver2Test, RegisterExternalDecoder) { diff --git a/video/BUILD.gn b/video/BUILD.gn index fdbe2fa2c6..0f81b79f2b 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -119,6 +119,7 @@ rtc_library("video") { "../common_video", "../common_video:corruption_detection_converters", "../common_video:corruption_detection_message", + "../common_video:corruption_score_calculator", "../common_video:frame_instrumentation_data", "../media:media_constants", "../media:rtc_sdp_video_format_utils", @@ -168,6 +169,7 @@ rtc_library("video") { "../system_wrappers:field_trial", "../system_wrappers:metrics", "../video/config:encoder_config", + "../video/corruption_detection:frame_instrumentation_evaluation", "adaptation:video_adaptation", "render:incoming_video_stream", "//third_party/abseil-cpp/absl/algorithm:container", diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc index 4372fdabdd..1dedc3e149 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -31,11 +31,13 @@ #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/encoded_image.h" +#include "api/video/video_frame.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_decoder_factory.h" #include "call/rtp_stream_receiver_controller_interface.h" #include "call/rtx_receive_stream.h" +#include "common_video/frame_instrumentation_data.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/include/video_coding_defines.h" #include "modules/video_coding/include/video_error_codes.h" @@ -51,6 +53,7 @@ #include "rtc_base/trace_event.h" #include "system_wrappers/include/clock.h" #include "video/call_stats2.h" +#include "video/corruption_detection/frame_instrumentation_evaluation.h" #include "video/frame_dumping_decoder.h" #include "video/receive_statistics_proxy.h" #include "video/render/incoming_video_stream.h" @@ -198,7 +201,7 @@ VideoReceiveStream2::VideoReceiveStream2( stats_proxy_(remote_ssrc(), &env_.clock(), call->worker_thread()), rtp_receive_statistics_(ReceiveStatistics::Create(&env_.clock())), timing_(std::move(timing)), - video_receiver_(&env_.clock(), timing_.get(), env_.field_trials()), + video_receiver_(&env_.clock(), timing_.get(), env_.field_trials(), this), rtp_video_stream_receiver_(env_, call->worker_thread(), &transport_adapter_, @@ -604,6 +607,12 @@ void VideoReceiveStream2::UpdateHistograms() { stats_proxy_.UpdateHistograms(fraction_lost, rtp_stats, nullptr); } +std::optional VideoReceiveStream2::CalculateCorruptionScore( + const VideoFrame& frame, + const FrameInstrumentationData& frame_instrumentation_data) { + return GetCorruptionScore(frame_instrumentation_data, frame); +} + bool VideoReceiveStream2::SetBaseMinimumPlayoutDelayMs(int delay_ms) { RTC_DCHECK_RUN_ON(&worker_sequence_checker_); TimeDelta delay = TimeDelta::Millis(delay_ms); diff --git a/video/video_receive_stream2.h b/video/video_receive_stream2.h index 2c7eb4c536..89309cfea0 100644 --- a/video/video_receive_stream2.h +++ b/video/video_receive_stream2.h @@ -24,10 +24,13 @@ #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/recordable_encoded_frame.h" +#include "api/video/video_frame.h" #include "call/call.h" #include "call/rtp_packet_sink_interface.h" #include "call/syncable.h" #include "call/video_receive_stream.h" +#include "common_video/frame_instrumentation_data.h" +#include "common_video/include/corruption_score_calculator.h" #include "modules/rtp_rtcp/source/source_tracker.h" #include "modules/video_coding/nack_requester.h" #include "modules/video_coding/video_receiver2.h" @@ -87,7 +90,8 @@ class VideoReceiveStream2 public RtpVideoStreamReceiver2::OnCompleteFrameCallback, public Syncable, public CallStatsObserver, - public FrameSchedulingReceiver { + public FrameSchedulingReceiver, + public CorruptionScoreCalculator { public: // The maximum number of buffered encoded frames when encoded output is // configured. @@ -239,6 +243,9 @@ class VideoReceiveStream2 RTC_RUN_ON(decode_sequence_checker_); void UpdateHistograms(); + std::optional CalculateCorruptionScore( + const VideoFrame& frame, + const FrameInstrumentationData& frame_instrumentation_data) override; const Environment env_;