diff --git a/video/BUILD.gn b/video/BUILD.gn index 2f4d83329a..0eb6d6f1e1 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -425,6 +425,7 @@ rtc_library("video_stream_encoder_impl") { "../api/video_codecs:video_codecs_api", "../call/adaptation:resource_adaptation", "../common_video", + "../common_video:frame_instrumentation_data", "../media:media_channel", "../modules:module_api_public", "../modules/video_coding", @@ -461,9 +462,11 @@ rtc_library("video_stream_encoder_impl") { "adaptation:video_adaptation", "config:encoder_config", "config:streams_config", + "corruption_detection:frame_instrumentation_generator", "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/cleanup", "//third_party/abseil-cpp/absl/container:inlined_vector", + "//third_party/abseil-cpp/absl/types:variant", ] } @@ -873,6 +876,7 @@ if (rtc_include_tests) { "../call/adaptation:resource_adaptation", "../call/adaptation:resource_adaptation_test_utilities", "../common_video", + "../common_video:frame_instrumentation_data", "../common_video/test:utilities", "../media:codec", "../media:media_constants", diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index da42b2a137..4fc7771f0f 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -20,6 +20,7 @@ #include "absl/algorithm/container.h" #include "absl/cleanup/cleanup.h" +#include "absl/types/variant.h" #include "api/field_trials_view.h" #include "api/sequence_checker.h" #include "api/task_queue/task_queue_base.h" @@ -30,13 +31,16 @@ #include "api/video/video_bitrate_allocator_factory.h" #include "api/video/video_codec_constants.h" #include "api/video/video_layers_allocation.h" +#include "api/video/video_stream_encoder_settings.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_encoder.h" #include "call/adaptation/resource_adaptation_processor.h" #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_adapter.h" +#include "common_video/frame_instrumentation_data.h" #include "media/base/media_channel.h" #include "modules/video_coding/include/video_codec_initializer.h" +#include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/svc/scalability_mode_util.h" #include "modules/video_coding/svc/svc_rate_allocator.h" #include "modules/video_coding/utility/vp8_constants.h" @@ -54,6 +58,8 @@ #include "video/adaptation/video_stream_encoder_resource_manager.h" #include "video/alignment_adjuster.h" #include "video/config/encoder_stream_factory.h" +#include "video/config/video_encoder_config.h" +#include "video/corruption_detection/frame_instrumentation_generator.h" #include "video/frame_cadence_adapter.h" #include "video/frame_dumping_encoder.h" @@ -757,6 +763,7 @@ void VideoStreamEncoder::Stop() { ReleaseEncoder(); encoder_ = nullptr; frame_cadence_adapter_ = nullptr; + frame_instrumentation_generator_ = nullptr; }); shutdown_event.Wait(rtc::Event::kForever); } @@ -934,6 +941,12 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, max_data_payload_length_ = max_data_payload_length; pending_encoder_reconfiguration_ = true; + if (settings_.enable_frame_instrumentation_generator) { + frame_instrumentation_generator_ = + std::make_unique( + encoder_config_.codec_type); + } + // Reconfigure the encoder now if the frame resolution is known. // Otherwise, the reconfiguration is deferred until the next frame to // minimize the number of reconfigurations. The codec configuration @@ -1532,6 +1545,9 @@ void VideoStreamEncoder::OnFrame(Timestamp post_time, encoder_stats_observer_->OnIncomingFrame(incoming_frame.width(), incoming_frame.height()); + if (frame_instrumentation_generator_) { + frame_instrumentation_generator_->OnCapturedFrame(incoming_frame); + } ++captured_frame_count_; bool cwnd_frame_drop = cwnd_frame_drop_interval_ && @@ -2164,6 +2180,22 @@ EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage( // running in parallel on different threads. encoder_stats_observer_->OnSendEncodedImage(image_copy, codec_specific_info); + std::unique_ptr codec_specific_info_copy; + if (codec_specific_info && frame_instrumentation_generator_) { + std::optional< + absl::variant> + frame_instrumentation_data = + frame_instrumentation_generator_->OnEncodedImage(image_copy); + RTC_CHECK(!codec_specific_info->frame_instrumentation_data.has_value()) + << "CodecSpecificInfo must not have frame_instrumentation_data set."; + if (frame_instrumentation_data.has_value()) { + codec_specific_info_copy = + std::make_unique(*codec_specific_info); + codec_specific_info_copy->frame_instrumentation_data = + frame_instrumentation_data; + codec_specific_info = codec_specific_info_copy.get(); + } + } EncodedImageCallback::Result result = sink_->OnEncodedImage(image_copy, codec_specific_info); diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index dd423c32cf..475a9e65b0 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -44,6 +44,7 @@ #include "rtc_base/rate_statistics.h" #include "rtc_base/thread_annotations.h" #include "video/adaptation/video_stream_encoder_resource_manager.h" +#include "video/corruption_detection/frame_instrumentation_generator.h" #include "video/encoder_bitrate_adjuster.h" #include "video/frame_cadence_adapter.h" #include "video/frame_encode_metadata_writer.h" @@ -447,6 +448,10 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, ScopedTaskSafety task_safety_; std::unique_ptr encoder_queue_; + + // Required for automatic corruption detection. + std::unique_ptr + frame_instrumentation_generator_; }; } // namespace webrtc diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 9e0626c83c..5fe7373351 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -1,4 +1,3 @@ - /* * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. * @@ -14,10 +13,12 @@ #include #include #include +#include #include #include #include "absl/memory/memory.h" +#include "absl/types/variant.h" #include "api/environment/environment.h" #include "api/environment/environment_factory.h" #include "api/field_trials_view.h" @@ -31,6 +32,7 @@ #include "api/units/data_rate.h" #include "api/units/time_delta.h" #include "api/video/builtin_video_bitrate_allocator_factory.h" +#include "api/video/encoded_image.h" #include "api/video/i420_buffer.h" #include "api/video/nv12_buffer.h" #include "api/video/resolution.h" @@ -44,6 +46,7 @@ #include "api/video_codecs/vp8_temporal_layers_factory.h" #include "call/adaptation/test/fake_adaptation_constraint.h" #include "call/adaptation/test/fake_resource.h" +#include "common_video/frame_instrumentation_data.h" #include "common_video/h264/h264_common.h" #include "common_video/include/video_frame_buffer.h" #include "media/base/video_adapter.h" @@ -54,6 +57,7 @@ #include "modules/video_coding/codecs/vp9/include/vp9.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" #include "modules/video_coding/codecs/vp9/svc_config.h" +#include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/utility/quality_scaler.h" #include "modules/video_coding/utility/simulcast_rate_allocator.h" #include "modules/video_coding/utility/vp8_constants.h" @@ -63,6 +67,7 @@ #include "rtc_base/logging.h" #include "rtc_base/ref_counted_object.h" #include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" #include "system_wrappers/include/metrics.h" #include "test/encoder_settings.h" #include "test/fake_encoder.h" @@ -1513,6 +1518,13 @@ class VideoStreamEncoderTest : public ::testing::Test { return number_of_layers_allocations_; } + std::optional< + absl::variant> + GetLastFrameInstrumentationData() const { + MutexLock lock(&mutex_); + return last_frame_instrumentation_data_; + } + private: Result OnEncodedImage( const EncodedImage& encoded_image, @@ -1538,6 +1550,11 @@ class VideoStreamEncoderTest : public ::testing::Test { if (num_received_layers_ == num_expected_layers_) { encoded_frame_event_.Set(); } + if (codec_specific_info && + codec_specific_info->frame_instrumentation_data.has_value()) { + last_frame_instrumentation_data_ = + codec_specific_info->frame_instrumentation_data; + } return Result(Result::OK, last_timestamp_); } @@ -1597,6 +1614,9 @@ class VideoStreamEncoderTest : public ::testing::Test { int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0; VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_); int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0; + std::optional< + absl::variant> + last_frame_instrumentation_data_ RTC_GUARDED_BY(&mutex_); }; class VideoBitrateAllocatorProxyFactory @@ -1664,6 +1684,44 @@ TEST_F(VideoStreamEncoderTest, EncodeOneFrame) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, PopulatesFrameInstrumentationDataWhenSetTo) { + video_send_config_.encoder_settings.enable_frame_instrumentation_generator = + true; + ConfigureEncoder(video_encoder_config_.Copy()); + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); + + // We need a QP for the encoded frame. + fake_encoder_.SetEncodedImageData(EncodedImageBuffer::Create( + kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25))); + video_source_.IncomingCapturedFrame( + CreateFrame(1, codec_width_, codec_height_)); + WaitForEncodedFrame(1); + + EXPECT_TRUE(sink_.GetLastFrameInstrumentationData().has_value()); + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + DoesNotPopulateFrameInstrumentationDataWhenSetNotTo) { + video_send_config_.encoder_settings.enable_frame_instrumentation_generator = + false; + ConfigureEncoder(video_encoder_config_.Copy()); + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); + + // We need a QP for the encoded frame. + fake_encoder_.SetEncodedImageData(EncodedImageBuffer::Create( + kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25))); + rtc::Event frame_destroyed_event; + video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event)); + WaitForEncodedFrame(1); + + EXPECT_FALSE(sink_.GetLastFrameInstrumentationData().has_value()); + video_stream_encoder_->Stop(); + EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout)); +} + TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) { // Dropped since no target bitrate has been set. rtc::Event frame_destroyed_event;