From 20b3271b614b77877c9fdcea3d5c559b7dda5592 Mon Sep 17 00:00:00 2001 From: Tommi Date: Thu, 29 Sep 2022 14:13:15 +0200 Subject: [PATCH] Fork VCMDecoderDatabase for VideoReceiver. This is to keep the deprecated VideoReceiver separate from the implementation used by VideoReceiver2 before updating VCMDecoderDatabase to have ownership of the registered decoders. Fixing typo (DataBase->Database) in the name of the remaining class. Bug: webrtc:14486, webrtc:14497 Change-Id: I5ee755921454b0831b3af6d0161f5b48c7c60540 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276781 Commit-Queue: Tomas Gunnarsson Reviewed-by: Philip Eliasson Cr-Commit-Position: refs/heads/main@{#38247} --- modules/video_coding/decoder_database.cc | 18 +-- modules/video_coding/decoder_database.h | 10 +- .../video_coding/decoder_database_unittest.cc | 12 +- modules/video_coding/video_coding_impl.cc | 131 ++++++++++++++++++ modules/video_coding/video_coding_impl.h | 48 ++++++- modules/video_coding/video_receiver2.h | 2 +- 6 files changed, 198 insertions(+), 23 deletions(-) diff --git a/modules/video_coding/decoder_database.cc b/modules/video_coding/decoder_database.cc index 7998302705..6d78e70848 100644 --- a/modules/video_coding/decoder_database.cc +++ b/modules/video_coding/decoder_database.cc @@ -15,11 +15,11 @@ namespace webrtc { -VCMDecoderDataBase::VCMDecoderDataBase() { +VCMDecoderDatabase::VCMDecoderDatabase() { decoder_sequence_checker_.Detach(); } -VideoDecoder* VCMDecoderDataBase::DeregisterExternalDecoder( +VideoDecoder* VCMDecoderDatabase::DeregisterExternalDecoder( uint8_t payload_type) { RTC_DCHECK_RUN_ON(&decoder_sequence_checker_); auto it = decoders_.find(payload_type); @@ -41,7 +41,7 @@ VideoDecoder* VCMDecoderDataBase::DeregisterExternalDecoder( // Add the external decoder object to the list of external decoders. // Won't be registered as a receive codec until RegisterReceiveCodec is called. -void VCMDecoderDataBase::RegisterExternalDecoder( +void VCMDecoderDatabase::RegisterExternalDecoder( uint8_t payload_type, VideoDecoder* external_decoder) { RTC_DCHECK_RUN_ON(&decoder_sequence_checker_); @@ -52,13 +52,13 @@ void VCMDecoderDataBase::RegisterExternalDecoder( } } -bool VCMDecoderDataBase::IsExternalDecoderRegistered( +bool VCMDecoderDatabase::IsExternalDecoderRegistered( uint8_t payload_type) const { RTC_DCHECK_RUN_ON(&decoder_sequence_checker_); return decoders_.find(payload_type) != decoders_.end(); } -void VCMDecoderDataBase::RegisterReceiveCodec( +void VCMDecoderDatabase::RegisterReceiveCodec( uint8_t payload_type, const VideoDecoder::Settings& settings) { // If payload value already exists, erase old and insert new. @@ -68,7 +68,7 @@ void VCMDecoderDataBase::RegisterReceiveCodec( decoder_settings_[payload_type] = settings; } -bool VCMDecoderDataBase::DeregisterReceiveCodec(uint8_t payload_type) { +bool VCMDecoderDatabase::DeregisterReceiveCodec(uint8_t payload_type) { if (decoder_settings_.erase(payload_type) == 0) { return false; } @@ -79,12 +79,12 @@ bool VCMDecoderDataBase::DeregisterReceiveCodec(uint8_t payload_type) { return true; } -void VCMDecoderDataBase::DeregisterReceiveCodecs() { +void VCMDecoderDatabase::DeregisterReceiveCodecs() { current_payload_type_ = absl::nullopt; decoder_settings_.clear(); } -VCMGenericDecoder* VCMDecoderDataBase::GetDecoder( +VCMGenericDecoder* VCMDecoderDatabase::GetDecoder( const VCMEncodedFrame& frame, VCMDecodedFrameCallback* decoded_frame_callback) { RTC_DCHECK_RUN_ON(&decoder_sequence_checker_); @@ -116,7 +116,7 @@ VCMGenericDecoder* VCMDecoderDataBase::GetDecoder( return &*current_decoder_; } -void VCMDecoderDataBase::CreateAndInitDecoder(const VCMEncodedFrame& frame) { +void VCMDecoderDatabase::CreateAndInitDecoder(const VCMEncodedFrame& frame) { uint8_t payload_type = frame.PayloadType(); RTC_DLOG(LS_INFO) << "Initializing decoder with payload type '" << int{payload_type} << "'."; diff --git a/modules/video_coding/decoder_database.h b/modules/video_coding/decoder_database.h index 422978629f..8969feaf8f 100644 --- a/modules/video_coding/decoder_database.h +++ b/modules/video_coding/decoder_database.h @@ -23,12 +23,12 @@ namespace webrtc { -class VCMDecoderDataBase { +class VCMDecoderDatabase { public: - VCMDecoderDataBase(); - VCMDecoderDataBase(const VCMDecoderDataBase&) = delete; - VCMDecoderDataBase& operator=(const VCMDecoderDataBase&) = delete; - ~VCMDecoderDataBase() = default; + VCMDecoderDatabase(); + VCMDecoderDatabase(const VCMDecoderDatabase&) = delete; + VCMDecoderDatabase& operator=(const VCMDecoderDatabase&) = delete; + ~VCMDecoderDatabase() = default; // Returns a pointer to the previously registered decoder or nullptr if none // was registered for the `payload_type`. diff --git a/modules/video_coding/decoder_database_unittest.cc b/modules/video_coding/decoder_database_unittest.cc index c7453d8040..f2f250c332 100644 --- a/modules/video_coding/decoder_database_unittest.cc +++ b/modules/video_coding/decoder_database_unittest.cc @@ -20,8 +20,8 @@ namespace { using ::testing::NiceMock; // Test registering and unregistering an external decoder instance. -TEST(VCMDecoderDataBaseTest, RegisterExternalDecoder) { - VCMDecoderDataBase db; +TEST(VCMDecoderDatabaseTest, RegisterExternalDecoder) { + VCMDecoderDatabase db; constexpr int kPayloadType = 1; ASSERT_FALSE(db.IsExternalDecoderRegistered(kPayloadType)); @@ -32,8 +32,8 @@ TEST(VCMDecoderDataBaseTest, RegisterExternalDecoder) { EXPECT_FALSE(db.IsExternalDecoderRegistered(kPayloadType)); } -TEST(VCMDecoderDataBaseTest, RegisterReceiveCodec) { - VCMDecoderDataBase db; +TEST(VCMDecoderDatabaseTest, RegisterReceiveCodec) { + VCMDecoderDatabase db; constexpr int kPayloadType = 1; ASSERT_FALSE(db.DeregisterReceiveCodec(kPayloadType)); @@ -46,8 +46,8 @@ TEST(VCMDecoderDataBaseTest, RegisterReceiveCodec) { EXPECT_TRUE(db.DeregisterReceiveCodec(kPayloadType)); } -TEST(VCMDecoderDataBaseTest, DeregisterReceiveCodecs) { - VCMDecoderDataBase db; +TEST(VCMDecoderDatabaseTest, DeregisterReceiveCodecs) { + VCMDecoderDatabase db; constexpr int kPayloadType1 = 1; constexpr int kPayloadType2 = 2; ASSERT_FALSE(db.DeregisterReceiveCodec(kPayloadType1)); diff --git a/modules/video_coding/video_coding_impl.cc b/modules/video_coding/video_coding_impl.cc index e0ad033d7c..2eaecd5011 100644 --- a/modules/video_coding/video_coding_impl.cc +++ b/modules/video_coding/video_coding_impl.cc @@ -19,6 +19,7 @@ #include "api/video/encoded_image.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/timing/timing.h" +#include "rtc_base/logging.h" #include "rtc_base/memory/always_valid_pointer.h" #include "system_wrappers/include/clock.h" @@ -38,6 +39,136 @@ int64_t VCMProcessTimer::TimeUntilProcess() const { void VCMProcessTimer::Processed() { _latestMs = _clock->TimeInMilliseconds(); } + +DEPRECATED_VCMDecoderDataBase::DEPRECATED_VCMDecoderDataBase() { + decoder_sequence_checker_.Detach(); +} + +VideoDecoder* DEPRECATED_VCMDecoderDataBase::DeregisterExternalDecoder( + uint8_t payload_type) { + RTC_DCHECK_RUN_ON(&decoder_sequence_checker_); + auto it = decoders_.find(payload_type); + if (it == decoders_.end()) { + return nullptr; + } + + // We can't use payload_type to check if the decoder is currently in use, + // because payload type may be out of date (e.g. before we decode the first + // frame after RegisterReceiveCodec). + if (current_decoder_ && current_decoder_->IsSameDecoder(it->second)) { + // Release it if it was registered and in use. + current_decoder_ = absl::nullopt; + } + VideoDecoder* ret = it->second; + decoders_.erase(it); + return ret; +} + +// Add the external decoder object to the list of external decoders. +// Won't be registered as a receive codec until RegisterReceiveCodec is called. +void DEPRECATED_VCMDecoderDataBase::RegisterExternalDecoder( + uint8_t payload_type, + VideoDecoder* external_decoder) { + RTC_DCHECK_RUN_ON(&decoder_sequence_checker_); + // If payload value already exists, erase old and insert new. + DeregisterExternalDecoder(payload_type); + decoders_[payload_type] = external_decoder; +} + +bool DEPRECATED_VCMDecoderDataBase::IsExternalDecoderRegistered( + uint8_t payload_type) const { + RTC_DCHECK_RUN_ON(&decoder_sequence_checker_); + return payload_type == current_payload_type_ || + decoders_.find(payload_type) != decoders_.end(); +} + +void DEPRECATED_VCMDecoderDataBase::RegisterReceiveCodec( + uint8_t payload_type, + const VideoDecoder::Settings& settings) { + // If payload value already exists, erase old and insert new. + if (payload_type == current_payload_type_) { + current_payload_type_ = absl::nullopt; + } + decoder_settings_[payload_type] = settings; +} + +bool DEPRECATED_VCMDecoderDataBase::DeregisterReceiveCodec( + uint8_t payload_type) { + if (decoder_settings_.erase(payload_type) == 0) { + return false; + } + if (payload_type == current_payload_type_) { + // This codec is currently in use. + current_payload_type_ = absl::nullopt; + } + return true; +} + +VCMGenericDecoder* DEPRECATED_VCMDecoderDataBase::GetDecoder( + const VCMEncodedFrame& frame, + VCMDecodedFrameCallback* decoded_frame_callback) { + RTC_DCHECK_RUN_ON(&decoder_sequence_checker_); + RTC_DCHECK(decoded_frame_callback->UserReceiveCallback()); + uint8_t payload_type = frame.PayloadType(); + if (payload_type == current_payload_type_ || payload_type == 0) { + return current_decoder_.has_value() ? &*current_decoder_ : nullptr; + } + // If decoder exists - delete. + if (current_decoder_.has_value()) { + current_decoder_ = absl::nullopt; + current_payload_type_ = absl::nullopt; + } + + CreateAndInitDecoder(frame); + if (current_decoder_ == absl::nullopt) { + return nullptr; + } + + VCMReceiveCallback* callback = decoded_frame_callback->UserReceiveCallback(); + callback->OnIncomingPayloadType(payload_type); + if (current_decoder_->RegisterDecodeCompleteCallback(decoded_frame_callback) < + 0) { + current_decoder_ = absl::nullopt; + return nullptr; + } + + current_payload_type_ = payload_type; + return &*current_decoder_; +} + +void DEPRECATED_VCMDecoderDataBase::CreateAndInitDecoder( + const VCMEncodedFrame& frame) { + uint8_t payload_type = frame.PayloadType(); + RTC_LOG(LS_INFO) << "Initializing decoder with payload type '" + << int{payload_type} << "'."; + auto decoder_item = decoder_settings_.find(payload_type); + if (decoder_item == decoder_settings_.end()) { + RTC_LOG(LS_ERROR) << "Can't find a decoder associated with payload type: " + << int{payload_type}; + return; + } + auto external_dec_item = decoders_.find(payload_type); + if (external_dec_item == decoders_.end()) { + RTC_LOG(LS_ERROR) << "No decoder of this type exists."; + return; + } + current_decoder_.emplace(external_dec_item->second); + + // Copy over input resolutions to prevent codec reinitialization due to + // the first frame being of a different resolution than the database values. + // This is best effort, since there's no guarantee that width/height have been + // parsed yet (and may be zero). + RenderResolution frame_resolution(frame.EncodedImage()._encodedWidth, + frame.EncodedImage()._encodedHeight); + if (frame_resolution.Valid()) { + decoder_item->second.set_max_render_resolution(frame_resolution); + } + if (!current_decoder_->Configure(decoder_item->second)) { + current_decoder_ = absl::nullopt; + RTC_LOG(LS_ERROR) << "Failed to initialize decoder."; + } +} + } // namespace vcm namespace { diff --git a/modules/video_coding/video_coding_impl.h b/modules/video_coding/video_coding_impl.h index 22237ca78e..927b2da4b8 100644 --- a/modules/video_coding/video_coding_impl.h +++ b/modules/video_coding/video_coding_impl.h @@ -11,6 +11,7 @@ #ifndef MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ #define MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ +#include #include #include #include @@ -18,7 +19,6 @@ #include "absl/types/optional.h" #include "api/field_trials_view.h" #include "api/sequence_checker.h" -#include "modules/video_coding/decoder_database.h" #include "modules/video_coding/frame_buffer.h" #include "modules/video_coding/generic_decoder.h" #include "modules/video_coding/include/video_coding.h" @@ -55,6 +55,50 @@ class VCMProcessTimer { int64_t _latestMs; }; +class DEPRECATED_VCMDecoderDataBase { + public: + DEPRECATED_VCMDecoderDataBase(); + DEPRECATED_VCMDecoderDataBase(const DEPRECATED_VCMDecoderDataBase&) = delete; + DEPRECATED_VCMDecoderDataBase& operator=( + const DEPRECATED_VCMDecoderDataBase&) = delete; + ~DEPRECATED_VCMDecoderDataBase() = default; + + // Returns a pointer to the previously registered decoder or nullptr if none + // was registered for the `payload_type`. + VideoDecoder* DeregisterExternalDecoder(uint8_t payload_type); + void RegisterExternalDecoder(uint8_t payload_type, + VideoDecoder* external_decoder); + bool IsExternalDecoderRegistered(uint8_t payload_type) const; + + void RegisterReceiveCodec(uint8_t payload_type, + const VideoDecoder::Settings& settings); + bool DeregisterReceiveCodec(uint8_t payload_type); + + // Returns a decoder specified by frame.PayloadType. The decoded frame + // callback of the decoder is set to `decoded_frame_callback`. If no such + // decoder already exists an instance will be created and initialized. + // nullptr is returned if no decoder with the specified payload type was found + // and the function failed to create one. + VCMGenericDecoder* GetDecoder( + const VCMEncodedFrame& frame, + VCMDecodedFrameCallback* decoded_frame_callback); + + private: + void CreateAndInitDecoder(const VCMEncodedFrame& frame) + RTC_RUN_ON(decoder_sequence_checker_); + + SequenceChecker decoder_sequence_checker_; + + absl::optional current_payload_type_; + absl::optional current_decoder_ + RTC_GUARDED_BY(decoder_sequence_checker_); + // Initialization paramaters for decoders keyed by payload type. + std::map decoder_settings_; + // Decoders keyed by payload type. + std::map decoders_ + RTC_GUARDED_BY(decoder_sequence_checker_); +}; + class VideoReceiver { public: VideoReceiver(Clock* clock, @@ -122,7 +166,7 @@ class VideoReceiver { // Callbacks are set before the decoder thread starts. // Once the decoder thread has been started, usage of `_codecDataBase` moves // over to the decoder thread. - VCMDecoderDataBase _codecDataBase; + DEPRECATED_VCMDecoderDataBase _codecDataBase; VCMProcessTimer _retransmissionTimer RTC_GUARDED_BY(module_thread_checker_); VCMProcessTimer _keyRequestTimer RTC_GUARDED_BY(module_thread_checker_); diff --git a/modules/video_coding/video_receiver2.h b/modules/video_coding/video_receiver2.h index f9b59939ad..67cb9afd48 100644 --- a/modules/video_coding/video_receiver2.h +++ b/modules/video_coding/video_receiver2.h @@ -63,7 +63,7 @@ class VideoReceiver2 { // Callbacks are set before the decoder thread starts. // Once the decoder thread has been started, usage of `_codecDataBase` moves // over to the decoder thread. - VCMDecoderDataBase codec_database_; + VCMDecoderDatabase codec_database_; }; } // namespace webrtc