From c12f6259384d5d7fa313296f5cf7f7998458984a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Spr=C3=A5ng?= Date: Wed, 13 Jan 2021 21:49:59 +0100 Subject: [PATCH] Adds VideoDecoder::GetDecoderInfo() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a new way to poll decoder metadata. A default implementation still delegates to the old methods. Root call site is updates to not use the olds methods. Follow-ups will dismantle usage of the olds methods in wrappers. Bug: webrtc:12271 Change-Id: Id0fa6863c96ff9e3b849da452d6540e7c5da4512 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/196520 Commit-Queue: Erik Språng Reviewed-by: Per Kjellander Reviewed-by: Philip Eliasson Cr-Commit-Position: refs/heads/master@{#32976} --- api/video_codecs/video_decoder.cc | 25 +++++++++++++++++ api/video_codecs/video_decoder.h | 17 ++++++++++- ...video_decoder_software_fallback_wrapper.cc | 20 +++++++++++-- .../codecs/av1/libaom_av1_decoder.cc | 8 ++++++ .../test/videocodec_test_fixture_impl.cc | 2 +- .../codecs/vp8/libvpx_vp8_decoder.cc | 7 +++++ .../codecs/vp8/libvpx_vp8_decoder.h | 1 + .../codecs/vp9/libvpx_vp9_decoder.cc | 7 +++++ .../codecs/vp9/libvpx_vp9_decoder.h | 1 + modules/video_coding/generic_decoder.cc | 28 +++++++++++++------ modules/video_coding/generic_decoder.h | 2 +- test/fake_decoder.cc | 6 ++++ test/fake_decoder.h | 1 + test/fake_vp8_decoder.cc | 7 +++++ test/fake_vp8_decoder.h | 2 +- .../video/quality_analyzing_video_decoder.cc | 6 ++++ .../video/quality_analyzing_video_decoder.h | 1 + test/video_decoder_proxy_factory.h | 3 ++ video/frame_dumping_decoder.cc | 5 ++++ video/video_receive_stream.cc | 5 ++++ video/video_receive_stream_unittest.cc | 1 + 21 files changed, 140 insertions(+), 15 deletions(-) diff --git a/api/video_codecs/video_decoder.cc b/api/video_codecs/video_decoder.cc index b181323911..fee3ec6d42 100644 --- a/api/video_codecs/video_decoder.cc +++ b/api/video_codecs/video_decoder.cc @@ -10,6 +10,8 @@ #include "api/video_codecs/video_decoder.h" +#include "rtc_base/strings/string_builder.h" + namespace webrtc { int32_t DecodedImageCallback::Decoded(VideoFrame& decodedImage, @@ -24,6 +26,12 @@ void DecodedImageCallback::Decoded(VideoFrame& decodedImage, Decoded(decodedImage, decode_time_ms.value_or(-1)); } +VideoDecoder::DecoderInfo VideoDecoder::GetDecoderInfo() const { + DecoderInfo info; + info.implementation_name = ImplementationName(); + return info; +} + bool VideoDecoder::PrefersLateDecoding() const { return true; } @@ -32,4 +40,21 @@ const char* VideoDecoder::ImplementationName() const { return "unknown"; } +std::string VideoDecoder::DecoderInfo::ToString() const { + char string_buf[2048]; + rtc::SimpleStringBuilder oss(string_buf); + + oss << "DecoderInfo { " + << "prefers_late_decoding = " + << "implementation_name = '" << implementation_name << "', " + << "is_hardware_accelerated = " + << (is_hardware_accelerated ? "true" : "false") << " }"; + return oss.str(); +} + +bool VideoDecoder::DecoderInfo::operator==(const DecoderInfo& rhs) const { + return is_hardware_accelerated == rhs.is_hardware_accelerated && + implementation_name == rhs.implementation_name; +} + } // namespace webrtc diff --git a/api/video_codecs/video_decoder.h b/api/video_codecs/video_decoder.h index 155e76c2d1..a6af3f22e9 100644 --- a/api/video_codecs/video_decoder.h +++ b/api/video_codecs/video_decoder.h @@ -42,6 +42,18 @@ class RTC_EXPORT DecodedImageCallback { class RTC_EXPORT VideoDecoder { public: + struct DecoderInfo { + // Descriptive name of the decoder implementation. + std::string implementation_name; + + // True if the decoder is backed by hardware acceleration. + bool is_hardware_accelerated = false; + + std::string ToString() const; + bool operator==(const DecoderInfo& rhs) const; + bool operator!=(const DecoderInfo& rhs) const { return !(*this == rhs); } + }; + virtual ~VideoDecoder() {} virtual int32_t InitDecode(const VideoCodec* codec_settings, @@ -56,12 +68,15 @@ class RTC_EXPORT VideoDecoder { virtual int32_t Release() = 0; + virtual DecoderInfo GetDecoderInfo() const; + + // Deprecated, use GetDecoderInfo().prefers_late_decoding instead. // Returns true if the decoder prefer to decode frames late. // That is, it can not decode infinite number of frames before the decoded // frame is consumed. // TODO(bugs.webrtc.org/12271): Remove when downstream has been updated. virtual bool PrefersLateDecoding() const; - + // Deprecated, use GetDecoderInfo().implementation_name instead. virtual const char* ImplementationName() const; }; diff --git a/api/video_codecs/video_decoder_software_fallback_wrapper.cc b/api/video_codecs/video_decoder_software_fallback_wrapper.cc index 32941d267a..e5743b3a2e 100644 --- a/api/video_codecs/video_decoder_software_fallback_wrapper.cc +++ b/api/video_codecs/video_decoder_software_fallback_wrapper.cc @@ -51,6 +51,7 @@ class VideoDecoderSoftwareFallbackWrapper final : public VideoDecoder { int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; const char* ImplementationName() const override; private: @@ -261,10 +262,23 @@ int32_t VideoDecoderSoftwareFallbackWrapper::Release() { return status; } +VideoDecoder::DecoderInfo VideoDecoderSoftwareFallbackWrapper::GetDecoderInfo() + const { + DecoderInfo info = active_decoder().GetDecoderInfo(); + if (decoder_type_ == DecoderType::kFallback) { + // Cached "A (fallback from B)" string. + info.implementation_name = fallback_implementation_name_; + } + return info; +} + const char* VideoDecoderSoftwareFallbackWrapper::ImplementationName() const { - return decoder_type_ == DecoderType::kFallback - ? fallback_implementation_name_.c_str() - : hw_decoder_->ImplementationName(); + if (decoder_type_ == DecoderType::kFallback) { + // Cached "A (fallback from B)" string. + return fallback_implementation_name_.c_str(); + } else { + return hw_decoder_->ImplementationName(); + } } VideoDecoder& VideoDecoderSoftwareFallbackWrapper::active_decoder() const { diff --git a/modules/video_coding/codecs/av1/libaom_av1_decoder.cc b/modules/video_coding/codecs/av1/libaom_av1_decoder.cc index bedb51937a..c187c72026 100644 --- a/modules/video_coding/codecs/av1/libaom_av1_decoder.cc +++ b/modules/video_coding/codecs/av1/libaom_av1_decoder.cc @@ -53,6 +53,7 @@ class LibaomAv1Decoder final : public VideoDecoder { int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; const char* ImplementationName() const override; private: @@ -182,6 +183,13 @@ int32_t LibaomAv1Decoder::Release() { return WEBRTC_VIDEO_CODEC_OK; } +VideoDecoder::DecoderInfo LibaomAv1Decoder::GetDecoderInfo() const { + DecoderInfo info; + info.implementation_name = "libaom"; + info.is_hardware_accelerated = false; + return info; +} + const char* LibaomAv1Decoder::ImplementationName() const { return "libaom"; } diff --git a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc index f6770d631e..7bd1ba35e0 100644 --- a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc +++ b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc @@ -748,7 +748,7 @@ void VideoCodecTestFixtureImpl::PrintSettings( task_queue->SendTask( [this, &encoder_name, &decoder_name] { encoder_name = encoder_->GetEncoderInfo().implementation_name; - decoder_name = decoders_.at(0)->ImplementationName(); + decoder_name = decoders_.at(0)->GetDecoderInfo().implementation_name; }, RTC_FROM_HERE); RTC_LOG(LS_INFO) << "enc_impl_name: " << encoder_name; diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc index af48c92535..979ded9a63 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc @@ -397,6 +397,13 @@ int LibvpxVp8Decoder::Release() { return ret_val; } +VideoDecoder::DecoderInfo LibvpxVp8Decoder::GetDecoderInfo() const { + DecoderInfo info; + info.implementation_name = "libvpx"; + info.is_hardware_accelerated = false; + return info; +} + const char* LibvpxVp8Decoder::ImplementationName() const { return "libvpx"; } diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h b/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h index cf699f1833..8d84b67ce3 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h @@ -38,6 +38,7 @@ class LibvpxVp8Decoder : public VideoDecoder { int RegisterDecodeCompleteCallback(DecodedImageCallback* callback) override; int Release() override; + DecoderInfo GetDecoderInfo() const override; const char* ImplementationName() const override; struct DeblockParams { diff --git a/modules/video_coding/codecs/vp9/libvpx_vp9_decoder.cc b/modules/video_coding/codecs/vp9/libvpx_vp9_decoder.cc index 45e0a0b4ab..0a99c6a46e 100644 --- a/modules/video_coding/codecs/vp9/libvpx_vp9_decoder.cc +++ b/modules/video_coding/codecs/vp9/libvpx_vp9_decoder.cc @@ -391,6 +391,13 @@ int LibvpxVp9Decoder::Release() { return ret_val; } +VideoDecoder::DecoderInfo LibvpxVp9Decoder::GetDecoderInfo() const { + DecoderInfo info; + info.implementation_name = "libvpx"; + info.is_hardware_accelerated = false; + return info; +} + const char* LibvpxVp9Decoder::ImplementationName() const { return "libvpx"; } diff --git a/modules/video_coding/codecs/vp9/libvpx_vp9_decoder.h b/modules/video_coding/codecs/vp9/libvpx_vp9_decoder.h index 59d207aec2..f26f42700c 100644 --- a/modules/video_coding/codecs/vp9/libvpx_vp9_decoder.h +++ b/modules/video_coding/codecs/vp9/libvpx_vp9_decoder.h @@ -40,6 +40,7 @@ class LibvpxVp9Decoder : public VP9Decoder { int Release() override; + DecoderInfo GetDecoderInfo() const override; const char* ImplementationName() const override; private: diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc index d237d0c67c..28c97f08f5 100644 --- a/modules/video_coding/generic_decoder.cc +++ b/modules/video_coding/generic_decoder.cc @@ -238,8 +238,12 @@ int32_t VCMGenericDecoder::InitDecode(const VideoCodec* settings, _codecType = settings->codecType; int err = decoder_->InitDecode(settings, numberOfCores); - implementation_name_ = decoder_->ImplementationName(); - RTC_LOG(LS_INFO) << "Decoder implementation: " << implementation_name_; + decoder_info_ = decoder_->GetDecoderInfo(); + RTC_LOG(LS_INFO) << "Decoder implementation: " << decoder_info_.ToString(); + if (_callback) { + _callback->OnDecoderImplementationName( + decoder_info_.implementation_name.c_str()); + } return err; } @@ -269,13 +273,16 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) { _nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength; int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(), frame.RenderTimeMs()); - const char* new_implementation_name = decoder_->ImplementationName(); - if (new_implementation_name != implementation_name_) { - implementation_name_ = new_implementation_name; + VideoDecoder::DecoderInfo decoder_info = decoder_->GetDecoderInfo(); + if (decoder_info != decoder_info_) { RTC_LOG(LS_INFO) << "Changed decoder implementation to: " - << new_implementation_name; + << decoder_info.ToString(); + + _callback->OnDecoderImplementationName( + decoder_info.implementation_name.empty() + ? "unknown" + : decoder_info.implementation_name.c_str()); } - _callback->OnDecoderImplementationName(implementation_name_.c_str()); if (ret < WEBRTC_VIDEO_CODEC_OK) { RTC_LOG(LS_WARNING) << "Failed to decode frame with timestamp " << frame.Timestamp() << ", error code: " << ret; @@ -291,7 +298,12 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) { int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback( VCMDecodedFrameCallback* callback) { _callback = callback; - return decoder_->RegisterDecodeCompleteCallback(callback); + int32_t ret = decoder_->RegisterDecodeCompleteCallback(callback); + if (callback && !decoder_info_.implementation_name.empty()) { + callback->OnDecoderImplementationName( + decoder_info_.implementation_name.c_str()); + } + return ret; } } // namespace webrtc diff --git a/modules/video_coding/generic_decoder.h b/modules/video_coding/generic_decoder.h index bd3620d933..9524bab99b 100644 --- a/modules/video_coding/generic_decoder.h +++ b/modules/video_coding/generic_decoder.h @@ -123,7 +123,7 @@ class VCMGenericDecoder { VideoCodecType _codecType; const bool _isExternal; VideoContentType _last_keyframe_content_type; - std::string implementation_name_; + VideoDecoder::DecoderInfo decoder_info_; }; } // namespace webrtc diff --git a/test/fake_decoder.cc b/test/fake_decoder.cc index e80c31cc40..e229bbb2a1 100644 --- a/test/fake_decoder.cc +++ b/test/fake_decoder.cc @@ -99,6 +99,12 @@ int32_t FakeDecoder::Release() { } const char* FakeDecoder::kImplementationName = "fake_decoder"; +VideoDecoder::DecoderInfo FakeDecoder::GetDecoderInfo() const { + DecoderInfo info; + info.implementation_name = kImplementationName; + info.is_hardware_accelerated = false; + return info; +} const char* FakeDecoder::ImplementationName() const { return kImplementationName; } diff --git a/test/fake_decoder.h b/test/fake_decoder.h index 055c55beca..2ac2045bc0 100644 --- a/test/fake_decoder.h +++ b/test/fake_decoder.h @@ -41,6 +41,7 @@ class FakeDecoder : public VideoDecoder { int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; const char* ImplementationName() const override; static const char* kImplementationName; diff --git a/test/fake_vp8_decoder.cc b/test/fake_vp8_decoder.cc index faaa554259..ec636dca11 100644 --- a/test/fake_vp8_decoder.cc +++ b/test/fake_vp8_decoder.cc @@ -79,6 +79,13 @@ int32_t FakeVp8Decoder::Release() { } const char* FakeVp8Decoder::kImplementationName = "fake_vp8_decoder"; +VideoDecoder::DecoderInfo FakeVp8Decoder::GetDecoderInfo() const { + DecoderInfo info; + info.implementation_name = kImplementationName; + info.is_hardware_accelerated = false; + return info; +} + const char* FakeVp8Decoder::ImplementationName() const { return kImplementationName; } diff --git a/test/fake_vp8_decoder.h b/test/fake_vp8_decoder.h index 4f0fa3d8a9..2e469a17f3 100644 --- a/test/fake_vp8_decoder.h +++ b/test/fake_vp8_decoder.h @@ -38,8 +38,8 @@ class FakeVp8Decoder : public VideoDecoder { int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; const char* ImplementationName() const override; - static const char* kImplementationName; private: diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc index 5cd961e787..27b9af50bb 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc @@ -123,6 +123,12 @@ int32_t QualityAnalyzingVideoDecoder::Release() { return result; } +VideoDecoder::DecoderInfo QualityAnalyzingVideoDecoder::GetDecoderInfo() const { + DecoderInfo info = delegate_->GetDecoderInfo(); + info.implementation_name = implementation_name_; + return info; +} + const char* QualityAnalyzingVideoDecoder::ImplementationName() const { return implementation_name_.c_str(); } diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h index 79ca68d868..a26ccbe1ee 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h @@ -69,6 +69,7 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder { int32_t RegisterDecodeCompleteCallback( DecodedImageCallback* callback) override; int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; const char* ImplementationName() const override; private: diff --git a/test/video_decoder_proxy_factory.h b/test/video_decoder_proxy_factory.h index 2027f71556..303d209dbd 100644 --- a/test/video_decoder_proxy_factory.h +++ b/test/video_decoder_proxy_factory.h @@ -61,6 +61,9 @@ class VideoDecoderProxyFactory final : public VideoDecoderFactory { return decoder_->RegisterDecodeCompleteCallback(callback); } int32_t Release() override { return decoder_->Release(); } + DecoderInfo GetDecoderInfo() const override { + return decoder_->GetDecoderInfo(); + } const char* ImplementationName() const override { return decoder_->ImplementationName(); } diff --git a/video/frame_dumping_decoder.cc b/video/frame_dumping_decoder.cc index c27a653ee2..59202dd03c 100644 --- a/video/frame_dumping_decoder.cc +++ b/video/frame_dumping_decoder.cc @@ -32,6 +32,7 @@ class FrameDumpingDecoder : public VideoDecoder { int32_t RegisterDecodeCompleteCallback( DecodedImageCallback* callback) override; int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; const char* ImplementationName() const override; private: @@ -72,6 +73,10 @@ int32_t FrameDumpingDecoder::Release() { return decoder_->Release(); } +VideoDecoder::DecoderInfo FrameDumpingDecoder::GetDecoderInfo() const { + return decoder_->GetDecoderInfo(); +} + const char* FrameDumpingDecoder::ImplementationName() const { return decoder_->ImplementationName(); } diff --git a/video/video_receive_stream.cc b/video/video_receive_stream.cc index fbe994f445..c46868d749 100644 --- a/video/video_receive_stream.cc +++ b/video/video_receive_stream.cc @@ -167,6 +167,11 @@ class NullVideoDecoder : public webrtc::VideoDecoder { int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; } + DecoderInfo GetDecoderInfo() const override { + DecoderInfo info; + info.implementation_name = "NullVideoDecoder"; + return info; + } const char* ImplementationName() const override { return "NullVideoDecoder"; } }; diff --git a/video/video_receive_stream_unittest.cc b/video/video_receive_stream_unittest.cc index 503c96c093..9ac640ba1b 100644 --- a/video/video_receive_stream_unittest.cc +++ b/video/video_receive_stream_unittest.cc @@ -72,6 +72,7 @@ class MockVideoDecoder : public VideoDecoder { (DecodedImageCallback*), (override)); MOCK_METHOD(int32_t, Release, (), (override)); + MOCK_METHOD(VideoDecoder::DecoderInfo, GetDetcoderInfo, (), (const override)); const char* ImplementationName() const { return "MockVideoDecoder"; } };