From e3aa88bbd5accadec73fa7e38584dfbf6aabe8a9 Mon Sep 17 00:00:00 2001 From: tommi Date: Tue, 4 Apr 2017 03:53:02 -0700 Subject: [PATCH] Deliver video frames on Android, on the decode thread. VideoCoding * Adding a method for polling for frames on Android only until the capture implementation takes care of this (longer term plan). CodecDatabase * Add an accessor for the current decoder * Use std::unique_ptr<> for ownership. * Remove "Release()" and "ReleaseDecoder()". Instead just delete. * Remove |friend| relationship between CodecDatabase and VCMGenericDecoder. VCMDecodedFrameCallback * DCHECKs for thread correctness. * Remove |lock_| now that a threading model has been established and verified. VCMGenericDecoder * All methods now have thread checks. * Variable access associated with thread checkers. VideoReceiver * Added two notification methods, DecoderThreadStarting() and DecoderThreadStopped() * Allows us to establish a period when the decoder thread is not running and it is safe to modify variables such as callbacks, that are only read when the decoder thread is running. * Allows us to DCHECK thread guarantees. * Allows synchronizing callbacks from the module process thread and have them only active while the decoder thread is running. * The above, allows us to establish two modes for the thread, single-threaded-mutable and multi-threaded-const. * Using that knowledge, we can remove |receive_crit_| as well as locking for a number of member variables. MediaCodecVideoDecoder * Removed frame polling code from this class, since this is now done from the root thread function in VideoReceiveStream. VideoReceiveStream * On Android: Polls for decoded frames every 10ms (same interval as previously in MediaCodecVideoDecoder) * [Un]Registers the |video_receiver_| with the module thread only around the time the decoder thread is started/stopped. * Notifies the receiver of start/stop events of the decoder thread. * Changed the decoder thread to use the new PlatformThread callback type. BUG=webrtc:7361, 695438 Review-Url: https://codereview.webrtc.org/2764573002 Cr-Commit-Position: refs/heads/master@{#17527} --- .../videodecodersoftwarefallbackwrapper.cc | 8 + .../videodecodersoftwarefallbackwrapper.h | 5 + webrtc/modules/video_coding/codec_database.cc | 89 ++++--- webrtc/modules/video_coding/codec_database.h | 16 +- .../modules/video_coding/generic_decoder.cc | 113 +++++---- webrtc/modules/video_coding/generic_decoder.h | 45 ++-- .../modules/video_coding/video_coding_impl.h | 66 ++++-- webrtc/modules/video_coding/video_receiver.cc | 217 ++++++++++++------ .../src/jni/androidmediadecoder_jni.cc | 194 +++++++++------- webrtc/video/video_receive_stream.cc | 70 ++++-- webrtc/video/video_receive_stream.h | 2 +- webrtc/video_decoder.h | 7 + 12 files changed, 511 insertions(+), 321 deletions(-) diff --git a/webrtc/media/engine/videodecodersoftwarefallbackwrapper.cc b/webrtc/media/engine/videodecodersoftwarefallbackwrapper.cc index 34d851e013..9fb4952051 100644 --- a/webrtc/media/engine/videodecodersoftwarefallbackwrapper.cc +++ b/webrtc/media/engine/videodecodersoftwarefallbackwrapper.cc @@ -138,6 +138,14 @@ bool VideoDecoderSoftwareFallbackWrapper::PrefersLateDecoding() const { return decoder_->PrefersLateDecoding(); } +#if defined(WEBRTC_ANDROID) +void VideoDecoderSoftwareFallbackWrapper::PollDecodedFrames() { + if (fallback_decoder_) + return fallback_decoder_->PollDecodedFrames(); + return decoder_->PollDecodedFrames(); +} +#endif + const char* VideoDecoderSoftwareFallbackWrapper::ImplementationName() const { if (fallback_decoder_) return fallback_implementation_name_.c_str(); diff --git a/webrtc/media/engine/videodecodersoftwarefallbackwrapper.h b/webrtc/media/engine/videodecodersoftwarefallbackwrapper.h index 80d91f4663..a5fedc319d 100644 --- a/webrtc/media/engine/videodecodersoftwarefallbackwrapper.h +++ b/webrtc/media/engine/videodecodersoftwarefallbackwrapper.h @@ -41,6 +41,11 @@ class VideoDecoderSoftwareFallbackWrapper : public webrtc::VideoDecoder { int32_t Release() override; bool PrefersLateDecoding() const override; +#if defined(WEBRTC_ANDROID) + // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7361 + void PollDecodedFrames() override; +#endif + const char* ImplementationName() const override; private: diff --git a/webrtc/modules/video_coding/codec_database.cc b/webrtc/modules/video_coding/codec_database.cc index 4d5e20215c..29899fccaa 100644 --- a/webrtc/modules/video_coding/codec_database.cc +++ b/webrtc/modules/video_coding/codec_database.cc @@ -71,6 +71,31 @@ VideoCodecH264 VideoEncoder::GetDefaultH264Settings() { return h264_settings; } +// Create an internal Decoder given a codec type +static std::unique_ptr CreateDecoder(VideoCodecType type) { + switch (type) { + case kVideoCodecVP8: + return std::unique_ptr( + new VCMGenericDecoder(VP8Decoder::Create())); + case kVideoCodecVP9: + return std::unique_ptr( + new VCMGenericDecoder(VP9Decoder::Create())); + case kVideoCodecI420: + return std::unique_ptr( + new VCMGenericDecoder(new I420Decoder())); + case kVideoCodecH264: + if (H264Decoder::IsSupported()) { + return std::unique_ptr( + new VCMGenericDecoder(H264Decoder::Create())); + } + break; + default: + break; + } + LOG(LS_WARNING) << "No internal decoder of this type exists."; + return std::unique_ptr(); +} + VCMDecoderMapItem::VCMDecoderMapItem(VideoCodec* settings, int number_of_cores, bool require_key_frame) @@ -98,13 +123,12 @@ VCMCodecDataBase::VCMCodecDataBase( external_encoder_(nullptr), internal_source_(false), encoded_frame_callback_(encoded_frame_callback), - ptr_decoder_(nullptr), dec_map_(), dec_external_map_() {} VCMCodecDataBase::~VCMCodecDataBase() { DeleteEncoder(); - ReleaseDecoder(ptr_decoder_); + ptr_decoder_.reset(); for (auto& kv : dec_map_) delete kv.second; for (auto& kv : dec_external_map_) @@ -391,11 +415,10 @@ bool VCMCodecDataBase::DeregisterExternalDecoder(uint8_t payload_type) { // We can't use payload_type to check if the decoder is currently in use, // because payload type may be out of date (e.g. before we decode the first // frame after RegisterReceiveCodec) - if (ptr_decoder_ != nullptr && - ptr_decoder_->_decoder == (*it).second->external_decoder_instance) { + if (ptr_decoder_ && + ptr_decoder_->IsSameDecoder((*it).second->external_decoder_instance)) { // Release it if it was registered and in use. - ReleaseDecoder(ptr_decoder_); - ptr_decoder_ = nullptr; + ptr_decoder_.reset(); } DeregisterReceiveCodec(payload_type); delete it->second; @@ -455,12 +478,11 @@ VCMGenericDecoder* VCMCodecDataBase::GetDecoder( RTC_DCHECK(decoded_frame_callback->UserReceiveCallback()); uint8_t payload_type = frame.PayloadType(); if (payload_type == receive_codec_.plType || payload_type == 0) { - return ptr_decoder_; + return ptr_decoder_.get(); } // Check for exisitng decoder, if exists - delete. if (ptr_decoder_) { - ReleaseDecoder(ptr_decoder_); - ptr_decoder_ = nullptr; + ptr_decoder_.reset(); memset(&receive_codec_, 0, sizeof(VideoCodec)); } ptr_decoder_ = CreateAndInitDecoder(frame, &receive_codec_); @@ -471,36 +493,26 @@ VCMGenericDecoder* VCMCodecDataBase::GetDecoder( callback->OnIncomingPayloadType(receive_codec_.plType); if (ptr_decoder_->RegisterDecodeCompleteCallback(decoded_frame_callback) < 0) { - ReleaseDecoder(ptr_decoder_); - ptr_decoder_ = nullptr; + ptr_decoder_.reset(); memset(&receive_codec_, 0, sizeof(VideoCodec)); return nullptr; } - return ptr_decoder_; + return ptr_decoder_.get(); } -void VCMCodecDataBase::ReleaseDecoder(VCMGenericDecoder* decoder) const { - if (decoder) { - RTC_DCHECK(decoder->_decoder); - decoder->Release(); - if (!decoder->External()) { - delete decoder->_decoder; - } - delete decoder; - } +VCMGenericDecoder* VCMCodecDataBase::GetCurrentDecoder() { + return ptr_decoder_.get(); } bool VCMCodecDataBase::PrefersLateDecoding() const { - if (!ptr_decoder_) - return true; - return ptr_decoder_->PrefersLateDecoding(); + return ptr_decoder_ ? ptr_decoder_->PrefersLateDecoding() : true; } bool VCMCodecDataBase::MatchesCurrentResolution(int width, int height) const { return send_codec_.width == width && send_codec_.height == height; } -VCMGenericDecoder* VCMCodecDataBase::CreateAndInitDecoder( +std::unique_ptr VCMCodecDataBase::CreateAndInitDecoder( const VCMEncodedFrame& frame, VideoCodec* new_codec) const { uint8_t payload_type = frame.PayloadType(); @@ -513,13 +525,13 @@ VCMGenericDecoder* VCMCodecDataBase::CreateAndInitDecoder( << static_cast(payload_type); return nullptr; } - VCMGenericDecoder* ptr_decoder = nullptr; + std::unique_ptr ptr_decoder; const VCMExtDecoderMapItem* external_dec_item = FindExternalDecoderItem(payload_type); if (external_dec_item) { // External codec. - ptr_decoder = new VCMGenericDecoder( - external_dec_item->external_decoder_instance, true); + ptr_decoder.reset(new VCMGenericDecoder( + external_dec_item->external_decoder_instance, true)); } else { // Create decoder. ptr_decoder = CreateDecoder(decoder_item->settings->codecType); @@ -538,7 +550,6 @@ VCMGenericDecoder* VCMCodecDataBase::CreateAndInitDecoder( } if (ptr_decoder->InitDecode(decoder_item->settings.get(), decoder_item->number_of_cores) < 0) { - ReleaseDecoder(ptr_decoder); return nullptr; } memcpy(new_codec, decoder_item->settings.get(), sizeof(VideoCodec)); @@ -552,26 +563,6 @@ void VCMCodecDataBase::DeleteEncoder() { ptr_encoder_.reset(); } -VCMGenericDecoder* VCMCodecDataBase::CreateDecoder(VideoCodecType type) const { - switch (type) { - case kVideoCodecVP8: - return new VCMGenericDecoder(VP8Decoder::Create()); - case kVideoCodecVP9: - return new VCMGenericDecoder(VP9Decoder::Create()); - case kVideoCodecI420: - return new VCMGenericDecoder(new I420Decoder()); - case kVideoCodecH264: - if (H264Decoder::IsSupported()) { - return new VCMGenericDecoder(H264Decoder::Create()); - } - break; - default: - break; - } - LOG(LS_WARNING) << "No internal decoder of this type exists."; - return nullptr; -} - const VCMDecoderMapItem* VCMCodecDataBase::FindDecoderItem( uint8_t payload_type) const { DecoderMap::const_iterator it = dec_map_.find(payload_type); diff --git a/webrtc/modules/video_coding/codec_database.h b/webrtc/modules/video_coding/codec_database.h index 1317e2262c..5f8656df8d 100644 --- a/webrtc/modules/video_coding/codec_database.h +++ b/webrtc/modules/video_coding/codec_database.h @@ -107,9 +107,9 @@ class VCMCodecDataBase { const VCMEncodedFrame& frame, VCMDecodedFrameCallback* decoded_frame_callback); - // Deletes the memory of the decoder instance |decoder|. Used to delete - // deep copies returned by CreateDecoderCopy(). - void ReleaseDecoder(VCMGenericDecoder* decoder) const; + // Returns the current decoder (i.e. the same value as was last returned from + // GetDecoder(); + VCMGenericDecoder* GetCurrentDecoder(); // Returns true if the currently active decoder prefer to decode frames late. // That means that frames must be decoded near the render times stamp. @@ -121,8 +121,9 @@ class VCMCodecDataBase { typedef std::map DecoderMap; typedef std::map ExternalDecoderMap; - VCMGenericDecoder* CreateAndInitDecoder(const VCMEncodedFrame& frame, - VideoCodec* new_codec) const; + std::unique_ptr CreateAndInitDecoder( + const VCMEncodedFrame& frame, + VideoCodec* new_codec) const; // Determines whether a new codec has to be created or not. // Checks every setting apart from maxFramerate and startBitrate. @@ -130,9 +131,6 @@ class VCMCodecDataBase { void DeleteEncoder(); - // Create an internal Decoder given a codec type - VCMGenericDecoder* CreateDecoder(VideoCodecType type) const; - const VCMDecoderMapItem* FindDecoderItem(uint8_t payload_type) const; const VCMExtDecoderMapItem* FindExternalDecoderItem( @@ -149,7 +147,7 @@ class VCMCodecDataBase { bool internal_source_; VCMEncodedFrameCallback* const encoded_frame_callback_; std::unique_ptr ptr_encoder_; - VCMGenericDecoder* ptr_decoder_; + std::unique_ptr ptr_decoder_; DecoderMap dec_map_; ExternalDecoderMap dec_external_map_; }; // VCMCodecDataBase diff --git a/webrtc/modules/video_coding/generic_decoder.cc b/webrtc/modules/video_coding/generic_decoder.cc index 2121ab6306..6530a07d4d 100644 --- a/webrtc/modules/video_coding/generic_decoder.cc +++ b/webrtc/modules/video_coding/generic_decoder.cc @@ -8,11 +8,12 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "webrtc/modules/video_coding/generic_decoder.h" + #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/base/trace_event.h" #include "webrtc/modules/video_coding/include/video_coding.h" -#include "webrtc/modules/video_coding/generic_decoder.h" #include "webrtc/modules/video_coding/internal_defines.h" #include "webrtc/system_wrappers/include/clock.h" @@ -23,9 +24,12 @@ VCMDecodedFrameCallback::VCMDecodedFrameCallback(VCMTiming* timing, : _clock(clock), _timing(timing), _timestampMap(kDecoderFrameMemoryLength), - _lastReceivedPictureID(0) {} + _lastReceivedPictureID(0) { + decoder_thread_.DetachFromThread(); +} VCMDecodedFrameCallback::~VCMDecodedFrameCallback() { + RTC_DCHECK(construction_thread_.CalledOnValidThread()); } void VCMDecodedFrameCallback::SetUserReceiveCallback( @@ -37,6 +41,7 @@ void VCMDecodedFrameCallback::SetUserReceiveCallback( } VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() { + RTC_DCHECK_RUN_ON(&decoder_thread_); // Called on the decode thread via VCMCodecDataBase::GetDecoder. // The callback must always have been set before this happens. RTC_DCHECK(_receiveCallback); @@ -59,16 +64,14 @@ int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, rtc::Optional decode_time_ms, rtc::Optional qp) { + RTC_DCHECK_RUN_ON(&decoder_thread_); RTC_DCHECK(_receiveCallback) << "Callback must not be null at this point"; + TRACE_EVENT_INSTANT1("webrtc", "VCMDecodedFrameCallback::Decoded", "timestamp", decodedImage.timestamp()); // TODO(holmer): We should improve this so that we can handle multiple // callbacks from one call to Decode(). - VCMFrameInformation* frameInfo; - { - rtc::CritScope cs(&lock_); - frameInfo = _timestampMap.Pop(decodedImage.timestamp()); - } + VCMFrameInformation* frameInfo = _timestampMap.Pop(decodedImage.timestamp()); if (frameInfo == NULL) { LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping " @@ -92,101 +95,115 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, int32_t VCMDecodedFrameCallback::ReceivedDecodedReferenceFrame( const uint64_t pictureId) { + RTC_DCHECK_RUN_ON(&decoder_thread_); return _receiveCallback->ReceivedDecodedReferenceFrame(pictureId); } int32_t VCMDecodedFrameCallback::ReceivedDecodedFrame( const uint64_t pictureId) { + RTC_DCHECK_RUN_ON(&decoder_thread_); _lastReceivedPictureID = pictureId; return 0; } uint64_t VCMDecodedFrameCallback::LastReceivedPictureID() const { + RTC_DCHECK_RUN_ON(&decoder_thread_); return _lastReceivedPictureID; } void VCMDecodedFrameCallback::OnDecoderImplementationName( const char* implementation_name) { + RTC_DCHECK_RUN_ON(&decoder_thread_); _receiveCallback->OnDecoderImplementationName(implementation_name); } void VCMDecodedFrameCallback::Map(uint32_t timestamp, VCMFrameInformation* frameInfo) { - rtc::CritScope cs(&lock_); + RTC_DCHECK_RUN_ON(&decoder_thread_); _timestampMap.Add(timestamp, frameInfo); } int32_t VCMDecodedFrameCallback::Pop(uint32_t timestamp) { - rtc::CritScope cs(&lock_); - if (_timestampMap.Pop(timestamp) == NULL) { - return VCM_GENERAL_ERROR; - } - return VCM_OK; + RTC_DCHECK_RUN_ON(&decoder_thread_); + return _timestampMap.Pop(timestamp) == nullptr ? VCM_GENERAL_ERROR : VCM_OK; } VCMGenericDecoder::VCMGenericDecoder(VideoDecoder* decoder, bool isExternal) : _callback(NULL), _frameInfos(), _nextFrameInfoIdx(0), - _decoder(decoder), + decoder_(decoder), _codecType(kVideoCodecUnknown), - _isExternal(isExternal), - _keyFrameDecoded(false) {} + _isExternal(isExternal) {} -VCMGenericDecoder::~VCMGenericDecoder() {} +VCMGenericDecoder::~VCMGenericDecoder() { + decoder_->Release(); + if (_isExternal) + decoder_.release(); + + RTC_DCHECK(_isExternal || !decoder_); +} int32_t VCMGenericDecoder::InitDecode(const VideoCodec* settings, int32_t numberOfCores) { + RTC_DCHECK_RUN_ON(&decoder_thread_); TRACE_EVENT0("webrtc", "VCMGenericDecoder::InitDecode"); _codecType = settings->codecType; - return _decoder->InitDecode(settings, numberOfCores); + return decoder_->InitDecode(settings, numberOfCores); } int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, int64_t nowMs) { - TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp", - frame.EncodedImage()._timeStamp); - _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs; - _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs(); - _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation(); - _callback->Map(frame.TimeStamp(), &_frameInfos[_nextFrameInfoIdx]); + RTC_DCHECK_RUN_ON(&decoder_thread_); + TRACE_EVENT2("webrtc", "VCMGenericDecoder::Decode", "timestamp", + frame.EncodedImage()._timeStamp, "decoder", + decoder_->ImplementationName()); + _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs; + _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs(); + _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation(); + _callback->Map(frame.TimeStamp(), &_frameInfos[_nextFrameInfoIdx]); - _nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength; - const RTPFragmentationHeader dummy_header; - int32_t ret = _decoder->Decode(frame.EncodedImage(), frame.MissingFrame(), - &dummy_header, - frame.CodecSpecific(), frame.RenderTimeMs()); + _nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength; + const RTPFragmentationHeader dummy_header; + int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(), + &dummy_header, frame.CodecSpecific(), + frame.RenderTimeMs()); - _callback->OnDecoderImplementationName(_decoder->ImplementationName()); + // TODO(tommi): Necessary every time? + // Maybe this should be the first thing the function does, and only the first + // time around? + _callback->OnDecoderImplementationName(decoder_->ImplementationName()); + + if (ret != WEBRTC_VIDEO_CODEC_OK) { if (ret < WEBRTC_VIDEO_CODEC_OK) { - LOG(LS_WARNING) << "Failed to decode frame with timestamp " - << frame.TimeStamp() << ", error code: " << ret; - _callback->Pop(frame.TimeStamp()); - return ret; - } else if (ret == WEBRTC_VIDEO_CODEC_NO_OUTPUT || - ret == WEBRTC_VIDEO_CODEC_REQUEST_SLI) { - // No output - _callback->Pop(frame.TimeStamp()); + LOG(LS_WARNING) << "Failed to decode frame with timestamp " + << frame.TimeStamp() << ", error code: " << ret; } - return ret; -} + // We pop the frame for all non-'OK', failure or success codes such as + // WEBRTC_VIDEO_CODEC_NO_OUTPUT and WEBRTC_VIDEO_CODEC_REQUEST_SLI. + _callback->Pop(frame.TimeStamp()); + } -int32_t VCMGenericDecoder::Release() { - return _decoder->Release(); + return ret; } int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback( VCMDecodedFrameCallback* callback) { + RTC_DCHECK_RUN_ON(&decoder_thread_); _callback = callback; - return _decoder->RegisterDecodeCompleteCallback(callback); -} - -bool VCMGenericDecoder::External() const { - return _isExternal; + return decoder_->RegisterDecodeCompleteCallback(callback); } bool VCMGenericDecoder::PrefersLateDecoding() const { - return _decoder->PrefersLateDecoding(); + RTC_DCHECK_RUN_ON(&decoder_thread_); + return decoder_->PrefersLateDecoding(); } +#if defined(WEBRTC_ANDROID) +void VCMGenericDecoder::PollDecodedFrames() { + RTC_DCHECK_RUN_ON(&decoder_thread_); + decoder_->PollDecodedFrames(); +} +#endif + } // namespace webrtc diff --git a/webrtc/modules/video_coding/generic_decoder.h b/webrtc/modules/video_coding/generic_decoder.h index 891ec893ff..1df2060247 100644 --- a/webrtc/modules/video_coding/generic_decoder.h +++ b/webrtc/modules/video_coding/generic_decoder.h @@ -11,7 +11,8 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_GENERIC_DECODER_H_ #define WEBRTC_MODULES_VIDEO_CODING_GENERIC_DECODER_H_ -#include "webrtc/base/criticalsection.h" +#include + #include "webrtc/base/thread_checker.h" #include "webrtc/modules/include/module_common_types.h" #include "webrtc/modules/video_coding/encoded_frame.h" @@ -36,6 +37,7 @@ class VCMDecodedFrameCallback : public DecodedImageCallback { public: VCMDecodedFrameCallback(VCMTiming* timing, Clock* clock); ~VCMDecodedFrameCallback() override; + void SetUserReceiveCallback(VCMReceiveCallback* receiveCallback); VCMReceiveCallback* UserReceiveCallback(); @@ -55,7 +57,7 @@ class VCMDecodedFrameCallback : public DecodedImageCallback { private: rtc::ThreadChecker construction_thread_; - // Protect |_timestampMap|. + rtc::ThreadChecker decoder_thread_; Clock* const _clock; // This callback must be set before the decoder thread starts running // and must only be unset when external threads (e.g decoder thread) @@ -63,15 +65,12 @@ class VCMDecodedFrameCallback : public DecodedImageCallback { // while there are more than one threads involved, it must be set // from the same thread, and therfore a lock is not required to access it. VCMReceiveCallback* _receiveCallback = nullptr; - VCMTiming* _timing; - rtc::CriticalSection lock_; - VCMTimestampMap _timestampMap GUARDED_BY(lock_); - uint64_t _lastReceivedPictureID; + VCMTiming* _timing ACCESS_ON(decoder_thread_); + VCMTimestampMap _timestampMap ACCESS_ON(decoder_thread_); + uint64_t _lastReceivedPictureID ACCESS_ON(decoder_thread_); }; class VCMGenericDecoder { - friend class VCMCodecDataBase; - public: explicit VCMGenericDecoder(VideoDecoder* decoder, bool isExternal = false); ~VCMGenericDecoder(); @@ -88,27 +87,31 @@ class VCMGenericDecoder { */ int32_t Decode(const VCMEncodedFrame& inputFrame, int64_t nowMs); - /** - * Free the decoder memory - */ - int32_t Release(); - /** * Set decode callback. Deregistering while decoding is illegal. */ int32_t RegisterDecodeCompleteCallback(VCMDecodedFrameCallback* callback); - bool External() const; bool PrefersLateDecoding() const; +#if defined(WEBRTC_ANDROID) + // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7361 + void PollDecodedFrames(); +#endif + + bool IsSameDecoder(VideoDecoder* decoder) const { + return decoder_.get() == decoder; + } + private: - VCMDecodedFrameCallback* _callback; - VCMFrameInformation _frameInfos[kDecoderFrameMemoryLength]; - uint32_t _nextFrameInfoIdx; - VideoDecoder* const _decoder; - VideoCodecType _codecType; - bool _isExternal; - bool _keyFrameDecoded; + rtc::ThreadChecker decoder_thread_; + VCMDecodedFrameCallback* _callback ACCESS_ON(decoder_thread_); + VCMFrameInformation _frameInfos[kDecoderFrameMemoryLength] ACCESS_ON( + decoder_thread_); + uint32_t _nextFrameInfoIdx ACCESS_ON(decoder_thread_); + std::unique_ptr decoder_; + VideoCodecType _codecType ACCESS_ON(decoder_thread_); + const bool _isExternal; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/video_coding_impl.h b/webrtc/modules/video_coding/video_coding_impl.h index 5e547df07a..d6e722db3e 100644 --- a/webrtc/modules/video_coding/video_coding_impl.h +++ b/webrtc/modules/video_coding/video_coding_impl.h @@ -35,6 +35,7 @@ namespace webrtc { +class ProcessThread; class VideoBitrateAllocator; class VideoBitrateAllocationObserver; @@ -150,7 +151,7 @@ class VideoReceiver : public Module { VCMTiming* timing, NackSender* nack_sender = nullptr, KeyFrameRequestSender* keyframe_request_sender = nullptr); - ~VideoReceiver(); + ~VideoReceiver() override; int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec, int32_t numberOfCores, @@ -168,8 +169,10 @@ class VideoReceiver : public Module { int32_t Decode(const webrtc::VCMEncodedFrame* frame); - // Called on the decoder thread when thread is exiting. - void DecodingStopped(); +#if defined(WEBRTC_ANDROID) + // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7361 + void PollDecodedFrames(); +#endif int32_t IncomingPacket(const uint8_t* incomingPayload, size_t payloadLength, @@ -195,39 +198,66 @@ class VideoReceiver : public Module { int64_t TimeUntilNextProcess() override; void Process() override; + void ProcessThreadAttached(ProcessThread* process_thread) override; void TriggerDecoderShutdown(); + void DecoderThreadStarting(); + void DecoderThreadStopped(); protected: - int32_t Decode(const webrtc::VCMEncodedFrame& frame) - EXCLUSIVE_LOCKS_REQUIRED(receive_crit_); + int32_t Decode(const webrtc::VCMEncodedFrame& frame); int32_t RequestKeyFrame(); private: + // Used for DCHECKing thread correctness. + // In build where DCHECKs are enabled, will return false before + // DecoderThreadStarting is called, then true until DecoderThreadStopped + // is called. + // In builds where DCHECKs aren't enabled, it will return true. + bool IsDecoderThreadRunning(); + rtc::ThreadChecker construction_thread_; + rtc::ThreadChecker decoder_thread_; + rtc::ThreadChecker module_thread_; Clock* const clock_; rtc::CriticalSection process_crit_; - rtc::CriticalSection receive_crit_; VCMTiming* _timing; VCMReceiver _receiver; VCMDecodedFrameCallback _decodedFrameCallback; - VCMFrameTypeCallback* _frameTypeCallback GUARDED_BY(process_crit_); - VCMReceiveStatisticsCallback* _receiveStatsCallback GUARDED_BY(process_crit_); - VCMPacketRequestCallback* _packetRequestCallback GUARDED_BY(process_crit_); - VCMFrameBuffer _frameFromFile; + // These callbacks are set on the construction thread before being attached + // to the module thread or decoding started, so a lock is not required. + VCMFrameTypeCallback* _frameTypeCallback; + VCMReceiveStatisticsCallback* _receiveStatsCallback; + VCMPacketRequestCallback* _packetRequestCallback; + + // Used on both the module and decoder thread. bool _scheduleKeyRequest GUARDED_BY(process_crit_); bool drop_frames_until_keyframe_ GUARDED_BY(process_crit_); - size_t max_nack_list_size_ GUARDED_BY(process_crit_); - VCMCodecDataBase _codecDataBase GUARDED_BY(receive_crit_); - EncodedImageCallback* pre_decode_image_callback_; + // Modified on the construction thread while not attached to the process + // thread. Once attached to the process thread, its value is only read + // so a lock is not required. + size_t max_nack_list_size_; - VCMProcessTimer _receiveStatsTimer; - VCMProcessTimer _retransmissionTimer; - VCMProcessTimer _keyRequestTimer; - QpParser qp_parser_; - ThreadUnsafeOneTimeEvent first_frame_received_; + // Callbacks are set before the decoder thread starts. + // Once the decoder thread has been started, usage of |_codecDataBase| moves + // over to the decoder thread. + VCMCodecDataBase _codecDataBase; + EncodedImageCallback* const pre_decode_image_callback_; + + VCMProcessTimer _receiveStatsTimer ACCESS_ON(module_thread_); + VCMProcessTimer _retransmissionTimer ACCESS_ON(module_thread_); + VCMProcessTimer _keyRequestTimer ACCESS_ON(module_thread_); + QpParser qp_parser_ ACCESS_ON(decoder_thread_); + ThreadUnsafeOneTimeEvent first_frame_received_ ACCESS_ON(decoder_thread_); + // Modified on the construction thread. Can be read without a lock and assumed + // to be non-null on the module and decoder threads. + ProcessThread* process_thread_ = nullptr; + bool is_attached_to_process_thread_ ACCESS_ON(construction_thread_) = false; +#if RTC_DCHECK_IS_ON + bool decoder_thread_is_running_ = false; +#endif }; } // namespace vcm diff --git a/webrtc/modules/video_coding/video_receiver.cc b/webrtc/modules/video_coding/video_receiver.cc index 84f5af521f..f1503ba89e 100644 --- a/webrtc/modules/video_coding/video_receiver.cc +++ b/webrtc/modules/video_coding/video_receiver.cc @@ -9,12 +9,14 @@ */ #include "webrtc/base/checks.h" +#include "webrtc/base/location.h" #include "webrtc/base/logging.h" #include "webrtc/base/trace_event.h" #include "webrtc/common_types.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" -#include "webrtc/modules/video_coding/include/video_codec_interface.h" +#include "webrtc/modules/utility/include/process_thread.h" #include "webrtc/modules/video_coding/encoded_frame.h" +#include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/jitter_buffer.h" #include "webrtc/modules/video_coding/packet.h" #include "webrtc/modules/video_coding/video_coding_impl.h" @@ -40,7 +42,6 @@ VideoReceiver::VideoReceiver(Clock* clock, _frameTypeCallback(nullptr), _receiveStatsCallback(nullptr), _packetRequestCallback(nullptr), - _frameFromFile(), _scheduleKeyRequest(false), drop_frames_until_keyframe_(false), max_nack_list_size_(0), @@ -48,18 +49,23 @@ VideoReceiver::VideoReceiver(Clock* clock, pre_decode_image_callback_(pre_decode_image_callback), _receiveStatsTimer(1000, clock_), _retransmissionTimer(10, clock_), - _keyRequestTimer(500, clock_) {} + _keyRequestTimer(500, clock_) { + decoder_thread_.DetachFromThread(); + module_thread_.DetachFromThread(); +} -VideoReceiver::~VideoReceiver() {} +VideoReceiver::~VideoReceiver() { + RTC_DCHECK_RUN_ON(&construction_thread_); +} void VideoReceiver::Process() { + RTC_DCHECK_RUN_ON(&module_thread_); // Receive-side statistics // TODO(philipel): Remove this if block when we know what to do with // ReceiveStatisticsProxy::QualitySample. if (_receiveStatsTimer.TimeUntilProcess() == 0) { _receiveStatsTimer.Processed(); - rtc::CritScope cs(&process_crit_); if (_receiveStatsCallback != nullptr) { _receiveStatsCallback->OnReceiveRatesUpdated(0, 0); } @@ -68,10 +74,10 @@ void VideoReceiver::Process() { // Key frame requests if (_keyRequestTimer.TimeUntilProcess() == 0) { _keyRequestTimer.Processed(); - bool request_key_frame = false; - { + bool request_key_frame = _frameTypeCallback != nullptr; + if (request_key_frame) { rtc::CritScope cs(&process_crit_); - request_key_frame = _scheduleKeyRequest && _frameTypeCallback != nullptr; + request_key_frame = _scheduleKeyRequest; } if (request_key_frame) RequestKeyFrame(); @@ -82,13 +88,8 @@ void VideoReceiver::Process() { // disabled when NACK is off. if (_retransmissionTimer.TimeUntilProcess() == 0) { _retransmissionTimer.Processed(); - bool callback_registered = false; - uint16_t length; - { - rtc::CritScope cs(&process_crit_); - length = max_nack_list_size_; - callback_registered = _packetRequestCallback != nullptr; - } + bool callback_registered = _packetRequestCallback != nullptr; + uint16_t length = max_nack_list_size_; if (callback_registered && length > 0) { // Collect sequence numbers from the default receiver. bool request_key_frame = false; @@ -98,7 +99,6 @@ void VideoReceiver::Process() { ret = RequestKeyFrame(); } if (ret == VCM_OK && !nackList.empty()) { - rtc::CritScope cs(&process_crit_); if (_packetRequestCallback != nullptr) { _packetRequestCallback->ResendPackets(&nackList[0], nackList.size()); } @@ -107,7 +107,18 @@ void VideoReceiver::Process() { } } +void VideoReceiver::ProcessThreadAttached(ProcessThread* process_thread) { + RTC_DCHECK_RUN_ON(&construction_thread_); + if (process_thread) { + is_attached_to_process_thread_ = true; + process_thread_ = process_thread; + } else { + is_attached_to_process_thread_ = false; + } +} + int64_t VideoReceiver::TimeUntilNextProcess() { + RTC_DCHECK_RUN_ON(&module_thread_); int64_t timeUntilNextProcess = _receiveStatsTimer.TimeUntilProcess(); if (_receiver.NackMode() != kNoNack) { // We need a Process call more often if we are relying on @@ -122,7 +133,7 @@ int64_t VideoReceiver::TimeUntilNextProcess() { } int32_t VideoReceiver::SetReceiveChannelParameters(int64_t rtt) { - rtc::CritScope cs(&receive_crit_); + RTC_DCHECK_RUN_ON(&module_thread_); _receiver.UpdateRtt(rtt); return 0; } @@ -142,7 +153,6 @@ int32_t VideoReceiver::SetVideoProtection(VCMVideoProtection videoProtection, } case kProtectionNackFEC: { - rtc::CritScope cs(&receive_crit_); RTC_DCHECK(enable); _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, @@ -165,20 +175,22 @@ int32_t VideoReceiver::SetVideoProtection(VCMVideoProtection videoProtection, // ready for rendering. int32_t VideoReceiver::RegisterReceiveCallback( VCMReceiveCallback* receiveCallback) { - RTC_DCHECK(construction_thread_.CalledOnValidThread()); - // TODO(tommi): Callback may be null, but only after the decoder thread has - // been stopped. Use the signal we now get that tells us when the decoder - // thread isn't running, to DCHECK that the method is never called while it - // is. Once we're confident, we can remove the lock. - rtc::CritScope cs(&receive_crit_); + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(!IsDecoderThreadRunning()); + // This value is set before the decoder thread starts and unset after + // the decoder thread has been stopped. _decodedFrameCallback.SetUserReceiveCallback(receiveCallback); return VCM_OK; } int32_t VideoReceiver::RegisterReceiveStatisticsCallback( VCMReceiveStatisticsCallback* receiveStats) { - RTC_DCHECK(construction_thread_.CalledOnValidThread()); - rtc::CritScope cs(&process_crit_); + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(!IsDecoderThreadRunning() && !is_attached_to_process_thread_); + // |_receiver| is used on both the decoder and module threads. + // However, since we make sure that we never do anything on the module thread + // when the decoder thread is not running, we don't need a lock for the + // |_receiver| or |_receiveStatsCallback| here. _receiver.RegisterStatsCallback(receiveStats); _receiveStatsCallback = receiveStats; return VCM_OK; @@ -187,10 +199,8 @@ int32_t VideoReceiver::RegisterReceiveStatisticsCallback( // Register an externally defined decoder object. void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder, uint8_t payloadType) { - RTC_DCHECK(construction_thread_.CalledOnValidThread()); - // TODO(tommi): This method must be called when the decoder thread is not - // running. Do we need a lock in that case? - rtc::CritScope cs(&receive_crit_); + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(!IsDecoderThreadRunning()); if (externalDecoder == nullptr) { RTC_CHECK(_codecDataBase.DeregisterExternalDecoder(payloadType)); return; @@ -201,53 +211,87 @@ void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder, // Register a frame type request callback. int32_t VideoReceiver::RegisterFrameTypeCallback( VCMFrameTypeCallback* frameTypeCallback) { - rtc::CritScope cs(&process_crit_); + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(!IsDecoderThreadRunning() && !is_attached_to_process_thread_); + // This callback is used on the module thread, but since we don't get + // callbacks on the module thread while the decoder thread isn't running + // (and this function must not be called when the decoder is running), + // we don't need a lock here. _frameTypeCallback = frameTypeCallback; return VCM_OK; } int32_t VideoReceiver::RegisterPacketRequestCallback( VCMPacketRequestCallback* callback) { - rtc::CritScope cs(&process_crit_); + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(!IsDecoderThreadRunning() && !is_attached_to_process_thread_); + // This callback is used on the module thread, but since we don't get + // callbacks on the module thread while the decoder thread isn't running + // (and this function must not be called when the decoder is running), + // we don't need a lock here. _packetRequestCallback = callback; return VCM_OK; } void VideoReceiver::TriggerDecoderShutdown() { - RTC_DCHECK(construction_thread_.CalledOnValidThread()); + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(IsDecoderThreadRunning()); _receiver.TriggerDecoderShutdown(); } +void VideoReceiver::DecoderThreadStarting() { + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(!IsDecoderThreadRunning()); + if (process_thread_ && !is_attached_to_process_thread_) { + process_thread_->RegisterModule(this, RTC_FROM_HERE); + } +#if RTC_DCHECK_IS_ON + decoder_thread_is_running_ = true; +#endif +} + +void VideoReceiver::DecoderThreadStopped() { + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(IsDecoderThreadRunning()); + if (process_thread_ && is_attached_to_process_thread_) { + process_thread_->DeRegisterModule(this); + } +#if RTC_DCHECK_IS_ON + decoder_thread_is_running_ = false; + decoder_thread_.DetachFromThread(); +#endif +} + // Decode next frame, blocking. // Should be called as often as possible to get the most out of the decoder. int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) { - bool prefer_late_decoding = false; - { - // TODO(tommi): Chances are that this lock isn't required. - rtc::CritScope cs(&receive_crit_); - prefer_late_decoding = _codecDataBase.PrefersLateDecoding(); - } - - VCMEncodedFrame* frame = - _receiver.FrameForDecoding(maxWaitTimeMs, prefer_late_decoding); + RTC_DCHECK_RUN_ON(&decoder_thread_); + VCMEncodedFrame* frame = _receiver.FrameForDecoding( + maxWaitTimeMs, _codecDataBase.PrefersLateDecoding()); if (!frame) return VCM_FRAME_NOT_READY; + bool drop_frame = false; { rtc::CritScope cs(&process_crit_); if (drop_frames_until_keyframe_) { // Still getting delta frames, schedule another keyframe request as if // decode failed. if (frame->FrameType() != kVideoFrameKey) { + drop_frame = true; _scheduleKeyRequest = true; - _receiver.ReleaseFrame(frame); - return VCM_FRAME_NOT_READY; + } else { + drop_frames_until_keyframe_ = false; } - drop_frames_until_keyframe_ = false; } } + if (drop_frame) { + _receiver.ReleaseFrame(frame); + return VCM_FRAME_NOT_READY; + } + if (pre_decode_image_callback_) { EncodedImage encoded_image(frame->EncodedImage()); int qp = -1; @@ -258,7 +302,6 @@ int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) { frame->CodecSpecific(), nullptr); } - rtc::CritScope cs(&receive_crit_); // If this frame was too late, we should adjust the delay accordingly _timing->UpdateCurrentDelay(frame->RenderTimeMs(), clock_->TimeInMilliseconds()); @@ -278,7 +321,7 @@ int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) { // TODO(philipel): Clean up among the Decode functions as we replace // VCMEncodedFrame with FrameObject. int32_t VideoReceiver::Decode(const webrtc::VCMEncodedFrame* frame) { - rtc::CritScope lock(&receive_crit_); + RTC_DCHECK_RUN_ON(&decoder_thread_); if (pre_decode_image_callback_) { EncodedImage encoded_image(frame->EncodedImage()); int qp = -1; @@ -291,19 +334,20 @@ int32_t VideoReceiver::Decode(const webrtc::VCMEncodedFrame* frame) { return Decode(*frame); } -void VideoReceiver::DecodingStopped() { - // No further calls to Decode() will be made after this point. - // TODO(tommi): Make use of this to clarify and check threading model. -} - int32_t VideoReceiver::RequestKeyFrame() { + RTC_DCHECK_RUN_ON(&module_thread_); + + // Since we deregister from the module thread when the decoder thread isn't + // running, we should get no calls here if decoding isn't being done. + RTC_DCHECK(IsDecoderThreadRunning()); + TRACE_EVENT0("webrtc", "RequestKeyFrame"); - rtc::CritScope cs(&process_crit_); if (_frameTypeCallback != nullptr) { const int32_t ret = _frameTypeCallback->RequestKeyFrame(); if (ret < 0) { return ret; } + rtc::CritScope cs(&process_crit_); _scheduleKeyRequest = false; } else { return VCM_MISSING_CALLBACK; @@ -313,6 +357,7 @@ int32_t VideoReceiver::RequestKeyFrame() { // Must be called from inside the receive side critical section. int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) { + RTC_DCHECK_RUN_ON(&decoder_thread_); TRACE_EVENT0("webrtc", "VideoReceiver::Decode"); // Change decoder if payload type has changed VCMGenericDecoder* decoder = @@ -324,31 +369,41 @@ int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) { int32_t ret = decoder->Decode(frame, clock_->TimeInMilliseconds()); // Check for failed decoding, run frame type request callback if needed. - bool request_key_frame = false; - if (ret < 0) { - request_key_frame = true; - } + bool request_key_frame = (ret < 0); if (!frame.Complete() || frame.MissingFrame()) { request_key_frame = true; ret = VCM_OK; } + if (request_key_frame) { rtc::CritScope cs(&process_crit_); - _scheduleKeyRequest = true; + if (!_scheduleKeyRequest) { + _scheduleKeyRequest = true; + // TODO(tommi): Consider if we could instead post a task to the module + // thread and call RequestKeyFrame directly. Here we call WakeUp so that + // TimeUntilNextProcess() gets called straight away. + process_thread_->WakeUp(this); + } } return ret; } +#if defined(WEBRTC_ANDROID) +void VideoReceiver::PollDecodedFrames() { + RTC_DCHECK_RUN_ON(&decoder_thread_); + auto* current_decoder = _codecDataBase.GetCurrentDecoder(); + if (current_decoder) + current_decoder->PollDecodedFrames(); +} +#endif + // Register possible receive codecs, can be called multiple times int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec, int32_t numberOfCores, bool requireKeyFrame) { - RTC_DCHECK(construction_thread_.CalledOnValidThread()); - // TODO(tommi): This method must only be called when the decoder thread - // is not running. Do we need a lock? If not, it looks like we might not need - // a lock at all for |_codecDataBase|. - rtc::CritScope cs(&receive_crit_); + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(!IsDecoderThreadRunning()); if (receiveCodec == nullptr) { return VCM_PARAMETER_ERROR; } @@ -363,6 +418,7 @@ int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec, int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload, size_t payloadLength, const WebRtcRTPHeader& rtpInfo) { + RTC_DCHECK_RUN_ON(&module_thread_); if (rtpInfo.frameType == kVideoFrameKey) { TRACE_EVENT1("webrtc", "VCM::PacketKeyFrame", "seqnum", rtpInfo.header.sequenceNumber); @@ -394,6 +450,7 @@ int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload, // to sync with audio. Not included in VideoCodingModule::Delay() // Defaults to 0 ms. int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) { + RTC_DCHECK_RUN_ON(&module_thread_); _timing->set_min_playout_delay(minPlayoutDelayMs); return VCM_OK; } @@ -401,22 +458,24 @@ int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) { // The estimated delay caused by rendering, defaults to // kDefaultRenderDelayMs = 10 ms int32_t VideoReceiver::SetRenderDelay(uint32_t timeMS) { + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(!IsDecoderThreadRunning()); _timing->set_render_delay(timeMS); return VCM_OK; } // Current video delay int32_t VideoReceiver::Delay() const { + RTC_DCHECK_RUN_ON(&module_thread_); return _timing->TargetVideoDelay(); } +// Only used by VCMRobustnessTest. int VideoReceiver::SetReceiverRobustnessMode( VideoCodingModule::ReceiverRobustness robustnessMode, VCMDecodeErrorMode decode_error_mode) { - RTC_DCHECK(construction_thread_.CalledOnValidThread()); - // TODO(tommi): This method must only be called when the decoder thread - // is not running and we don't need to hold this lock. - rtc::CritScope cs(&receive_crit_); + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(!IsDecoderThreadRunning()); switch (robustnessMode) { case VideoCodingModule::kNone: _receiver.SetNackMode(kNoNack, -1, -1); @@ -434,24 +493,40 @@ int VideoReceiver::SetReceiverRobustnessMode( } void VideoReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) { - rtc::CritScope cs(&receive_crit_); + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(!IsDecoderThreadRunning()); _receiver.SetDecodeErrorMode(decode_error_mode); } void VideoReceiver::SetNackSettings(size_t max_nack_list_size, int max_packet_age_to_nack, int max_incomplete_time_ms) { - if (max_nack_list_size != 0) { - rtc::CritScope cs(&process_crit_); + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(!IsDecoderThreadRunning()); + + if (max_nack_list_size != 0) max_nack_list_size_ = max_nack_list_size; - } _receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack, max_incomplete_time_ms); } int VideoReceiver::SetMinReceiverDelay(int desired_delay_ms) { + RTC_DCHECK_RUN_ON(&construction_thread_); + RTC_DCHECK(!IsDecoderThreadRunning()); + // TODO(tommi): Is the method only used by tests? Maybe could be offered + // via a test only subclass? + // Info from Stefan: If it is indeed only used by tests I think it's just that + // it hasn't been cleaned up when the calling code was cleaned up. return _receiver.SetMinReceiverDelay(desired_delay_ms); } +bool VideoReceiver::IsDecoderThreadRunning() { +#if RTC_DCHECK_IS_ON + return decoder_thread_is_running_; +#else + return true; +#endif +} + } // namespace vcm } // namespace webrtc diff --git a/webrtc/sdk/android/src/jni/androidmediadecoder_jni.cc b/webrtc/sdk/android/src/jni/androidmediadecoder_jni.cc index 8f50fc2107..6bd749a3bc 100644 --- a/webrtc/sdk/android/src/jni/androidmediadecoder_jni.cc +++ b/webrtc/sdk/android/src/jni/androidmediadecoder_jni.cc @@ -36,6 +36,18 @@ #include "webrtc/sdk/android/src/jni/surfacetexturehelper_jni.h" #include "webrtc/system_wrappers/include/logcat_trace_context.h" +// Logging macros. +#define TAG_DECODER "MediaCodecVideoDecoder" +#ifdef TRACK_BUFFER_TIMING +#define ALOGV(...) +__android_log_print(ANDROID_LOG_VERBOSE, TAG_DECODER, __VA_ARGS__) +#else +#define ALOGV(...) +#endif +#define ALOGD LOG_TAG(rtc::LS_INFO, TAG_DECODER) +#define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_DECODER) +#define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_DECODER) + using rtc::Bind; using rtc::Thread; using rtc::ThreadManager; @@ -53,22 +65,7 @@ using webrtc::kVideoCodecVP9; namespace webrtc_jni { -// Logging macros. -#define TAG_DECODER "MediaCodecVideoDecoder" -#ifdef TRACK_BUFFER_TIMING -#define ALOGV(...) - __android_log_print(ANDROID_LOG_VERBOSE, TAG_DECODER, __VA_ARGS__) -#else -#define ALOGV(...) -#endif -#define ALOGD LOG_TAG(rtc::LS_INFO, TAG_DECODER) -#define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_DECODER) -#define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_DECODER) - -enum { kMaxWarningLogFrames = 2 }; - -class MediaCodecVideoDecoder : public webrtc::VideoDecoder, - public rtc::MessageHandler { +class MediaCodecVideoDecoder : public webrtc::VideoDecoder { public: explicit MediaCodecVideoDecoder( JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context); @@ -83,6 +80,8 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder, const CodecSpecificInfo* codecSpecificInfo = NULL, int64_t renderTimeMs = -1) override; + void PollDecodedFrames() override; + int32_t RegisterDecodeCompleteCallback(DecodedImageCallback* callback) override; @@ -90,22 +89,41 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder, bool PrefersLateDecoding() const override { return true; } - // rtc::MessageHandler implementation. - void OnMessage(rtc::Message* msg) override; - const char* ImplementationName() const override; private: - // CHECK-fail if not running on |codec_thread_|. - void CheckOnCodecThread(); + struct DecodedFrame { + DecodedFrame(VideoFrame frame, + int decode_time_ms, + int64_t timestamp, + int64_t ntp_timestamp, + rtc::Optional qp) + : frame(std::move(frame)), + decode_time_ms(decode_time_ms), + qp(std::move(qp)) { + frame.set_timestamp(timestamp); + frame.set_ntp_time_ms(ntp_timestamp); + } + + VideoFrame frame; + int decode_time_ms; + rtc::Optional qp; + }; + + // Returns true if running on |codec_thread_|. Used for DCHECKing. + bool IsOnCodecThread(); int32_t InitDecodeOnCodecThread(); int32_t ResetDecodeOnCodecThread(); int32_t ReleaseOnCodecThread(); - int32_t DecodeOnCodecThread(const EncodedImage& inputImage); + int32_t DecodeOnCodecThread(const EncodedImage& inputImage, + std::vector* frames); + void PollDecodedFramesOnCodecThread(std::vector* frames); // Deliver any outputs pending in the MediaCodec to our |callback_| and return // true on success. - bool DeliverPendingOutputs(JNIEnv* jni, int dequeue_timeout_us); + bool DeliverPendingOutputs(JNIEnv* jni, + int dequeue_timeout_us, + std::vector* frames); int32_t ProcessHWErrorOnCodecThread(); void EnableFrameLogOnWarning(); void ResetVariables(); @@ -179,6 +197,9 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder, // Global references; must be deleted in Release(). std::vector input_buffers_; + + // Added to on the codec thread, frames are delivered on the decoder thread. + std::vector decoded_frames_; }; MediaCodecVideoDecoder::MediaCodecVideoDecoder( @@ -200,7 +221,7 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder( "", "()V"))) { codec_thread_->SetName("MediaCodecVideoDecoder", NULL); - RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder"; + RTC_CHECK(codec_thread_->Start()); j_init_decode_method_ = GetMethodID( jni, *j_media_codec_video_decoder_class_, "initDecode", @@ -295,7 +316,7 @@ int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst, return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } // Factory should guard against other codecs being used with us. - RTC_CHECK(inst->codecType == codecType_) + RTC_DCHECK(inst->codecType == codecType_) << "Unsupported codec " << inst->codecType << " for " << codecType_; if (sw_fallback_required_) { @@ -316,7 +337,7 @@ int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst, } void MediaCodecVideoDecoder::ResetVariables() { - CheckOnCodecThread(); + RTC_DCHECK(IsOnCodecThread()); key_frame_required_ = true; frames_received_ = 0; @@ -331,7 +352,7 @@ void MediaCodecVideoDecoder::ResetVariables() { } int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { - CheckOnCodecThread(); + RTC_DCHECK(IsOnCodecThread()); JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); ALOGD << "InitDecodeOnCodecThread Type: " << (int)codecType_ << ". " @@ -405,13 +426,11 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { } } - codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this); - return WEBRTC_VIDEO_CODEC_OK; } int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() { - CheckOnCodecThread(); + RTC_DCHECK(IsOnCodecThread()); JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); ALOGD << "ResetDecodeOnCodecThread Type: " << (int)codecType_ << ". " @@ -420,7 +439,6 @@ int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() { ". Frames decoded: " << frames_decoded_; inited_ = false; - rtc::MessageQueueManager::Clear(this); ResetVariables(); jni->CallVoidMethod( @@ -436,8 +454,6 @@ int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() { } inited_ = true; - codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this); - return WEBRTC_VIDEO_CODEC_OK; } @@ -448,10 +464,10 @@ int32_t MediaCodecVideoDecoder::Release() { } int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() { + RTC_DCHECK(IsOnCodecThread()); if (!inited_) { return WEBRTC_VIDEO_CODEC_OK; } - CheckOnCodecThread(); JNIEnv* jni = AttachCurrentThreadIfNeeded(); ALOGD << "DecoderReleaseOnCodecThread: Frames received: " << frames_received_ << ". Frames decoded: " << frames_decoded_; @@ -463,7 +479,6 @@ int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() { jni->CallVoidMethod(*j_media_codec_video_decoder_, j_release_method_); surface_texture_helper_ = nullptr; inited_ = false; - rtc::MessageQueueManager::Clear(this); if (CheckException(jni)) { ALOGE << "Decoder release exception"; return WEBRTC_VIDEO_CODEC_ERROR; @@ -472,19 +487,19 @@ int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() { return WEBRTC_VIDEO_CODEC_OK; } -void MediaCodecVideoDecoder::CheckOnCodecThread() { - RTC_CHECK(codec_thread_.get() == ThreadManager::Instance()->CurrentThread()) - << "Running on wrong thread!"; +bool MediaCodecVideoDecoder::IsOnCodecThread() { + return codec_thread_.get() == ThreadManager::Instance()->CurrentThread(); } void MediaCodecVideoDecoder::EnableFrameLogOnWarning() { // Log next 2 output frames. + static const int kMaxWarningLogFrames = 2; frames_decoded_logged_ = std::max( frames_decoded_logged_, frames_decoded_ + kMaxWarningLogFrames); } int32_t MediaCodecVideoDecoder::ProcessHWErrorOnCodecThread() { - CheckOnCodecThread(); + RTC_DCHECK(IsOnCodecThread()); int ret_val = ReleaseOnCodecThread(); if (ret_val < 0) { ALOGE << "ProcessHWError: Release failure"; @@ -515,22 +530,17 @@ int32_t MediaCodecVideoDecoder::Decode( const RTPFragmentationHeader* fragmentation, const CodecSpecificInfo* codecSpecificInfo, int64_t renderTimeMs) { + RTC_DCHECK(callback_); + RTC_DCHECK(inited_); + if (sw_fallback_required_) { ALOGE << "Decode() - fallback to SW codec"; return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; } - if (callback_ == NULL) { - ALOGE << "Decode() - callback_ is NULL"; - return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } if (inputImage._buffer == NULL && inputImage._length > 0) { ALOGE << "Decode() - inputImage is incorrect"; return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } - if (!inited_) { - ALOGE << "Decode() - decoder is not initialized"; - return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } // Check if encoded frame dimension has changed. if ((inputImage._encodedWidth * inputImage._encodedHeight > 0) && @@ -575,14 +585,32 @@ int32_t MediaCodecVideoDecoder::Decode( return WEBRTC_VIDEO_CODEC_ERROR; } - return codec_thread_->Invoke( + std::vector frames; + int32_t ret = codec_thread_->Invoke( + RTC_FROM_HERE, Bind(&MediaCodecVideoDecoder::DecodeOnCodecThread, this, + inputImage, &frames)); + for (auto& f : frames) + callback_->Decoded(f.frame, rtc::Optional(f.decode_time_ms), f.qp); + return ret; +} + +void MediaCodecVideoDecoder::PollDecodedFrames() { + RTC_DCHECK(callback_); + + std::vector frames; + codec_thread_->Invoke( RTC_FROM_HERE, - Bind(&MediaCodecVideoDecoder::DecodeOnCodecThread, this, inputImage)); + Bind(&MediaCodecVideoDecoder::PollDecodedFramesOnCodecThread, this, + &frames)); + + for (auto& f : frames) + callback_->Decoded(f.frame, rtc::Optional(f.decode_time_ms), f.qp); } int32_t MediaCodecVideoDecoder::DecodeOnCodecThread( - const EncodedImage& inputImage) { - CheckOnCodecThread(); + const EncodedImage& inputImage, + std::vector* frames) { + RTC_DCHECK(IsOnCodecThread()); JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); @@ -598,7 +626,7 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread( const int64 drain_start = rtc::TimeMillis(); while ((frames_received_ > frames_decoded_ + max_pending_frames_) && (rtc::TimeMillis() - drain_start) < kMediaCodecTimeoutMs) { - if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) { + if (!DeliverPendingOutputs(jni, kMediaCodecPollMs, frames)) { ALOGE << "DeliverPendingOutputs error. Frames received: " << frames_received_ << ". Frames decoded: " << frames_decoded_; return ProcessHWErrorOnCodecThread(); @@ -618,7 +646,7 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread( ". Retry DeliverPendingOutputs."; EnableFrameLogOnWarning(); // Try to drain the decoder. - if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) { + if (!DeliverPendingOutputs(jni, kMediaCodecPollMs, frames)) { ALOGE << "DeliverPendingOutputs error. Frames received: " << frames_received_ << ". Frames decoded: " << frames_decoded_; return ProcessHWErrorOnCodecThread(); @@ -636,7 +664,7 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread( jobject j_input_buffer = input_buffers_[j_input_buffer_index]; uint8_t* buffer = reinterpret_cast(jni->GetDirectBufferAddress(j_input_buffer)); - RTC_CHECK(buffer) << "Indirect buffer??"; + RTC_DCHECK(buffer) << "Indirect buffer??"; int64_t buffer_capacity = jni->GetDirectBufferCapacity(j_input_buffer); if (CheckException(jni) || buffer_capacity < inputImage._length) { ALOGE << "Input frame size "<< inputImage._length << @@ -689,7 +717,7 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread( } // Try to drain the decoder - if (!DeliverPendingOutputs(jni, 0)) { + if (!DeliverPendingOutputs(jni, 0, frames)) { ALOGE << "DeliverPendingOutputs error"; return ProcessHWErrorOnCodecThread(); } @@ -697,9 +725,26 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread( return WEBRTC_VIDEO_CODEC_OK; } +void MediaCodecVideoDecoder::PollDecodedFramesOnCodecThread( + std::vector* frames) { + RTC_DCHECK(IsOnCodecThread()); + + JNIEnv* jni = AttachCurrentThreadIfNeeded(); + ScopedLocalRefFrame local_ref_frame(jni); + + if (!DeliverPendingOutputs(jni, 0, frames)) { + ALOGE << "PollDecodedFramesOnCodecThread: DeliverPendingOutputs error"; + ProcessHWErrorOnCodecThread(); + } +} + bool MediaCodecVideoDecoder::DeliverPendingOutputs( - JNIEnv* jni, int dequeue_timeout_ms) { - CheckOnCodecThread(); + JNIEnv* jni, + int dequeue_timeout_ms, + std::vector* frames) { + RTC_DCHECK(IsOnCodecThread()); + RTC_DCHECK(frames); + if (frames_received_ <= frames_decoded_) { // No need to query for output buffers - decoder is drained. return true; @@ -808,7 +853,7 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs( rtc::scoped_refptr i420_buffer = decoded_frame_pool_.CreateBuffer(width, height); if (color_format == COLOR_FormatYUV420Planar) { - RTC_CHECK_EQ(0, stride % 2); + RTC_DCHECK_EQ(0, stride % 2); const int uv_stride = stride / 2; const uint8_t* y_ptr = payload; const uint8_t* u_ptr = y_ptr + stride * slice_height; @@ -834,7 +879,7 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs( i420_buffer->MutableDataV(), i420_buffer->StrideV(), chroma_width, chroma_height); if (slice_height % 2 == 1) { - RTC_CHECK_EQ(height, slice_height); + RTC_DCHECK_EQ(height, slice_height); // Duplicate the last chroma rows. uint8_t* u_last_row_ptr = i420_buffer->MutableDataU() + chroma_height * i420_buffer->StrideU(); @@ -909,9 +954,16 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs( rtc::Optional qp = pending_frame_qps_.front(); pending_frame_qps_.pop_front(); - callback_->Decoded(decoded_frame, rtc::Optional(decode_time_ms), - qp); + decoded_frames_.push_back(DecodedFrame(std::move(decoded_frame), + decode_time_ms, output_timestamps_ms, + output_ntp_timestamps_ms, qp)); } + + frames->reserve(frames->size() + decoded_frames_.size()); + std::move(decoded_frames_.begin(), decoded_frames_.end(), + std::back_inserter(*frames)); + decoded_frames_.clear(); + return true; } @@ -921,26 +973,6 @@ int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback( return WEBRTC_VIDEO_CODEC_OK; } -void MediaCodecVideoDecoder::OnMessage(rtc::Message* msg) { - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - ScopedLocalRefFrame local_ref_frame(jni); - if (!inited_) { - return; - } - // We only ever send one message to |this| directly (not through a Bind()'d - // functor), so expect no ID/data. - RTC_CHECK(!msg->message_id) << "Unexpected message!"; - RTC_CHECK(!msg->pdata) << "Unexpected message!"; - CheckOnCodecThread(); - - if (!DeliverPendingOutputs(jni, 0)) { - ALOGE << "OnMessage: DeliverPendingOutputs error"; - ProcessHWErrorOnCodecThread(); - return; - } - codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this); -} - MediaCodecVideoDecoderFactory::MediaCodecVideoDecoderFactory() : egl_context_(nullptr) { ALOGD << "MediaCodecVideoDecoderFactory ctor"; diff --git a/webrtc/video/video_receive_stream.cc b/webrtc/video/video_receive_stream.cc index c59af899a4..1549147e2f 100644 --- a/webrtc/video/video_receive_stream.cc +++ b/webrtc/video/video_receive_stream.cc @@ -20,6 +20,7 @@ #include "webrtc/base/location.h" #include "webrtc/base/logging.h" #include "webrtc/base/optional.h" +#include "webrtc/base/timeutils.h" #include "webrtc/base/trace_event.h" #include "webrtc/common_types.h" #include "webrtc/common_video/h264/profile_level_id.h" @@ -168,19 +169,21 @@ VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) { namespace internal { -VideoReceiveStream::VideoReceiveStream( - int num_cpu_cores, - PacketRouter* packet_router, - VideoReceiveStream::Config config, - ProcessThread* process_thread, - CallStats* call_stats, - VieRemb* remb) +VideoReceiveStream::VideoReceiveStream(int num_cpu_cores, + PacketRouter* packet_router, + VideoReceiveStream::Config config, + ProcessThread* process_thread, + CallStats* call_stats, + VieRemb* remb) : transport_adapter_(config.rtcp_send_transport), config_(std::move(config)), num_cpu_cores_(num_cpu_cores), process_thread_(process_thread), clock_(Clock::GetRealTimeClock()), - decode_thread_(DecodeThreadFunction, this, "DecodingThread"), + decode_thread_(&DecodeThreadFunction, + this, + "DecodingThread", + rtc::kHighestPriority), call_stats_(call_stats), timing_(new VCMTiming(clock_)), video_receiver_(clock_, nullptr, this, timing_.get(), this, this), @@ -221,7 +224,6 @@ VideoReceiveStream::VideoReceiveStream( frame_buffer_.reset(new video_coding::FrameBuffer( clock_, jitter_estimator_.get(), timing_.get(), &stats_proxy_)); - process_thread_->RegisterModule(&video_receiver_, RTC_FROM_HERE); process_thread_->RegisterModule(&rtp_stream_sync_, RTC_FROM_HERE); } @@ -231,7 +233,6 @@ VideoReceiveStream::~VideoReceiveStream() { Stop(); process_thread_->DeRegisterModule(&rtp_stream_sync_); - process_thread_->DeRegisterModule(&video_receiver_); } void VideoReceiveStream::SignalNetworkState(NetworkState state) { @@ -239,7 +240,6 @@ void VideoReceiveStream::SignalNetworkState(NetworkState state) { rtp_stream_receiver_.SignalNetworkState(state); } - bool VideoReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) { return rtp_stream_receiver_.DeliverRtcp(packet, length); } @@ -302,25 +302,31 @@ void VideoReceiveStream::Start() { &stats_proxy_, renderer)); // Register the channel to receive stats updates. call_stats_->RegisterStatsObserver(video_stream_decoder_.get()); + + video_receiver_.DecoderThreadStarting(); + process_thread_->RegisterModule(&video_receiver_, RTC_FROM_HERE); + // Start the decode thread decode_thread_.Start(); - decode_thread_.SetPriority(rtc::kHighestPriority); rtp_stream_receiver_.StartReceive(); } void VideoReceiveStream::Stop() { RTC_DCHECK_RUN_ON(&worker_thread_checker_); rtp_stream_receiver_.StopReceive(); - // TriggerDecoderShutdown will release any waiting decoder thread and make it - // stop immediately, instead of waiting for a timeout. Needs to be called - // before joining the decoder thread thread. - video_receiver_.TriggerDecoderShutdown(); frame_buffer_->Stop(); call_stats_->DeregisterStatsObserver(&rtp_stream_receiver_); + process_thread_->DeRegisterModule(&video_receiver_); if (decode_thread_.IsRunning()) { + // TriggerDecoderShutdown will release any waiting decoder thread and make + // it stop immediately, instead of waiting for a timeout. Needs to be called + // before joining the decoder thread thread. + video_receiver_.TriggerDecoderShutdown(); + decode_thread_.Stop(); + video_receiver_.DecoderThreadStopped(); // Deregister external decoders so they are no longer running during // destruction. This effectively stops the VCM since the decoder thread is // stopped, the VCM is deregistered and no asynchronous decoder threads are @@ -464,30 +470,48 @@ void VideoReceiveStream::SetMinimumPlayoutDelay(int delay_ms) { video_receiver_.SetMinimumPlayoutDelay(delay_ms); } -bool VideoReceiveStream::DecodeThreadFunction(void* ptr) { - return static_cast(ptr)->Decode(); +void VideoReceiveStream::DecodeThreadFunction(void* ptr) { + while (static_cast(ptr)->Decode()) { + } } bool VideoReceiveStream::Decode() { TRACE_EVENT0("webrtc", "VideoReceiveStream::Decode"); static const int kMaxWaitForFrameMs = 3000; std::unique_ptr frame; - video_coding::FrameBuffer::ReturnReason res = - frame_buffer_->NextFrame(kMaxWaitForFrameMs, &frame); - if (res == video_coding::FrameBuffer::ReturnReason::kStopped) { - video_receiver_.DecodingStopped(); + video_coding::FrameBuffer::ReturnReason res; +#if defined(WEBRTC_ANDROID) + // This is a temporary workaround for video capture on Android in order to + // deliver asynchronously delivered frames, on the decoder thread. + // More details here: + // https://bugs.chromium.org/p/webrtc/issues/detail?id=7361 + static const int kPollIntervalMs = 10; + int time_remaining = kMaxWaitForFrameMs; + do { + res = frame_buffer_->NextFrame(kPollIntervalMs, &frame); + if (res != video_coding::FrameBuffer::ReturnReason::kTimeout) + break; + time_remaining -= kPollIntervalMs; + video_receiver_.PollDecodedFrames(); + } while (time_remaining > 0); +#else + res = frame_buffer_->NextFrame(kMaxWaitForFrameMs, &frame); +#endif + + if (res == video_coding::FrameBuffer::ReturnReason::kStopped) return false; - } if (frame) { if (video_receiver_.Decode(frame.get()) == VCM_OK) rtp_stream_receiver_.FrameDecoded(frame->picture_id); } else { + RTC_DCHECK_EQ(res, video_coding::FrameBuffer::ReturnReason::kTimeout); LOG(LS_WARNING) << "No decodable frame in " << kMaxWaitForFrameMs << " ms, requesting keyframe."; RequestKeyFrame(); } + return true; } } // namespace internal diff --git a/webrtc/video/video_receive_stream.h b/webrtc/video/video_receive_stream.h index cb2c73fe28..3248a4fb27 100644 --- a/webrtc/video/video_receive_stream.h +++ b/webrtc/video/video_receive_stream.h @@ -109,7 +109,7 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, void SetMinimumPlayoutDelay(int delay_ms) override; private: - static bool DecodeThreadFunction(void* ptr); + static void DecodeThreadFunction(void* ptr); bool Decode(); rtc::ThreadChecker worker_thread_checker_; diff --git a/webrtc/video_decoder.h b/webrtc/video_decoder.h index 70c09129f7..fd487849ce 100644 --- a/webrtc/video_decoder.h +++ b/webrtc/video_decoder.h @@ -68,6 +68,13 @@ class VideoDecoder { const CodecSpecificInfo* codec_specific_info = NULL, int64_t render_time_ms = -1) = 0; +#if defined(WEBRTC_ANDROID) + // This is a temporary remedy while the Android capture implementation is + // being changed to deliver frames on the decoder thread without polling. + // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7361 + virtual void PollDecodedFrames() {} +#endif + virtual int32_t RegisterDecodeCompleteCallback( DecodedImageCallback* callback) = 0;