From f7eb75be1abe1c45d94881106828eef8cb4d8556 Mon Sep 17 00:00:00 2001 From: "andresp@webrtc.org" Date: Sat, 14 Sep 2013 00:25:28 +0000 Subject: [PATCH] Split VideoCodingModuleImpl into VideoSender and VideoReceiver. Only implmentation is changed the interface to the module is unchanged for now. R=mikhal@webrtc.org Review URL: https://webrtc-codereview.appspot.com/2200008 git-svn-id: http://webrtc.googlecode.com/svn/trunk@4746 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../main/source/video_coding.gypi | 2 + .../main/source/video_coding_impl.cc | 1557 ++++------------- .../main/source/video_coding_impl.h | 457 ++--- .../main/source/video_receiver.cc | 775 ++++++++ .../video_coding/main/source/video_sender.cc | 420 +++++ 5 files changed, 1674 insertions(+), 1537 deletions(-) create mode 100644 webrtc/modules/video_coding/main/source/video_receiver.cc create mode 100644 webrtc/modules/video_coding/main/source/video_sender.cc diff --git a/webrtc/modules/video_coding/main/source/video_coding.gypi b/webrtc/modules/video_coding/main/source/video_coding.gypi index f4ecc5bfdd..0c8644bc52 100644 --- a/webrtc/modules/video_coding/main/source/video_coding.gypi +++ b/webrtc/modules/video_coding/main/source/video_coding.gypi @@ -88,6 +88,8 @@ 'timestamp_map.cc', 'timing.cc', 'video_coding_impl.cc', + 'video_sender.cc', + 'video_receiver.cc', ], # source # TODO(jschuh): Bug 1348: fix size_t to int truncations. 'msvs_disabled_warnings': [ 4267, ], diff --git a/webrtc/modules/video_coding/main/source/video_coding_impl.cc b/webrtc/modules/video_coding/main/source/video_coding_impl.cc index 81398d83d7..098c4fb44f 100644 --- a/webrtc/modules/video_coding/main/source/video_coding_impl.cc +++ b/webrtc/modules/video_coding/main/source/video_coding_impl.cc @@ -20,8 +20,7 @@ #include "webrtc/system_wrappers/interface/trace_event.h" namespace webrtc { - -// #define DEBUG_DECODER_BIT_STREAM +namespace vcm { uint32_t VCMProcessTimer::Period() const { @@ -43,1263 +42,311 @@ void VCMProcessTimer::Processed() { _latestMs = _clock->TimeInMilliseconds(); } +} // namespace vcm -VideoCodingModuleImpl::VideoCodingModuleImpl(const int32_t id, +namespace { +class VideoCodingModuleImpl : public VideoCodingModule { + public: + VideoCodingModuleImpl(const int32_t id, + Clock* clock, + EventFactory* event_factory, + bool owns_event_factory) + : VideoCodingModule(), + sender_(new vcm::VideoSender(id, clock)), + receiver_(new vcm::VideoReceiver(id, clock, event_factory)), + own_event_factory_(owns_event_factory ? event_factory : NULL) {} + + virtual ~VideoCodingModuleImpl() { + sender_.reset(); + receiver_.reset(); + own_event_factory_.reset(); + } + + virtual int32_t TimeUntilNextProcess() OVERRIDE { + int32_t sender_time = sender_->TimeUntilNextProcess(); + int32_t receiver_time = receiver_->TimeUntilNextProcess(); + assert(sender_time >= 0); + assert(receiver_time >= 0); + return VCM_MIN(sender_time, receiver_time); + } + + virtual int32_t Process() OVERRIDE { + int32_t sender_return = sender_->Process(); + int32_t receiver_return = receiver_->Process(); + if (sender_return != VCM_OK) + return sender_return; + return receiver_return; + } + + virtual int32_t InitializeSender() OVERRIDE { + return sender_->InitializeSender(); + } + + virtual int32_t RegisterSendCodec(const VideoCodec* sendCodec, + uint32_t numberOfCores, + uint32_t maxPayloadSize) OVERRIDE { + return sender_->RegisterSendCodec(sendCodec, numberOfCores, maxPayloadSize); + } + + virtual int32_t SendCodec(VideoCodec* currentSendCodec) const OVERRIDE { + return sender_->SendCodec(currentSendCodec); + } + + virtual VideoCodecType SendCodec() const OVERRIDE { + return sender_->SendCodec(); + } + + virtual int32_t RegisterExternalEncoder(VideoEncoder* externalEncoder, + uint8_t payloadType, + bool internalSource) OVERRIDE { + return sender_->RegisterExternalEncoder( + externalEncoder, payloadType, internalSource); + } + + virtual int32_t CodecConfigParameters(uint8_t* buffer, int32_t size) + OVERRIDE { + return sender_->CodecConfigParameters(buffer, size); + } + + virtual int Bitrate(unsigned int* bitrate) const OVERRIDE { + return sender_->Bitrate(bitrate); + } + + virtual int FrameRate(unsigned int* framerate) const OVERRIDE { + return sender_->FrameRate(framerate); + } + + virtual int32_t SetChannelParameters(uint32_t target_bitrate, // bits/s. + uint8_t lossRate, + uint32_t rtt) OVERRIDE { + return sender_->SetChannelParameters(target_bitrate, lossRate, rtt); + } + + virtual int32_t RegisterTransportCallback(VCMPacketizationCallback* transport) + OVERRIDE { + return sender_->RegisterTransportCallback(transport); + } + + virtual int32_t RegisterSendStatisticsCallback( + VCMSendStatisticsCallback* sendStats) OVERRIDE { + return sender_->RegisterSendStatisticsCallback(sendStats); + } + + virtual int32_t RegisterVideoQMCallback( + VCMQMSettingsCallback* videoQMSettings) OVERRIDE { + return sender_->RegisterVideoQMCallback(videoQMSettings); + } + + virtual int32_t RegisterProtectionCallback(VCMProtectionCallback* protection) + OVERRIDE { + return sender_->RegisterProtectionCallback(protection); + } + + virtual int32_t SetVideoProtection(VCMVideoProtection videoProtection, + bool enable) OVERRIDE { + int32_t sender_return = + sender_->SetVideoProtection(videoProtection, enable); + int32_t receiver_return = + receiver_->SetVideoProtection(videoProtection, enable); + if (sender_return == VCM_OK) + return receiver_return; + return sender_return; + } + + virtual int32_t AddVideoFrame(const I420VideoFrame& videoFrame, + const VideoContentMetrics* contentMetrics, + const CodecSpecificInfo* codecSpecificInfo) + OVERRIDE { + return sender_->AddVideoFrame( + videoFrame, contentMetrics, codecSpecificInfo); + } + + virtual int32_t IntraFrameRequest(int stream_index) OVERRIDE { + return sender_->IntraFrameRequest(stream_index); + } + + virtual int32_t EnableFrameDropper(bool enable) OVERRIDE { + return sender_->EnableFrameDropper(enable); + } + + virtual int32_t SentFrameCount(VCMFrameCount& frameCount) const OVERRIDE { + return sender_->SentFrameCount(&frameCount); + } + + virtual int SetSenderNackMode(SenderNackMode mode) OVERRIDE { + return sender_->SetSenderNackMode(mode); + } + + virtual int SetSenderReferenceSelection(bool enable) OVERRIDE { + return sender_->SetSenderReferenceSelection(enable); + } + + virtual int SetSenderFEC(bool enable) OVERRIDE { + return sender_->SetSenderFEC(enable); + } + + virtual int SetSenderKeyFramePeriod(int periodMs) OVERRIDE { + return sender_->SetSenderKeyFramePeriod(periodMs); + } + + virtual int StartDebugRecording(const char* file_name_utf8) OVERRIDE { + return sender_->StartDebugRecording(file_name_utf8); + } + + virtual int StopDebugRecording() OVERRIDE { + return sender_->StopDebugRecording(); + } + + virtual int32_t InitializeReceiver() OVERRIDE { + return receiver_->InitializeReceiver(); + } + + virtual int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec, + int32_t numberOfCores, + bool requireKeyFrame) OVERRIDE { + return receiver_->RegisterReceiveCodec( + receiveCodec, numberOfCores, requireKeyFrame); + } + + virtual int32_t RegisterExternalDecoder(VideoDecoder* externalDecoder, + uint8_t payloadType, + bool internalRenderTiming) OVERRIDE { + return receiver_->RegisterExternalDecoder( + externalDecoder, payloadType, internalRenderTiming); + } + + virtual int32_t RegisterReceiveCallback(VCMReceiveCallback* receiveCallback) + OVERRIDE { + return receiver_->RegisterReceiveCallback(receiveCallback); + } + + virtual int32_t RegisterReceiveStatisticsCallback( + VCMReceiveStatisticsCallback* receiveStats) OVERRIDE { + return receiver_->RegisterReceiveStatisticsCallback(receiveStats); + } + + virtual int32_t RegisterFrameTypeCallback( + VCMFrameTypeCallback* frameTypeCallback) OVERRIDE { + return receiver_->RegisterFrameTypeCallback(frameTypeCallback); + } + + virtual int32_t RegisterPacketRequestCallback( + VCMPacketRequestCallback* callback) OVERRIDE { + return receiver_->RegisterPacketRequestCallback(callback); + } + + virtual int RegisterRenderBufferSizeCallback( + VCMRenderBufferSizeCallback* callback) OVERRIDE { + return receiver_->RegisterRenderBufferSizeCallback(callback); + } + + virtual int32_t Decode(uint16_t maxWaitTimeMs) OVERRIDE { + return receiver_->Decode(maxWaitTimeMs); + } + + virtual int32_t DecodeDualFrame(uint16_t maxWaitTimeMs) OVERRIDE { + return receiver_->DecodeDualFrame(maxWaitTimeMs); + } + + virtual int32_t ResetDecoder() OVERRIDE { return receiver_->ResetDecoder(); } + + virtual int32_t ReceiveCodec(VideoCodec* currentReceiveCodec) const { + return receiver_->ReceiveCodec(currentReceiveCodec); + } + + virtual VideoCodecType ReceiveCodec() const OVERRIDE { + return receiver_->ReceiveCodec(); + } + + virtual int32_t IncomingPacket(const uint8_t* incomingPayload, + uint32_t payloadLength, + const WebRtcRTPHeader& rtpInfo) OVERRIDE { + return receiver_->IncomingPacket(incomingPayload, payloadLength, rtpInfo); + } + + virtual int32_t SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) OVERRIDE { + return receiver_->SetMinimumPlayoutDelay(minPlayoutDelayMs); + } + + virtual int32_t SetRenderDelay(uint32_t timeMS) OVERRIDE { + return receiver_->SetRenderDelay(timeMS); + } + + virtual int32_t Delay() const OVERRIDE { return receiver_->Delay(); } + + virtual int32_t ReceivedFrameCount(VCMFrameCount& frameCount) const OVERRIDE { + return receiver_->ReceivedFrameCount(&frameCount); + } + + virtual uint32_t DiscardedPackets() const OVERRIDE { + return receiver_->DiscardedPackets(); + } + + virtual int SetReceiverRobustnessMode(ReceiverRobustness robustnessMode, + VCMDecodeErrorMode errorMode) OVERRIDE { + return receiver_->SetReceiverRobustnessMode(robustnessMode, errorMode); + } + + virtual void SetNackSettings(size_t max_nack_list_size, + int max_packet_age_to_nack, + int max_incomplete_time_ms) OVERRIDE { + return receiver_->SetNackSettings( + max_nack_list_size, max_packet_age_to_nack, max_incomplete_time_ms); + } + + void SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) OVERRIDE { + return receiver_->SetDecodeErrorMode(decode_error_mode); + } + + virtual int SetMinReceiverDelay(int desired_delay_ms) OVERRIDE { + return receiver_->SetMinReceiverDelay(desired_delay_ms); + } + + virtual int32_t SetReceiveChannelParameters(uint32_t rtt) OVERRIDE { + return receiver_->SetReceiveChannelParameters(rtt); + } + + private: + scoped_ptr sender_; + scoped_ptr receiver_; + scoped_ptr own_event_factory_; +}; +} // namespace + +uint8_t VideoCodingModule::NumberOfCodecs() { + return VCMCodecDataBase::NumberOfCodecs(); +} + +int32_t VideoCodingModule::Codec(uint8_t listId, VideoCodec* codec) { + if (codec == NULL) { + return VCM_PARAMETER_ERROR; + } + return VCMCodecDataBase::Codec(listId, codec) ? 0 : -1; +} + +int32_t VideoCodingModule::Codec(VideoCodecType codecType, VideoCodec* codec) { + if (codec == NULL) { + return VCM_PARAMETER_ERROR; + } + return VCMCodecDataBase::Codec(codecType, codec) ? 0 : -1; +} + +VideoCodingModule* VideoCodingModule::Create(const int32_t id) { + return new VideoCodingModuleImpl( + id, Clock::GetRealTimeClock(), new EventFactoryImpl, true); +} + +VideoCodingModule* VideoCodingModule::Create(const int32_t id, Clock* clock, - EventFactory* event_factory, - bool owns_event_factory) - : _id(id), - clock_(clock), - _receiveCritSect(CriticalSectionWrapper::CreateCriticalSection()), - _receiverInited(false), - _timing(clock_, id, 1), - _dualTiming(clock_, id, 2, &_timing), - _receiver(&_timing, clock_, event_factory, id, 1, true), - _dualReceiver(&_dualTiming, clock_, event_factory, id, 2, false), - _decodedFrameCallback(_timing, clock_), - _dualDecodedFrameCallback(_dualTiming, clock_), - _frameTypeCallback(NULL), - _receiveStatsCallback(NULL), - _packetRequestCallback(NULL), - render_buffer_callback_(NULL), - _decoder(NULL), - _dualDecoder(NULL), -#ifdef DEBUG_DECODER_BIT_STREAM - _bitStreamBeforeDecoder(NULL), -#endif - _frameFromFile(), - _keyRequestMode(kKeyOnError), - _scheduleKeyRequest(false), - max_nack_list_size_(0), - _sendCritSect(CriticalSectionWrapper::CreateCriticalSection()), - _encoder(), - _encodedFrameCallback(), - _nextFrameTypes(1, kVideoFrameDelta), - _mediaOpt(id, clock_), - _sendCodecType(kVideoCodecUnknown), - _sendStatsCallback(NULL), - _encoderInputFile(NULL), - _codecDataBase(id), - _receiveStatsTimer(1000, clock_), - _sendStatsTimer(1000, clock_), - _retransmissionTimer(10, clock_), - _keyRequestTimer(500, clock_), - event_factory_(event_factory), - owns_event_factory_(owns_event_factory), - frame_dropper_enabled_(true) { - assert(clock_); -#ifdef DEBUG_DECODER_BIT_STREAM - _bitStreamBeforeDecoder = fopen("decoderBitStream.bit", "wb"); -#endif + EventFactory* event_factory) { + assert(clock); + assert(event_factory); + return new VideoCodingModuleImpl(id, clock, event_factory, false); } -VideoCodingModuleImpl::~VideoCodingModuleImpl() { - if (_dualDecoder != NULL) { - _codecDataBase.ReleaseDecoder(_dualDecoder); - } - delete _receiveCritSect; - delete _sendCritSect; - if (owns_event_factory_) { - delete event_factory_; - } -#ifdef DEBUG_DECODER_BIT_STREAM - fclose(_bitStreamBeforeDecoder); -#endif - if (_encoderInputFile != NULL) { - fclose(_encoderInputFile); - } -} - -VideoCodingModule* -VideoCodingModule::Create(const int32_t id) { - return new VideoCodingModuleImpl(id, Clock::GetRealTimeClock(), - new EventFactoryImpl, true); -} - -VideoCodingModule* -VideoCodingModule::Create(const int32_t id, Clock* clock, - EventFactory* event_factory) { - assert(clock); - assert(event_factory); - return new VideoCodingModuleImpl(id, clock, event_factory, false); -} - -void -VideoCodingModule::Destroy(VideoCodingModule* module) { - if (module != NULL) { - delete static_cast(module); - } -} - -int32_t -VideoCodingModuleImpl::Process() { - int32_t returnValue = VCM_OK; - - // Receive-side statistics - if (_receiveStatsTimer.TimeUntilProcess() == 0) { - _receiveStatsTimer.Processed(); - if (_receiveStatsCallback != NULL) { - uint32_t bitRate; - uint32_t frameRate; - _receiver.ReceiveStatistics(&bitRate, &frameRate); - _receiveStatsCallback->OnReceiveStatisticsUpdate(bitRate, - frameRate); - } - - // Size of render buffer. - if (render_buffer_callback_) { - int buffer_size_ms = _receiver.RenderBufferSizeMs(); - render_buffer_callback_->RenderBufferSizeMs(buffer_size_ms); - } - } - - // Send-side statistics - if (_sendStatsTimer.TimeUntilProcess() == 0) { - _sendStatsTimer.Processed(); - if (_sendStatsCallback != NULL) { - uint32_t bitRate; - uint32_t frameRate; - { - CriticalSectionScoped cs(_sendCritSect); - bitRate = _mediaOpt.SentBitRate(); - frameRate = _mediaOpt.SentFrameRate(); - } - _sendStatsCallback->SendStatistics(bitRate, frameRate); - } - } - - // Packet retransmission requests - // TODO(holmer): Add API for changing Process interval and make sure it's - // disabled when NACK is off. - if (_retransmissionTimer.TimeUntilProcess() == 0) { - _retransmissionTimer.Processed(); - CriticalSectionScoped cs(_receiveCritSect); - if (_packetRequestCallback != NULL) { - uint16_t length = max_nack_list_size_; - std::vector nackList(length); - const int32_t ret = NackList(&nackList[0], length); - if (ret != VCM_OK && returnValue == VCM_OK) { - returnValue = ret; - } - if (length > 0) { - _packetRequestCallback->ResendPackets(&nackList[0], length); - } - } - } - - // Key frame requests - if (_keyRequestTimer.TimeUntilProcess() == 0) { - _keyRequestTimer.Processed(); - if (_scheduleKeyRequest && _frameTypeCallback != NULL) { - const int32_t ret = RequestKeyFrame(); - if (ret != VCM_OK && returnValue == VCM_OK) { - returnValue = ret; - } - } - } - - return returnValue; -} - -int32_t -VideoCodingModuleImpl::Id() const { - CriticalSectionScoped receiveCs(_receiveCritSect); - { - CriticalSectionScoped sendCs(_sendCritSect); - return _id; - } -} - -// Change the unique identifier of this object -int32_t -VideoCodingModuleImpl::ChangeUniqueId(const int32_t id) { - CriticalSectionScoped receiveCs(_receiveCritSect); - { - CriticalSectionScoped sendCs(_sendCritSect); - _id = id; - return VCM_OK; - } -} - -// Returns the number of milliseconds until the module wants a worker thread to -// call Process -int32_t -VideoCodingModuleImpl::TimeUntilNextProcess() { - uint32_t timeUntilNextProcess = VCM_MIN( - _receiveStatsTimer.TimeUntilProcess(), - _sendStatsTimer.TimeUntilProcess()); - if ((_receiver.NackMode() != kNoNack) || - (_dualReceiver.State() != kPassive)) { - // We need a Process call more often if we are relying on - // retransmissions - timeUntilNextProcess = VCM_MIN(timeUntilNextProcess, - _retransmissionTimer.TimeUntilProcess()); - } - timeUntilNextProcess = VCM_MIN(timeUntilNextProcess, - _keyRequestTimer.TimeUntilProcess()); - - return timeUntilNextProcess; -} - -// Get number of supported codecs -uint8_t -VideoCodingModule::NumberOfCodecs() { - return VCMCodecDataBase::NumberOfCodecs(); -} - -// Get supported codec with id -int32_t -VideoCodingModule::Codec(uint8_t listId, VideoCodec* codec) { - if (codec == NULL) { - return VCM_PARAMETER_ERROR; - } - return VCMCodecDataBase::Codec(listId, codec) ? 0 : -1; -} - -// Get supported codec with type -int32_t -VideoCodingModule::Codec(VideoCodecType codecType, VideoCodec* codec) { - if (codec == NULL) { - return VCM_PARAMETER_ERROR; - } - return VCMCodecDataBase::Codec(codecType, codec) ? 0 : -1; -} - -/* -* Sender -*/ - -// Reset send side to initial state - all components -int32_t -VideoCodingModuleImpl::InitializeSender() { - CriticalSectionScoped cs(_sendCritSect); - _codecDataBase.ResetSender(); - _encoder = NULL; - _encodedFrameCallback.SetTransportCallback(NULL); - // setting default bitRate and frameRate to 0 - _mediaOpt.SetEncodingData(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0); - _mediaOpt.Reset(); // Resetting frame dropper - return VCM_OK; -} - -// Register the send codec to be used. -int32_t -VideoCodingModuleImpl::RegisterSendCodec(const VideoCodec* sendCodec, - uint32_t numberOfCores, - uint32_t maxPayloadSize) { - CriticalSectionScoped cs(_sendCritSect); - if (sendCodec == NULL) { - return VCM_PARAMETER_ERROR; - } - - bool ret = _codecDataBase.SetSendCodec(sendCodec, numberOfCores, - maxPayloadSize, - &_encodedFrameCallback); - if (!ret) { - WEBRTC_TRACE(webrtc::kTraceError, - webrtc::kTraceVideoCoding, - VCMId(_id), - "Failed to initialize encoder"); - return VCM_CODEC_ERROR; - } - - _encoder = _codecDataBase.GetEncoder(); - _sendCodecType = sendCodec->codecType; - int numLayers = (_sendCodecType != kVideoCodecVP8) ? 1 : - sendCodec->codecSpecific.VP8.numberOfTemporalLayers; - // If we have screensharing and we have layers, we disable frame dropper. - bool disable_frame_dropper = - numLayers > 1 && sendCodec->mode == kScreensharing; - if (disable_frame_dropper) { - _mediaOpt.EnableFrameDropper(false); - } else if (frame_dropper_enabled_) { - _mediaOpt.EnableFrameDropper(true); - } - _nextFrameTypes.clear(); - _nextFrameTypes.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1), - kVideoFrameDelta); - - _mediaOpt.SetEncodingData(_sendCodecType, - sendCodec->maxBitrate * 1000, - sendCodec->maxFramerate * 1000, - sendCodec->startBitrate * 1000, - sendCodec->width, - sendCodec->height, - numLayers); - _mediaOpt.SetMtu(maxPayloadSize); - - return VCM_OK; -} - -// Get current send codec -int32_t -VideoCodingModuleImpl::SendCodec(VideoCodec* currentSendCodec) const { - CriticalSectionScoped cs(_sendCritSect); - - if (currentSendCodec == NULL) { - return VCM_PARAMETER_ERROR; - } - return _codecDataBase.SendCodec(currentSendCodec) ? 0 : -1; -} - -// Get the current send codec type -VideoCodecType -VideoCodingModuleImpl::SendCodec() const { - CriticalSectionScoped cs(_sendCritSect); - - return _codecDataBase.SendCodec(); -} - -// Register an external decoder object. -// This can not be used together with external decoder callbacks. -int32_t -VideoCodingModuleImpl::RegisterExternalEncoder(VideoEncoder* externalEncoder, - uint8_t payloadType, - bool internalSource /*= false*/) { - CriticalSectionScoped cs(_sendCritSect); - - if (externalEncoder == NULL) { - bool wasSendCodec = false; - const bool ret = _codecDataBase.DeregisterExternalEncoder( - payloadType, &wasSendCodec); - if (wasSendCodec) { - // Make sure the VCM doesn't use the de-registered codec - _encoder = NULL; - } - return ret ? 0 : -1; - } - _codecDataBase.RegisterExternalEncoder(externalEncoder, payloadType, - internalSource); - return 0; -} - -// Get codec config parameters -int32_t -VideoCodingModuleImpl::CodecConfigParameters(uint8_t* buffer, - int32_t size) { - CriticalSectionScoped cs(_sendCritSect); - if (_encoder != NULL) { - return _encoder->CodecConfigParameters(buffer, size); - } - return VCM_UNINITIALIZED; -} - -// Get encode bitrate -int VideoCodingModuleImpl::Bitrate(unsigned int* bitrate) const { - CriticalSectionScoped cs(_sendCritSect); - // return the bit rate which the encoder is set to - if (!_encoder) { - return VCM_UNINITIALIZED; +void VideoCodingModule::Destroy(VideoCodingModule* module) { + if (module != NULL) { + delete static_cast(module); } - *bitrate = _encoder->BitRate(); - return 0; } - -// Get encode frame rate -int VideoCodingModuleImpl::FrameRate(unsigned int* framerate) const { - CriticalSectionScoped cs(_sendCritSect); - // input frame rate, not compensated - if (!_encoder) { - return VCM_UNINITIALIZED; - } - *framerate = _encoder->FrameRate(); - return 0; -} - -// Set channel parameters -int32_t -VideoCodingModuleImpl::SetChannelParameters(uint32_t target_bitrate, - uint8_t lossRate, - uint32_t rtt) { - int32_t ret = 0; - { - CriticalSectionScoped sendCs(_sendCritSect); - uint32_t targetRate = _mediaOpt.SetTargetRates(target_bitrate, - lossRate, - rtt); - if (_encoder != NULL) { - ret = _encoder->SetChannelParameters(lossRate, rtt); - if (ret < 0) { - return ret; - } - ret = (int32_t)_encoder->SetRates(targetRate, - _mediaOpt.InputFrameRate()); - if (ret < 0) { - return ret; - } - } else { - return VCM_UNINITIALIZED; - } // encoder - } // send side - return VCM_OK; -} - -int32_t -VideoCodingModuleImpl::SetReceiveChannelParameters(uint32_t rtt) { - CriticalSectionScoped receiveCs(_receiveCritSect); - _receiver.UpdateRtt(rtt); - return 0; -} - -// Register a transport callback which will be called to deliver the encoded -// buffers -int32_t -VideoCodingModuleImpl::RegisterTransportCallback( - VCMPacketizationCallback* transport) { - CriticalSectionScoped cs(_sendCritSect); - _encodedFrameCallback.SetMediaOpt(&_mediaOpt); - _encodedFrameCallback.SetTransportCallback(transport); - return VCM_OK; -} - -// Register video output information callback which will be called to deliver -// information about the video stream produced by the encoder, for instance the -// average frame rate and bit rate. -int32_t -VideoCodingModuleImpl::RegisterSendStatisticsCallback( - VCMSendStatisticsCallback* sendStats) { - CriticalSectionScoped cs(_sendCritSect); - _sendStatsCallback = sendStats; - return VCM_OK; -} - -// Register a video quality settings callback which will be called when frame -// rate/dimensions need to be updated for video quality optimization -int32_t -VideoCodingModuleImpl::RegisterVideoQMCallback( - VCMQMSettingsCallback* videoQMSettings) { - CriticalSectionScoped cs(_sendCritSect); - return _mediaOpt.RegisterVideoQMCallback(videoQMSettings); -} - - -// Register a video protection callback which will be called to deliver the -// requested FEC rate and NACK status (on/off). -int32_t -VideoCodingModuleImpl::RegisterProtectionCallback( - VCMProtectionCallback* protection) { - CriticalSectionScoped cs(_sendCritSect); - _mediaOpt.RegisterProtectionCallback(protection); - return VCM_OK; -} - -// Enable or disable a video protection method. -// Note: This API should be deprecated, as it does not offer a distinction -// between the protection method and decoding with or without errors. If such a -// behavior is desired, use the following API: SetReceiverRobustnessMode. -int32_t -VideoCodingModuleImpl::SetVideoProtection(VCMVideoProtection videoProtection, - bool enable) { - // By default, do not decode with errors. - _receiver.SetDecodeErrorMode(kNoErrors); - // The dual decoder should always be error free. - _dualReceiver.SetDecodeErrorMode(kNoErrors); - switch (videoProtection) { - case kProtectionNack: - { - // Both send-side and receive-side - SetVideoProtection(kProtectionNackSender, enable); - SetVideoProtection(kProtectionNackReceiver, enable); - break; - } - - case kProtectionNackSender: - { - CriticalSectionScoped cs(_sendCritSect); - _mediaOpt.EnableProtectionMethod(enable, media_optimization::kNack); - break; - } - - case kProtectionNackReceiver: - { - CriticalSectionScoped cs(_receiveCritSect); - if (enable) { - // Enable NACK and always wait for retransmits. - _receiver.SetNackMode(kNack, -1, -1); - } else { - _receiver.SetNackMode(kNoNack, -1, -1); - } - break; - } - - case kProtectionDualDecoder: - { - CriticalSectionScoped cs(_receiveCritSect); - if (enable) { - // Enable NACK but don't wait for retransmissions and don't - // add any extra delay. - _receiver.SetNackMode(kNack, 0, 0); - // Enable NACK and always wait for retransmissions and - // compensate with extra delay. - _dualReceiver.SetNackMode(kNack, -1, -1); - _receiver.SetDecodeErrorMode(kWithErrors); - } else { - _dualReceiver.SetNackMode(kNoNack, -1, -1); - } - break; - } - - case kProtectionKeyOnLoss: - { - CriticalSectionScoped cs(_receiveCritSect); - if (enable) { - _keyRequestMode = kKeyOnLoss; - _receiver.SetDecodeErrorMode(kWithErrors); - } else if (_keyRequestMode == kKeyOnLoss) { - _keyRequestMode = kKeyOnError; // default mode - } else { - return VCM_PARAMETER_ERROR; - } - break; - } - - case kProtectionKeyOnKeyLoss: - { - CriticalSectionScoped cs(_receiveCritSect); - if (enable) { - _keyRequestMode = kKeyOnKeyLoss; - } else if (_keyRequestMode == kKeyOnKeyLoss) { - _keyRequestMode = kKeyOnError; // default mode - } else { - return VCM_PARAMETER_ERROR; - } - break; - } - - case kProtectionNackFEC: - { - { - // Receive side - CriticalSectionScoped cs(_receiveCritSect); - if (enable) { - // Enable hybrid NACK/FEC. Always wait for retransmissions - // and don't add extra delay when RTT is above - // kLowRttNackMs. - _receiver.SetNackMode(kNack, - media_optimization::kLowRttNackMs, - -1); - _receiver.SetDecodeErrorMode(kNoErrors); - _receiver.SetDecodeErrorMode(kNoErrors); - } else { - _receiver.SetNackMode(kNoNack, -1, -1); - } - } - // Send Side - { - CriticalSectionScoped cs(_sendCritSect); - _mediaOpt.EnableProtectionMethod(enable, - media_optimization::kNackFec); - } - break; - } - - case kProtectionFEC: - { - CriticalSectionScoped cs(_sendCritSect); - _mediaOpt.EnableProtectionMethod(enable, media_optimization::kFec); - break; - } - - case kProtectionPeriodicKeyFrames: - { - CriticalSectionScoped cs(_sendCritSect); - return _codecDataBase.SetPeriodicKeyFrames(enable) ? 0 : -1; - break; - } - } - return VCM_OK; -} - -// Add one raw video frame to the encoder, blocking. -int32_t -VideoCodingModuleImpl::AddVideoFrame(const I420VideoFrame& videoFrame, - const VideoContentMetrics* contentMetrics, - const CodecSpecificInfo* codecSpecificInfo) { - CriticalSectionScoped cs(_sendCritSect); - if (_encoder == NULL) { - return VCM_UNINITIALIZED; - } - // TODO(holmer): Add support for dropping frames per stream. Currently we - // only have one frame dropper for all streams. - if (_nextFrameTypes[0] == kFrameEmpty) { - return VCM_OK; - } - _mediaOpt.UpdateIncomingFrameRate(); - - if (_mediaOpt.DropFrame()) { - WEBRTC_TRACE(webrtc::kTraceStream, - webrtc::kTraceVideoCoding, - VCMId(_id), - "Drop frame due to bitrate"); - } else { - _mediaOpt.UpdateContentData(contentMetrics); - int32_t ret = _encoder->Encode(videoFrame, - codecSpecificInfo, - _nextFrameTypes); - if (_encoderInputFile != NULL) { - if (PrintI420VideoFrame(videoFrame, _encoderInputFile) < 0) { - return -1; - } - } - if (ret < 0) { - WEBRTC_TRACE(webrtc::kTraceError, - webrtc::kTraceVideoCoding, - VCMId(_id), - "Encode error: %d", ret); - return ret; - } - for (size_t i = 0; i < _nextFrameTypes.size(); ++i) { - _nextFrameTypes[i] = kVideoFrameDelta; // Default frame type. - } - } - return VCM_OK; -} - -int32_t VideoCodingModuleImpl::IntraFrameRequest(int stream_index) { - CriticalSectionScoped cs(_sendCritSect); - if (stream_index < 0 || - static_cast(stream_index) >= _nextFrameTypes.size()) { - return -1; - } - _nextFrameTypes[stream_index] = kVideoFrameKey; - if (_encoder != NULL && _encoder->InternalSource()) { - // Try to request the frame if we have an external encoder with - // internal source since AddVideoFrame never will be called. - if (_encoder->RequestFrame(_nextFrameTypes) == - WEBRTC_VIDEO_CODEC_OK) { - _nextFrameTypes[stream_index] = kVideoFrameDelta; - } - } - return VCM_OK; -} - -int32_t -VideoCodingModuleImpl::EnableFrameDropper(bool enable) { - CriticalSectionScoped cs(_sendCritSect); - frame_dropper_enabled_ = enable; - _mediaOpt.EnableFrameDropper(enable); - return VCM_OK; -} - - -int32_t -VideoCodingModuleImpl::SentFrameCount(VCMFrameCount &frameCount) const { - CriticalSectionScoped cs(_sendCritSect); - return _mediaOpt.SentFrameCount(frameCount); -} - -// Initialize receiver, resets codec database etc -int32_t -VideoCodingModuleImpl::InitializeReceiver() { - CriticalSectionScoped cs(_receiveCritSect); - int32_t ret = _receiver.Initialize(); - if (ret < 0) { - return ret; - } - - ret = _dualReceiver.Initialize(); - if (ret < 0) { - return ret; - } - _codecDataBase.ResetReceiver(); - _timing.Reset(); - - _decoder = NULL; - _decodedFrameCallback.SetUserReceiveCallback(NULL); - _receiverInited = true; - _frameTypeCallback = NULL; - _receiveStatsCallback = NULL; - _packetRequestCallback = NULL; - _keyRequestMode = kKeyOnError; - _scheduleKeyRequest = false; - - return VCM_OK; -} - -// Register a receive callback. Will be called whenever there is a new frame -// ready for rendering. -int32_t -VideoCodingModuleImpl::RegisterReceiveCallback( - VCMReceiveCallback* receiveCallback) { - CriticalSectionScoped cs(_receiveCritSect); - _decodedFrameCallback.SetUserReceiveCallback(receiveCallback); - return VCM_OK; -} - -int32_t -VideoCodingModuleImpl::RegisterReceiveStatisticsCallback( - VCMReceiveStatisticsCallback* receiveStats) { - CriticalSectionScoped cs(_receiveCritSect); - _receiveStatsCallback = receiveStats; - return VCM_OK; -} - -// Register an externally defined decoder/render object. -// Can be a decoder only or a decoder coupled with a renderer. -int32_t -VideoCodingModuleImpl::RegisterExternalDecoder(VideoDecoder* externalDecoder, - uint8_t payloadType, - bool internalRenderTiming) { - CriticalSectionScoped cs(_receiveCritSect); - if (externalDecoder == NULL) { - // Make sure the VCM updates the decoder next time it decodes. - _decoder = NULL; - return _codecDataBase.DeregisterExternalDecoder(payloadType) ? 0 : -1; - } - return _codecDataBase.RegisterExternalDecoder( - externalDecoder, payloadType, internalRenderTiming) ? 0 : -1; -} - -// Register a frame type request callback. -int32_t -VideoCodingModuleImpl::RegisterFrameTypeCallback( - VCMFrameTypeCallback* frameTypeCallback) { - CriticalSectionScoped cs(_receiveCritSect); - _frameTypeCallback = frameTypeCallback; - return VCM_OK; -} - -int32_t -VideoCodingModuleImpl::RegisterPacketRequestCallback( - VCMPacketRequestCallback* callback) { - CriticalSectionScoped cs(_receiveCritSect); - _packetRequestCallback = callback; - return VCM_OK; -} - -int VideoCodingModuleImpl::RegisterRenderBufferSizeCallback( - VCMRenderBufferSizeCallback* callback) { - CriticalSectionScoped cs(_receiveCritSect); - render_buffer_callback_ = callback; - return VCM_OK; -} - -// Decode next frame, blocking. -// Should be called as often as possible to get the most out of the decoder. -int32_t -VideoCodingModuleImpl::Decode(uint16_t maxWaitTimeMs) { - int64_t nextRenderTimeMs; - { - CriticalSectionScoped cs(_receiveCritSect); - if (!_receiverInited) { - return VCM_UNINITIALIZED; - } - if (!_codecDataBase.DecoderRegistered()) { - return VCM_NO_CODEC_REGISTERED; - } - } - - const bool dualReceiverEnabledNotReceiving = - (_dualReceiver.State() != kReceiving && - _dualReceiver.NackMode() == kNack); - - VCMEncodedFrame* frame = _receiver.FrameForDecoding( - maxWaitTimeMs, - nextRenderTimeMs, - _codecDataBase.SupportsRenderScheduling(), - &_dualReceiver); - - if (dualReceiverEnabledNotReceiving - && _dualReceiver.State() == kReceiving) { - // Dual receiver is enabled (kNACK enabled), but was not receiving - // before the call to FrameForDecoding(). After the call the state - // changed to receiving, and therefore we must copy the primary decoder - // state to the dual decoder to make it possible for the dual decoder to - // start decoding retransmitted frames and recover. - CriticalSectionScoped cs(_receiveCritSect); - if (_dualDecoder != NULL) { - _codecDataBase.ReleaseDecoder(_dualDecoder); - } - _dualDecoder = _codecDataBase.CreateDecoderCopy(); - if (_dualDecoder != NULL) { - _dualDecoder->RegisterDecodeCompleteCallback( - &_dualDecodedFrameCallback); - } else { - _dualReceiver.Reset(); - } - } - - if (frame == NULL) { - return VCM_FRAME_NOT_READY; - } else { - CriticalSectionScoped cs(_receiveCritSect); - - // If this frame was too late, we should adjust the delay accordingly - _timing.UpdateCurrentDelay(frame->RenderTimeMs(), - clock_->TimeInMilliseconds()); - -#ifdef DEBUG_DECODER_BIT_STREAM - if (_bitStreamBeforeDecoder != NULL) { - // Write bit stream to file for debugging purposes - if (fwrite(frame->Buffer(), 1, frame->Length(), - _bitStreamBeforeDecoder) != frame->Length()) { - return -1; - } - } -#endif - const int32_t ret = Decode(*frame); - _receiver.ReleaseFrame(frame); - frame = NULL; - if (ret != VCM_OK) { - return ret; - } - } - return VCM_OK; -} - -int32_t -VideoCodingModuleImpl::RequestSliceLossIndication( - const uint64_t pictureID) const { - TRACE_EVENT1("webrtc", "RequestSLI", "picture_id", pictureID); - if (_frameTypeCallback != NULL) { - const int32_t ret = - _frameTypeCallback->SliceLossIndicationRequest(pictureID); - if (ret < 0) { - WEBRTC_TRACE(webrtc::kTraceError, - webrtc::kTraceVideoCoding, - VCMId(_id), - "Failed to request key frame"); - return ret; - } - } else { - WEBRTC_TRACE(webrtc::kTraceWarning, - webrtc::kTraceVideoCoding, - VCMId(_id), - "No frame type request callback registered"); - return VCM_MISSING_CALLBACK; - } - return VCM_OK; -} - -int32_t -VideoCodingModuleImpl::RequestKeyFrame() { - TRACE_EVENT0("webrtc", "RequestKeyFrame"); - if (_frameTypeCallback != NULL) { - const int32_t ret = _frameTypeCallback->RequestKeyFrame(); - if (ret < 0) { - WEBRTC_TRACE(webrtc::kTraceError, - webrtc::kTraceVideoCoding, - VCMId(_id), - "Failed to request key frame"); - return ret; - } - _scheduleKeyRequest = false; - } else { - WEBRTC_TRACE(webrtc::kTraceWarning, - webrtc::kTraceVideoCoding, - VCMId(_id), - "No frame type request callback registered"); - return VCM_MISSING_CALLBACK; - } - return VCM_OK; -} - -int32_t -VideoCodingModuleImpl::DecodeDualFrame(uint16_t maxWaitTimeMs) { - CriticalSectionScoped cs(_receiveCritSect); - if (_dualReceiver.State() != kReceiving || - _dualReceiver.NackMode() != kNack) { - // The dual receiver is currently not receiving or - // dual decoder mode is disabled. - return VCM_OK; - } - int64_t dummyRenderTime; - int32_t decodeCount = 0; - // The dual decoder's state is copied from the main decoder, which may - // decode with errors. Make sure that the dual decoder does not introduce - // error. - _dualReceiver.SetDecodeErrorMode(kNoErrors); - VCMEncodedFrame* dualFrame = _dualReceiver.FrameForDecoding( - maxWaitTimeMs, - dummyRenderTime); - if (dualFrame != NULL && _dualDecoder != NULL) { - WEBRTC_TRACE(webrtc::kTraceStream, - webrtc::kTraceVideoCoding, - VCMId(_id), - "Decoding frame %u with dual decoder", - dualFrame->TimeStamp()); - // Decode dualFrame and try to catch up - int32_t ret = _dualDecoder->Decode(*dualFrame, - clock_->TimeInMilliseconds()); - if (ret != WEBRTC_VIDEO_CODEC_OK) { - WEBRTC_TRACE(webrtc::kTraceWarning, - webrtc::kTraceVideoCoding, - VCMId(_id), - "Failed to decode frame with dual decoder"); - _dualReceiver.ReleaseFrame(dualFrame); - return VCM_CODEC_ERROR; - } - if (_receiver.DualDecoderCaughtUp(dualFrame, _dualReceiver)) { - // Copy the complete decoder state of the dual decoder - // to the primary decoder. - WEBRTC_TRACE(webrtc::kTraceStream, - webrtc::kTraceVideoCoding, - VCMId(_id), - "Dual decoder caught up"); - _codecDataBase.CopyDecoder(*_dualDecoder); - _codecDataBase.ReleaseDecoder(_dualDecoder); - _dualDecoder = NULL; - } - decodeCount++; - } - _dualReceiver.ReleaseFrame(dualFrame); - return decodeCount; -} - - -// Must be called from inside the receive side critical section. -int32_t -VideoCodingModuleImpl::Decode(const VCMEncodedFrame& frame) { - TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame.TimeStamp(), - "Decode", "type", frame.FrameType()); - // Change decoder if payload type has changed - const bool renderTimingBefore = _codecDataBase.SupportsRenderScheduling(); - _decoder = _codecDataBase.GetDecoder(frame.PayloadType(), - &_decodedFrameCallback); - if (renderTimingBefore != _codecDataBase.SupportsRenderScheduling()) { - // Make sure we reset the decode time estimate since it will - // be zero for codecs without render timing. - _timing.ResetDecodeTime(); - } - if (_decoder == NULL) { - return VCM_NO_CODEC_REGISTERED; - } - // Decode a frame - int32_t ret = _decoder->Decode(frame, clock_->TimeInMilliseconds()); - - // Check for failed decoding, run frame type request callback if needed. - if (ret < 0) { - if (ret == VCM_ERROR_REQUEST_SLI) { - return RequestSliceLossIndication( - _decodedFrameCallback.LastReceivedPictureID() + 1); - } else { - WEBRTC_TRACE(webrtc::kTraceError, - webrtc::kTraceVideoCoding, - VCMId(_id), - "Failed to decode frame %u, requesting key frame", - frame.TimeStamp()); - ret = RequestKeyFrame(); - } - } else if (ret == VCM_REQUEST_SLI) { - ret = RequestSliceLossIndication( - _decodedFrameCallback.LastReceivedPictureID() + 1); - } - if (!frame.Complete() || frame.MissingFrame()) { - switch (_keyRequestMode) { - case kKeyOnKeyLoss: - { - if (frame.FrameType() == kVideoFrameKey) { - _scheduleKeyRequest = true; - return VCM_OK; - } - break; - } - case kKeyOnLoss: - { - _scheduleKeyRequest = true; - return VCM_OK; - } - default: - break; - } - } - TRACE_EVENT_ASYNC_END0("webrtc", "Video", frame.TimeStamp()); - return ret; -} - -// Reset the decoder state -int32_t -VideoCodingModuleImpl::ResetDecoder() { - CriticalSectionScoped cs(_receiveCritSect); - if (_decoder != NULL) { - _receiver.Initialize(); - _timing.Reset(); - _scheduleKeyRequest = false; - _decoder->Reset(); - } - if (_dualReceiver.State() != kPassive) { - _dualReceiver.Initialize(); - } - if (_dualDecoder != NULL) { - _codecDataBase.ReleaseDecoder(_dualDecoder); - _dualDecoder = NULL; - } - return VCM_OK; -} - -// Register possible receive codecs, can be called multiple times -int32_t -VideoCodingModuleImpl::RegisterReceiveCodec(const VideoCodec* receiveCodec, - int32_t numberOfCores, - bool requireKeyFrame) { - CriticalSectionScoped cs(_receiveCritSect); - if (receiveCodec == NULL) { - return VCM_PARAMETER_ERROR; - } - if (!_codecDataBase.RegisterReceiveCodec(receiveCodec, numberOfCores, - requireKeyFrame)) { - return -1; - } - return 0; -} - -// Get current received codec -int32_t -VideoCodingModuleImpl::ReceiveCodec(VideoCodec* currentReceiveCodec) const { - CriticalSectionScoped cs(_receiveCritSect); - if (currentReceiveCodec == NULL) { - return VCM_PARAMETER_ERROR; - } - return _codecDataBase.ReceiveCodec(currentReceiveCodec) ? 0 : -1; -} - -// Get current received codec -VideoCodecType -VideoCodingModuleImpl::ReceiveCodec() const { - CriticalSectionScoped cs(_receiveCritSect); - return _codecDataBase.ReceiveCodec(); -} - -// Incoming packet from network parsed and ready for decode, non blocking. -int32_t -VideoCodingModuleImpl::IncomingPacket(const uint8_t* incomingPayload, - uint32_t payloadLength, - const WebRtcRTPHeader& rtpInfo) { - if (rtpInfo.frameType == kVideoFrameKey) { - TRACE_EVENT1("webrtc", "VCM::PacketKeyFrame", - "seqnum", rtpInfo.header.sequenceNumber); - } - if (incomingPayload == NULL) { - // The jitter buffer doesn't handle non-zero payload lengths for packets - // without payload. - // TODO(holmer): We should fix this in the jitter buffer. - payloadLength = 0; - } - const VCMPacket packet(incomingPayload, payloadLength, rtpInfo); - int32_t ret; - if (_dualReceiver.State() != kPassive) { - ret = _dualReceiver.InsertPacket(packet, - rtpInfo.type.Video.width, - rtpInfo.type.Video.height); - if (ret == VCM_FLUSH_INDICATOR) { - RequestKeyFrame(); - ResetDecoder(); - } else if (ret < 0) { - return ret; - } - } - ret = _receiver.InsertPacket(packet, - rtpInfo.type.Video.width, - rtpInfo.type.Video.height); - // TODO(holmer): Investigate if this somehow should use the key frame - // request scheduling to throttle the requests. - if (ret == VCM_FLUSH_INDICATOR) { - RequestKeyFrame(); - ResetDecoder(); - } else if (ret < 0) { - return ret; - } - return VCM_OK; -} - -// Minimum playout delay (used for lip-sync). This is the minimum delay required -// to sync with audio. Not included in VideoCodingModule::Delay() -// Defaults to 0 ms. -int32_t -VideoCodingModuleImpl::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) { - _timing.set_min_playout_delay(minPlayoutDelayMs); - return VCM_OK; -} - -// The estimated delay caused by rendering, defaults to -// kDefaultRenderDelayMs = 10 ms -int32_t -VideoCodingModuleImpl::SetRenderDelay(uint32_t timeMS) { - _timing.set_render_delay(timeMS); - return VCM_OK; -} - -// Current video delay -int32_t -VideoCodingModuleImpl::Delay() const { - return _timing.TargetVideoDelay(); -} - -// Nack list -int32_t -VideoCodingModuleImpl::NackList(uint16_t* nackList, uint16_t& size) { - VCMNackStatus nackStatus = kNackOk; - uint16_t nack_list_length = 0; - // Collect sequence numbers from the default receiver - // if in normal nack mode. Otherwise collect them from - // the dual receiver if the dual receiver is receiving. - if (_receiver.NackMode() != kNoNack) { - nackStatus = _receiver.NackList(nackList, size, &nack_list_length); - } - if (nack_list_length == 0 && _dualReceiver.State() != kPassive) { - nackStatus = _dualReceiver.NackList(nackList, size, &nack_list_length); - } - size = nack_list_length; - - switch (nackStatus) { - case kNackNeedMoreMemory: - { - WEBRTC_TRACE(webrtc::kTraceError, - webrtc::kTraceVideoCoding, - VCMId(_id), - "Out of memory"); - return VCM_MEMORY; - } - case kNackKeyFrameRequest: - { - CriticalSectionScoped cs(_receiveCritSect); - WEBRTC_TRACE(webrtc::kTraceWarning, - webrtc::kTraceVideoCoding, - VCMId(_id), - "Failed to get NACK list, requesting key frame"); - return RequestKeyFrame(); - } - default: - break; - } - return VCM_OK; -} - -int32_t -VideoCodingModuleImpl::ReceivedFrameCount(VCMFrameCount& frameCount) const { - _receiver.ReceivedFrameCount(&frameCount); - return VCM_OK; -} - -uint32_t VideoCodingModuleImpl::DiscardedPackets() const { - return _receiver.DiscardedPackets(); -} - -int VideoCodingModuleImpl::SetSenderNackMode(SenderNackMode mode) { - CriticalSectionScoped cs(_sendCritSect); - - switch (mode) { - case kNackNone: - _mediaOpt.EnableProtectionMethod(false, media_optimization::kNack); - break; - case kNackAll: - _mediaOpt.EnableProtectionMethod(true, media_optimization::kNack); - break; - case kNackSelective: - return VCM_NOT_IMPLEMENTED; - break; - } - return VCM_OK; -} - -int VideoCodingModuleImpl::SetSenderReferenceSelection(bool enable) { - return VCM_NOT_IMPLEMENTED; -} - -int VideoCodingModuleImpl::SetSenderFEC(bool enable) { - CriticalSectionScoped cs(_sendCritSect); - _mediaOpt.EnableProtectionMethod(enable, media_optimization::kFec); - return VCM_OK; -} - -int VideoCodingModuleImpl::SetSenderKeyFramePeriod(int periodMs) { - return VCM_NOT_IMPLEMENTED; -} - -int VideoCodingModuleImpl::SetReceiverRobustnessMode( - ReceiverRobustness robustnessMode, - VCMDecodeErrorMode decode_error_mode) { - CriticalSectionScoped cs(_receiveCritSect); - switch (robustnessMode) { - case kNone: - _receiver.SetNackMode(kNoNack, -1, -1); - _dualReceiver.SetNackMode(kNoNack, -1, -1); - if (decode_error_mode == kNoErrors) { - _keyRequestMode = kKeyOnLoss; - } else { - _keyRequestMode = kKeyOnError; - } - break; - case kHardNack: - // Always wait for retransmissions (except when decoding with errors). - _receiver.SetNackMode(kNack, -1, -1); - _dualReceiver.SetNackMode(kNoNack, -1, -1); - _keyRequestMode = kKeyOnError; // TODO(hlundin): On long NACK list? - break; - case kSoftNack: - assert(false); // TODO(hlundin): Not completed. - return VCM_NOT_IMPLEMENTED; - // Enable hybrid NACK/FEC. Always wait for retransmissions and don't add - // extra delay when RTT is above kLowRttNackMs. - _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1); - _dualReceiver.SetNackMode(kNoNack, -1, -1); - _keyRequestMode = kKeyOnError; - break; - case kDualDecoder: - if (decode_error_mode == kNoErrors) { - return VCM_PARAMETER_ERROR; - } - // Enable NACK but don't wait for retransmissions and don't add any extra - // delay. - _receiver.SetNackMode(kNack, 0, 0); - // Enable NACK, compensate with extra delay and wait for retransmissions. - _dualReceiver.SetNackMode(kNack, -1, -1); - _keyRequestMode = kKeyOnError; - break; - case kReferenceSelection: - assert(false); // TODO(hlundin): Not completed. - return VCM_NOT_IMPLEMENTED; - if (decode_error_mode == kNoErrors) { - return VCM_PARAMETER_ERROR; - } - _receiver.SetNackMode(kNoNack, -1, -1); - _dualReceiver.SetNackMode(kNoNack, -1, -1); - break; - } - _receiver.SetDecodeErrorMode(decode_error_mode); - // The dual decoder should never decode with errors. - _dualReceiver.SetDecodeErrorMode(kNoErrors); - return VCM_OK; -} - -void VideoCodingModuleImpl::SetDecodeErrorMode( - VCMDecodeErrorMode decode_error_mode) { - CriticalSectionScoped cs(_receiveCritSect); - _receiver.SetDecodeErrorMode(decode_error_mode); -} - -void VideoCodingModuleImpl::SetNackSettings(size_t max_nack_list_size, - int max_packet_age_to_nack, - int max_incomplete_time_ms) { - if (max_nack_list_size != 0) { - CriticalSectionScoped cs(_receiveCritSect); - max_nack_list_size_ = max_nack_list_size; - } - _receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack, - max_incomplete_time_ms); - _dualReceiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack, - max_incomplete_time_ms); -} - -int VideoCodingModuleImpl::SetMinReceiverDelay(int desired_delay_ms) { - return _receiver.SetMinReceiverDelay(desired_delay_ms); -} - -int VideoCodingModuleImpl::StartDebugRecording(const char* file_name_utf8) { - CriticalSectionScoped cs(_sendCritSect); - _encoderInputFile = fopen(file_name_utf8, "wb"); - if (_encoderInputFile == NULL) - return VCM_GENERAL_ERROR; - return VCM_OK; -} - -int VideoCodingModuleImpl::StopDebugRecording() { - CriticalSectionScoped cs(_sendCritSect); - if (_encoderInputFile != NULL) { - fclose(_encoderInputFile); - _encoderInputFile = NULL; - } - return VCM_OK; -} - } // namespace webrtc diff --git a/webrtc/modules/video_coding/main/source/video_coding_impl.h b/webrtc/modules/video_coding/main/source/video_coding_impl.h index 6af45c02fc..21f5c32d91 100644 --- a/webrtc/modules/video_coding/main/source/video_coding_impl.h +++ b/webrtc/modules/video_coding/main/source/video_coding_impl.h @@ -26,300 +26,193 @@ #include "webrtc/system_wrappers/interface/clock.h" #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" -namespace webrtc -{ +namespace webrtc { +namespace vcm { -class VCMProcessTimer -{ -public: - VCMProcessTimer(uint32_t periodMs, Clock* clock) - : _clock(clock), - _periodMs(periodMs), - _latestMs(_clock->TimeInMilliseconds()) {} - uint32_t Period() const; - uint32_t TimeUntilProcess() const; - void Processed(); +class VCMProcessTimer { + public: + VCMProcessTimer(uint32_t periodMs, Clock* clock) + : _clock(clock), + _periodMs(periodMs), + _latestMs(_clock->TimeInMilliseconds()) {} + uint32_t Period() const; + uint32_t TimeUntilProcess() const; + void Processed(); -private: - Clock* _clock; - uint32_t _periodMs; - int64_t _latestMs; + private: + Clock* _clock; + uint32_t _periodMs; + int64_t _latestMs; }; -enum VCMKeyRequestMode -{ +class VideoSender { + public: + typedef VideoCodingModule::SenderNackMode SenderNackMode; + + VideoSender(const int32_t id, Clock* clock); + ~VideoSender(); + + int32_t InitializeSender(); + + // Register the send codec to be used. + int32_t RegisterSendCodec(const VideoCodec* sendCodec, + uint32_t numberOfCores, + uint32_t maxPayloadSize); + + int32_t SendCodec(VideoCodec* currentSendCodec) const; + VideoCodecType SendCodec() const; + int32_t RegisterExternalEncoder(VideoEncoder* externalEncoder, + uint8_t payloadType, + bool internalSource); + + int32_t CodecConfigParameters(uint8_t* buffer, int32_t size); + int Bitrate(unsigned int* bitrate) const; + int FrameRate(unsigned int* framerate) const; + + int32_t SetChannelParameters(uint32_t target_bitrate, // bits/s. + uint8_t lossRate, + uint32_t rtt); + + int32_t RegisterTransportCallback(VCMPacketizationCallback* transport); + int32_t RegisterSendStatisticsCallback(VCMSendStatisticsCallback* sendStats); + int32_t RegisterVideoQMCallback(VCMQMSettingsCallback* videoQMSettings); + int32_t RegisterProtectionCallback(VCMProtectionCallback* protection); + int32_t SetVideoProtection(VCMVideoProtection videoProtection, bool enable); + + int32_t AddVideoFrame(const I420VideoFrame& videoFrame, + const VideoContentMetrics* _contentMetrics, + const CodecSpecificInfo* codecSpecificInfo); + + int32_t IntraFrameRequest(int stream_index); + int32_t EnableFrameDropper(bool enable); + int32_t SentFrameCount(VCMFrameCount* frameCount) const; + + int SetSenderNackMode(SenderNackMode mode); + int SetSenderReferenceSelection(bool enable); + int SetSenderFEC(bool enable); + int SetSenderKeyFramePeriod(int periodMs); + + int StartDebugRecording(const char* file_name_utf8); + int StopDebugRecording(); + + int32_t TimeUntilNextProcess(); + int32_t Process(); + + private: + int32_t _id; + Clock* clock_; + + CriticalSectionWrapper* _sendCritSect; + VCMGenericEncoder* _encoder; + VCMEncodedFrameCallback _encodedFrameCallback; + std::vector _nextFrameTypes; + media_optimization::VCMMediaOptimization _mediaOpt; + VideoCodecType _sendCodecType; + VCMSendStatisticsCallback* _sendStatsCallback; + FILE* _encoderInputFile; + VCMCodecDataBase _codecDataBase; + bool frame_dropper_enabled_; + VCMProcessTimer _sendStatsTimer; +}; + +class VideoReceiver { + public: + typedef VideoCodingModule::ReceiverRobustness ReceiverRobustness; + + VideoReceiver(const int32_t id, Clock* clock, EventFactory* event_factory); + ~VideoReceiver(); + + int32_t InitializeReceiver(); + int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec, + int32_t numberOfCores, + bool requireKeyFrame); + + int32_t RegisterExternalDecoder(VideoDecoder* externalDecoder, + uint8_t payloadType, + bool internalRenderTiming); + int32_t RegisterReceiveCallback(VCMReceiveCallback* receiveCallback); + int32_t RegisterReceiveStatisticsCallback( + VCMReceiveStatisticsCallback* receiveStats); + int32_t RegisterFrameTypeCallback(VCMFrameTypeCallback* frameTypeCallback); + int32_t RegisterPacketRequestCallback(VCMPacketRequestCallback* callback); + int RegisterRenderBufferSizeCallback(VCMRenderBufferSizeCallback* callback); + + int32_t Decode(uint16_t maxWaitTimeMs); + int32_t DecodeDualFrame(uint16_t maxWaitTimeMs); + int32_t ResetDecoder(); + + int32_t ReceiveCodec(VideoCodec* currentReceiveCodec) const; + VideoCodecType ReceiveCodec() const; + + int32_t IncomingPacket(const uint8_t* incomingPayload, + uint32_t payloadLength, + const WebRtcRTPHeader& rtpInfo); + int32_t SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs); + int32_t SetRenderDelay(uint32_t timeMS); + int32_t Delay() const; + int32_t ReceivedFrameCount(VCMFrameCount* frameCount) const; + uint32_t DiscardedPackets() const; + + int SetReceiverRobustnessMode(ReceiverRobustness robustnessMode, + VCMDecodeErrorMode errorMode); + void SetNackSettings(size_t max_nack_list_size, + int max_packet_age_to_nack, + int max_incomplete_time_ms); + + void SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode); + int SetMinReceiverDelay(int desired_delay_ms); + + int32_t SetReceiveChannelParameters(uint32_t rtt); + int32_t SetVideoProtection(VCMVideoProtection videoProtection, bool enable); + + int32_t TimeUntilNextProcess(); + int32_t Process(); + + protected: + int32_t Decode(const webrtc::VCMEncodedFrame& frame); + int32_t RequestKeyFrame(); + int32_t RequestSliceLossIndication(const uint64_t pictureID) const; + int32_t NackList(uint16_t* nackList, uint16_t* size); + + private: + enum VCMKeyRequestMode { kKeyOnError, // Normal mode, request key frames on decoder error kKeyOnKeyLoss, // Request key frames on decoder error and on packet loss // in key frames. kKeyOnLoss, // Request key frames on decoder error and on packet loss // in any frame -}; + }; -class VideoCodingModuleImpl : public VideoCodingModule -{ -public: - VideoCodingModuleImpl(const int32_t id, Clock* clock, - EventFactory* event_factory, bool owns_event_factory); - - virtual ~VideoCodingModuleImpl(); - - int32_t Id() const; - - // Change the unique identifier of this object - virtual int32_t ChangeUniqueId(const int32_t id); - - // Returns the number of milliseconds until the module want a worker thread - // to call Process - virtual int32_t TimeUntilNextProcess(); - - virtual int32_t Process(); - - /* - * Sender - */ - - // Initialize send codec - virtual int32_t InitializeSender(); - - // Register the send codec to be used. - virtual int32_t RegisterSendCodec(const VideoCodec* sendCodec, - uint32_t numberOfCores, - uint32_t maxPayloadSize); - - // Get current send codec - virtual int32_t SendCodec(VideoCodec* currentSendCodec) const; - - // Get current send codec type - virtual VideoCodecType SendCodec() const; - - // Register an external encoder object. - virtual int32_t RegisterExternalEncoder(VideoEncoder* externalEncoder, - uint8_t payloadType, - bool internalSource = false); - - // Get codec config parameters - virtual int32_t CodecConfigParameters(uint8_t* buffer, - int32_t size); - - // Get encode bitrate - virtual int Bitrate(unsigned int* bitrate) const; - - // Get encode frame rate - virtual int FrameRate(unsigned int* framerate) const; - - // Set channel parameters - virtual int32_t SetChannelParameters( - uint32_t target_bitrate, // bits/s. - uint8_t lossRate, - uint32_t rtt); - - // Set receive channel parameters. - virtual int32_t SetReceiveChannelParameters(uint32_t rtt); - - // Register a transport callback which will be called to deliver the - // encoded buffers - virtual int32_t RegisterTransportCallback( - VCMPacketizationCallback* transport); - - // Register a send statistics callback which will be called to deliver - // information about the video stream produced by the encoder, - // for instance the average frame rate and bit rate. - virtual int32_t RegisterSendStatisticsCallback( - VCMSendStatisticsCallback* sendStats); - - // Register a video quality settings callback which will be called when - // frame rate/dimensions need to be updated for video quality optimization - virtual int32_t RegisterVideoQMCallback( - VCMQMSettingsCallback* videoQMSettings); - - // Register a video protection callback which will be called to deliver - // the requested FEC rate and NACK status (on/off). - virtual int32_t RegisterProtectionCallback( - VCMProtectionCallback* protection); - - // Enable or disable a video protection method. - virtual int32_t SetVideoProtection(VCMVideoProtection videoProtection, - bool enable); - - // Add one raw video frame to the encoder, blocking. - virtual int32_t AddVideoFrame( - const I420VideoFrame& videoFrame, - const VideoContentMetrics* _contentMetrics = NULL, - const CodecSpecificInfo* codecSpecificInfo = NULL); - - virtual int32_t IntraFrameRequest(int stream_index); - - //Enable frame dropper - virtual int32_t EnableFrameDropper(bool enable); - - // Sent frame counters - virtual int32_t SentFrameCount(VCMFrameCount& frameCount) const; - - /* - * Receiver - */ - - // Initialize receiver, resets codec database etc - virtual int32_t InitializeReceiver(); - - // Register possible reveive codecs, can be called multiple times - virtual int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec, - int32_t numberOfCores, - bool requireKeyFrame = false); - - // Register an externally defined decoder/render object. - // Can be a decoder only or a decoder coupled with a renderer. - virtual int32_t RegisterExternalDecoder(VideoDecoder* externalDecoder, - uint8_t payloadType, - bool internalRenderTiming); - - // Register a receive callback. Will be called whenever there are a new - // frame ready for rendering. - virtual int32_t RegisterReceiveCallback( - VCMReceiveCallback* receiveCallback); - - // Register a receive statistics callback which will be called to deliver - // information about the video stream received by the receiving side of the - // VCM, for instance the average frame rate and bit rate. - virtual int32_t RegisterReceiveStatisticsCallback( - VCMReceiveStatisticsCallback* receiveStats); - - // Register a frame type request callback. - virtual int32_t RegisterFrameTypeCallback( - VCMFrameTypeCallback* frameTypeCallback); - - // Nack callback - virtual int32_t RegisterPacketRequestCallback( - VCMPacketRequestCallback* callback); - - // Render buffer size callback. - virtual int RegisterRenderBufferSizeCallback( - VCMRenderBufferSizeCallback* callback); - - // Decode next frame, blocks for a maximum of maxWaitTimeMs milliseconds. - // Should be called as often as possible to get the most out of the decoder. - virtual int32_t Decode(uint16_t maxWaitTimeMs = 200); - - // Decode next dual frame, blocks for a maximum of maxWaitTimeMs - // milliseconds. - virtual int32_t DecodeDualFrame(uint16_t maxWaitTimeMs = 200); - - // Reset the decoder state - virtual int32_t ResetDecoder(); - - // Get current received codec - virtual int32_t ReceiveCodec(VideoCodec* currentReceiveCodec) const; - - // Get current received codec type - virtual VideoCodecType ReceiveCodec() const; - - // Incoming packet from network parsed and ready for decode, non blocking. - virtual int32_t IncomingPacket(const uint8_t* incomingPayload, - uint32_t payloadLength, - const WebRtcRTPHeader& rtpInfo); - - // Minimum playout delay (Used for lip-sync). This is the minimum delay - // required to sync with audio. Not included in VideoCodingModule::Delay() - // Defaults to 0 ms. - virtual int32_t SetMinimumPlayoutDelay( - uint32_t minPlayoutDelayMs); - - // The estimated delay caused by rendering - virtual int32_t SetRenderDelay(uint32_t timeMS); - - // Current delay - virtual int32_t Delay() const; - - // Received frame counters - virtual int32_t ReceivedFrameCount(VCMFrameCount& frameCount) const; - - // Returns the number of packets discarded by the jitter buffer. - virtual uint32_t DiscardedPackets() const; - - - // Robustness APIs - - // Set the sender RTX/NACK mode. - virtual int SetSenderNackMode(SenderNackMode mode); - - // Set the sender reference picture selection (RPS) mode. - virtual int SetSenderReferenceSelection(bool enable); - - // Set the sender forward error correction (FEC) mode. - virtual int SetSenderFEC(bool enable); - - // Set the key frame period, or disable periodic key frames (I-frames). - virtual int SetSenderKeyFramePeriod(int periodMs); - - // Set the receiver robustness mode. - virtual int SetReceiverRobustnessMode(ReceiverRobustness robustnessMode, - VCMDecodeErrorMode errorMode); - - virtual void SetNackSettings(size_t max_nack_list_size, - int max_packet_age_to_nack, - int max_incomplete_time_ms); - - // Sets jitter buffer decode error mode. - void SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode); - - - // Set the video delay for the receiver (default = 0). - virtual int SetMinReceiverDelay(int desired_delay_ms); - - // Enables recording of debugging information. - virtual int StartDebugRecording(const char* file_name_utf8); - - // Disables recording of debugging information. - virtual int StopDebugRecording(); - -protected: - int32_t Decode(const webrtc::VCMEncodedFrame& frame); - int32_t RequestKeyFrame(); - int32_t RequestSliceLossIndication( - const uint64_t pictureID) const; - int32_t NackList(uint16_t* nackList, uint16_t& size); - -private: - int32_t _id; - Clock* clock_; - CriticalSectionWrapper* _receiveCritSect; - bool _receiverInited; - VCMTiming _timing; - VCMTiming _dualTiming; - VCMReceiver _receiver; - VCMReceiver _dualReceiver; - VCMDecodedFrameCallback _decodedFrameCallback; - VCMDecodedFrameCallback _dualDecodedFrameCallback; - VCMFrameTypeCallback* _frameTypeCallback; - VCMReceiveStatisticsCallback* _receiveStatsCallback; - VCMPacketRequestCallback* _packetRequestCallback; - VCMRenderBufferSizeCallback* render_buffer_callback_; - VCMGenericDecoder* _decoder; - VCMGenericDecoder* _dualDecoder; + int32_t _id; + Clock* clock_; + CriticalSectionWrapper* _receiveCritSect; + bool _receiverInited; + VCMTiming _timing; + VCMTiming _dualTiming; + VCMReceiver _receiver; + VCMReceiver _dualReceiver; + VCMDecodedFrameCallback _decodedFrameCallback; + VCMDecodedFrameCallback _dualDecodedFrameCallback; + VCMFrameTypeCallback* _frameTypeCallback; + VCMReceiveStatisticsCallback* _receiveStatsCallback; + VCMPacketRequestCallback* _packetRequestCallback; + VCMRenderBufferSizeCallback* render_buffer_callback_; + VCMGenericDecoder* _decoder; + VCMGenericDecoder* _dualDecoder; #ifdef DEBUG_DECODER_BIT_STREAM - FILE* _bitStreamBeforeDecoder; + FILE* _bitStreamBeforeDecoder; #endif - VCMFrameBuffer _frameFromFile; - VCMKeyRequestMode _keyRequestMode; - bool _scheduleKeyRequest; - size_t max_nack_list_size_; + VCMFrameBuffer _frameFromFile; + VCMKeyRequestMode _keyRequestMode; + bool _scheduleKeyRequest; + size_t max_nack_list_size_; - CriticalSectionWrapper* _sendCritSect; // Critical section for send side - VCMGenericEncoder* _encoder; - VCMEncodedFrameCallback _encodedFrameCallback; - std::vector _nextFrameTypes; - media_optimization::VCMMediaOptimization _mediaOpt; - VideoCodecType _sendCodecType; - VCMSendStatisticsCallback* _sendStatsCallback; - FILE* _encoderInputFile; - VCMCodecDataBase _codecDataBase; - VCMProcessTimer _receiveStatsTimer; - VCMProcessTimer _sendStatsTimer; - VCMProcessTimer _retransmissionTimer; - VCMProcessTimer _keyRequestTimer; - EventFactory* event_factory_; - bool owns_event_factory_; - bool frame_dropper_enabled_; + VCMCodecDataBase _codecDataBase; + VCMProcessTimer _receiveStatsTimer; + VCMProcessTimer _retransmissionTimer; + VCMProcessTimer _keyRequestTimer; }; + +} // namespace vcm } // namespace webrtc -#endif // WEBRTC_MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ +#endif // WEBRTC_MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ diff --git a/webrtc/modules/video_coding/main/source/video_receiver.cc b/webrtc/modules/video_coding/main/source/video_receiver.cc new file mode 100644 index 0000000000..03f6ab1eab --- /dev/null +++ b/webrtc/modules/video_coding/main/source/video_receiver.cc @@ -0,0 +1,775 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/common_types.h" +#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" +#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" +#include "webrtc/modules/video_coding/main/source/encoded_frame.h" +#include "webrtc/modules/video_coding/main/source/jitter_buffer.h" +#include "webrtc/modules/video_coding/main/source/packet.h" +#include "webrtc/modules/video_coding/main/source/video_coding_impl.h" +#include "webrtc/system_wrappers/interface/clock.h" +#include "webrtc/system_wrappers/interface/trace.h" +#include "webrtc/system_wrappers/interface/trace_event.h" + +// #define DEBUG_DECODER_BIT_STREAM + +namespace webrtc { +namespace vcm { + +VideoReceiver::VideoReceiver(const int32_t id, + Clock* clock, + EventFactory* event_factory) + : _id(id), + clock_(clock), + _receiveCritSect(CriticalSectionWrapper::CreateCriticalSection()), + _receiverInited(false), + _timing(clock_, id, 1), + _dualTiming(clock_, id, 2, &_timing), + _receiver(&_timing, clock_, event_factory, id, 1, true), + _dualReceiver(&_dualTiming, clock_, event_factory, id, 2, false), + _decodedFrameCallback(_timing, clock_), + _dualDecodedFrameCallback(_dualTiming, clock_), + _frameTypeCallback(NULL), + _receiveStatsCallback(NULL), + _packetRequestCallback(NULL), + render_buffer_callback_(NULL), + _decoder(NULL), + _dualDecoder(NULL), +#ifdef DEBUG_DECODER_BIT_STREAM + _bitStreamBeforeDecoder(NULL), +#endif + _frameFromFile(), + _keyRequestMode(kKeyOnError), + _scheduleKeyRequest(false), + max_nack_list_size_(0), + _codecDataBase(id), + _receiveStatsTimer(1000, clock_), + _retransmissionTimer(10, clock_), + _keyRequestTimer(500, clock_) { + assert(clock_); +#ifdef DEBUG_DECODER_BIT_STREAM + _bitStreamBeforeDecoder = fopen("decoderBitStream.bit", "wb"); +#endif +} + +VideoReceiver::~VideoReceiver() { + if (_dualDecoder != NULL) { + _codecDataBase.ReleaseDecoder(_dualDecoder); + } + delete _receiveCritSect; +#ifdef DEBUG_DECODER_BIT_STREAM + fclose(_bitStreamBeforeDecoder); +#endif +} + +int32_t VideoReceiver::Process() { + int32_t returnValue = VCM_OK; + + // Receive-side statistics + if (_receiveStatsTimer.TimeUntilProcess() == 0) { + _receiveStatsTimer.Processed(); + if (_receiveStatsCallback != NULL) { + uint32_t bitRate; + uint32_t frameRate; + _receiver.ReceiveStatistics(&bitRate, &frameRate); + _receiveStatsCallback->OnReceiveStatisticsUpdate(bitRate, frameRate); + } + + // Size of render buffer. + if (render_buffer_callback_) { + int buffer_size_ms = _receiver.RenderBufferSizeMs(); + render_buffer_callback_->RenderBufferSizeMs(buffer_size_ms); + } + } + + // Key frame requests + if (_keyRequestTimer.TimeUntilProcess() == 0) { + _keyRequestTimer.Processed(); + if (_scheduleKeyRequest && _frameTypeCallback != NULL) { + const int32_t ret = RequestKeyFrame(); + if (ret != VCM_OK && returnValue == VCM_OK) { + returnValue = ret; + } + } + } + + // Packet retransmission requests + // TODO(holmer): Add API for changing Process interval and make sure it's + // disabled when NACK is off. + if (_retransmissionTimer.TimeUntilProcess() == 0) { + _retransmissionTimer.Processed(); + CriticalSectionScoped cs(_receiveCritSect); + if (_packetRequestCallback != NULL) { + uint16_t length = max_nack_list_size_; + std::vector nackList(length); + const int32_t ret = NackList(&nackList[0], &length); + if (ret != VCM_OK && returnValue == VCM_OK) { + returnValue = ret; + } + if (length > 0) { + _packetRequestCallback->ResendPackets(&nackList[0], length); + } + } + } + + return returnValue; +} + +int32_t VideoReceiver::TimeUntilNextProcess() { + uint32_t timeUntilNextProcess = _receiveStatsTimer.TimeUntilProcess(); + if ((_receiver.NackMode() != kNoNack) || + (_dualReceiver.State() != kPassive)) { + // We need a Process call more often if we are relying on + // retransmissions + timeUntilNextProcess = + VCM_MIN(timeUntilNextProcess, _retransmissionTimer.TimeUntilProcess()); + } + timeUntilNextProcess = + VCM_MIN(timeUntilNextProcess, _keyRequestTimer.TimeUntilProcess()); + + return timeUntilNextProcess; +} + +int32_t VideoReceiver::SetReceiveChannelParameters(uint32_t rtt) { + CriticalSectionScoped receiveCs(_receiveCritSect); + _receiver.UpdateRtt(rtt); + return 0; +} + +// Enable or disable a video protection method. +// Note: This API should be deprecated, as it does not offer a distinction +// between the protection method and decoding with or without errors. If such a +// behavior is desired, use the following API: SetReceiverRobustnessMode. +int32_t VideoReceiver::SetVideoProtection(VCMVideoProtection videoProtection, + bool enable) { + // By default, do not decode with errors. + _receiver.SetDecodeErrorMode(kNoErrors); + // The dual decoder should always be error free. + _dualReceiver.SetDecodeErrorMode(kNoErrors); + switch (videoProtection) { + case kProtectionNack: + case kProtectionNackReceiver: { + CriticalSectionScoped cs(_receiveCritSect); + if (enable) { + // Enable NACK and always wait for retransmits. + _receiver.SetNackMode(kNack, -1, -1); + } else { + _receiver.SetNackMode(kNoNack, -1, -1); + } + break; + } + + case kProtectionDualDecoder: { + CriticalSectionScoped cs(_receiveCritSect); + if (enable) { + // Enable NACK but don't wait for retransmissions and don't + // add any extra delay. + _receiver.SetNackMode(kNack, 0, 0); + // Enable NACK and always wait for retransmissions and + // compensate with extra delay. + _dualReceiver.SetNackMode(kNack, -1, -1); + _receiver.SetDecodeErrorMode(kWithErrors); + } else { + _dualReceiver.SetNackMode(kNoNack, -1, -1); + } + break; + } + + case kProtectionKeyOnLoss: { + CriticalSectionScoped cs(_receiveCritSect); + if (enable) { + _keyRequestMode = kKeyOnLoss; + _receiver.SetDecodeErrorMode(kWithErrors); + } else if (_keyRequestMode == kKeyOnLoss) { + _keyRequestMode = kKeyOnError; // default mode + } else { + return VCM_PARAMETER_ERROR; + } + break; + } + + case kProtectionKeyOnKeyLoss: { + CriticalSectionScoped cs(_receiveCritSect); + if (enable) { + _keyRequestMode = kKeyOnKeyLoss; + } else if (_keyRequestMode == kKeyOnKeyLoss) { + _keyRequestMode = kKeyOnError; // default mode + } else { + return VCM_PARAMETER_ERROR; + } + break; + } + + case kProtectionNackFEC: { + CriticalSectionScoped cs(_receiveCritSect); + if (enable) { + // Enable hybrid NACK/FEC. Always wait for retransmissions + // and don't add extra delay when RTT is above + // kLowRttNackMs. + _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1); + _receiver.SetDecodeErrorMode(kNoErrors); + _receiver.SetDecodeErrorMode(kNoErrors); + } else { + _receiver.SetNackMode(kNoNack, -1, -1); + } + break; + } + case kProtectionNackSender: + case kProtectionFEC: + case kProtectionPeriodicKeyFrames: + // Ignore encoder modes. + return VCM_OK; + } + return VCM_OK; +} + +// Initialize receiver, resets codec database etc +int32_t VideoReceiver::InitializeReceiver() { + CriticalSectionScoped cs(_receiveCritSect); + int32_t ret = _receiver.Initialize(); + if (ret < 0) { + return ret; + } + + ret = _dualReceiver.Initialize(); + if (ret < 0) { + return ret; + } + _codecDataBase.ResetReceiver(); + _timing.Reset(); + + _decoder = NULL; + _decodedFrameCallback.SetUserReceiveCallback(NULL); + _receiverInited = true; + _frameTypeCallback = NULL; + _receiveStatsCallback = NULL; + _packetRequestCallback = NULL; + _keyRequestMode = kKeyOnError; + _scheduleKeyRequest = false; + + return VCM_OK; +} + +// Register a receive callback. Will be called whenever there is a new frame +// ready for rendering. +int32_t VideoReceiver::RegisterReceiveCallback( + VCMReceiveCallback* receiveCallback) { + CriticalSectionScoped cs(_receiveCritSect); + _decodedFrameCallback.SetUserReceiveCallback(receiveCallback); + return VCM_OK; +} + +int32_t VideoReceiver::RegisterReceiveStatisticsCallback( + VCMReceiveStatisticsCallback* receiveStats) { + CriticalSectionScoped cs(_receiveCritSect); + _receiveStatsCallback = receiveStats; + return VCM_OK; +} + +// Register an externally defined decoder/render object. +// Can be a decoder only or a decoder coupled with a renderer. +int32_t VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder, + uint8_t payloadType, + bool internalRenderTiming) { + CriticalSectionScoped cs(_receiveCritSect); + if (externalDecoder == NULL) { + // Make sure the VCM updates the decoder next time it decodes. + _decoder = NULL; + return _codecDataBase.DeregisterExternalDecoder(payloadType) ? 0 : -1; + } + return _codecDataBase.RegisterExternalDecoder( + externalDecoder, payloadType, internalRenderTiming) + ? 0 + : -1; +} + +// Register a frame type request callback. +int32_t VideoReceiver::RegisterFrameTypeCallback( + VCMFrameTypeCallback* frameTypeCallback) { + CriticalSectionScoped cs(_receiveCritSect); + _frameTypeCallback = frameTypeCallback; + return VCM_OK; +} + +int32_t VideoReceiver::RegisterPacketRequestCallback( + VCMPacketRequestCallback* callback) { + CriticalSectionScoped cs(_receiveCritSect); + _packetRequestCallback = callback; + return VCM_OK; +} + +int VideoReceiver::RegisterRenderBufferSizeCallback( + VCMRenderBufferSizeCallback* callback) { + CriticalSectionScoped cs(_receiveCritSect); + render_buffer_callback_ = callback; + return VCM_OK; +} + +// Decode next frame, blocking. +// Should be called as often as possible to get the most out of the decoder. +int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) { + int64_t nextRenderTimeMs; + { + CriticalSectionScoped cs(_receiveCritSect); + if (!_receiverInited) { + return VCM_UNINITIALIZED; + } + if (!_codecDataBase.DecoderRegistered()) { + return VCM_NO_CODEC_REGISTERED; + } + } + + const bool dualReceiverEnabledNotReceiving = ( + _dualReceiver.State() != kReceiving && _dualReceiver.NackMode() == kNack); + + VCMEncodedFrame* frame = + _receiver.FrameForDecoding(maxWaitTimeMs, + nextRenderTimeMs, + _codecDataBase.SupportsRenderScheduling(), + &_dualReceiver); + + if (dualReceiverEnabledNotReceiving && _dualReceiver.State() == kReceiving) { + // Dual receiver is enabled (kNACK enabled), but was not receiving + // before the call to FrameForDecoding(). After the call the state + // changed to receiving, and therefore we must copy the primary decoder + // state to the dual decoder to make it possible for the dual decoder to + // start decoding retransmitted frames and recover. + CriticalSectionScoped cs(_receiveCritSect); + if (_dualDecoder != NULL) { + _codecDataBase.ReleaseDecoder(_dualDecoder); + } + _dualDecoder = _codecDataBase.CreateDecoderCopy(); + if (_dualDecoder != NULL) { + _dualDecoder->RegisterDecodeCompleteCallback(&_dualDecodedFrameCallback); + } else { + _dualReceiver.Reset(); + } + } + + if (frame == NULL) { + return VCM_FRAME_NOT_READY; + } else { + CriticalSectionScoped cs(_receiveCritSect); + + // If this frame was too late, we should adjust the delay accordingly + _timing.UpdateCurrentDelay(frame->RenderTimeMs(), + clock_->TimeInMilliseconds()); + +#ifdef DEBUG_DECODER_BIT_STREAM + if (_bitStreamBeforeDecoder != NULL) { + // Write bit stream to file for debugging purposes + if (fwrite( + frame->Buffer(), 1, frame->Length(), _bitStreamBeforeDecoder) != + frame->Length()) { + return -1; + } + } +#endif + const int32_t ret = Decode(*frame); + _receiver.ReleaseFrame(frame); + frame = NULL; + if (ret != VCM_OK) { + return ret; + } + } + return VCM_OK; +} + +int32_t VideoReceiver::RequestSliceLossIndication( + const uint64_t pictureID) const { + TRACE_EVENT1("webrtc", "RequestSLI", "picture_id", pictureID); + if (_frameTypeCallback != NULL) { + const int32_t ret = + _frameTypeCallback->SliceLossIndicationRequest(pictureID); + if (ret < 0) { + WEBRTC_TRACE(webrtc::kTraceError, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Failed to request key frame"); + return ret; + } + } else { + WEBRTC_TRACE(webrtc::kTraceWarning, + webrtc::kTraceVideoCoding, + VCMId(_id), + "No frame type request callback registered"); + return VCM_MISSING_CALLBACK; + } + return VCM_OK; +} + +int32_t VideoReceiver::RequestKeyFrame() { + TRACE_EVENT0("webrtc", "RequestKeyFrame"); + if (_frameTypeCallback != NULL) { + const int32_t ret = _frameTypeCallback->RequestKeyFrame(); + if (ret < 0) { + WEBRTC_TRACE(webrtc::kTraceError, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Failed to request key frame"); + return ret; + } + _scheduleKeyRequest = false; + } else { + WEBRTC_TRACE(webrtc::kTraceWarning, + webrtc::kTraceVideoCoding, + VCMId(_id), + "No frame type request callback registered"); + return VCM_MISSING_CALLBACK; + } + return VCM_OK; +} + +int32_t VideoReceiver::DecodeDualFrame(uint16_t maxWaitTimeMs) { + CriticalSectionScoped cs(_receiveCritSect); + if (_dualReceiver.State() != kReceiving || + _dualReceiver.NackMode() != kNack) { + // The dual receiver is currently not receiving or + // dual decoder mode is disabled. + return VCM_OK; + } + int64_t dummyRenderTime; + int32_t decodeCount = 0; + // The dual decoder's state is copied from the main decoder, which may + // decode with errors. Make sure that the dual decoder does not introduce + // error. + _dualReceiver.SetDecodeErrorMode(kNoErrors); + VCMEncodedFrame* dualFrame = + _dualReceiver.FrameForDecoding(maxWaitTimeMs, dummyRenderTime); + if (dualFrame != NULL && _dualDecoder != NULL) { + WEBRTC_TRACE(webrtc::kTraceStream, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Decoding frame %u with dual decoder", + dualFrame->TimeStamp()); + // Decode dualFrame and try to catch up + int32_t ret = + _dualDecoder->Decode(*dualFrame, clock_->TimeInMilliseconds()); + if (ret != WEBRTC_VIDEO_CODEC_OK) { + WEBRTC_TRACE(webrtc::kTraceWarning, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Failed to decode frame with dual decoder"); + _dualReceiver.ReleaseFrame(dualFrame); + return VCM_CODEC_ERROR; + } + if (_receiver.DualDecoderCaughtUp(dualFrame, _dualReceiver)) { + // Copy the complete decoder state of the dual decoder + // to the primary decoder. + WEBRTC_TRACE(webrtc::kTraceStream, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Dual decoder caught up"); + _codecDataBase.CopyDecoder(*_dualDecoder); + _codecDataBase.ReleaseDecoder(_dualDecoder); + _dualDecoder = NULL; + } + decodeCount++; + } + _dualReceiver.ReleaseFrame(dualFrame); + return decodeCount; +} + +// Must be called from inside the receive side critical section. +int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) { + TRACE_EVENT_ASYNC_STEP1("webrtc", + "Video", + frame.TimeStamp(), + "Decode", + "type", + frame.FrameType()); + // Change decoder if payload type has changed + const bool renderTimingBefore = _codecDataBase.SupportsRenderScheduling(); + _decoder = + _codecDataBase.GetDecoder(frame.PayloadType(), &_decodedFrameCallback); + if (renderTimingBefore != _codecDataBase.SupportsRenderScheduling()) { + // Make sure we reset the decode time estimate since it will + // be zero for codecs without render timing. + _timing.ResetDecodeTime(); + } + if (_decoder == NULL) { + return VCM_NO_CODEC_REGISTERED; + } + // Decode a frame + int32_t ret = _decoder->Decode(frame, clock_->TimeInMilliseconds()); + + // Check for failed decoding, run frame type request callback if needed. + if (ret < 0) { + if (ret == VCM_ERROR_REQUEST_SLI) { + return RequestSliceLossIndication( + _decodedFrameCallback.LastReceivedPictureID() + 1); + } else { + WEBRTC_TRACE(webrtc::kTraceError, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Failed to decode frame %u, requesting key frame", + frame.TimeStamp()); + ret = RequestKeyFrame(); + } + } else if (ret == VCM_REQUEST_SLI) { + ret = RequestSliceLossIndication( + _decodedFrameCallback.LastReceivedPictureID() + 1); + } + if (!frame.Complete() || frame.MissingFrame()) { + switch (_keyRequestMode) { + case kKeyOnKeyLoss: { + if (frame.FrameType() == kVideoFrameKey) { + _scheduleKeyRequest = true; + return VCM_OK; + } + break; + } + case kKeyOnLoss: { + _scheduleKeyRequest = true; + return VCM_OK; + } + default: + break; + } + } + TRACE_EVENT_ASYNC_END0("webrtc", "Video", frame.TimeStamp()); + return ret; +} + +// Reset the decoder state +int32_t VideoReceiver::ResetDecoder() { + CriticalSectionScoped cs(_receiveCritSect); + if (_decoder != NULL) { + _receiver.Initialize(); + _timing.Reset(); + _scheduleKeyRequest = false; + _decoder->Reset(); + } + if (_dualReceiver.State() != kPassive) { + _dualReceiver.Initialize(); + } + if (_dualDecoder != NULL) { + _codecDataBase.ReleaseDecoder(_dualDecoder); + _dualDecoder = NULL; + } + return VCM_OK; +} + +// Register possible receive codecs, can be called multiple times +int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec, + int32_t numberOfCores, + bool requireKeyFrame) { + CriticalSectionScoped cs(_receiveCritSect); + if (receiveCodec == NULL) { + return VCM_PARAMETER_ERROR; + } + if (!_codecDataBase.RegisterReceiveCodec( + receiveCodec, numberOfCores, requireKeyFrame)) { + return -1; + } + return 0; +} + +// Get current received codec +int32_t VideoReceiver::ReceiveCodec(VideoCodec* currentReceiveCodec) const { + CriticalSectionScoped cs(_receiveCritSect); + if (currentReceiveCodec == NULL) { + return VCM_PARAMETER_ERROR; + } + return _codecDataBase.ReceiveCodec(currentReceiveCodec) ? 0 : -1; +} + +// Get current received codec +VideoCodecType VideoReceiver::ReceiveCodec() const { + CriticalSectionScoped cs(_receiveCritSect); + return _codecDataBase.ReceiveCodec(); +} + +// Incoming packet from network parsed and ready for decode, non blocking. +int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload, + uint32_t payloadLength, + const WebRtcRTPHeader& rtpInfo) { + if (rtpInfo.frameType == kVideoFrameKey) { + TRACE_EVENT1("webrtc", + "VCM::PacketKeyFrame", + "seqnum", + rtpInfo.header.sequenceNumber); + } + if (incomingPayload == NULL) { + // The jitter buffer doesn't handle non-zero payload lengths for packets + // without payload. + // TODO(holmer): We should fix this in the jitter buffer. + payloadLength = 0; + } + const VCMPacket packet(incomingPayload, payloadLength, rtpInfo); + int32_t ret; + if (_dualReceiver.State() != kPassive) { + ret = _dualReceiver.InsertPacket( + packet, rtpInfo.type.Video.width, rtpInfo.type.Video.height); + if (ret == VCM_FLUSH_INDICATOR) { + RequestKeyFrame(); + ResetDecoder(); + } else if (ret < 0) { + return ret; + } + } + ret = _receiver.InsertPacket( + packet, rtpInfo.type.Video.width, rtpInfo.type.Video.height); + // TODO(holmer): Investigate if this somehow should use the key frame + // request scheduling to throttle the requests. + if (ret == VCM_FLUSH_INDICATOR) { + RequestKeyFrame(); + ResetDecoder(); + } else if (ret < 0) { + return ret; + } + return VCM_OK; +} + +// Minimum playout delay (used for lip-sync). This is the minimum delay required +// to sync with audio. Not included in VideoCodingModule::Delay() +// Defaults to 0 ms. +int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) { + _timing.set_min_playout_delay(minPlayoutDelayMs); + return VCM_OK; +} + +// The estimated delay caused by rendering, defaults to +// kDefaultRenderDelayMs = 10 ms +int32_t VideoReceiver::SetRenderDelay(uint32_t timeMS) { + _timing.set_render_delay(timeMS); + return VCM_OK; +} + +// Current video delay +int32_t VideoReceiver::Delay() const { return _timing.TargetVideoDelay(); } + +// Nack list +int32_t VideoReceiver::NackList(uint16_t* nackList, uint16_t* size) { + VCMNackStatus nackStatus = kNackOk; + uint16_t nack_list_length = 0; + // Collect sequence numbers from the default receiver + // if in normal nack mode. Otherwise collect them from + // the dual receiver if the dual receiver is receiving. + if (_receiver.NackMode() != kNoNack) { + nackStatus = _receiver.NackList(nackList, *size, &nack_list_length); + } + if (nack_list_length == 0 && _dualReceiver.State() != kPassive) { + nackStatus = _dualReceiver.NackList(nackList, *size, &nack_list_length); + } + *size = nack_list_length; + + switch (nackStatus) { + case kNackNeedMoreMemory: { + WEBRTC_TRACE(webrtc::kTraceError, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Out of memory"); + return VCM_MEMORY; + } + case kNackKeyFrameRequest: { + CriticalSectionScoped cs(_receiveCritSect); + WEBRTC_TRACE(webrtc::kTraceWarning, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Failed to get NACK list, requesting key frame"); + return RequestKeyFrame(); + } + default: + break; + } + return VCM_OK; +} + +int32_t VideoReceiver::ReceivedFrameCount(VCMFrameCount* frameCount) const { + _receiver.ReceivedFrameCount(frameCount); + return VCM_OK; +} + +uint32_t VideoReceiver::DiscardedPackets() const { + return _receiver.DiscardedPackets(); +} + +int VideoReceiver::SetReceiverRobustnessMode( + ReceiverRobustness robustnessMode, + VCMDecodeErrorMode decode_error_mode) { + CriticalSectionScoped cs(_receiveCritSect); + switch (robustnessMode) { + case VideoCodingModule::kNone: + _receiver.SetNackMode(kNoNack, -1, -1); + _dualReceiver.SetNackMode(kNoNack, -1, -1); + if (decode_error_mode == kNoErrors) { + _keyRequestMode = kKeyOnLoss; + } else { + _keyRequestMode = kKeyOnError; + } + break; + case VideoCodingModule::kHardNack: + // Always wait for retransmissions (except when decoding with errors). + _receiver.SetNackMode(kNack, -1, -1); + _dualReceiver.SetNackMode(kNoNack, -1, -1); + _keyRequestMode = kKeyOnError; // TODO(hlundin): On long NACK list? + break; + case VideoCodingModule::kSoftNack: + assert(false); // TODO(hlundin): Not completed. + return VCM_NOT_IMPLEMENTED; + // Enable hybrid NACK/FEC. Always wait for retransmissions and don't add + // extra delay when RTT is above kLowRttNackMs. + _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1); + _dualReceiver.SetNackMode(kNoNack, -1, -1); + _keyRequestMode = kKeyOnError; + break; + case VideoCodingModule::kDualDecoder: + if (decode_error_mode == kNoErrors) { + return VCM_PARAMETER_ERROR; + } + // Enable NACK but don't wait for retransmissions and don't add any extra + // delay. + _receiver.SetNackMode(kNack, 0, 0); + // Enable NACK, compensate with extra delay and wait for retransmissions. + _dualReceiver.SetNackMode(kNack, -1, -1); + _keyRequestMode = kKeyOnError; + break; + case VideoCodingModule::kReferenceSelection: + assert(false); // TODO(hlundin): Not completed. + return VCM_NOT_IMPLEMENTED; + if (decode_error_mode == kNoErrors) { + return VCM_PARAMETER_ERROR; + } + _receiver.SetNackMode(kNoNack, -1, -1); + _dualReceiver.SetNackMode(kNoNack, -1, -1); + break; + } + _receiver.SetDecodeErrorMode(decode_error_mode); + // The dual decoder should never decode with errors. + _dualReceiver.SetDecodeErrorMode(kNoErrors); + return VCM_OK; +} + +void VideoReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) { + CriticalSectionScoped cs(_receiveCritSect); + _receiver.SetDecodeErrorMode(decode_error_mode); +} + +void VideoReceiver::SetNackSettings(size_t max_nack_list_size, + int max_packet_age_to_nack, + int max_incomplete_time_ms) { + if (max_nack_list_size != 0) { + CriticalSectionScoped cs(_receiveCritSect); + max_nack_list_size_ = max_nack_list_size; + } + _receiver.SetNackSettings( + max_nack_list_size, max_packet_age_to_nack, max_incomplete_time_ms); + _dualReceiver.SetNackSettings( + max_nack_list_size, max_packet_age_to_nack, max_incomplete_time_ms); +} + +int VideoReceiver::SetMinReceiverDelay(int desired_delay_ms) { + return _receiver.SetMinReceiverDelay(desired_delay_ms); +} + +} // namespace vcm +} // namespace webrtc diff --git a/webrtc/modules/video_coding/main/source/video_sender.cc b/webrtc/modules/video_coding/main/source/video_sender.cc new file mode 100644 index 0000000000..73ee856230 --- /dev/null +++ b/webrtc/modules/video_coding/main/source/video_sender.cc @@ -0,0 +1,420 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/common_types.h" + +#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" +#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" +#include "webrtc/modules/video_coding/main/source/encoded_frame.h" +#include "webrtc/modules/video_coding/main/source/video_coding_impl.h" +#include "webrtc/system_wrappers/interface/clock.h" + +namespace webrtc { +namespace vcm { + +VideoSender::VideoSender(const int32_t id, Clock* clock) + : _id(id), + clock_(clock), + _sendCritSect(CriticalSectionWrapper::CreateCriticalSection()), + _encoder(), + _encodedFrameCallback(), + _nextFrameTypes(1, kVideoFrameDelta), + _mediaOpt(id, clock_), + _sendCodecType(kVideoCodecUnknown), + _sendStatsCallback(NULL), + _encoderInputFile(NULL), + _codecDataBase(id), + frame_dropper_enabled_(true), + _sendStatsTimer(1000, clock_) {} + +VideoSender::~VideoSender() { + delete _sendCritSect; + if (_encoderInputFile != NULL) { + fclose(_encoderInputFile); + } +} + +int32_t VideoSender::Process() { + int32_t returnValue = VCM_OK; + + if (_sendStatsTimer.TimeUntilProcess() == 0) { + _sendStatsTimer.Processed(); + if (_sendStatsCallback != NULL) { + uint32_t bitRate; + uint32_t frameRate; + { + CriticalSectionScoped cs(_sendCritSect); + bitRate = _mediaOpt.SentBitRate(); + frameRate = _mediaOpt.SentFrameRate(); + } + _sendStatsCallback->SendStatistics(bitRate, frameRate); + } + } + + return returnValue; +} + +// Reset send side to initial state - all components +int32_t VideoSender::InitializeSender() { + CriticalSectionScoped cs(_sendCritSect); + _codecDataBase.ResetSender(); + _encoder = NULL; + _encodedFrameCallback.SetTransportCallback(NULL); + // setting default bitRate and frameRate to 0 + _mediaOpt.SetEncodingData(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0); + _mediaOpt.Reset(); // Resetting frame dropper + return VCM_OK; +} + +int32_t VideoSender::TimeUntilNextProcess() { + return _sendStatsTimer.TimeUntilProcess(); +} + +// Register the send codec to be used. +int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec, + uint32_t numberOfCores, + uint32_t maxPayloadSize) { + CriticalSectionScoped cs(_sendCritSect); + if (sendCodec == NULL) { + return VCM_PARAMETER_ERROR; + } + + bool ret = _codecDataBase.SetSendCodec( + sendCodec, numberOfCores, maxPayloadSize, &_encodedFrameCallback); + if (!ret) { + WEBRTC_TRACE(webrtc::kTraceError, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Failed to initialize encoder"); + return VCM_CODEC_ERROR; + } + + _encoder = _codecDataBase.GetEncoder(); + _sendCodecType = sendCodec->codecType; + int numLayers = (_sendCodecType != kVideoCodecVP8) + ? 1 + : sendCodec->codecSpecific.VP8.numberOfTemporalLayers; + // If we have screensharing and we have layers, we disable frame dropper. + bool disable_frame_dropper = + numLayers > 1 && sendCodec->mode == kScreensharing; + if (disable_frame_dropper) { + _mediaOpt.EnableFrameDropper(false); + } else if (frame_dropper_enabled_) { + _mediaOpt.EnableFrameDropper(true); + } + _nextFrameTypes.clear(); + _nextFrameTypes.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1), + kVideoFrameDelta); + + _mediaOpt.SetEncodingData(_sendCodecType, + sendCodec->maxBitrate * 1000, + sendCodec->maxFramerate * 1000, + sendCodec->startBitrate * 1000, + sendCodec->width, + sendCodec->height, + numLayers); + _mediaOpt.SetMtu(maxPayloadSize); + + return VCM_OK; +} + +// Get current send codec +int32_t VideoSender::SendCodec(VideoCodec* currentSendCodec) const { + CriticalSectionScoped cs(_sendCritSect); + + if (currentSendCodec == NULL) { + return VCM_PARAMETER_ERROR; + } + return _codecDataBase.SendCodec(currentSendCodec) ? 0 : -1; +} + +// Get the current send codec type +VideoCodecType VideoSender::SendCodec() const { + CriticalSectionScoped cs(_sendCritSect); + + return _codecDataBase.SendCodec(); +} + +// Register an external decoder object. +// This can not be used together with external decoder callbacks. +int32_t VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder, + uint8_t payloadType, + bool internalSource /*= false*/) { + CriticalSectionScoped cs(_sendCritSect); + + if (externalEncoder == NULL) { + bool wasSendCodec = false; + const bool ret = + _codecDataBase.DeregisterExternalEncoder(payloadType, &wasSendCodec); + if (wasSendCodec) { + // Make sure the VCM doesn't use the de-registered codec + _encoder = NULL; + } + return ret ? 0 : -1; + } + _codecDataBase.RegisterExternalEncoder( + externalEncoder, payloadType, internalSource); + return 0; +} + +// Get codec config parameters +int32_t VideoSender::CodecConfigParameters(uint8_t* buffer, int32_t size) { + CriticalSectionScoped cs(_sendCritSect); + if (_encoder != NULL) { + return _encoder->CodecConfigParameters(buffer, size); + } + return VCM_UNINITIALIZED; +} + +// Get encode bitrate +int VideoSender::Bitrate(unsigned int* bitrate) const { + CriticalSectionScoped cs(_sendCritSect); + // return the bit rate which the encoder is set to + if (!_encoder) { + return VCM_UNINITIALIZED; + } + *bitrate = _encoder->BitRate(); + return 0; +} + +// Get encode frame rate +int VideoSender::FrameRate(unsigned int* framerate) const { + CriticalSectionScoped cs(_sendCritSect); + // input frame rate, not compensated + if (!_encoder) { + return VCM_UNINITIALIZED; + } + *framerate = _encoder->FrameRate(); + return 0; +} + +// Set channel parameters +int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate, + uint8_t lossRate, + uint32_t rtt) { + int32_t ret = 0; + { + CriticalSectionScoped sendCs(_sendCritSect); + uint32_t targetRate = + _mediaOpt.SetTargetRates(target_bitrate, lossRate, rtt); + if (_encoder != NULL) { + ret = _encoder->SetChannelParameters(lossRate, rtt); + if (ret < 0) { + return ret; + } + ret = (int32_t)_encoder->SetRates(targetRate, _mediaOpt.InputFrameRate()); + if (ret < 0) { + return ret; + } + } else { + return VCM_UNINITIALIZED; + } // encoder + } // send side + return VCM_OK; +} + +int32_t VideoSender::RegisterTransportCallback( + VCMPacketizationCallback* transport) { + CriticalSectionScoped cs(_sendCritSect); + _encodedFrameCallback.SetMediaOpt(&_mediaOpt); + _encodedFrameCallback.SetTransportCallback(transport); + return VCM_OK; +} + +// Register video output information callback which will be called to deliver +// information about the video stream produced by the encoder, for instance the +// average frame rate and bit rate. +int32_t VideoSender::RegisterSendStatisticsCallback( + VCMSendStatisticsCallback* sendStats) { + CriticalSectionScoped cs(_sendCritSect); + _sendStatsCallback = sendStats; + return VCM_OK; +} + +// Register a video quality settings callback which will be called when frame +// rate/dimensions need to be updated for video quality optimization +int32_t VideoSender::RegisterVideoQMCallback( + VCMQMSettingsCallback* videoQMSettings) { + CriticalSectionScoped cs(_sendCritSect); + return _mediaOpt.RegisterVideoQMCallback(videoQMSettings); +} + +// Register a video protection callback which will be called to deliver the +// requested FEC rate and NACK status (on/off). +int32_t VideoSender::RegisterProtectionCallback( + VCMProtectionCallback* protection) { + CriticalSectionScoped cs(_sendCritSect); + _mediaOpt.RegisterProtectionCallback(protection); + return VCM_OK; +} + +// Enable or disable a video protection method. +// Note: This API should be deprecated, as it does not offer a distinction +// between the protection method and decoding with or without errors. If such a +// behavior is desired, use the following API: SetReceiverRobustnessMode. +int32_t VideoSender::SetVideoProtection(VCMVideoProtection videoProtection, + bool enable) { + switch (videoProtection) { + case kProtectionNack: + case kProtectionNackSender: { + CriticalSectionScoped cs(_sendCritSect); + _mediaOpt.EnableProtectionMethod(enable, media_optimization::kNack); + break; + } + + case kProtectionNackFEC: { + CriticalSectionScoped cs(_sendCritSect); + _mediaOpt.EnableProtectionMethod(enable, media_optimization::kNackFec); + break; + } + + case kProtectionFEC: { + CriticalSectionScoped cs(_sendCritSect); + _mediaOpt.EnableProtectionMethod(enable, media_optimization::kFec); + break; + } + + case kProtectionPeriodicKeyFrames: { + CriticalSectionScoped cs(_sendCritSect); + return _codecDataBase.SetPeriodicKeyFrames(enable) ? 0 : -1; + break; + } + case kProtectionNackReceiver: + case kProtectionDualDecoder: + case kProtectionKeyOnLoss: + case kProtectionKeyOnKeyLoss: + // Ignore decoder modes. + return VCM_OK; + } + return VCM_OK; +} +// Add one raw video frame to the encoder, blocking. +int32_t VideoSender::AddVideoFrame(const I420VideoFrame& videoFrame, + const VideoContentMetrics* contentMetrics, + const CodecSpecificInfo* codecSpecificInfo) { + CriticalSectionScoped cs(_sendCritSect); + if (_encoder == NULL) { + return VCM_UNINITIALIZED; + } + // TODO(holmer): Add support for dropping frames per stream. Currently we + // only have one frame dropper for all streams. + if (_nextFrameTypes[0] == kFrameEmpty) { + return VCM_OK; + } + _mediaOpt.UpdateIncomingFrameRate(); + + if (_mediaOpt.DropFrame()) { + WEBRTC_TRACE(webrtc::kTraceStream, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Drop frame due to bitrate"); + } else { + _mediaOpt.UpdateContentData(contentMetrics); + int32_t ret = + _encoder->Encode(videoFrame, codecSpecificInfo, _nextFrameTypes); + if (_encoderInputFile != NULL) { + if (PrintI420VideoFrame(videoFrame, _encoderInputFile) < 0) { + return -1; + } + } + if (ret < 0) { + WEBRTC_TRACE(webrtc::kTraceError, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Encode error: %d", + ret); + return ret; + } + for (size_t i = 0; i < _nextFrameTypes.size(); ++i) { + _nextFrameTypes[i] = kVideoFrameDelta; // Default frame type. + } + } + return VCM_OK; +} + +int32_t VideoSender::IntraFrameRequest(int stream_index) { + CriticalSectionScoped cs(_sendCritSect); + if (stream_index < 0 || + static_cast(stream_index) >= _nextFrameTypes.size()) { + return -1; + } + _nextFrameTypes[stream_index] = kVideoFrameKey; + if (_encoder != NULL && _encoder->InternalSource()) { + // Try to request the frame if we have an external encoder with + // internal source since AddVideoFrame never will be called. + if (_encoder->RequestFrame(_nextFrameTypes) == WEBRTC_VIDEO_CODEC_OK) { + _nextFrameTypes[stream_index] = kVideoFrameDelta; + } + } + return VCM_OK; +} + +int32_t VideoSender::EnableFrameDropper(bool enable) { + CriticalSectionScoped cs(_sendCritSect); + frame_dropper_enabled_ = enable; + _mediaOpt.EnableFrameDropper(enable); + return VCM_OK; +} + +int32_t VideoSender::SentFrameCount(VCMFrameCount* frameCount) const { + CriticalSectionScoped cs(_sendCritSect); + return _mediaOpt.SentFrameCount(*frameCount); +} + +int VideoSender::SetSenderNackMode(SenderNackMode mode) { + CriticalSectionScoped cs(_sendCritSect); + + switch (mode) { + case VideoCodingModule::kNackNone: + _mediaOpt.EnableProtectionMethod(false, media_optimization::kNack); + break; + case VideoCodingModule::kNackAll: + _mediaOpt.EnableProtectionMethod(true, media_optimization::kNack); + break; + case VideoCodingModule::kNackSelective: + return VCM_NOT_IMPLEMENTED; + break; + } + return VCM_OK; +} + +int VideoSender::SetSenderReferenceSelection(bool enable) { + return VCM_NOT_IMPLEMENTED; +} + +int VideoSender::SetSenderFEC(bool enable) { + CriticalSectionScoped cs(_sendCritSect); + _mediaOpt.EnableProtectionMethod(enable, media_optimization::kFec); + return VCM_OK; +} + +int VideoSender::SetSenderKeyFramePeriod(int periodMs) { + return VCM_NOT_IMPLEMENTED; +} + +int VideoSender::StartDebugRecording(const char* file_name_utf8) { + CriticalSectionScoped cs(_sendCritSect); + _encoderInputFile = fopen(file_name_utf8, "wb"); + if (_encoderInputFile == NULL) + return VCM_GENERAL_ERROR; + return VCM_OK; +} + +int VideoSender::StopDebugRecording() { + CriticalSectionScoped cs(_sendCritSect); + if (_encoderInputFile != NULL) { + fclose(_encoderInputFile); + _encoderInputFile = NULL; + } + return VCM_OK; +} + +} // namespace vcm +} // namespace webrtc