From 54d072ea20594165c8f124ef382a3de0d111f938 Mon Sep 17 00:00:00 2001 From: "guoweis@webrtc.org" Date: Tue, 17 Mar 2015 21:54:50 +0000 Subject: [PATCH] Add CVO support to video_coding layer. CVO is, instead of rotating frame on the capture side, to have renderer rotate the frame based on a new rtp header extension. The change includes 1. encoder side needs to pass this from raw frame to the encoded frame. 2. decoder needs to copy it from rtp packet (only the last packet of a frame has this info) to decoded frame. R=mflodman@webrtc.org TBR=stefan@webrtc.org BUG=4145 Review URL: https://webrtc-codereview.appspot.com/46429006 Cr-Commit-Position: refs/heads/master@{#8767} git-svn-id: http://webrtc.googlecode.com/svn/trunk@8767 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../video_coding/main/source/encoded_frame.cc | 51 ++++++++++--------- .../video_coding/main/source/encoded_frame.h | 10 ++++ .../video_coding/main/source/frame_buffer.cc | 13 +++++ .../main/source/generic_decoder.cc | 2 + .../main/source/generic_decoder.h | 1 + .../main/source/generic_encoder.cc | 33 +++++++++--- .../main/source/generic_encoder.h | 5 ++ 7 files changed, 83 insertions(+), 32 deletions(-) diff --git a/webrtc/modules/video_coding/main/source/encoded_frame.cc b/webrtc/modules/video_coding/main/source/encoded_frame.cc index 0d07955555..0cf4874c25 100644 --- a/webrtc/modules/video_coding/main/source/encoded_frame.cc +++ b/webrtc/modules/video_coding/main/source/encoded_frame.cc @@ -16,26 +16,26 @@ namespace webrtc { VCMEncodedFrame::VCMEncodedFrame() -: -webrtc::EncodedImage(), -_renderTimeMs(-1), -_payloadType(0), -_missingFrame(false), -_codec(kVideoCodecUnknown), -_fragmentation() -{ + : webrtc::EncodedImage(), + _renderTimeMs(-1), + _payloadType(0), + _missingFrame(false), + _codec(kVideoCodecUnknown), + _fragmentation(), + _rotation(kVideoRotation_0), + _rotation_set(false) { _codecSpecificInfo.codecType = kVideoCodecUnknown; } VCMEncodedFrame::VCMEncodedFrame(const webrtc::EncodedImage& rhs) -: -webrtc::EncodedImage(rhs), -_renderTimeMs(-1), -_payloadType(0), -_missingFrame(false), -_codec(kVideoCodecUnknown), -_fragmentation() -{ + : webrtc::EncodedImage(rhs), + _renderTimeMs(-1), + _payloadType(0), + _missingFrame(false), + _codec(kVideoCodecUnknown), + _fragmentation(), + _rotation(kVideoRotation_0), + _rotation_set(false) { _codecSpecificInfo.codecType = kVideoCodecUnknown; _buffer = NULL; _size = 0; @@ -48,14 +48,15 @@ _fragmentation() } VCMEncodedFrame::VCMEncodedFrame(const VCMEncodedFrame& rhs) - : - webrtc::EncodedImage(rhs), - _renderTimeMs(rhs._renderTimeMs), - _payloadType(rhs._payloadType), - _missingFrame(rhs._missingFrame), - _codecSpecificInfo(rhs._codecSpecificInfo), - _codec(rhs._codec), - _fragmentation() { + : webrtc::EncodedImage(rhs), + _renderTimeMs(rhs._renderTimeMs), + _payloadType(rhs._payloadType), + _missingFrame(rhs._missingFrame), + _codecSpecificInfo(rhs._codecSpecificInfo), + _codec(rhs._codec), + _fragmentation(), + _rotation(rhs._rotation), + _rotation_set(rhs._rotation_set) { _buffer = NULL; _size = 0; _length = 0; @@ -96,6 +97,8 @@ void VCMEncodedFrame::Reset() _length = 0; _codecSpecificInfo.codecType = kVideoCodecUnknown; _codec = kVideoCodecUnknown; + _rotation = kVideoRotation_0; + _rotation_set = false; } void VCMEncodedFrame::CopyCodecSpecific(const RTPVideoHeader* header) diff --git a/webrtc/modules/video_coding/main/source/encoded_frame.h b/webrtc/modules/video_coding/main/source/encoded_frame.h index 4be4e6b50a..d8589070d4 100644 --- a/webrtc/modules/video_coding/main/source/encoded_frame.h +++ b/webrtc/modules/video_coding/main/source/encoded_frame.h @@ -70,6 +70,10 @@ public: */ webrtc::FrameType FrameType() const {return ConvertFrameType(_frameType);} /** + * Get frame rotation + */ + VideoRotation rotation() const { return _rotation; } + /** * True if this frame is complete, false otherwise */ bool Complete() const { return _completeFrame; } @@ -116,6 +120,12 @@ protected: CodecSpecificInfo _codecSpecificInfo; webrtc::VideoCodecType _codec; RTPFragmentationHeader _fragmentation; + VideoRotation _rotation; + + // Video rotation is only set along with the last packet for each frame + // (same as marker bit). This |_rotation_set| is only for debugging purpose + // to ensure we don't set it twice for a frame. + bool _rotation_set; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/main/source/frame_buffer.cc b/webrtc/modules/video_coding/main/source/frame_buffer.cc index 94b06f1ff7..8bd375893d 100644 --- a/webrtc/modules/video_coding/main/source/frame_buffer.cc +++ b/webrtc/modules/video_coding/main/source/frame_buffer.cc @@ -13,6 +13,7 @@ #include #include +#include "webrtc/base/checks.h" #include "webrtc/modules/video_coding/main/source/packet.h" #include "webrtc/system_wrappers/interface/logging.h" @@ -146,6 +147,18 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet, _latestPacketTimeMs = timeInMs; + // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ + // ts_126114v120700p.pdf Section 7.4.5. + // The MTSI client shall add the payload bytes as defined in this clause + // onto the last RTP packet in each group of packets which make up a key + // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265 + // (HEVC)). + if (packet.markerBit) { + DCHECK(!_rotation_set); + _rotation = packet.codecSpecificHeader.rotation; + _rotation_set = true; + } + if (_sessionInfo.complete()) { SetState(kStateComplete); return kCompleteSession; diff --git a/webrtc/modules/video_coding/main/source/generic_decoder.cc b/webrtc/modules/video_coding/main/source/generic_decoder.cc index 4f8a7ca7d1..88bc75ae91 100644 --- a/webrtc/modules/video_coding/main/source/generic_decoder.cc +++ b/webrtc/modules/video_coding/main/source/generic_decoder.cc @@ -74,6 +74,7 @@ int32_t VCMDecodedFrameCallback::Decoded(I420VideoFrame& decodedImage) if (callback != NULL) { decodedImage.set_render_time_ms(frameInfo->renderTimeMs); + decodedImage.set_rotation(frameInfo->rotation); callback->FrameToRender(decodedImage); } return WEBRTC_VIDEO_CODEC_OK; @@ -148,6 +149,7 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, { _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs; _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs(); + _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation(); _callback->Map(frame.TimeStamp(), &_frameInfos[_nextFrameInfoIdx]); _nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength; diff --git a/webrtc/modules/video_coding/main/source/generic_decoder.h b/webrtc/modules/video_coding/main/source/generic_decoder.h index 846d4d3e11..fab94bc87f 100644 --- a/webrtc/modules/video_coding/main/source/generic_decoder.h +++ b/webrtc/modules/video_coding/main/source/generic_decoder.h @@ -29,6 +29,7 @@ struct VCMFrameInformation int64_t renderTimeMs; int64_t decodeStartTimeMs; void* userData; + VideoRotation rotation; }; class VCMDecodedFrameCallback : public DecodedImageCallback diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.cc b/webrtc/modules/video_coding/main/source/generic_encoder.cc index dcddf2f5ce..0cc2ec4e8d 100644 --- a/webrtc/modules/video_coding/main/source/generic_encoder.cc +++ b/webrtc/modules/video_coding/main/source/generic_encoder.cc @@ -60,9 +60,11 @@ VCMGenericEncoder::VCMGenericEncoder(VideoEncoder* encoder, bool internalSource) : encoder_(encoder), rate_observer_(rate_observer), + vcm_encoded_frame_callback_(nullptr), bit_rate_(0), frame_rate_(0), - internal_source_(internalSource) { + internal_source_(internalSource), + rotation_(kVideoRotation_0) { } VCMGenericEncoder::~VCMGenericEncoder() @@ -75,6 +77,7 @@ int32_t VCMGenericEncoder::Release() rtc::CritScope lock(&rates_lock_); bit_rate_ = 0; frame_rate_ = 0; + vcm_encoded_frame_callback_ = nullptr; } return encoder_->Release(); @@ -106,6 +109,16 @@ VCMGenericEncoder::Encode(const I420VideoFrame& inputFrame, std::vector video_frame_types(frameTypes.size(), kDeltaFrame); VCMEncodedFrame::ConvertFrameTypes(frameTypes, &video_frame_types); + + rotation_ = inputFrame.rotation(); + + if (vcm_encoded_frame_callback_) { + // Keep track of the current frame rotation and apply to the output of the + // encoder. There might not be exact as the encoder could have one frame + // delay but it should be close enough. + vcm_encoded_frame_callback_->SetRotation(rotation_); + } + return encoder_->Encode(inputFrame, codecSpecificInfo, &video_frame_types); } @@ -178,6 +191,7 @@ int32_t VCMGenericEncoder::RegisterEncodeCallback(VCMEncodedFrameCallback* VCMencodedFrameCallback) { VCMencodedFrameCallback->SetInternalSource(internal_source_); + vcm_encoded_frame_callback_ = VCMencodedFrameCallback; return encoder_->RegisterEncodeCompleteCallback(VCMencodedFrameCallback); } @@ -191,14 +205,16 @@ VCMGenericEncoder::InternalSource() const * Callback Implementation ***************************/ VCMEncodedFrameCallback::VCMEncodedFrameCallback( - EncodedImageCallback* post_encode_callback): -_sendCallback(), -_mediaOpt(NULL), -_payloadType(0), -_internalSource(false), -post_encode_callback_(post_encode_callback) + EncodedImageCallback* post_encode_callback) + : _sendCallback(), + _mediaOpt(NULL), + _payloadType(0), + _internalSource(false), + _rotation(kVideoRotation_0), + post_encode_callback_(post_encode_callback) #ifdef DEBUG_ENCODER_BIT_STREAM -, _bitStreamAfterEncoder(NULL) + , + _bitStreamAfterEncoder(NULL) #endif { #ifdef DEBUG_ENCODER_BIT_STREAM @@ -241,6 +257,7 @@ int32_t VCMEncodedFrameCallback::Encoded( memset(&rtpVideoHeader, 0, sizeof(RTPVideoHeader)); RTPVideoHeader* rtpVideoHeaderPtr = &rtpVideoHeader; CopyCodecSpecific(codecSpecificInfo, &rtpVideoHeaderPtr); + rtpVideoHeader.rotation = _rotation; int32_t callbackReturn = _sendCallback->SendData( _payloadType, encodedImage, *fragmentationHeader, rtpVideoHeaderPtr); diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.h b/webrtc/modules/video_coding/main/source/generic_encoder.h index eba77150be..ce037d1ea1 100644 --- a/webrtc/modules/video_coding/main/source/generic_encoder.h +++ b/webrtc/modules/video_coding/main/source/generic_encoder.h @@ -54,11 +54,14 @@ public: void SetPayloadType(uint8_t payloadType) { _payloadType = payloadType; }; void SetInternalSource(bool internalSource) { _internalSource = internalSource; }; + void SetRotation(VideoRotation rotation) { _rotation = rotation; } + private: VCMPacketizationCallback* _sendCallback; media_optimization::MediaOptimization* _mediaOpt; uint8_t _payloadType; bool _internalSource; + VideoRotation _rotation; EncodedImageCallback* post_encode_callback_; @@ -136,10 +139,12 @@ public: private: VideoEncoder* const encoder_; VideoEncoderRateObserver* const rate_observer_; + VCMEncodedFrameCallback* vcm_encoded_frame_callback_; uint32_t bit_rate_; uint32_t frame_rate_; const bool internal_source_; mutable rtc::CriticalSection rates_lock_; + VideoRotation rotation_; }; // end of VCMGenericEncoder class } // namespace webrtc