diff --git a/DEPS b/DEPS index 7b3d0295bd..1e56c874cc 100644 --- a/DEPS +++ b/DEPS @@ -42,6 +42,7 @@ include_rules = [ '+third_party', '+unicode', '+webrtc', + '+vpx', ] # checkdeps.py shouldn't check include paths for files in these dirs: diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index bf04adfe7e..405a173bd1 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -182,6 +182,8 @@ source_set("webrtc_vp9") { if (rtc_build_vp9) { sources = [ "codecs/vp9/include/vp9.h", + "codecs/vp9/vp9_frame_buffer_pool.cc", + "codecs/vp9/vp9_frame_buffer_pool.h", "codecs/vp9/vp9_impl.cc", "codecs/vp9/vp9_impl.h", ] diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9.gyp b/webrtc/modules/video_coding/codecs/vp9/vp9.gyp index 795db62a30..ac7e67a79c 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9.gyp +++ b/webrtc/modules/video_coding/codecs/vp9/vp9.gyp @@ -28,6 +28,8 @@ ['build_vp9==1', { 'sources': [ 'include/vp9.h', + 'vp9_frame_buffer_pool.cc', + 'vp9_frame_buffer_pool.h', 'vp9_impl.cc', 'vp9_impl.h', ], diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc new file mode 100644 index 0000000000..6e16bc1468 --- /dev/null +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#include "webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h" + +#include "vpx/vpx_codec.h" +#include "vpx/vpx_decoder.h" +#include "vpx/vpx_frame_buffer.h" + +#include "webrtc/base/checks.h" +#include "webrtc/system_wrappers/interface/logging.h" + +namespace webrtc { + +uint8_t* Vp9FrameBufferPool::Vp9FrameBuffer::GetData() { + return data_.data(); +} + +size_t Vp9FrameBufferPool::Vp9FrameBuffer::GetDataSize() const { + return data_.size(); +} + +void Vp9FrameBufferPool::Vp9FrameBuffer::SetSize(size_t size) { + data_.SetSize(size); +} + +bool Vp9FrameBufferPool::InitializeVpxUsePool( + vpx_codec_ctx* vpx_codec_context) { + DCHECK(vpx_codec_context); + // Tell libvpx to use this pool. + if (vpx_codec_set_frame_buffer_functions( + // In which context to use these callback functions. + vpx_codec_context, + // Called by libvpx when it needs another frame buffer. + &Vp9FrameBufferPool::VpxGetFrameBuffer, + // Called by libvpx when it no longer uses a frame buffer. + &Vp9FrameBufferPool::VpxReleaseFrameBuffer, + // |this| will be passed as |user_priv| to VpxGetFrameBuffer. + this)) { + // Failed to configure libvpx to use Vp9FrameBufferPool. + return false; + } + return true; +} + +rtc::scoped_refptr +Vp9FrameBufferPool::GetFrameBuffer(size_t min_size) { + DCHECK_GT(min_size, 0u); + rtc::scoped_refptr available_buffer = nullptr; + { + rtc::CritScope cs(&buffers_lock_); + // Do we have a buffer we can recycle? + for (const auto& buffer : allocated_buffers_) { + if (buffer->HasOneRef()) { + available_buffer = buffer; + break; + } + } + // Otherwise create one. + if (available_buffer == nullptr) { + available_buffer = new rtc::RefCountedObject(); + allocated_buffers_.push_back(available_buffer); + if (allocated_buffers_.size() > max_num_buffers_) { + LOG(LS_WARNING) + << allocated_buffers_.size() << " Vp9FrameBuffers have been " + << "allocated by a Vp9FrameBufferPool (exceeding what is " + << "considered reasonable, " << max_num_buffers_ << ")."; + RTC_NOTREACHED(); + } + } + } + + available_buffer->SetSize(min_size); + return available_buffer; +} + +int Vp9FrameBufferPool::GetNumBuffersInUse() const { + int num_buffers_in_use = 0; + rtc::CritScope cs(&buffers_lock_); + for (const auto& buffer : allocated_buffers_) { + if (!buffer->HasOneRef()) + ++num_buffers_in_use; + } + return num_buffers_in_use; +} + +void Vp9FrameBufferPool::ClearPool() { + rtc::CritScope cs(&buffers_lock_); + allocated_buffers_.clear(); +} + +// static +int32 Vp9FrameBufferPool::VpxGetFrameBuffer(void* user_priv, + size_t min_size, + vpx_codec_frame_buffer* fb) { + DCHECK(user_priv); + DCHECK(fb); + Vp9FrameBufferPool* pool = static_cast(user_priv); + + rtc::scoped_refptr buffer = pool->GetFrameBuffer(min_size); + fb->data = buffer->GetData(); + fb->size = buffer->GetDataSize(); + // Store Vp9FrameBuffer* in |priv| for use in VpxReleaseFrameBuffer. + // This also makes vpx_codec_get_frame return images with their |fb_priv| set + // to |buffer| which is important for external reference counting. + // Release from refptr so that the buffer's |ref_count_| remains 1 when + // |buffer| goes out of scope. + fb->priv = static_cast(buffer.release()); + return 0; +} + +// static +int32 Vp9FrameBufferPool::VpxReleaseFrameBuffer(void* user_priv, + vpx_codec_frame_buffer* fb) { + DCHECK(user_priv); + DCHECK(fb); + Vp9FrameBuffer* buffer = static_cast(fb->priv); + if (buffer != nullptr) { + buffer->Release(); + // When libvpx fails to decode and you continue to try to decode (and fail) + // libvpx can for some reason try to release the same buffer multiple times. + // Setting |priv| to null protects against trying to Release multiple times. + fb->priv = nullptr; + } + return 0; +} + +} // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h b/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h new file mode 100644 index 0000000000..68ebbee5ec --- /dev/null +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_FRAME_BUFFER_POOL_H_ +#define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_FRAME_BUFFER_POOL_H_ + +#include + +#include "webrtc/base/basictypes.h" +#include "webrtc/base/buffer.h" +#include "webrtc/base/criticalsection.h" +#include "webrtc/base/refcount.h" +#include "webrtc/base/scoped_ref_ptr.h" + +struct vpx_codec_ctx; +struct vpx_codec_frame_buffer; + +namespace webrtc { + +// This memory pool is used to serve buffers to libvpx for decoding purposes in +// VP9, which is set up in InitializeVPXUsePool. After the initialization any +// time libvpx wants to decode a frame it will use buffers provided and released +// through VpxGetFrameBuffer and VpxReleaseFrameBuffer. +// The benefit of owning the pool that libvpx relies on for decoding is that the +// decoded frames returned by libvpx (from vpx_codec_get_frame) use parts of our +// buffers for the decoded image data. By retaining ownership of this buffer +// using scoped_refptr, the image buffer can be reused by I420VideoFrames and no +// frame copy has to occur during decoding and frame delivery. +// +// Pseudo example usage case: +// Vp9FrameBufferPool pool; +// pool.InitializeVpxUsePool(decoder_ctx); +// ... +// +// // During decoding, libvpx will get and release buffers from the pool. +// vpx_codec_decode(decoder_ctx, ...); +// +// vpx_image_t* img = vpx_codec_get_frame(decoder_ctx, &iter); +// // Important to use scoped_refptr to protect it against being recycled by +// // the pool. +// scoped_refptr img_buffer = (Vp9FrameBuffer*)img->fb_priv; +// ... +// +// // Destroying the codec will make libvpx release any buffers it was using. +// vpx_codec_destroy(decoder_ctx); +class Vp9FrameBufferPool { + public: + class Vp9FrameBuffer : public rtc::RefCountInterface { + public: + uint8_t* GetData(); + size_t GetDataSize() const; + void SetSize(size_t size); + + virtual bool HasOneRef() const = 0; + + private: + // Data as an easily resizable buffer. + rtc::Buffer data_; + }; + + // Configures libvpx to, in the specified context, use this memory pool for + // buffers used to decompress frames. This is only supported for VP9. + bool InitializeVpxUsePool(vpx_codec_ctx* vpx_codec_context); + + // Gets a frame buffer of at least |min_size|, recycling an available one or + // creating a new one. When no longer referenced from the outside the buffer + // becomes recyclable. + rtc::scoped_refptr GetFrameBuffer(size_t min_size); + // Gets the number of buffers currently in use (not ready to be recycled). + int GetNumBuffersInUse() const; + // Releases allocated buffers, deleting available buffers. Buffers in use are + // not deleted until they are no longer referenced. + void ClearPool(); + + // InitializeVpxUsePool configures libvpx to call this function when it needs + // a new frame buffer. Parameters: + // |user_priv| Private data passed to libvpx, InitializeVpxUsePool sets it up + // to be a pointer to the pool. + // |min_size| Minimum size needed by libvpx (to decompress a frame). + // |fb| Pointer to the libvpx frame buffer object, this is updated to + // use the pool's buffer. + // Returns 0 on success. Returns < 0 on failure. + static int32 VpxGetFrameBuffer(void* user_priv, + size_t min_size, + vpx_codec_frame_buffer* fb); + + // InitializeVpxUsePool configures libvpx to call this function when it has + // finished using one of the pool's frame buffer. Parameters: + // |user_priv| Private data passed to libvpx, InitializeVpxUsePool sets it up + // to be a pointer to the pool. + // |fb| Pointer to the libvpx frame buffer object, its |priv| will be + // a pointer to one of the pool's Vp9FrameBuffer. + static int32 VpxReleaseFrameBuffer(void* user_priv, + vpx_codec_frame_buffer* fb); + + private: + // Protects |allocated_buffers_|. + mutable rtc::CriticalSection buffers_lock_; + // All buffers, in use or ready to be recycled. + std::vector> allocated_buffers_ + GUARDED_BY(buffers_lock_); + // If more buffers than this are allocated we print warnings, and crash if + // in debug mode. + static const size_t max_num_buffers_ = 10; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_FRAME_BUFFER_POOL_H_ diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc index 310e53af5e..02f91ec3c3 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -21,13 +21,25 @@ #include "vpx/vp8cx.h" #include "vpx/vp8dx.h" +#include "webrtc/base/bind.h" #include "webrtc/base/checks.h" #include "webrtc/common.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/interface/module_common_types.h" +#include "webrtc/system_wrappers/interface/logging.h" #include "webrtc/system_wrappers/interface/tick_util.h" #include "webrtc/system_wrappers/interface/trace_event.h" +namespace { + +// VP9DecoderImpl::ReturnFrame helper function used with WrappedI420Buffer. +static void WrappedI420BufferNoLongerUsedCb( + webrtc::Vp9FrameBufferPool::Vp9FrameBuffer* img_buffer) { + img_buffer->Release(); +} + +} // anonymous namespace + namespace webrtc { VP9Encoder* VP9Encoder::Create() { @@ -388,6 +400,14 @@ VP9DecoderImpl::VP9DecoderImpl() VP9DecoderImpl::~VP9DecoderImpl() { inited_ = true; // in order to do the actual release Release(); + int num_buffers_in_use = frame_buffer_pool_.GetNumBuffersInUse(); + if (num_buffers_in_use > 0) { + // The frame buffers are reference counted and frames are exposed after + // decoding. There may be valid usage cases where previous frames are still + // referenced after ~VP9DecoderImpl that is not a leak. + LOG(LS_INFO) << num_buffers_in_use << " Vp9FrameBuffers are still " + << "referenced during ~VP9DecoderImpl."; + } } int VP9DecoderImpl::Reset() { @@ -421,6 +441,11 @@ int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) { // Save VideoCodec instance for later; mainly for duplicating the decoder. codec_ = *inst; } + + if (!frame_buffer_pool_.InitializeVpxUsePool(decoder_)) { + return WEBRTC_VIDEO_CODEC_MEMORY; + } + inited_ = true; // Always start with a complete key frame. key_frame_required_ = true; @@ -455,6 +480,8 @@ int VP9DecoderImpl::Decode(const EncodedImage& input_image, if (input_image._length == 0) { buffer = NULL; // Triggers full frame concealment. } + // During decode libvpx may get and release buffers from |frame_buffer_pool_|. + // In practice libvpx keeps a few (~3-4) buffers alive at a time. if (vpx_codec_decode(decoder_, buffer, static_cast(input_image._length), @@ -462,6 +489,9 @@ int VP9DecoderImpl::Decode(const EncodedImage& input_image, VPX_DL_REALTIME)) { return WEBRTC_VIDEO_CODEC_ERROR; } + // |img->fb_priv| contains the image data, a reference counted Vp9FrameBuffer. + // It may be released by libvpx during future vpx_codec_decode or + // vpx_codec_destroy calls. img = vpx_codec_get_frame(decoder_, &iter); int ret = ReturnFrame(img, input_image._timeStamp); if (ret != 0) { @@ -475,15 +505,32 @@ int VP9DecoderImpl::ReturnFrame(const vpx_image_t* img, uint32_t timestamp) { // Decoder OK and NULL image => No show frame. return WEBRTC_VIDEO_CODEC_NO_OUTPUT; } - decoded_image_.CreateFrame(img->planes[VPX_PLANE_Y], - img->planes[VPX_PLANE_U], - img->planes[VPX_PLANE_V], - img->d_w, img->d_h, - img->stride[VPX_PLANE_Y], - img->stride[VPX_PLANE_U], - img->stride[VPX_PLANE_V]); - decoded_image_.set_timestamp(timestamp); - int ret = decode_complete_callback_->Decoded(decoded_image_); + + // This buffer contains all of |img|'s image data, a reference counted + // Vp9FrameBuffer. Performing AddRef/Release ensures it is not released and + // recycled during use (libvpx is done with the buffers after a few + // vpx_codec_decode calls or vpx_codec_destroy). + Vp9FrameBufferPool::Vp9FrameBuffer* img_buffer = + static_cast(img->fb_priv); + img_buffer->AddRef(); + // The buffer can be used directly by the I420VideoFrame (without copy) by + // using a WrappedI420Buffer. + rtc::scoped_refptr img_wrapped_buffer( + new rtc::RefCountedObject( + img->d_w, img->d_h, + img->d_w, img->d_h, + img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y], + img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U], + img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V], + // WrappedI420Buffer's mechanism for allowing the release of its frame + // buffer is through a callback function. This is where we should + // release |img_buffer|. + rtc::Bind(&WrappedI420BufferNoLongerUsedCb, img_buffer))); + + I420VideoFrame decoded_image; + decoded_image.set_video_frame_buffer(img_wrapped_buffer); + decoded_image.set_timestamp(timestamp); + int ret = decode_complete_callback_->Decoded(decoded_image); if (ret != 0) return ret; return WEBRTC_VIDEO_CODEC_OK; @@ -497,12 +544,18 @@ int VP9DecoderImpl::RegisterDecodeCompleteCallback( int VP9DecoderImpl::Release() { if (decoder_ != NULL) { + // When a codec is destroyed libvpx will release any buffers of + // |frame_buffer_pool_| it is currently using. if (vpx_codec_destroy(decoder_)) { return WEBRTC_VIDEO_CODEC_MEMORY; } delete decoder_; decoder_ = NULL; } + // Releases buffers from the pool. Any buffers not in use are deleted. Buffers + // still referenced externally are deleted once fully released, not returning + // to the pool. + frame_buffer_pool_.ClearPool(); inited_ = false; return WEBRTC_VIDEO_CODEC_OK; } diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h index 28b9ecc8da..bd7c7f58c9 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h @@ -13,6 +13,7 @@ #define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP9_IMPL_H_ #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" +#include "webrtc/modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h" #include "vpx/vpx_decoder.h" #include "vpx/vpx_encoder.h" @@ -101,7 +102,8 @@ class VP9DecoderImpl : public VP9Decoder { private: int ReturnFrame(const vpx_image_t* img, uint32_t timeStamp); - I420VideoFrame decoded_image_; + // Memory pool used to share buffers between libvpx and webrtc. + Vp9FrameBufferPool frame_buffer_pool_; DecodedImageCallback* decode_complete_callback_; bool inited_; vpx_codec_ctx_t* decoder_;