diff --git a/api/BUILD.gn b/api/BUILD.gn index 581f1c2849..72078b48a1 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -304,6 +304,23 @@ if (rtc_include_tests) { ] } + rtc_source_set("mock_video_codec_factory") { + testonly = true + sources = [ + "test/mock_video_decoder_factory.h", + "test/mock_video_encoder_factory.h", + ] + + public_deps = [ + "../api/video_codecs:video_codecs_api", + ] + + deps = [ + "../test:test_support", + "//testing/gmock", + ] + } + rtc_source_set("fakemetricsobserver") { testonly = true sources = [ diff --git a/api/test/mock_video_decoder_factory.h b/api/test/mock_video_decoder_factory.h new file mode 100644 index 0000000000..915e3911f0 --- /dev/null +++ b/api/test/mock_video_decoder_factory.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_VIDEO_DECODER_FACTORY_H_ +#define API_TEST_MOCK_VIDEO_DECODER_FACTORY_H_ + +#include +#include + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockVideoDecoderFactory : public webrtc::VideoDecoderFactory { + public: + MOCK_CONST_METHOD0(GetSupportedFormats, + std::vector()); + + // We need to proxy to a return type that is copyable. + std::unique_ptr CreateVideoDecoder( + const webrtc::SdpVideoFormat& format) { + return std::unique_ptr( + CreateVideoDecoderProxy(format)); + } + MOCK_METHOD1(CreateVideoDecoderProxy, + webrtc::VideoDecoder*(const webrtc::SdpVideoFormat&)); + + MOCK_METHOD0(Die, void()); + ~MockVideoDecoderFactory() { Die(); } +}; +} // namespace webrtc + +#endif // API_TEST_MOCK_VIDEO_DECODER_FACTORY_H_ diff --git a/api/test/mock_video_encoder_factory.h b/api/test/mock_video_encoder_factory.h new file mode 100644 index 0000000000..a694b636e0 --- /dev/null +++ b/api/test/mock_video_encoder_factory.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_VIDEO_ENCODER_FACTORY_H_ +#define API_TEST_MOCK_VIDEO_ENCODER_FACTORY_H_ + +#include +#include + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockVideoEncoderFactory : public webrtc::VideoEncoderFactory { + public: + MOCK_CONST_METHOD0(GetSupportedFormats, + std::vector()); + MOCK_CONST_METHOD1(QueryVideoEncoder, + CodecInfo(const webrtc::SdpVideoFormat&)); + + // We need to proxy to a return type that is copyable. + std::unique_ptr CreateVideoEncoder( + const webrtc::SdpVideoFormat& format) { + return std::unique_ptr( + CreateVideoEncoderProxy(format)); + } + MOCK_METHOD1(CreateVideoEncoderProxy, + webrtc::VideoEncoder*(const webrtc::SdpVideoFormat&)); + + MOCK_METHOD0(Die, void()); + ~MockVideoEncoderFactory() { Die(); } +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_VIDEO_ENCODER_FACTORY_H_ diff --git a/media/BUILD.gn b/media/BUILD.gn index e98001a4b7..2e46ee583a 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -523,6 +523,7 @@ if (rtc_include_tests) { ":rtc_media", ":rtc_media_base", ":rtc_media_tests_utils", + "../api:mock_video_codec_factory", "../api:video_frame_api", "../api/audio_codecs:builtin_audio_decoder_factory", "../api/audio_codecs:builtin_audio_encoder_factory", diff --git a/media/engine/webrtcvideoengine_unittest.cc b/media/engine/webrtcvideoengine_unittest.cc index 7fc5924c92..c3d3446724 100644 --- a/media/engine/webrtcvideoengine_unittest.cc +++ b/media/engine/webrtcvideoengine_unittest.cc @@ -13,6 +13,8 @@ #include #include +#include "api/test/mock_video_decoder_factory.h" +#include "api/test/mock_video_encoder_factory.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_encoder.h" @@ -863,44 +865,6 @@ TEST_F(WebRtcVideoEngineTest, RegisterExternalH264DecoderIfSupported) { ASSERT_EQ(1u, decoder_factory_->decoders().size()); } -class MockVideoEncoderFactory : public webrtc::VideoEncoderFactory { - public: - MOCK_CONST_METHOD0(GetSupportedFormats, - std::vector()); - MOCK_CONST_METHOD1(QueryVideoEncoder, - CodecInfo(const webrtc::SdpVideoFormat&)); - - // We need to proxy to a return type that is copyable. - std::unique_ptr CreateVideoEncoder( - const webrtc::SdpVideoFormat& format) { - return std::unique_ptr( - CreateVideoEncoderProxy(format)); - } - MOCK_METHOD1(CreateVideoEncoderProxy, - webrtc::VideoEncoder*(const webrtc::SdpVideoFormat&)); - - MOCK_METHOD0(Die, void()); - ~MockVideoEncoderFactory() { Die(); } -}; - -class MockVideoDecoderFactory : public webrtc::VideoDecoderFactory { - public: - MOCK_CONST_METHOD0(GetSupportedFormats, - std::vector()); - - // We need to proxy to a return type that is copyable. - std::unique_ptr CreateVideoDecoder( - const webrtc::SdpVideoFormat& format) { - return std::unique_ptr( - CreateVideoDecoderProxy(format)); - } - MOCK_METHOD1(CreateVideoDecoderProxy, - webrtc::VideoDecoder*(const webrtc::SdpVideoFormat&)); - - MOCK_METHOD0(Die, void()); - ~MockVideoDecoderFactory() { Die(); } -}; - TEST(WebRtcVideoEngineNewVideoCodecFactoryTest, NullFactories) { std::unique_ptr encoder_factory; std::unique_ptr decoder_factory; @@ -911,8 +875,10 @@ TEST(WebRtcVideoEngineNewVideoCodecFactoryTest, NullFactories) { TEST(WebRtcVideoEngineNewVideoCodecFactoryTest, EmptyFactories) { // |engine| take ownership of the factories. - MockVideoEncoderFactory* encoder_factory = new MockVideoEncoderFactory(); - MockVideoDecoderFactory* decoder_factory = new MockVideoDecoderFactory(); + webrtc::MockVideoEncoderFactory* encoder_factory = + new webrtc::MockVideoEncoderFactory(); + webrtc::MockVideoDecoderFactory* decoder_factory = + new webrtc::MockVideoDecoderFactory(); WebRtcVideoEngine engine( (std::unique_ptr(encoder_factory)), (std::unique_ptr(decoder_factory))); @@ -928,8 +894,10 @@ TEST(WebRtcVideoEngineNewVideoCodecFactoryTest, EmptyFactories) { // new factories. TEST(WebRtcVideoEngineNewVideoCodecFactoryTest, Vp8) { // |engine| take ownership of the factories. - MockVideoEncoderFactory* encoder_factory = new MockVideoEncoderFactory(); - MockVideoDecoderFactory* decoder_factory = new MockVideoDecoderFactory(); + webrtc::MockVideoEncoderFactory* encoder_factory = + new webrtc::MockVideoEncoderFactory(); + webrtc::MockVideoDecoderFactory* decoder_factory = + new webrtc::MockVideoDecoderFactory(); WebRtcVideoEngine engine( (std::unique_ptr(encoder_factory)), (std::unique_ptr(decoder_factory))); diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index eb0a419298..d1e3311f0d 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -92,6 +92,7 @@ rtc_static_library("video_coding") { ":video_coding_utility", ":webrtc_h264", ":webrtc_i420", + ":webrtc_stereo", ":webrtc_vp8", ":webrtc_vp9", "..:module_api", @@ -233,6 +234,30 @@ rtc_static_library("webrtc_i420") { ] } +rtc_static_library("webrtc_stereo") { + sources = [ + "codecs/stereo/include/stereo_decoder_adapter.h", + "codecs/stereo/include/stereo_encoder_adapter.h", + "codecs/stereo/stereo_decoder_adapter.cc", + "codecs/stereo/stereo_encoder_adapter.cc", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + + deps = [ + ":video_coding_utility", + "..:module_api", + "../..:webrtc_common", + "../../api/video_codecs:video_codecs_api", + "../../common_video:common_video", + "../../rtc_base:rtc_base_approved", + "../../system_wrappers", + ] +} + rtc_static_library("webrtc_vp8") { sources = [ "codecs/vp8/default_temporal_layers.cc", @@ -423,6 +448,7 @@ if (rtc_include_tests) { } sources = [ "codecs/h264/test/h264_impl_unittest.cc", + "codecs/stereo/test/stereo_adapter_unittest.cc", "codecs/test/videoprocessor_integrationtest.cc", "codecs/test/videoprocessor_integrationtest.h", "codecs/test/videoprocessor_integrationtest_libvpx.cc", @@ -437,9 +463,11 @@ if (rtc_include_tests) { ":video_coding", ":video_coding_utility", ":webrtc_h264", + ":webrtc_stereo", ":webrtc_vp8", ":webrtc_vp9", "../..:webrtc_common", + "../../api:mock_video_codec_factory", "../../api:optional", "../../api:video_frame_api", "../../common_video", diff --git a/modules/video_coding/codecs/stereo/OWNERS b/modules/video_coding/codecs/stereo/OWNERS new file mode 100644 index 0000000000..6b72be3336 --- /dev/null +++ b/modules/video_coding/codecs/stereo/OWNERS @@ -0,0 +1 @@ +emircan@webrtc.org diff --git a/modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h b/modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h new file mode 100644 index 0000000000..729517fb5d --- /dev/null +++ b/modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_STEREO_INCLUDE_STEREO_DECODER_ADAPTER_H_ +#define MODULES_VIDEO_CODING_CODECS_STEREO_INCLUDE_STEREO_DECODER_ADAPTER_H_ + +#include +#include +#include + +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h" + +namespace webrtc { + +class StereoDecoderAdapter : public VideoDecoder { + public: + // |factory| is not owned and expected to outlive this class' lifetime. + explicit StereoDecoderAdapter(VideoDecoderFactory* factory); + virtual ~StereoDecoderAdapter(); + + // Implements VideoDecoder + int32_t InitDecode(const VideoCodec* codec_settings, + int32_t number_of_cores) override; + int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + const RTPFragmentationHeader* fragmentation, + const CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) override; + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override; + int32_t Release() override; + + void Decoded(AlphaCodecStream stream_idx, + VideoFrame* decoded_image, + rtc::Optional decode_time_ms, + rtc::Optional qp); + + private: + // Wrapper class that redirects Decoded() calls. + class AdapterDecodedImageCallback; + + // Holds the decoded image output of a frame. + struct DecodedImageData; + + void MergeAlphaImages(VideoFrame* decoded_image, + const rtc::Optional& decode_time_ms, + const rtc::Optional& qp, + VideoFrame* stereo_decoded_image, + const rtc::Optional& stereo_decode_time_ms, + const rtc::Optional& stereo_qp); + + VideoDecoderFactory* const factory_; + std::vector> decoders_; + std::vector> adapter_callbacks_; + DecodedImageCallback* decoded_complete_callback_; + + // Holds YUV or AXX decode output of a frame that is identified by timestamp. + std::map decoded_data_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_STEREO_INCLUDE_STEREO_DECODER_ADAPTER_H_ diff --git a/modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h b/modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h new file mode 100644 index 0000000000..ef1e9e112e --- /dev/null +++ b/modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_STEREO_INCLUDE_STEREO_ENCODER_ADAPTER_H_ +#define MODULES_VIDEO_CODING_CODECS_STEREO_INCLUDE_STEREO_ENCODER_ADAPTER_H_ + +#include +#include + +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/video_coding/include/video_codec_interface.h" + +namespace webrtc { + +enum AlphaCodecStream { + kYUVStream = 0, + kAXXStream = 1, + kAlphaCodecStreams = 2, +}; + +class StereoEncoderAdapter : public VideoEncoder { + public: + // |factory| is not owned and expected to outlive this class' lifetime. + explicit StereoEncoderAdapter(VideoEncoderFactory* factory); + virtual ~StereoEncoderAdapter(); + + // Implements VideoEncoder + int InitEncode(const VideoCodec* inst, + int number_of_cores, + size_t max_payload_size) override; + int Encode(const VideoFrame& input_image, + const CodecSpecificInfo* codec_specific_info, + const std::vector* frame_types) override; + int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; + int SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; + int SetRateAllocation(const BitrateAllocation& bitrate, + uint32_t new_framerate) override; + int Release() override; + const char* ImplementationName() const override; + + EncodedImageCallback::Result OnEncodedImage( + AlphaCodecStream stream_idx, + const EncodedImage& encodedImage, + const CodecSpecificInfo* codecSpecificInfo, + const RTPFragmentationHeader* fragmentation); + + private: + // Wrapper class that redirects OnEncodedImage() calls. + class AdapterEncodedImageCallback; + + // Holds the encoded image output of a frame. + struct EncodedImageData; + + VideoEncoderFactory* const factory_; + std::vector> encoders_; + std::vector> adapter_callbacks_; + EncodedImageCallback* encoded_complete_callback_; + + uint64_t picture_index_ = 0; + std::vector stereo_dummy_planes_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_STEREO_INCLUDE_STEREO_ENCODER_ADAPTER_H_ diff --git a/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc b/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc new file mode 100644 index 0000000000..5f8b4c5989 --- /dev/null +++ b/modules/video_coding/codecs/stereo/stereo_decoder_adapter.cc @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h" + +#include "api/video/i420_buffer.h" +#include "api/video_codecs/sdp_video_format.h" +#include "common_video/include/video_frame.h" +#include "common_video/include/video_frame_buffer.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/keep_ref_until_done.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +class StereoDecoderAdapter::AdapterDecodedImageCallback + : public webrtc::DecodedImageCallback { + public: + AdapterDecodedImageCallback(webrtc::StereoDecoderAdapter* adapter, + AlphaCodecStream stream_idx) + : adapter_(adapter), stream_idx_(stream_idx) {} + + void Decoded(VideoFrame& decodedImage, + rtc::Optional decode_time_ms, + rtc::Optional qp) override { + if (!adapter_) + return; + adapter_->Decoded(stream_idx_, &decodedImage, decode_time_ms, qp); + } + int32_t Decoded(VideoFrame& decodedImage) override { + RTC_NOTREACHED(); + return WEBRTC_VIDEO_CODEC_OK; + } + int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override { + RTC_NOTREACHED(); + return WEBRTC_VIDEO_CODEC_OK; + } + + private: + StereoDecoderAdapter* adapter_; + const AlphaCodecStream stream_idx_; +}; + +struct StereoDecoderAdapter::DecodedImageData { + explicit DecodedImageData(AlphaCodecStream stream_idx) + : stream_idx_(stream_idx), + decodedImage_(I420Buffer::Create(1 /* width */, 1 /* height */), + 0, + 0, + kVideoRotation_0) { + RTC_DCHECK_EQ(kAXXStream, stream_idx); + } + DecodedImageData(AlphaCodecStream stream_idx, + const VideoFrame& decodedImage, + const rtc::Optional& decode_time_ms, + const rtc::Optional& qp) + : stream_idx_(stream_idx), + decodedImage_(decodedImage), + decode_time_ms_(decode_time_ms), + qp_(qp) {} + const AlphaCodecStream stream_idx_; + VideoFrame decodedImage_; + const rtc::Optional decode_time_ms_; + const rtc::Optional qp_; + + private: + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(DecodedImageData); +}; + +StereoDecoderAdapter::StereoDecoderAdapter(VideoDecoderFactory* factory) + : factory_(factory) {} + +StereoDecoderAdapter::~StereoDecoderAdapter() { + Release(); +} + +int32_t StereoDecoderAdapter::InitDecode(const VideoCodec* codec_settings, + int32_t number_of_cores) { + VideoCodec settings = *codec_settings; + settings.codecType = kVideoCodecVP9; + for (size_t i = 0; i < kAlphaCodecStreams; ++i) { + const SdpVideoFormat format("VP9"); + std::unique_ptr decoder = + factory_->CreateVideoDecoder(format); + const int32_t rv = decoder->InitDecode(&settings, number_of_cores); + if (rv) + return rv; + adapter_callbacks_.emplace_back( + new StereoDecoderAdapter::AdapterDecodedImageCallback( + this, static_cast(i))); + decoder->RegisterDecodeCompleteCallback(adapter_callbacks_.back().get()); + decoders_.emplace_back(std::move(decoder)); + } + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t StereoDecoderAdapter::Decode( + const EncodedImage& input_image, + bool missing_frames, + const RTPFragmentationHeader* /*fragmentation*/, + const CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) { + // TODO(emircan): Read |codec_specific_info->stereoInfo| to split frames. + int32_t rv = + decoders_[kYUVStream]->Decode(input_image, missing_frames, nullptr, + codec_specific_info, render_time_ms); + if (rv) + return rv; + rv = decoders_[kAXXStream]->Decode(input_image, missing_frames, nullptr, + codec_specific_info, render_time_ms); + return rv; +} + +int32_t StereoDecoderAdapter::RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) { + decoded_complete_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t StereoDecoderAdapter::Release() { + for (auto& decoder : decoders_) { + const int32_t rv = decoder->Release(); + if (rv) + return rv; + } + decoders_.clear(); + adapter_callbacks_.clear(); + return WEBRTC_VIDEO_CODEC_OK; +} + +void StereoDecoderAdapter::Decoded(AlphaCodecStream stream_idx, + VideoFrame* decoded_image, + rtc::Optional decode_time_ms, + rtc::Optional qp) { + const auto& other_decoded_data_it = + decoded_data_.find(decoded_image->timestamp()); + if (other_decoded_data_it != decoded_data_.end()) { + auto& other_image_data = other_decoded_data_it->second; + if (stream_idx == kYUVStream) { + RTC_DCHECK_EQ(kAXXStream, other_image_data.stream_idx_); + MergeAlphaImages(decoded_image, decode_time_ms, qp, + &other_image_data.decodedImage_, + other_image_data.decode_time_ms_, other_image_data.qp_); + } else { + RTC_DCHECK_EQ(kYUVStream, other_image_data.stream_idx_); + RTC_DCHECK_EQ(kAXXStream, stream_idx); + MergeAlphaImages(&other_image_data.decodedImage_, + other_image_data.decode_time_ms_, other_image_data.qp_, + decoded_image, decode_time_ms, qp); + } + decoded_data_.erase(decoded_data_.begin(), other_decoded_data_it); + return; + } + RTC_DCHECK(decoded_data_.find(decoded_image->timestamp()) == + decoded_data_.end()); + decoded_data_.emplace( + std::piecewise_construct, + std::forward_as_tuple(decoded_image->timestamp()), + std::forward_as_tuple(stream_idx, *decoded_image, decode_time_ms, qp)); +} + +void StereoDecoderAdapter::MergeAlphaImages( + VideoFrame* decodedImage, + const rtc::Optional& decode_time_ms, + const rtc::Optional& qp, + VideoFrame* alpha_decodedImage, + const rtc::Optional& alpha_decode_time_ms, + const rtc::Optional& alpha_qp) { + // TODO(emircan): Merge the output and put in a VideoFrame container that can + // transport I420A. + decoded_complete_callback_->Decoded(*decodedImage, decode_time_ms, qp); + decoded_complete_callback_->Decoded(*alpha_decodedImage, alpha_decode_time_ms, + alpha_qp); +} + +} // namespace webrtc diff --git a/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc b/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc new file mode 100644 index 0000000000..f6a04d0b5b --- /dev/null +++ b/modules/video_coding/codecs/stereo/stereo_encoder_adapter.cc @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h" + +#include "api/video_codecs/sdp_video_format.h" +#include "common_video/include/video_frame.h" +#include "common_video/include/video_frame_buffer.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "modules/include/module_common_types.h" +#include "rtc_base/keep_ref_until_done.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +// Callback wrapper that helps distinguish returned results from |encoders_| +// instances. +class StereoEncoderAdapter::AdapterEncodedImageCallback + : public webrtc::EncodedImageCallback { + public: + AdapterEncodedImageCallback(webrtc::StereoEncoderAdapter* adapter, + AlphaCodecStream stream_idx) + : adapter_(adapter), stream_idx_(stream_idx) {} + + EncodedImageCallback::Result OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation) override { + if (!adapter_) + return Result(Result::OK); + return adapter_->OnEncodedImage(stream_idx_, encoded_image, + codec_specific_info, fragmentation); + } + + private: + StereoEncoderAdapter* adapter_; + const AlphaCodecStream stream_idx_; +}; + +StereoEncoderAdapter::StereoEncoderAdapter(VideoEncoderFactory* factory) + : factory_(factory), encoded_complete_callback_(nullptr) {} + +StereoEncoderAdapter::~StereoEncoderAdapter() { + Release(); +} + +int StereoEncoderAdapter::InitEncode(const VideoCodec* inst, + int number_of_cores, + size_t max_payload_size) { + const size_t buffer_size = + CalcBufferSize(VideoType::kI420, inst->width, inst->height); + stereo_dummy_planes_.resize(buffer_size); + // It is more expensive to encode 0x00, so use 0x80 instead. + std::fill(stereo_dummy_planes_.begin(), stereo_dummy_planes_.end(), 0x80); + + for (size_t i = 0; i < kAlphaCodecStreams; ++i) { + const SdpVideoFormat format("VP9"); + std::unique_ptr encoder = + factory_->CreateVideoEncoder(format); + const int rv = encoder->InitEncode(inst, number_of_cores, max_payload_size); + if (rv) { + LOG(LS_ERROR) << "Failed to create stere codec index " << i; + return rv; + } + adapter_callbacks_.emplace_back(new AdapterEncodedImageCallback( + this, static_cast(i))); + encoder->RegisterEncodeCompleteCallback(adapter_callbacks_.back().get()); + encoders_.emplace_back(std::move(encoder)); + } + return WEBRTC_VIDEO_CODEC_OK; +} + +int StereoEncoderAdapter::Encode(const VideoFrame& input_image, + const CodecSpecificInfo* codec_specific_info, + const std::vector* frame_types) { + if (!encoded_complete_callback_) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + // TODO(emircan): Extract alpha and create an alpha frame with dummy planes. + // Since we don't have a way of transporting alpha yet, put a dummy output for + // alpha consisting of YXX. + + // Encode AXX + rtc::scoped_refptr yuva_buffer = + input_image.video_frame_buffer()->ToI420(); + rtc::scoped_refptr alpha_buffer( + new rtc::RefCountedObject( + input_image.width(), input_image.height(), yuva_buffer->DataY(), + yuva_buffer->StrideY(), stereo_dummy_planes_.data(), + yuva_buffer->StrideU(), stereo_dummy_planes_.data(), + yuva_buffer->StrideV(), + rtc::KeepRefUntilDone(input_image.video_frame_buffer()))); + VideoFrame alpha_image(alpha_buffer, input_image.timestamp(), + input_image.render_time_ms(), input_image.rotation()); + encoders_[kAXXStream]->Encode(alpha_image, codec_specific_info, frame_types); + + // Encode YUV + int rv = encoders_[kYUVStream]->Encode(input_image, codec_specific_info, + frame_types); + return rv; +} + +int StereoEncoderAdapter::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + encoded_complete_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +int StereoEncoderAdapter::SetChannelParameters(uint32_t packet_loss, + int64_t rtt) { + for (auto& encoder : encoders_) { + const int rv = encoder->SetChannelParameters(packet_loss, rtt); + if (rv) + return rv; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +int StereoEncoderAdapter::SetRateAllocation(const BitrateAllocation& bitrate, + uint32_t framerate) { + for (auto& encoder : encoders_) { + // TODO(emircan): |new_framerate| is used to calculate duration for encoder + // instances. We report the total frame rate to keep real time for now. + // Remove this after refactoring duration logic. + const int rv = + encoder->SetRateAllocation(bitrate, encoders_.size() * framerate); + if (rv) + return rv; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +int StereoEncoderAdapter::Release() { + for (auto& encoder : encoders_) { + const int rv = encoder->Release(); + if (rv) + return rv; + } + encoders_.clear(); + adapter_callbacks_.clear(); + return WEBRTC_VIDEO_CODEC_OK; +} + +const char* StereoEncoderAdapter::ImplementationName() const { + return "StereoEncoderAdapter"; +} + +EncodedImageCallback::Result StereoEncoderAdapter::OnEncodedImage( + AlphaCodecStream stream_idx, + const EncodedImage& encodedImage, + const CodecSpecificInfo* codecSpecificInfo, + const RTPFragmentationHeader* fragmentation) { + // TODO(emircan): Fill |codec_specific_info| with stereo parameters. + encoded_complete_callback_->OnEncodedImage(encodedImage, codecSpecificInfo, + fragmentation); + return EncodedImageCallback::Result(EncodedImageCallback::Result::OK); +} + +} // namespace webrtc diff --git a/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc b/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc new file mode 100644 index 0000000000..a8913e7e52 --- /dev/null +++ b/modules/video_coding/codecs/stereo/test/stereo_adapter_unittest.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/mock_video_decoder_factory.h" +#include "api/test/mock_video_encoder_factory.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h" +#include "modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h" +#include "modules/video_coding/codecs/test/video_codec_test.h" +#include "modules/video_coding/codecs/vp9/include/vp9.h" + +using testing::_; +using testing::Return; + +namespace webrtc { + +class TestStereoAdapter : public VideoCodecTest { + public: + TestStereoAdapter() + : decoder_factory_(new webrtc::MockVideoDecoderFactory), + encoder_factory_(new webrtc::MockVideoEncoderFactory) {} + + protected: + VideoDecoder* CreateDecoder() override { + return new StereoDecoderAdapter(decoder_factory_.get()); + } + + VideoEncoder* CreateEncoder() override { + return new StereoEncoderAdapter(encoder_factory_.get()); + } + + VideoCodec codec_settings() override { + VideoCodec codec_settings; + codec_settings.codecType = webrtc::kVideoCodecVP9; + codec_settings.VP9()->numberOfTemporalLayers = 1; + codec_settings.VP9()->numberOfSpatialLayers = 1; + return codec_settings; + } + + private: + void SetUp() override { + EXPECT_CALL(*decoder_factory_, Die()); + VideoDecoder* decoder1 = VP9Decoder::Create(); + VideoDecoder* decoder2 = VP9Decoder::Create(); + EXPECT_CALL(*decoder_factory_, CreateVideoDecoderProxy(_)) + .WillOnce(Return(decoder1)) + .WillOnce(Return(decoder2)); + + EXPECT_CALL(*encoder_factory_, Die()); + VideoEncoder* encoder1 = VP9Encoder::Create(); + VideoEncoder* encoder2 = VP9Encoder::Create(); + EXPECT_CALL(*encoder_factory_, CreateVideoEncoderProxy(_)) + .WillOnce(Return(encoder1)) + .WillOnce(Return(encoder2)); + + VideoCodecTest::SetUp(); + } + + const std::unique_ptr decoder_factory_; + const std::unique_ptr encoder_factory_; +}; + +// TODO(emircan): Currently VideoCodecTest tests do a complete setup +// step that goes beyond constructing |decoder_|. Simplify these tests to do +// less. +TEST_F(TestStereoAdapter, ConstructAndDestructDecoder) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release()); +} + +TEST_F(TestStereoAdapter, ConstructAndDestructEncoder) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); +} + +TEST_F(TestStereoAdapter, EncodeDecodeI420Frame) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->Encode(*input_frame_, nullptr, nullptr)); + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info)); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + decoder_->Decode(encoded_frame, false, nullptr)); + std::unique_ptr decoded_frame; + rtc::Optional decoded_qp; + ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp)); + ASSERT_TRUE(decoded_frame); + EXPECT_GT(I420PSNR(input_frame_.get(), decoded_frame.get()), 36); +} + +} // namespace webrtc