From 4d71edef45afa38b3f68b6af0519ac0f21d327df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Bostr=C3=B6m?= Date: Tue, 19 May 2015 23:09:35 +0200 Subject: [PATCH] Add HW fallback option to software encoding. Permits falling back to software encoding for unsupported resolutions. BUG=chromium:475116, chromium:487934 R=mflodman@webrtc.org, stefan@webrtc.org Review URL: https://webrtc-codereview.appspot.com/46279004 Cr-Commit-Position: refs/heads/master@{#9227} --- talk/media/webrtc/webrtcvideoengine2.cc | 20 ++- talk/media/webrtc/webrtcvideoengine2.h | 4 +- webrtc/video/BUILD.gn | 1 + webrtc/video/call.cc | 10 -- webrtc/video/video_encoder.cc | 125 +++++++++++++++ webrtc/video/video_encoder_unittest.cc | 200 ++++++++++++++++++++++++ webrtc/video/webrtc_video.gypi | 1 + webrtc/video_encoder.h | 32 ++++ webrtc/webrtc_tests.gypi | 1 + 9 files changed, 379 insertions(+), 15 deletions(-) create mode 100644 webrtc/video/video_encoder.cc create mode 100644 webrtc/video/video_encoder_unittest.cc diff --git a/talk/media/webrtc/webrtcvideoengine2.cc b/talk/media/webrtc/webrtcvideoengine2.cc index a7fc6bdc07..7d8cc026f3 100644 --- a/talk/media/webrtc/webrtcvideoengine2.cc +++ b/talk/media/webrtc/webrtcvideoengine2.cc @@ -1634,6 +1634,21 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::VideoSendStreamParameters:: codec_settings(codec_settings) { } +WebRtcVideoChannel2::WebRtcVideoSendStream::AllocatedEncoder::AllocatedEncoder( + webrtc::VideoEncoder* encoder, + webrtc::VideoCodecType type, + bool external) + : encoder(encoder), + external_encoder(nullptr), + type(type), + external(external) { + if (external) { + external_encoder = encoder; + this->encoder = + new webrtc::VideoEncoderSoftwareFallbackWrapper(type, encoder); + } +} + WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream( webrtc::Call* call, WebRtcVideoEncoderFactory* external_encoder_factory, @@ -1885,10 +1900,9 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoder( void WebRtcVideoChannel2::WebRtcVideoSendStream::DestroyVideoEncoder( AllocatedEncoder* encoder) { if (encoder->external) { - external_encoder_factory_->DestroyVideoEncoder(encoder->encoder); - } else { - delete encoder->encoder; + external_encoder_factory_->DestroyVideoEncoder(encoder->external_encoder); } + delete encoder->encoder; } void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodecAndOptions( diff --git a/talk/media/webrtc/webrtcvideoengine2.h b/talk/media/webrtc/webrtcvideoengine2.h index 8d24f29d41..58973094db 100644 --- a/talk/media/webrtc/webrtcvideoengine2.h +++ b/talk/media/webrtc/webrtcvideoengine2.h @@ -321,9 +321,9 @@ class WebRtcVideoChannel2 : public rtc::MessageHandler, struct AllocatedEncoder { AllocatedEncoder(webrtc::VideoEncoder* encoder, webrtc::VideoCodecType type, - bool external) - : encoder(encoder), type(type), external(external) {} + bool external); webrtc::VideoEncoder* encoder; + webrtc::VideoEncoder* external_encoder; webrtc::VideoCodecType type; bool external; }; diff --git a/webrtc/video/BUILD.gn b/webrtc/video/BUILD.gn index 02c56a2dad..0504dbe88c 100644 --- a/webrtc/video/BUILD.gn +++ b/webrtc/video/BUILD.gn @@ -22,6 +22,7 @@ source_set("video") { "transport_adapter.cc", "transport_adapter.h", "video_decoder.cc", + "video_encoder.cc", "video_receive_stream.cc", "video_receive_stream.h", "video_send_stream.cc", diff --git a/webrtc/video/call.cc b/webrtc/video/call.cc index f19c3b4962..37f9f2d60a 100644 --- a/webrtc/video/call.cc +++ b/webrtc/video/call.cc @@ -36,16 +36,6 @@ #include "webrtc/video/video_send_stream.h" namespace webrtc { -VideoEncoder* VideoEncoder::Create(VideoEncoder::EncoderType codec_type) { - switch (codec_type) { - case kVp8: - return VP8Encoder::Create(); - case kVp9: - return VP9Encoder::Create(); - } - RTC_NOTREACHED(); - return nullptr; -} const int Call::Config::kDefaultStartBitrateBps = 300000; diff --git a/webrtc/video/video_encoder.cc b/webrtc/video/video_encoder.cc new file mode 100644 index 0000000000..6931856819 --- /dev/null +++ b/webrtc/video/video_encoder.cc @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/video_encoder.h" + +#include "webrtc/base/checks.h" +#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" +#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" +#include "webrtc/system_wrappers/interface/logging.h" + +namespace webrtc { +VideoEncoder* VideoEncoder::Create(VideoEncoder::EncoderType codec_type) { + switch (codec_type) { + case kVp8: + return VP8Encoder::Create(); + case kVp9: + return VP9Encoder::Create(); + case kUnsupportedCodec: + RTC_NOTREACHED(); + return nullptr; + } + RTC_NOTREACHED(); + return nullptr; +} + +VideoEncoder::EncoderType CodecToEncoderType(VideoCodecType codec_type) { + switch (codec_type) { + case kVideoCodecVP8: + return VideoEncoder::kVp8; + case kVideoCodecVP9: + return VideoEncoder::kVp9; + default: + return VideoEncoder::kUnsupportedCodec; + } +} + +VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper( + VideoCodecType codec_type, + webrtc::VideoEncoder* encoder) + : encoder_type_(CodecToEncoderType(codec_type)), + encoder_(encoder), + callback_(nullptr) { +} + +int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode( + const VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size) { + int32_t ret = + encoder_->InitEncode(codec_settings, number_of_cores, max_payload_size); + if (ret == WEBRTC_VIDEO_CODEC_OK || encoder_type_ == kUnsupportedCodec) { + fallback_encoder_.reset(); + if (callback_) + encoder_->RegisterEncodeCompleteCallback(callback_); + return ret; + } + // Try to instantiate software codec. + fallback_encoder_.reset(VideoEncoder::Create(encoder_type_)); + if (fallback_encoder_->InitEncode(codec_settings, number_of_cores, + max_payload_size) == + WEBRTC_VIDEO_CODEC_OK) { + if (callback_) + fallback_encoder_->RegisterEncodeCompleteCallback(callback_); + return WEBRTC_VIDEO_CODEC_OK; + } + // Software encoder failed, reset and use original return code. + fallback_encoder_.reset(); + return ret; +} + +int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + callback_ = callback; + int32_t ret = encoder_->RegisterEncodeCompleteCallback(callback); + if (fallback_encoder_) + return fallback_encoder_->RegisterEncodeCompleteCallback(callback); + return ret; +} + +int32_t VideoEncoderSoftwareFallbackWrapper::Release() { + if (fallback_encoder_) + return fallback_encoder_->Release(); + return encoder_->Release(); +} + +int32_t VideoEncoderSoftwareFallbackWrapper::Encode( + const I420VideoFrame& frame, + const CodecSpecificInfo* codec_specific_info, + const std::vector* frame_types) { + if (fallback_encoder_) + return fallback_encoder_->Encode(frame, codec_specific_info, frame_types); + return encoder_->Encode(frame, codec_specific_info, frame_types); +} + +int32_t VideoEncoderSoftwareFallbackWrapper::SetChannelParameters( + uint32_t packet_loss, + int64_t rtt) { + int32_t ret = encoder_->SetChannelParameters(packet_loss, rtt); + if (fallback_encoder_) + return fallback_encoder_->SetChannelParameters(packet_loss, rtt); + return ret; +} + +int32_t VideoEncoderSoftwareFallbackWrapper::SetRates(uint32_t bitrate, + uint32_t framerate) { + int32_t ret = encoder_->SetRates(bitrate, framerate); + if (fallback_encoder_) + return fallback_encoder_->SetRates(bitrate, framerate); + return ret; +} + +void VideoEncoderSoftwareFallbackWrapper::OnDroppedFrame() { + if (fallback_encoder_) + return fallback_encoder_->OnDroppedFrame(); + return encoder_->OnDroppedFrame(); +} + +} // namespace webrtc diff --git a/webrtc/video/video_encoder_unittest.cc b/webrtc/video/video_encoder_unittest.cc new file mode 100644 index 0000000000..acfc4de23d --- /dev/null +++ b/webrtc/video/video_encoder_unittest.cc @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/video_encoder.h" + +#include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/modules/video_coding/codecs/interface/video_error_codes.h" + +namespace webrtc { + +const size_t kMaxPayloadSize = 800; + +class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { + protected: + VideoEncoderSoftwareFallbackWrapperTest() + : fallback_wrapper_(kVideoCodecVP8, &fake_encoder_) {} + + class CountingFakeEncoder : public VideoEncoder { + public: + int32_t InitEncode(const VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size) override { + ++init_encode_count_; + return init_encode_return_code_; + } + int32_t Encode(const I420VideoFrame& frame, + const CodecSpecificInfo* codec_specific_info, + const std::vector* frame_types) override { + ++encode_count_; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override { + encode_complete_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() override { + ++release_count_; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override { + ++set_channel_parameters_count_; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t SetRates(uint32_t bitrate, uint32_t framerate) override { + ++set_rates_count_; + return WEBRTC_VIDEO_CODEC_OK; + } + + void OnDroppedFrame() override { ++on_dropped_frame_count_; } + + int init_encode_count_ = 0; + int32_t init_encode_return_code_ = WEBRTC_VIDEO_CODEC_OK; + int encode_count_ = 0; + EncodedImageCallback* encode_complete_callback_ = nullptr; + int release_count_ = 0; + int set_channel_parameters_count_ = 0; + int set_rates_count_ = 0; + int on_dropped_frame_count_ = 0; + }; + + class FakeEncodedImageCallback : public EncodedImageCallback { + public: + int32_t Encoded(const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation) override { + return ++callback_count_; + } + int callback_count_ = 0; + }; + + void UtilizeFallbackEncoder(); + + FakeEncodedImageCallback callback_; + CountingFakeEncoder fake_encoder_; + VideoEncoderSoftwareFallbackWrapper fallback_wrapper_; + VideoCodec codec_ = {}; + I420VideoFrame frame_; +}; + +void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() { + static const int kWidth = 320; + static const int kHeight = 240; + fallback_wrapper_.RegisterEncodeCompleteCallback(&callback_); + EXPECT_EQ(&callback_, fake_encoder_.encode_complete_callback_); + + // Register with failing fake encoder. Should succeed with VP8 fallback. + codec_.codecType = kVideoCodecVP8; + codec_.maxFramerate = 30; + codec_.width = kWidth; + codec_.height = kHeight; + fake_encoder_.init_encode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_.InitEncode(&codec_, 2, kMaxPayloadSize)); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.SetRates(300, 30)); + + frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, (kWidth + 1) / 2, + (kWidth + 1) / 2); + memset(frame_.buffer(webrtc::kYPlane), 16, + frame_.allocated_size(webrtc::kYPlane)); + memset(frame_.buffer(webrtc::kUPlane), 128, + frame_.allocated_size(webrtc::kUPlane)); + memset(frame_.buffer(webrtc::kVPlane), 128, + frame_.allocated_size(webrtc::kVPlane)); + + std::vector types(1, kKeyFrame); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_.Encode(frame_, nullptr, &types)); + EXPECT_EQ(0, fake_encoder_.encode_count_); + EXPECT_GT(callback_.callback_count_, 0); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, InitializesEncoder) { + VideoCodec codec = {}; + fallback_wrapper_.InitEncode(&codec, 2, kMaxPayloadSize); + EXPECT_EQ(1, fake_encoder_.init_encode_count_); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, CanUtilizeFallbackEncoder) { + UtilizeFallbackEncoder(); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + InternalEncoderNotReleasedDuringFallback) { + UtilizeFallbackEncoder(); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); + EXPECT_EQ(0, fake_encoder_.release_count_); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + InternalEncoderNotEncodingDuringFallback) { + UtilizeFallbackEncoder(); + EXPECT_EQ(0, fake_encoder_.encode_count_); + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + CanRegisterCallbackWhileUsingFallbackEncoder) { + UtilizeFallbackEncoder(); + // Registering an encode-complete callback should still work when fallback + // encoder is being used. + FakeEncodedImageCallback callback2; + fallback_wrapper_.RegisterEncodeCompleteCallback(&callback2); + EXPECT_EQ(&callback2, fake_encoder_.encode_complete_callback_); + + // Encoding a frame using the fallback should arrive at the new callback. + std::vector types(1, kKeyFrame); + frame_.set_timestamp(frame_.timestamp() + 1000); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_.Encode(frame_, nullptr, &types)); + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + SetChannelParametersForwardedDuringFallback) { + UtilizeFallbackEncoder(); + EXPECT_EQ(0, fake_encoder_.set_channel_parameters_count_); + fallback_wrapper_.SetChannelParameters(1, 1); + EXPECT_EQ(1, fake_encoder_.set_channel_parameters_count_); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + SetRatesForwardedDuringFallback) { + UtilizeFallbackEncoder(); + EXPECT_EQ(1, fake_encoder_.set_rates_count_); + fallback_wrapper_.SetRates(1, 1); + EXPECT_EQ(2, fake_encoder_.set_rates_count_); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + OnDroppedFrameForwardedWithoutFallback) { + fallback_wrapper_.OnDroppedFrame(); + EXPECT_EQ(1, fake_encoder_.on_dropped_frame_count_); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + OnDroppedFrameNotForwardedDuringFallback) { + UtilizeFallbackEncoder(); + fallback_wrapper_.OnDroppedFrame(); + EXPECT_EQ(0, fake_encoder_.on_dropped_frame_count_); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); +} + +} // namespace webrtc diff --git a/webrtc/video/webrtc_video.gypi b/webrtc/video/webrtc_video.gypi index 37047c6455..545f284f68 100644 --- a/webrtc/video/webrtc_video.gypi +++ b/webrtc/video/webrtc_video.gypi @@ -23,6 +23,7 @@ 'video/transport_adapter.cc', 'video/transport_adapter.h', 'video/video_decoder.cc', + 'video/video_encoder.cc', 'video/video_receive_stream.cc', 'video/video_receive_stream.h', 'video/video_send_stream.cc', diff --git a/webrtc/video_encoder.h b/webrtc/video_encoder.h index ce2569bff9..832f40f56d 100644 --- a/webrtc/video_encoder.h +++ b/webrtc/video_encoder.h @@ -39,6 +39,7 @@ class VideoEncoder { enum EncoderType { kVp8, kVp9, + kUnsupportedCodec, }; static VideoEncoder* Create(EncoderType codec_type); @@ -125,5 +126,36 @@ class VideoEncoder { virtual void OnDroppedFrame() {}; }; +// Class used to wrap external VideoEncoders to provide a fallback option on +// software encoding when a hardware encoder fails to encode a stream due to +// hardware restrictions, such as max resolution. +class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder { + public: + VideoEncoderSoftwareFallbackWrapper(VideoCodecType codec_type, + webrtc::VideoEncoder* encoder); + + int32_t InitEncode(const VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size) override; + + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override; + + int32_t Release() override; + int32_t Encode(const I420VideoFrame& frame, + const CodecSpecificInfo* codec_specific_info, + const std::vector* frame_types) override; + int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; + + int32_t SetRates(uint32_t bitrate, uint32_t framerate) override; + void OnDroppedFrame() override; + + private: + const EncoderType encoder_type_; + webrtc::VideoEncoder* const encoder_; + + rtc::scoped_ptr fallback_encoder_; + EncodedImageCallback* callback_; +}; } // namespace webrtc #endif // WEBRTC_VIDEO_ENCODER_H_ diff --git a/webrtc/webrtc_tests.gypi b/webrtc/webrtc_tests.gypi index a47129a9d6..6a4dec9c38 100644 --- a/webrtc/webrtc_tests.gypi +++ b/webrtc/webrtc_tests.gypi @@ -151,6 +151,7 @@ 'video/end_to_end_tests.cc', 'video/send_statistics_proxy_unittest.cc', 'video/video_decoder_unittest.cc', + 'video/video_encoder_unittest.cc', 'video/video_send_stream_tests.cc', ], 'dependencies': [