diff --git a/webrtc/video/video_encoder.cc b/webrtc/video/video_encoder.cc index 3e6ce634f8..d116a1ea67 100644 --- a/webrtc/video/video_encoder.cc +++ b/webrtc/video/video_encoder.cc @@ -50,34 +50,67 @@ VideoEncoder::EncoderType CodecToEncoderType(VideoCodecType codec_type) { VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper( VideoCodecType codec_type, webrtc::VideoEncoder* encoder) - : encoder_type_(CodecToEncoderType(codec_type)), + : rates_set_(false), + channel_parameters_set_(false), + encoder_type_(CodecToEncoderType(codec_type)), encoder_(encoder), - callback_(nullptr) { + callback_(nullptr) {} + +bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder() { + RTC_CHECK(encoder_type_ != kUnsupportedCodec) + << "Encoder requesting fallback to codec not supported in software."; + fallback_encoder_.reset(VideoEncoder::Create(encoder_type_)); + if (fallback_encoder_->InitEncode(&codec_settings_, number_of_cores_, + max_payload_size_) != + WEBRTC_VIDEO_CODEC_OK) { + LOG(LS_ERROR) << "Failed to initialize software-encoder fallback."; + fallback_encoder_->Release(); + fallback_encoder_.reset(); + return false; + } + // Replay callback, rates, and channel parameters. + if (callback_) + fallback_encoder_->RegisterEncodeCompleteCallback(callback_); + if (rates_set_) + fallback_encoder_->SetRates(bitrate_, framerate_); + if (channel_parameters_set_) + fallback_encoder_->SetChannelParameters(packet_loss_, rtt_); + + // Since we're switching to the fallback encoder, Release the real encoder. It + // may be re-initialized via InitEncode later, and it will continue to get + // Set calls for rates and channel parameters in the meantime. + encoder_->Release(); + return true; } int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode( const VideoCodec* codec_settings, int32_t number_of_cores, size_t max_payload_size) { + // Store settings, in case we need to dynamically switch to the fallback + // encoder after a failed Encode call. + codec_settings_ = *codec_settings; + number_of_cores_ = number_of_cores; + max_payload_size_ = max_payload_size; + // Clear stored rate/channel parameters. + rates_set_ = false; + channel_parameters_set_ = false; + int32_t ret = encoder_->InitEncode(codec_settings, number_of_cores, max_payload_size); if (ret == WEBRTC_VIDEO_CODEC_OK || encoder_type_ == kUnsupportedCodec) { + if (fallback_encoder_) + fallback_encoder_->Release(); fallback_encoder_.reset(); if (callback_) encoder_->RegisterEncodeCompleteCallback(callback_); return ret; } // Try to instantiate software codec. - fallback_encoder_.reset(VideoEncoder::Create(encoder_type_)); - if (fallback_encoder_->InitEncode(codec_settings, number_of_cores, - max_payload_size) == - WEBRTC_VIDEO_CODEC_OK) { - if (callback_) - fallback_encoder_->RegisterEncodeCompleteCallback(callback_); + if (InitFallbackEncoder()) { return WEBRTC_VIDEO_CODEC_OK; } - // Software encoder failed, reset and use original return code. - fallback_encoder_.reset(); + // Software encoder failed, use original return code. return ret; } @@ -91,6 +124,9 @@ int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback( } int32_t VideoEncoderSoftwareFallbackWrapper::Release() { + // If the fallback_encoder_ is non-null, it means it was created via + // InitFallbackEncoder which has Release()d encoder_, so we should only ever + // need to Release() whichever one is active. if (fallback_encoder_) return fallback_encoder_->Release(); return encoder_->Release(); @@ -102,12 +138,21 @@ int32_t VideoEncoderSoftwareFallbackWrapper::Encode( const std::vector* frame_types) { if (fallback_encoder_) return fallback_encoder_->Encode(frame, codec_specific_info, frame_types); - return encoder_->Encode(frame, codec_specific_info, frame_types); + int32_t ret = encoder_->Encode(frame, codec_specific_info, frame_types); + // If requested, try a software fallback. + if (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE && InitFallbackEncoder()) { + // Fallback was successful, so start using it with this frame. + return fallback_encoder_->Encode(frame, codec_specific_info, frame_types); + } + return ret; } int32_t VideoEncoderSoftwareFallbackWrapper::SetChannelParameters( uint32_t packet_loss, int64_t rtt) { + channel_parameters_set_ = true; + packet_loss_ = packet_loss; + rtt_ = rtt; int32_t ret = encoder_->SetChannelParameters(packet_loss, rtt); if (fallback_encoder_) return fallback_encoder_->SetChannelParameters(packet_loss, rtt); @@ -116,6 +161,9 @@ int32_t VideoEncoderSoftwareFallbackWrapper::SetChannelParameters( int32_t VideoEncoderSoftwareFallbackWrapper::SetRates(uint32_t bitrate, uint32_t framerate) { + rates_set_ = true; + bitrate_ = bitrate; + framerate_ = framerate; int32_t ret = encoder_->SetRates(bitrate, framerate); if (fallback_encoder_) return fallback_encoder_->SetRates(bitrate, framerate); diff --git a/webrtc/video/video_encoder_unittest.cc b/webrtc/video/video_encoder_unittest.cc index 5f2c37a016..f84835620b 100644 --- a/webrtc/video/video_encoder_unittest.cc +++ b/webrtc/video/video_encoder_unittest.cc @@ -15,6 +15,8 @@ namespace webrtc { +const int kWidth = 320; +const int kHeight = 240; const size_t kMaxPayloadSize = 800; class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { @@ -34,7 +36,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { const CodecSpecificInfo* codec_specific_info, const std::vector* frame_types) override { ++encode_count_; - return WEBRTC_VIDEO_CODEC_OK; + return encode_return_code_; } int32_t RegisterEncodeCompleteCallback( @@ -67,6 +69,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { int init_encode_count_ = 0; int32_t init_encode_return_code_ = WEBRTC_VIDEO_CODEC_OK; + int32_t encode_return_code_ = WEBRTC_VIDEO_CODEC_OK; int encode_count_ = 0; EncodedImageCallback* encode_complete_callback_ = nullptr; int release_count_ = 0; @@ -87,6 +90,8 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { }; void UtilizeFallbackEncoder(); + void FallbackFromEncodeRequest(); + void EncodeFrame(); FakeEncodedImageCallback callback_; CountingFakeEncoder fake_encoder_; @@ -95,9 +100,22 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { VideoFrame frame_; }; +void VideoEncoderSoftwareFallbackWrapperTest::EncodeFrame() { + frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, (kWidth + 1) / 2, + (kWidth + 1) / 2); + memset(frame_.buffer(webrtc::kYPlane), 16, + frame_.allocated_size(webrtc::kYPlane)); + memset(frame_.buffer(webrtc::kUPlane), 128, + frame_.allocated_size(webrtc::kUPlane)); + memset(frame_.buffer(webrtc::kVPlane), 128, + frame_.allocated_size(webrtc::kVPlane)); + + std::vector types(1, kKeyFrame); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_.Encode(frame_, nullptr, &types)); +} + void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() { - static const int kWidth = 320; - static const int kHeight = 240; fallback_wrapper_.RegisterEncodeCompleteCallback(&callback_); EXPECT_EQ(&callback_, fake_encoder_.encode_complete_callback_); @@ -111,20 +129,31 @@ void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() { fallback_wrapper_.InitEncode(&codec_, 2, kMaxPayloadSize)); EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.SetRates(300, 30)); - frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, (kWidth + 1) / 2, - (kWidth + 1) / 2); - memset(frame_.buffer(webrtc::kYPlane), 16, - frame_.allocated_size(webrtc::kYPlane)); - memset(frame_.buffer(webrtc::kUPlane), 128, - frame_.allocated_size(webrtc::kUPlane)); - memset(frame_.buffer(webrtc::kVPlane), 128, - frame_.allocated_size(webrtc::kVPlane)); + int callback_count = callback_.callback_count_; + int encode_count = fake_encoder_.encode_count_; + EncodeFrame(); + EXPECT_EQ(encode_count, fake_encoder_.encode_count_); + EXPECT_EQ(callback_count + 1, callback_.callback_count_); +} - std::vector types(1, kKeyFrame); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - fallback_wrapper_.Encode(frame_, nullptr, &types)); - EXPECT_EQ(0, fake_encoder_.encode_count_); - EXPECT_GT(callback_.callback_count_, 0); +void VideoEncoderSoftwareFallbackWrapperTest::FallbackFromEncodeRequest() { + fallback_wrapper_.RegisterEncodeCompleteCallback(&callback_); + codec_.codecType = kVideoCodecVP8; + codec_.maxFramerate = 30; + codec_.width = kWidth; + codec_.height = kHeight; + fallback_wrapper_.InitEncode(&codec_, 2, kMaxPayloadSize); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.SetRates(300, 30)); + EXPECT_EQ(1, fake_encoder_.init_encode_count_); + + // Have the non-fallback encoder request a software fallback. + fake_encoder_.encode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + int callback_count = callback_.callback_count_; + int encode_count = fake_encoder_.encode_count_; + EncodeFrame(); + // Single encode request, which returned failure. + EXPECT_EQ(encode_count + 1, fake_encoder_.encode_count_); + EXPECT_EQ(callback_count + 1, callback_.callback_count_); } TEST_F(VideoEncoderSoftwareFallbackWrapperTest, InitializesEncoder) { @@ -133,22 +162,35 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest, InitializesEncoder) { EXPECT_EQ(1, fake_encoder_.init_encode_count_); } +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, EncodeRequestsFallback) { + FallbackFromEncodeRequest(); + // After fallback, further encodes shouldn't hit the fake encoder. + int encode_count = fake_encoder_.encode_count_; + EncodeFrame(); + EXPECT_EQ(encode_count, fake_encoder_.encode_count_); +} + TEST_F(VideoEncoderSoftwareFallbackWrapperTest, CanUtilizeFallbackEncoder) { UtilizeFallbackEncoder(); EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); } TEST_F(VideoEncoderSoftwareFallbackWrapperTest, - InternalEncoderNotReleasedDuringFallback) { - UtilizeFallbackEncoder(); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); + InternalEncoderReleasedDuringFallback) { EXPECT_EQ(0, fake_encoder_.release_count_); + UtilizeFallbackEncoder(); + EXPECT_EQ(1, fake_encoder_.release_count_); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); + // No extra release when the fallback is released. + EXPECT_EQ(1, fake_encoder_.release_count_); } TEST_F(VideoEncoderSoftwareFallbackWrapperTest, InternalEncoderNotEncodingDuringFallback) { UtilizeFallbackEncoder(); - EXPECT_EQ(0, fake_encoder_.encode_count_); + int encode_count = fake_encoder_.encode_count_; + EncodeFrame(); + EXPECT_EQ(encode_count, fake_encoder_.encode_count_); EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); } diff --git a/webrtc/video_encoder.h b/webrtc/video_encoder.h index 0858a71b09..609c07391f 100644 --- a/webrtc/video_encoder.h +++ b/webrtc/video_encoder.h @@ -156,6 +156,24 @@ class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder { bool SupportsNativeHandle() const override; private: + bool InitFallbackEncoder(); + + // Settings used in the last InitEncode call and used if a dynamic fallback to + // software is required. + VideoCodec codec_settings_; + int32_t number_of_cores_; + size_t max_payload_size_; + + // The last bitrate/framerate set, and a flag for noting they are set. + bool rates_set_; + uint32_t bitrate_; + uint32_t framerate_; + + // The last channel parameters set, and a flag for noting they are set. + bool channel_parameters_set_; + uint32_t packet_loss_; + int64_t rtt_; + const EncoderType encoder_type_; webrtc::VideoEncoder* const encoder_;