diff --git a/webrtc/api/android/jni/androidmediaencoder_jni.cc b/webrtc/api/android/jni/androidmediaencoder_jni.cc index e6eeaf7f9d..3361154a75 100644 --- a/webrtc/api/android/jni/androidmediaencoder_jni.cc +++ b/webrtc/api/android/jni/androidmediaencoder_jni.cc @@ -112,8 +112,7 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, int32_t Release() override; int32_t SetChannelParameters(uint32_t /* packet_loss */, int64_t /* rtt */) override; - int32_t SetRateAllocation(const webrtc::BitrateAllocation& rate_allocation, - uint32_t frame_rate) override; + int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override; // rtc::MessageHandler implementation. void OnMessage(rtc::Message* msg) override; @@ -466,12 +465,11 @@ int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */, return WEBRTC_VIDEO_CODEC_OK; } -int32_t MediaCodecVideoEncoder::SetRateAllocation( - const webrtc::BitrateAllocation& rate_allocation, - uint32_t frame_rate) { +int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate, + uint32_t frame_rate) { return codec_thread_->Invoke( RTC_FROM_HERE, Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread, this, - rate_allocation.get_sum_kbps(), frame_rate)); + new_bit_rate, frame_rate)); } void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) { diff --git a/webrtc/call/call_perf_tests.cc b/webrtc/call/call_perf_tests.cc index a141de85be..39c333e984 100644 --- a/webrtc/call/call_perf_tests.cc +++ b/webrtc/call/call_perf_tests.cc @@ -640,9 +640,8 @@ TEST_F(CallPerfTest, KeepsHighBitrateWhenReconfiguringSender) { FakeEncoder(Clock::GetRealTimeClock()), time_to_reconfigure_(false, false), encoder_inits_(0), - last_set_bitrate_kbps_(0), - send_stream_(nullptr), - frame_generator_(nullptr) {} + last_set_bitrate_(0), + send_stream_(nullptr) {} int32_t InitEncode(const VideoCodec* config, int32_t number_of_cores, @@ -652,9 +651,8 @@ TEST_F(CallPerfTest, KeepsHighBitrateWhenReconfiguringSender) { // First time initialization. Frame size is known. // |expected_bitrate| is affected by bandwidth estimation before the // first frame arrives to the encoder. - uint32_t expected_bitrate = last_set_bitrate_kbps_ > 0 - ? last_set_bitrate_kbps_ - : kInitialBitrateKbps; + uint32_t expected_bitrate = + last_set_bitrate_ > 0 ? last_set_bitrate_ : kInitialBitrateKbps; EXPECT_EQ(expected_bitrate, config->startBitrate) << "Encoder not initialized at expected bitrate."; EXPECT_EQ(kDefaultWidth, config->width); @@ -662,8 +660,9 @@ TEST_F(CallPerfTest, KeepsHighBitrateWhenReconfiguringSender) { } else if (encoder_inits_ == 2) { EXPECT_EQ(2 * kDefaultWidth, config->width); EXPECT_EQ(2 * kDefaultHeight, config->height); - EXPECT_GE(last_set_bitrate_kbps_, kReconfigureThresholdKbps); - EXPECT_NEAR(config->startBitrate, last_set_bitrate_kbps_, + EXPECT_GE(last_set_bitrate_, kReconfigureThresholdKbps); + EXPECT_NEAR(config->startBitrate, + last_set_bitrate_, kPermittedReconfiguredBitrateDiffKbps) << "Encoder reconfigured with bitrate too far away from last set."; observation_complete_.Set(); @@ -671,14 +670,14 @@ TEST_F(CallPerfTest, KeepsHighBitrateWhenReconfiguringSender) { return FakeEncoder::InitEncode(config, number_of_cores, max_payload_size); } - int32_t SetRateAllocation(const BitrateAllocation& rate_allocation, - uint32_t framerate) override { - last_set_bitrate_kbps_ = rate_allocation.get_sum_kbps(); + int32_t SetRates(uint32_t new_target_bitrate_kbps, + uint32_t framerate) override { + last_set_bitrate_ = new_target_bitrate_kbps; if (encoder_inits_ == 1 && - rate_allocation.get_sum_kbps() > kReconfigureThresholdKbps) { + new_target_bitrate_kbps > kReconfigureThresholdKbps) { time_to_reconfigure_.Set(); } - return FakeEncoder::SetRateAllocation(rate_allocation, framerate); + return FakeEncoder::SetRates(new_target_bitrate_kbps, framerate); } Call::Config GetSenderCallConfig() override { @@ -724,7 +723,7 @@ TEST_F(CallPerfTest, KeepsHighBitrateWhenReconfiguringSender) { private: rtc::Event time_to_reconfigure_; int encoder_inits_; - uint32_t last_set_bitrate_kbps_; + uint32_t last_set_bitrate_; VideoSendStream* send_stream_; test::FrameGeneratorCapturer* frame_generator_; VideoEncoderConfig encoder_config_; diff --git a/webrtc/common_types.cc b/webrtc/common_types.cc index f28089b9fe..8238443c56 100644 --- a/webrtc/common_types.cc +++ b/webrtc/common_types.cc @@ -11,7 +11,6 @@ #include "webrtc/base/checks.h" #include "webrtc/common_types.h" -#include #include namespace webrtc { @@ -60,7 +59,6 @@ VideoCodec::VideoCodec() simulcastStream(), spatialLayers(), mode(kRealtimeVideo), - expect_encode_from_texture(false), codec_specific_() {} VideoCodecVP8* VideoCodec::VP8() { @@ -93,89 +91,4 @@ const VideoCodecH264& VideoCodec::H264() const { return codec_specific_.H264; } -static const char* kPayloadNameVp8 = "VP8"; -static const char* kPayloadNameVp9 = "VP9"; -static const char* kPayloadNameH264 = "H264"; -static const char* kPayloadNameI420 = "I420"; -static const char* kPayloadNameRED = "RED"; -static const char* kPayloadNameULPFEC = "ULPFEC"; -static const char* kPayloadNameGeneric = "Generic"; - -rtc::Optional CodecTypeToPayloadName(VideoCodecType type) { - switch (type) { - case kVideoCodecVP8: - return rtc::Optional(kPayloadNameVp8); - case kVideoCodecVP9: - return rtc::Optional(kPayloadNameVp9); - case kVideoCodecH264: - return rtc::Optional(kPayloadNameH264); - case kVideoCodecI420: - return rtc::Optional(kPayloadNameI420); - case kVideoCodecRED: - return rtc::Optional(kPayloadNameRED); - case kVideoCodecULPFEC: - return rtc::Optional(kPayloadNameULPFEC); - case kVideoCodecGeneric: - return rtc::Optional(kPayloadNameGeneric); - default: - return rtc::Optional(); - } -} - -rtc::Optional PayloadNameToCodecType(const std::string& name) { - if (name == kPayloadNameVp8) - return rtc::Optional(kVideoCodecVP8); - if (name == kPayloadNameVp9) - return rtc::Optional(kVideoCodecVP9); - if (name == kPayloadNameH264) - return rtc::Optional(kVideoCodecH264); - if (name == kPayloadNameI420) - return rtc::Optional(kVideoCodecI420); - if (name == kPayloadNameRED) - return rtc::Optional(kVideoCodecRED); - if (name == kPayloadNameULPFEC) - return rtc::Optional(kVideoCodecULPFEC); - if (name == kPayloadNameGeneric) - return rtc::Optional(kVideoCodecGeneric); - return rtc::Optional(); -} - -const size_t BitrateAllocation::kMaxBitrateBps = - std::numeric_limits::max(); - -BitrateAllocation::BitrateAllocation() : sum_(0), bitrates_{} {} - -bool BitrateAllocation::SetBitrate(size_t spatial_index, - size_t temporal_index, - uint32_t bitrate_bps) { - RTC_DCHECK_LT(spatial_index, static_cast(kMaxSpatialLayers)); - RTC_DCHECK_LT(temporal_index, static_cast(kMaxTemporalStreams)); - RTC_DCHECK_LE(bitrates_[spatial_index][temporal_index], sum_); - uint64_t new_bitrate_sum_bps = sum_; - new_bitrate_sum_bps -= bitrates_[spatial_index][temporal_index]; - new_bitrate_sum_bps += bitrate_bps; - if (new_bitrate_sum_bps > kMaxBitrateBps) - return false; - - bitrates_[spatial_index][temporal_index] = bitrate_bps; - sum_ = static_cast(new_bitrate_sum_bps); - return true; -} - -uint32_t BitrateAllocation::GetBitrate(size_t spatial_index, - size_t temporal_index) const { - RTC_DCHECK_LT(spatial_index, static_cast(kMaxSpatialLayers)); - RTC_DCHECK_LT(temporal_index, static_cast(kMaxTemporalStreams)); - return bitrates_[spatial_index][temporal_index]; -} - -// Get the sum of all the temporal layer for a specific spatial layer. -uint32_t BitrateAllocation::GetSpatialLayerSum(size_t spatial_index) const { - RTC_DCHECK_LT(spatial_index, static_cast(kMaxSpatialLayers)); - uint32_t sum = 0; - for (int i = 0; i < kMaxTemporalStreams; ++i) - sum += bitrates_[spatial_index][i]; - return sum; -} - } // namespace webrtc diff --git a/webrtc/common_types.h b/webrtc/common_types.h index a9b472c5b7..d4f2f90830 100644 --- a/webrtc/common_types.h +++ b/webrtc/common_types.h @@ -18,7 +18,6 @@ #include #include -#include "webrtc/base/optional.h" #include "webrtc/common_video/rotation.h" #include "webrtc/typedefs.h" @@ -523,7 +522,7 @@ struct VideoCodecVP8 { bool automaticResizeOn; bool frameDroppingOn; int keyFrameInterval; - TemporalLayersFactory* tl_factory; + const TemporalLayersFactory* tl_factory; }; // VP9 specific. @@ -564,10 +563,6 @@ enum VideoCodecType { kVideoCodecUnknown }; -// Translates from name of codec to codec type and vice versa. -rtc::Optional CodecTypeToPayloadName(VideoCodecType type); -rtc::Optional PayloadNameToCodecType(const std::string& name); - union VideoCodecUnion { VideoCodecVP8 VP8; VideoCodecVP9 VP9; @@ -643,35 +638,6 @@ class VideoCodec { VideoCodecUnion codec_specific_; }; -class BitrateAllocation { - public: - static const size_t kMaxBitrateBps; - BitrateAllocation(); - - bool SetBitrate(size_t spatial_index, - size_t temporal_index, - uint32_t bitrate_bps); - - uint32_t GetBitrate(size_t spatial_index, size_t temporal_index) const; - - // Get the sum of all the temporal layer for a specific spatial layer. - uint32_t GetSpatialLayerSum(size_t spatial_index) const; - - uint32_t get_sum_bps() const { return sum_; } // Sum of all bitrates. - uint32_t get_sum_kbps() const { return (sum_ + 500) / 1000; } - - inline bool operator==(const BitrateAllocation& other) const { - return memcmp(bitrates_, other.bitrates_, sizeof(bitrates_)) == 0; - } - inline bool operator!=(const BitrateAllocation& other) const { - return !(*this == other); - } - - private: - uint32_t sum_; - uint32_t bitrates_[kMaxSpatialLayers][kMaxTemporalStreams]; -}; - // Bandwidth over-use detector options. These are used to drive // experimentation with bandwidth estimation parameters. // See modules/remote_bitrate_estimator/overuse_detector.h diff --git a/webrtc/common_video/BUILD.gn b/webrtc/common_video/BUILD.gn index 34cb6d6a3d..ed98e02611 100644 --- a/webrtc/common_video/BUILD.gn +++ b/webrtc/common_video/BUILD.gn @@ -35,7 +35,6 @@ rtc_static_library("common_video") { "include/frame_callback.h", "include/i420_buffer_pool.h", "include/incoming_video_stream.h", - "include/video_bitrate_allocator.h", "include/video_frame_buffer.h", "incoming_video_stream.cc", "libyuv/include/webrtc_libyuv.h", diff --git a/webrtc/common_video/common_video.gyp b/webrtc/common_video/common_video.gyp index 262b77a1ac..58b2462fa8 100644 --- a/webrtc/common_video/common_video.gyp +++ b/webrtc/common_video/common_video.gyp @@ -73,7 +73,6 @@ 'include/frame_callback.h', 'include/i420_buffer_pool.h', 'include/incoming_video_stream.h', - 'include/video_bitrate_allocator.h', 'include/video_frame_buffer.h', 'libyuv/include/webrtc_libyuv.h', 'libyuv/webrtc_libyuv.cc', diff --git a/webrtc/common_video/include/video_bitrate_allocator.h b/webrtc/common_video/include/video_bitrate_allocator.h deleted file mode 100644 index 66ff077be5..0000000000 --- a/webrtc/common_video/include/video_bitrate_allocator.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef WEBRTC_COMMON_VIDEO_INCLUDE_VIDEO_BITRATE_ALLOCATOR_H_ -#define WEBRTC_COMMON_VIDEO_INCLUDE_VIDEO_BITRATE_ALLOCATOR_H_ - -#include "webrtc/common_types.h" - -namespace webrtc { - -class VideoBitrateAllocator { - public: - VideoBitrateAllocator() {} - virtual ~VideoBitrateAllocator() {} - - virtual BitrateAllocation GetAllocation(uint32_t total_bitrate, - uint32_t framerate) = 0; - virtual uint32_t GetPreferredBitrateBps(uint32_t framerate) = 0; -}; - -} // namespace webrtc - -#endif // WEBRTC_COMMON_VIDEO_INCLUDE_VIDEO_BITRATE_ALLOCATOR_H_ diff --git a/webrtc/media/engine/fakewebrtcvideoengine.h b/webrtc/media/engine/fakewebrtcvideoengine.h index cb77072e2b..a6cfeb3edf 100644 --- a/webrtc/media/engine/fakewebrtcvideoengine.h +++ b/webrtc/media/engine/fakewebrtcvideoengine.h @@ -132,9 +132,9 @@ class FakeWebRtcVideoEncoder : public webrtc::VideoEncoder { FakeWebRtcVideoEncoder() : init_encode_event_(false, false), num_frames_encoded_(0) {} - int32_t InitEncode(const webrtc::VideoCodec* codecSettings, - int32_t numberOfCores, - size_t maxPayloadSize) override { + virtual int32_t InitEncode(const webrtc::VideoCodec* codecSettings, + int32_t numberOfCores, + size_t maxPayloadSize) { rtc::CritScope lock(&crit_); codec_settings_ = *codecSettings; init_encode_event_.Set(); @@ -148,28 +148,27 @@ class FakeWebRtcVideoEncoder : public webrtc::VideoEncoder { return codec_settings_; } - int32_t Encode(const webrtc::VideoFrame& inputImage, - const webrtc::CodecSpecificInfo* codecSpecificInfo, - const std::vector* frame_types) override { + virtual int32_t Encode(const webrtc::VideoFrame& inputImage, + const webrtc::CodecSpecificInfo* codecSpecificInfo, + const std::vector* frame_types) { rtc::CritScope lock(&crit_); ++num_frames_encoded_; init_encode_event_.Set(); return WEBRTC_VIDEO_CODEC_OK; } - int32_t RegisterEncodeCompleteCallback( - webrtc::EncodedImageCallback* callback) override { + virtual int32_t RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback* callback) { return WEBRTC_VIDEO_CODEC_OK; } - int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; } + virtual int32_t Release() { return WEBRTC_VIDEO_CODEC_OK; } - int32_t SetChannelParameters(uint32_t packetLoss, int64_t rtt) override { + virtual int32_t SetChannelParameters(uint32_t packetLoss, int64_t rtt) { return WEBRTC_VIDEO_CODEC_OK; } - int32_t SetRateAllocation(const webrtc::BitrateAllocation& allocation, - uint32_t framerate) override { + virtual int32_t SetRates(uint32_t newBitRate, uint32_t frameRate) { return WEBRTC_VIDEO_CODEC_OK; } diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index 54a1cdec3a..8321bcd501 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -507,7 +507,6 @@ if (rtc_include_tests) { "video_coding/test/stream_generator.cc", "video_coding/test/stream_generator.h", "video_coding/timing_unittest.cc", - "video_coding/utility/default_video_bitrate_allocator_unittest.cc", "video_coding/utility/frame_dropper_unittest.cc", "video_coding/utility/ivf_file_writer_unittest.cc", "video_coding/utility/moving_average_unittest.cc", diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index 8f413d9820..9bd746be75 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -33,7 +33,6 @@ rtc_static_library("video_coding") { "h264_sps_pps_tracker.h", "histogram.cc", "histogram.h", - "include/video_codec_initializer.h", "include/video_coding.h", "include/video_coding_defines.h", "inter_frame_delay.cc", @@ -71,7 +70,6 @@ rtc_static_library("video_coding") { "timestamp_map.h", "timing.cc", "timing.h", - "video_codec_initializer.cc", "video_coding_impl.cc", "video_coding_impl.h", "video_receiver.cc", @@ -100,8 +98,6 @@ rtc_static_library("video_coding") { rtc_static_library("video_coding_utility") { sources = [ - "utility/default_video_bitrate_allocator.cc", - "utility/default_video_bitrate_allocator.h", "utility/frame_dropper.cc", "utility/frame_dropper.h", "utility/ivf_file_writer.cc", diff --git a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc index d411b18bf8..5c0aa1bd45 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc +++ b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc @@ -152,14 +152,6 @@ static void RtpFragmentize(EncodedImage* encoded_image, H264EncoderImpl::H264EncoderImpl() : openh264_encoder_(nullptr), - width_(0), - height_(0), - max_frame_rate_(0.0f), - target_bps_(0), - max_bps_(0), - mode_(kRealtimeVideo), - frame_dropping_on_(false), - key_frame_interval_(0), number_of_cores_(0), encoded_image_callback_(nullptr), has_reported_init_(false), @@ -271,13 +263,11 @@ int32_t H264EncoderImpl::RegisterEncodeCompleteCallback( return WEBRTC_VIDEO_CODEC_OK; } -int32_t H264EncoderImpl::SetRateAllocation( - const BitrateAllocation& bitrate_allocation, - uint32_t framerate) { - if (bitrate_allocation.get_sum_bps() <= 0 || framerate <= 0) +int32_t H264EncoderImpl::SetRates(uint32_t bitrate, uint32_t framerate) { + if (bitrate <= 0 || framerate <= 0) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - - target_bps_ = bitrate_allocation.get_sum_bps(); + } + target_bps_ = bitrate * 1000; max_frame_rate_ = static_cast(framerate); quality_scaler_.ReportFramerate(framerate); diff --git a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h index ca28eb3250..1e461b992f 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h +++ b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h @@ -44,8 +44,7 @@ class H264EncoderImpl : public H264Encoder { int32_t RegisterEncodeCompleteCallback( EncodedImageCallback* callback) override; - int32_t SetRateAllocation(const BitrateAllocation& bitrate_allocation, - uint32_t framerate) override; + int32_t SetRates(uint32_t bitrate, uint32_t framerate) override; // The result of encoding - an EncodedImage and RTPFragmentationHeader - are // passed to the encode complete callback. @@ -75,8 +74,8 @@ class H264EncoderImpl : public H264Encoder { int width_; int height_; float max_frame_rate_; - uint32_t target_bps_; - uint32_t max_bps_; + unsigned int target_bps_; + unsigned int max_bps_; VideoCodecMode mode_; // H.264 specifc parameters bool frame_dropping_on_; diff --git a/webrtc/modules/video_coding/codecs/i420/include/i420.h b/webrtc/modules/video_coding/codecs/i420/include/i420.h index 38dcafd0c3..220aea4b86 100644 --- a/webrtc/modules/video_coding/codecs/i420/include/i420.h +++ b/webrtc/modules/video_coding/codecs/i420/include/i420.h @@ -13,7 +13,6 @@ #include -#include "webrtc/base/checks.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/typedefs.h" @@ -66,6 +65,10 @@ class I420Encoder : public VideoEncoder { // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. int Release() override; + int SetRates(uint32_t /*newBitRate*/, uint32_t /*frameRate*/) override { + return WEBRTC_VIDEO_CODEC_OK; + } + int SetChannelParameters(uint32_t /*packetLoss*/, int64_t /*rtt*/) override { return WEBRTC_VIDEO_CODEC_OK; } diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor.cc b/webrtc/modules/video_coding/codecs/test/videoprocessor.cc index 0449276b3f..8436eb0042 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor.cc +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor.cc @@ -10,18 +10,14 @@ #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" +#include #include #include #include -#include #include -#include "webrtc/base/checks.h" #include "webrtc/base/timeutils.h" -#include "webrtc/modules/video_coding/include/video_codec_initializer.h" -#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h" -#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include "webrtc/system_wrappers/include/cpu_info.h" namespace webrtc { @@ -39,7 +35,7 @@ TestConfig::TestConfig() frame_length_in_bytes(0), use_single_core(false), keyframe_interval(0), - codec_settings(nullptr), + codec_settings(NULL), verbose(true) {} TestConfig::~TestConfig() {} @@ -58,9 +54,8 @@ VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder, packet_manipulator_(packet_manipulator), config_(config), stats_(stats), - encode_callback_(nullptr), - decode_callback_(nullptr), - last_successful_frame_buffer_(nullptr), + encode_callback_(NULL), + decode_callback_(NULL), first_key_frame_has_been_excluded_(false), last_frame_missing_(false), initialized_(false), @@ -70,23 +65,13 @@ VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder, num_dropped_frames_(0), num_spatial_resizes_(0), last_encoder_frame_width_(0), - last_encoder_frame_height_(0), - bit_rate_factor_(0.0), - encode_start_ns_(0), - decode_start_ns_(0) { - std::unique_ptr tl_factory; - if (config_.codec_settings->codecType == VideoCodecType::kVideoCodecVP8) { - tl_factory.reset(new TemporalLayersFactory()); - config.codec_settings->VP8()->tl_factory = tl_factory.get(); - } - bitrate_allocator_ = VideoCodecInitializer::CreateBitrateAllocator( - *config.codec_settings, std::move(tl_factory)); - RTC_DCHECK(encoder); - RTC_DCHECK(decoder); - RTC_DCHECK(frame_reader); - RTC_DCHECK(frame_writer); - RTC_DCHECK(packet_manipulator); - RTC_DCHECK(stats); + last_encoder_frame_height_(0) { + assert(encoder); + assert(decoder); + assert(frame_reader); + assert(frame_writer); + assert(packet_manipulator); + assert(stats); } bool VideoProcessorImpl::Init() { @@ -164,10 +149,8 @@ VideoProcessorImpl::~VideoProcessorImpl() { } void VideoProcessorImpl::SetRates(int bit_rate, int frame_rate) { - int set_rates_result = encoder_->SetRateAllocation( - bitrate_allocator_->GetAllocation(bit_rate * 1000, frame_rate), - frame_rate); - RTC_CHECK_GE(set_rates_result, 0); + int set_rates_result = encoder_->SetRates(bit_rate, frame_rate); + assert(set_rates_result >= 0); if (set_rates_result < 0) { fprintf(stderr, "Failed to update encoder with new rate %d, " @@ -195,7 +178,7 @@ int VideoProcessorImpl::NumberSpatialResizes() { } bool VideoProcessorImpl::ProcessFrame(int frame_number) { - RTC_DCHECK_GE(frame_number, 0); + assert(frame_number >= 0); if (!initialized_) { fprintf(stderr, "Attempting to use uninitialized VideoProcessor!\n"); return false; @@ -289,7 +272,7 @@ void VideoProcessorImpl::FrameEncoded( exclude_this_frame = true; break; default: - RTC_NOTREACHED(); + assert(false); } } @@ -358,11 +341,11 @@ void VideoProcessorImpl::FrameDecoded(const VideoFrame& image) { CalcBufferSize(kI420, up_image->width(), up_image->height()); std::unique_ptr image_buffer(new uint8_t[length]); int extracted_length = ExtractBuffer(up_image, length, image_buffer.get()); - RTC_DCHECK_GT(extracted_length, 0); + assert(extracted_length > 0); // Update our copy of the last successful frame: memcpy(last_successful_frame_buffer_, image_buffer.get(), extracted_length); bool write_success = frame_writer_->WriteFrame(image_buffer.get()); - RTC_DCHECK(write_success); + assert(write_success); if (!write_success) { fprintf(stderr, "Failed to write frame %d to disk!", frame_number); } @@ -372,11 +355,11 @@ void VideoProcessorImpl::FrameDecoded(const VideoFrame& image) { size_t length = CalcBufferSize(kI420, image.width(), image.height()); std::unique_ptr image_buffer(new uint8_t[length]); int extracted_length = ExtractBuffer(image, length, image_buffer.get()); - RTC_DCHECK_GT(extracted_length, 0); + assert(extracted_length > 0); memcpy(last_successful_frame_buffer_, image_buffer.get(), extracted_length); bool write_success = frame_writer_->WriteFrame(image_buffer.get()); - RTC_DCHECK(write_success); + assert(write_success); if (!write_success) { fprintf(stderr, "Failed to write frame %d to disk!", frame_number); } @@ -386,8 +369,8 @@ void VideoProcessorImpl::FrameDecoded(const VideoFrame& image) { int VideoProcessorImpl::GetElapsedTimeMicroseconds(int64_t start, int64_t stop) { uint64_t encode_time = (stop - start) / rtc::kNumNanosecsPerMicrosec; - RTC_DCHECK_LT(encode_time, - static_cast(std::numeric_limits::max())); + assert(encode_time < + static_cast(std::numeric_limits::max())); return static_cast(encode_time); } @@ -398,7 +381,7 @@ const char* ExcludeFrameTypesToStr(ExcludeFrameTypes e) { case kExcludeAllKeyFrames: return "ExcludeAllKeyFrames"; default: - RTC_NOTREACHED(); + assert(false); return "Unknown"; } } @@ -416,7 +399,7 @@ const char* VideoCodecTypeToStr(webrtc::VideoCodecType e) { case kVideoCodecUnknown: return "Unknown"; default: - RTC_NOTREACHED(); + assert(false); return "Unknown"; } } diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor.h b/webrtc/modules/video_coding/codecs/test/videoprocessor.h index dba204f169..0887f4cc68 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor.h +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor.h @@ -11,7 +11,6 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ #define WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ -#include #include #include "webrtc/base/checks.h" @@ -24,9 +23,6 @@ #include "webrtc/video_frame.h" namespace webrtc { - -class VideoBitrateAllocator; - namespace test { // Defines which frame types shall be excluded from packet loss and when. @@ -195,7 +191,6 @@ class VideoProcessorImpl : public VideoProcessor { webrtc::VideoEncoder* encoder_; webrtc::VideoDecoder* decoder_; - std::unique_ptr bitrate_allocator_; FrameReader* frame_reader_; FrameWriter* frame_writer_; PacketManipulator* packet_manipulator_; diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index 2adcdaebff..9c99a0a973 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -15,7 +15,6 @@ #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" -#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/include/video_coding.h" @@ -114,7 +113,6 @@ class VideoProcessorIntegrationTest : public testing::Test { webrtc::test::TestConfig config_; VideoCodec codec_settings_; webrtc::test::VideoProcessor* processor_; - TemporalLayersFactory tl_factory_; // Quantities defined/updated for every encoder rate update. // Some quantities defined per temporal layer (at most 3 layers in this test). diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc index 4f04743fc5..e2ceb4a282 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -13,10 +13,6 @@ #include #include -#include -#include - -#include "webrtc/base/checks.h" #include "webrtc/modules/include/module_common_types.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" @@ -26,17 +22,16 @@ namespace webrtc { -DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers, +DefaultTemporalLayers::DefaultTemporalLayers(int numberOfTemporalLayers, uint8_t initial_tl0_pic_idx) - : number_of_temporal_layers_(number_of_temporal_layers), + : number_of_temporal_layers_(numberOfTemporalLayers), temporal_ids_length_(0), temporal_pattern_length_(0), tl0_pic_idx_(initial_tl0_pic_idx), pattern_idx_(255), timestamp_(0), last_base_layer_sync_(false) { - RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers); - RTC_CHECK_GE(number_of_temporal_layers, 0); + assert(kMaxTemporalStreams >= numberOfTemporalLayers); memset(temporal_ids_, 0, sizeof(temporal_ids_)); memset(temporal_pattern_, 0, sizeof(temporal_pattern_)); } @@ -48,50 +43,18 @@ int DefaultTemporalLayers::CurrentLayerId() const { return temporal_ids_[index]; } -std::vector DefaultTemporalLayers::OnRatesUpdated( - int bitrate_kbps, - int max_bitrate_kbps, - int framerate) { - std::vector bitrates; - const int num_layers = std::max(1, number_of_temporal_layers_); - for (int i = 0; i < num_layers; ++i) { - float layer_bitrate = - bitrate_kbps * kVp8LayerRateAlloction[num_layers - 1][i]; - bitrates.push_back(static_cast(layer_bitrate + 0.5)); - } - new_bitrates_kbps_ = rtc::Optional>(bitrates); - - // Allocation table is of aggregates, transform to individual rates. - uint32_t sum = 0; - for (int i = 0; i < num_layers; ++i) { - uint32_t layer_bitrate = bitrates[i]; - RTC_DCHECK_LE(sum, bitrates[i]); - bitrates[i] -= sum; - sum = layer_bitrate; - - if (sum >= static_cast(bitrate_kbps)) { - // Sum adds up; any subsequent layers will be 0. - bitrates.resize(i + 1); - break; - } - } - - return bitrates; -} - -bool DefaultTemporalLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { - if (!new_bitrates_kbps_) - return false; - +bool DefaultTemporalLayers::ConfigureBitrates(int bitrateKbit, + int max_bitrate_kbit, + int framerate, + vpx_codec_enc_cfg_t* cfg) { switch (number_of_temporal_layers_) { case 0: - FALLTHROUGH(); case 1: temporal_ids_length_ = 1; temporal_ids_[0] = 0; cfg->ts_number_layers = number_of_temporal_layers_; cfg->ts_periodicity = temporal_ids_length_; - cfg->ts_target_bitrate[0] = (*new_bitrates_kbps_)[0]; + cfg->ts_target_bitrate[0] = bitrateKbit; cfg->ts_rate_decimator[0] = 1; memcpy(cfg->ts_layer_id, temporal_ids_, sizeof(unsigned int) * temporal_ids_length_); @@ -106,8 +69,8 @@ bool DefaultTemporalLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { cfg->ts_periodicity = temporal_ids_length_; // Split stream 60% 40%. // Bitrate API for VP8 is the agregated bitrate for all lower layers. - cfg->ts_target_bitrate[0] = (*new_bitrates_kbps_)[0]; - cfg->ts_target_bitrate[1] = (*new_bitrates_kbps_)[1]; + cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[1][0]; + cfg->ts_target_bitrate[1] = bitrateKbit; cfg->ts_rate_decimator[0] = 2; cfg->ts_rate_decimator[1] = 1; memcpy(cfg->ts_layer_id, temporal_ids_, @@ -132,9 +95,9 @@ bool DefaultTemporalLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { cfg->ts_periodicity = temporal_ids_length_; // Split stream 40% 20% 40%. // Bitrate API for VP8 is the agregated bitrate for all lower layers. - cfg->ts_target_bitrate[0] = (*new_bitrates_kbps_)[0]; - cfg->ts_target_bitrate[1] = (*new_bitrates_kbps_)[1]; - cfg->ts_target_bitrate[2] = (*new_bitrates_kbps_)[2]; + cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[2][0]; + cfg->ts_target_bitrate[1] = bitrateKbit * kVp8LayerRateAlloction[2][1]; + cfg->ts_target_bitrate[2] = bitrateKbit; cfg->ts_rate_decimator[0] = 4; cfg->ts_rate_decimator[1] = 2; cfg->ts_rate_decimator[2] = 1; @@ -164,10 +127,10 @@ bool DefaultTemporalLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { // Bitrate API for VP8 is the agregated bitrate for all lower layers. cfg->ts_number_layers = 4; cfg->ts_periodicity = temporal_ids_length_; - cfg->ts_target_bitrate[0] = (*new_bitrates_kbps_)[0]; - cfg->ts_target_bitrate[1] = (*new_bitrates_kbps_)[1]; - cfg->ts_target_bitrate[2] = (*new_bitrates_kbps_)[2]; - cfg->ts_target_bitrate[3] = (*new_bitrates_kbps_)[3]; + cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[3][0]; + cfg->ts_target_bitrate[1] = bitrateKbit * kVp8LayerRateAlloction[3][1]; + cfg->ts_target_bitrate[2] = bitrateKbit * kVp8LayerRateAlloction[3][2]; + cfg->ts_target_bitrate[3] = bitrateKbit; cfg->ts_rate_decimator[0] = 8; cfg->ts_rate_decimator[1] = 4; cfg->ts_rate_decimator[2] = 2; @@ -193,12 +156,9 @@ bool DefaultTemporalLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { temporal_pattern_[15] = kTemporalUpdateNone; break; default: - RTC_NOTREACHED(); + assert(false); return false; } - - new_bitrates_kbps_ = rtc::Optional>(); - return true; } @@ -324,18 +284,8 @@ void DefaultTemporalLayers::PopulateCodecSpecific( } TemporalLayers* TemporalLayersFactory::Create( - int simulcast_id, int temporal_layers, uint8_t initial_tl0_pic_idx) const { - TemporalLayers* tl = - new DefaultTemporalLayers(temporal_layers, initial_tl0_pic_idx); - if (listener_) - listener_->OnTemporalLayersCreated(simulcast_id, tl); - return tl; + return new DefaultTemporalLayers(temporal_layers, initial_tl0_pic_idx); } - -void TemporalLayersFactory::SetListener(TemporalLayersListener* listener) { - listener_ = listener; -} - } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h index 0cd0b06451..19846ba5ff 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -12,12 +12,8 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_ #define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_ -#include - #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" -#include "webrtc/base/optional.h" - namespace webrtc { class DefaultTemporalLayers : public TemporalLayers { @@ -30,13 +26,10 @@ class DefaultTemporalLayers : public TemporalLayers { // and/or update the reference buffers. int EncodeFlags(uint32_t timestamp) override; - // Update state based on new bitrate target and incoming framerate. - // Returns the bitrate allocation for the active temporal layers. - std::vector OnRatesUpdated(int bitrate_kbps, - int max_bitrate_kbps, - int framerate) override; - - bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override; + bool ConfigureBitrates(int bitrate_kbit, + int max_bitrate_kbit, + int framerate, + vpx_codec_enc_cfg_t* cfg) override; void PopulateCodecSpecific(bool base_layer_sync, CodecSpecificInfoVP8* vp8_info, @@ -44,6 +37,8 @@ class DefaultTemporalLayers : public TemporalLayers { void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override {} + bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override { return false; } + int CurrentLayerId() const override; private: @@ -82,7 +77,7 @@ class DefaultTemporalLayers : public TemporalLayers { }; enum { kMaxTemporalPattern = 16 }; - const int number_of_temporal_layers_; + int number_of_temporal_layers_; int temporal_ids_length_; int temporal_ids_[kMaxTemporalPattern]; int temporal_pattern_length_; @@ -91,7 +86,6 @@ class DefaultTemporalLayers : public TemporalLayers { uint8_t pattern_idx_; uint32_t timestamp_; bool last_base_layer_sync_; - rtc::Optional> new_bitrates_kbps_; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc index e22c8b4675..1e36c44810 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc @@ -53,8 +53,7 @@ TEST(TemporalLayersTest, 2Layers) { DefaultTemporalLayers tl(2, 0); vpx_codec_enc_cfg_t cfg; CodecSpecificInfoVP8 vp8_info; - tl.OnRatesUpdated(500, 500, 30); - tl.UpdateConfiguration(&cfg); + tl.ConfigureBitrates(500, 500, 30, &cfg); int expected_flags[16] = { kTemporalUpdateLastAndGoldenRefAltRef, @@ -95,8 +94,7 @@ TEST(TemporalLayersTest, 3Layers) { DefaultTemporalLayers tl(3, 0); vpx_codec_enc_cfg_t cfg; CodecSpecificInfoVP8 vp8_info; - tl.OnRatesUpdated(500, 500, 30); - tl.UpdateConfiguration(&cfg); + tl.ConfigureBitrates(500, 500, 30, &cfg); int expected_flags[16] = { kTemporalUpdateLastAndGoldenRefAltRef, @@ -137,8 +135,7 @@ TEST(TemporalLayersTest, 4Layers) { DefaultTemporalLayers tl(4, 0); vpx_codec_enc_cfg_t cfg; CodecSpecificInfoVP8 vp8_info; - tl.OnRatesUpdated(500, 500, 30); - tl.UpdateConfiguration(&cfg); + tl.ConfigureBitrates(500, 500, 30, &cfg); int expected_flags[16] = { kTemporalUpdateLast, kTemporalUpdateNone, @@ -178,8 +175,7 @@ TEST(TemporalLayersTest, KeyFrame) { DefaultTemporalLayers tl(3, 0); vpx_codec_enc_cfg_t cfg; CodecSpecificInfoVP8 vp8_info; - tl.OnRatesUpdated(500, 500, 30); - tl.UpdateConfiguration(&cfg); + tl.ConfigureBitrates(500, 500, 30, &cfg); int expected_flags[8] = { kTemporalUpdateLastAndGoldenRefAltRef, diff --git a/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc index a3e65e4ce0..b9721cde1b 100644 --- a/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc @@ -12,8 +12,6 @@ #include "vpx/vpx_encoder.h" #include "vpx/vp8cx.h" -#include "webrtc/base/checks.h" -#include "webrtc/base/optional.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" @@ -97,20 +95,26 @@ class RealTimeTemporalLayers : public TemporalLayers { layer_ids_(NULL), encode_flags_length_(0), encode_flags_(NULL) { - RTC_CHECK_GE(max_temporal_layers_, 1); - RTC_CHECK_GE(max_temporal_layers_, 3); + assert(max_temporal_layers_ >= 1); + assert(max_temporal_layers_ <= 3); } virtual ~RealTimeTemporalLayers() {} - std::vector OnRatesUpdated(int bitrate_kbps, - int max_bitrate_kbps, - int framerate) override { + bool ConfigureBitrates(int bitrate_kbit, + int max_bitrate_kbit, + int framerate, + vpx_codec_enc_cfg_t* cfg) override { temporal_layers_ = CalculateNumberOfTemporalLayers(temporal_layers_, framerate); temporal_layers_ = std::min(temporal_layers_, max_temporal_layers_); - RTC_CHECK_GE(temporal_layers_, 1); - RTC_CHECK_LE(temporal_layers_, 3); + assert(temporal_layers_ >= 1 && temporal_layers_ <= 3); + + cfg->ts_number_layers = temporal_layers_; + for (int tl = 0; tl < temporal_layers_; ++tl) { + cfg->ts_target_bitrate[tl] = + bitrate_kbit * kVp8LayerRateAlloction[temporal_layers_ - 1][tl]; + } switch (temporal_layers_) { case 1: { @@ -121,6 +125,9 @@ class RealTimeTemporalLayers : public TemporalLayers { static const int encode_flags[] = {kTemporalUpdateLastRefAll}; encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids); encode_flags_ = encode_flags; + + cfg->ts_rate_decimator[0] = 1; + cfg->ts_periodicity = layer_ids_length_; } break; case 2: { @@ -139,6 +146,10 @@ class RealTimeTemporalLayers : public TemporalLayers { kTemporalUpdateNone}; encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids); encode_flags_ = encode_flags; + + cfg->ts_rate_decimator[0] = 2; + cfg->ts_rate_decimator[1] = 1; + cfg->ts_periodicity = layer_ids_length_; } break; case 3: { @@ -157,59 +168,19 @@ class RealTimeTemporalLayers : public TemporalLayers { kTemporalUpdateNone}; encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids); encode_flags_ = encode_flags; + + cfg->ts_rate_decimator[0] = 4; + cfg->ts_rate_decimator[1] = 2; + cfg->ts_rate_decimator[2] = 1; + cfg->ts_periodicity = layer_ids_length_; } break; default: - RTC_NOTREACHED(); - return std::vector(); + assert(false); + return false; } - - std::vector bitrates; - const int num_layers = std::max(1, temporal_layers_); - for (int i = 0; i < num_layers; ++i) { - float layer_bitrate = - bitrate_kbps * kVp8LayerRateAlloction[num_layers - 1][i]; - bitrates.push_back(static_cast(layer_bitrate + 0.5)); - } - new_bitrates_kbps_ = rtc::Optional>(bitrates); - - // Allocation table is of aggregates, transform to individual rates. - uint32_t sum = 0; - for (int i = 0; i < num_layers; ++i) { - uint32_t layer_bitrate = bitrates[i]; - RTC_DCHECK_LE(sum, bitrates[i]); - bitrates[i] -= sum; - sum += layer_bitrate; - - if (sum == static_cast(bitrate_kbps)) { - // Sum adds up; any subsequent layers will be 0. - bitrates.resize(i); - break; - } - } - - return bitrates; - } - - bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override { - if (!new_bitrates_kbps_) - return false; - - cfg->ts_number_layers = temporal_layers_; - for (int tl = 0; tl < temporal_layers_; ++tl) { - cfg->ts_target_bitrate[tl] = (*new_bitrates_kbps_)[tl]; - } - new_bitrates_kbps_ = rtc::Optional>(); - - cfg->ts_periodicity = layer_ids_length_; - int decimator = 1; - for (int i = temporal_layers_ - 1; i >= 0; --i, decimator *= 2) { - cfg->ts_rate_decimator[i] = decimator; - } - memcpy(cfg->ts_layer_id, layer_ids_, sizeof(unsigned int) * layer_ids_length_); - return true; } @@ -277,6 +248,8 @@ class RealTimeTemporalLayers : public TemporalLayers { void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override {} + bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override { return false; } + private: int temporal_layers_; int max_temporal_layers_; @@ -293,19 +266,12 @@ class RealTimeTemporalLayers : public TemporalLayers { // Pattern of encode flags. int encode_flags_length_; const int* encode_flags_; - - rtc::Optional> new_bitrates_kbps_; }; } // namespace TemporalLayers* RealTimeTemporalLayersFactory::Create( - int simulcast_id, int max_temporal_layers, uint8_t initial_tl0_pic_idx) const { - TemporalLayers* tl = - new RealTimeTemporalLayers(max_temporal_layers, initial_tl0_pic_idx); - if (listener_) - listener_->OnTemporalLayersCreated(simulcast_id, tl); - return tl; + return new RealTimeTemporalLayers(max_temporal_layers, initial_tl0_pic_idx); } } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc index 9c7a5b0d27..e821497440 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -65,8 +65,7 @@ ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, min_qp_(-1), max_qp_(-1), max_debt_bytes_(0), - framerate_(-1), - bitrate_updated_(false) { + frame_rate_(-1) { RTC_CHECK_GT(num_temporal_layers, 0); RTC_CHECK_LE(num_temporal_layers, 2); } @@ -137,7 +136,7 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { int64_t ts_diff; if (last_timestamp_ == -1) { - ts_diff = kOneSecond90Khz / (framerate_ <= 0 ? 5 : framerate_); + ts_diff = kOneSecond90Khz / (frame_rate_ <= 0 ? 5 : frame_rate_); } else { ts_diff = unwrapped_timestamp - last_timestamp_; } @@ -148,19 +147,47 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { return flags; } -std::vector ScreenshareLayers::OnRatesUpdated(int bitrate_kbps, - int max_bitrate_kbps, - int framerate) { +bool ScreenshareLayers::ConfigureBitrates(int bitrate_kbps, + int max_bitrate_kbps, + int framerate, + vpx_codec_enc_cfg_t* cfg) { layers_[0].target_rate_kbps_ = bitrate_kbps; layers_[1].target_rate_kbps_ = max_bitrate_kbps; - framerate_ = framerate; - bitrate_updated_ = true; - std::vector allocation; - allocation.push_back(bitrate_kbps); - if (max_bitrate_kbps > bitrate_kbps) - allocation.push_back(max_bitrate_kbps - bitrate_kbps); - return allocation; + int target_bitrate_kbps = bitrate_kbps; + + if (cfg != nullptr) { + if (number_of_temporal_layers_ > 1) { + // Calculate a codec target bitrate. This may be higher than TL0, gaining + // quality at the expense of frame rate at TL0. Constraints: + // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction. + // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate. + target_bitrate_kbps = + std::min(bitrate_kbps * kMaxTL0FpsReduction, + max_bitrate_kbps / kAcceptableTargetOvershoot); + + cfg->rc_target_bitrate = std::max(bitrate_kbps, target_bitrate_kbps); + } + + // Don't reconfigure qp limits during quality boost frames. + if (active_layer_ == -1 || + layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) { + min_qp_ = cfg->rc_min_quantizer; + max_qp_ = cfg->rc_max_quantizer; + // After a dropped frame, a frame with max qp will be encoded and the + // quality will then ramp up from there. To boost the speed of recovery, + // encode the next frame with lower max qp. TL0 is the most important to + // improve since the errors in this layer will propagate to TL1. + // Currently, reduce max qp by 20% for TL0 and 15% for TL1. + layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100); + layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100); + } + } + + int avg_frame_size = (target_bitrate_kbps * 1000) / (8 * framerate); + max_debt_bytes_ = 4 * avg_frame_size; + + return true; } void ScreenshareLayers::FrameEncoded(unsigned int size, @@ -252,52 +279,9 @@ bool ScreenshareLayers::TimeToSync(int64_t timestamp) const { } bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { - bool cfg_updated = false; - if (bitrate_updated_) { - uint32_t target_bitrate_kbps = layers_[0].target_rate_kbps_; - - if (number_of_temporal_layers_ > 1) { - // Calculate a codec target bitrate. This may be higher than TL0, gaining - // quality at the expense of frame rate at TL0. Constraints: - // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction. - // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate. - target_bitrate_kbps = - std::min(layers_[0].target_rate_kbps_ * kMaxTL0FpsReduction, - layers_[1].target_rate_kbps_ / kAcceptableTargetOvershoot); - - cfg->rc_target_bitrate = - std::max(layers_[0].target_rate_kbps_, target_bitrate_kbps); - } - - // Don't reconfigure qp limits during quality boost frames. - if (active_layer_ == -1 || - layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) { - min_qp_ = cfg->rc_min_quantizer; - max_qp_ = cfg->rc_max_quantizer; - // After a dropped frame, a frame with max qp will be encoded and the - // quality will then ramp up from there. To boost the speed of recovery, - // encode the next frame with lower max qp. TL0 is the most important to - // improve since the errors in this layer will propagate to TL1. - // Currently, reduce max qp by 20% for TL0 and 15% for TL1. - layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100); - layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100); - } - - if (framerate_ > 0) { - int avg_frame_size = (target_bitrate_kbps * 1000) / (8 * framerate_); - max_debt_bytes_ = 4 * avg_frame_size; - } - - bitrate_updated_ = false; - cfg_updated = true; - } - - // Don't try to update boosts state if not active yet. - if (active_layer_ == -1) - return cfg_updated; - if (max_qp_ == -1 || number_of_temporal_layers_ <= 1) - return cfg_updated; + return false; + RTC_DCHECK_NE(-1, active_layer_); // If layer is in the quality boost state (following a dropped frame), update // the configuration with the adjusted (lower) qp and set the state back to @@ -309,16 +293,15 @@ bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { layers_[active_layer_].state = TemporalLayer::State::kNormal; } else { if (max_qp_ == -1) - return cfg_updated; + return false; adjusted_max_qp = max_qp_; // Set the normal max qp. } if (adjusted_max_qp == cfg->rc_max_quantizer) - return cfg_updated; + return false; cfg->rc_max_quantizer = adjusted_max_qp; - cfg_updated = true; - return cfg_updated; + return true; } void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) { diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h index e4d3c76aab..8392bc4f35 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -9,7 +9,9 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ #define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ -#include +#include + +#include "vpx/vpx_encoder.h" #include "webrtc/base/timeutils.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" @@ -39,15 +41,10 @@ class ScreenshareLayers : public TemporalLayers { // and/or update the reference buffers. int EncodeFlags(uint32_t timestamp) override; - // Update state based on new bitrate target and incoming framerate. - // Returns the bitrate allocation for the active temporal layers. - std::vector OnRatesUpdated(int bitrate_kbps, - int max_bitrate_kbps, - int framerate) override; - - // Update the encoder configuration with target bitrates or other parameters. - // Returns true iff the configuration was actually modified. - bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override; + bool ConfigureBitrates(int bitrate_kbps, + int max_bitrate_kbps, + int framerate, + vpx_codec_enc_cfg_t* cfg) override; void PopulateCodecSpecific(bool base_layer_sync, CodecSpecificInfoVP8* vp8_info, @@ -57,6 +54,11 @@ class ScreenshareLayers : public TemporalLayers { int CurrentLayerId() const override; + // Allows the layers adapter to update the encoder configuration prior to a + // frame being encoded. Return true if the configuration should be updated + // and false if now change is needed. + bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override; + private: bool TimeToSync(int64_t timestamp) const; @@ -73,8 +75,7 @@ class ScreenshareLayers : public TemporalLayers { int min_qp_; int max_qp_; uint32_t max_debt_bytes_; - int framerate_; - bool bitrate_updated_; + int frame_rate_; static const int kMaxNumTemporalLayers = 2; struct TemporalLayer { diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index 9e8c1c36ff..e6316d6af9 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -22,7 +22,6 @@ #include "webrtc/test/gtest.h" using ::testing::_; -using ::testing::ElementsAre; using ::testing::NiceMock; using ::testing::Return; @@ -62,11 +61,8 @@ class ScreenshareLayerTest : public ::testing::Test { memset(&vpx_cfg, 0, sizeof(vpx_codec_enc_cfg_t)); vpx_cfg.rc_min_quantizer = min_qp_; vpx_cfg.rc_max_quantizer = max_qp_; - EXPECT_THAT(layers_->OnRatesUpdated(kDefaultTl0BitrateKbps, - kDefaultTl1BitrateKbps, kFrameRate), - ElementsAre(kDefaultTl0BitrateKbps, - kDefaultTl1BitrateKbps - kDefaultTl0BitrateKbps)); - EXPECT_TRUE(layers_->UpdateConfiguration(&vpx_cfg)); + EXPECT_TRUE(layers_->ConfigureBitrates( + kDefaultTl0BitrateKbps, kDefaultTl1BitrateKbps, kFrameRate, &vpx_cfg)); frame_size_ = ((vpx_cfg.rc_target_bitrate * 1000) / 8) / kFrameRate; } @@ -377,41 +373,27 @@ TEST_F(ScreenshareLayerTest, TooHighBitrate) { TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) { vpx_codec_enc_cfg_t cfg = GetConfig(); - const int kTl0_kbps = 100; - const int kTl1_kbps = 1000; - layers_->OnRatesUpdated(kTl0_kbps, kTl1_kbps, 5); - - EXPECT_THAT(layers_->OnRatesUpdated(kTl0_kbps, kTl1_kbps, 5), - ElementsAre(kTl0_kbps, kTl1_kbps - kTl0_kbps)); - EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); + layers_->ConfigureBitrates(100, 1000, 5, &cfg); EXPECT_EQ(static_cast( - ScreenshareLayers::kMaxTL0FpsReduction * kTl0_kbps + 0.5), + ScreenshareLayers::kMaxTL0FpsReduction * 100 + 0.5), cfg.rc_target_bitrate); } TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) { vpx_codec_enc_cfg_t cfg = GetConfig(); - const int kTl0_kbps = 100; - const int kTl1_kbps = 450; - EXPECT_THAT(layers_->OnRatesUpdated(kTl0_kbps, kTl1_kbps, 5), - ElementsAre(kTl0_kbps, kTl1_kbps - kTl0_kbps)); - EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); + layers_->ConfigureBitrates(100, 450, 5, &cfg); EXPECT_EQ(static_cast( - kTl1_kbps / ScreenshareLayers::kAcceptableTargetOvershoot), + 450 / ScreenshareLayers::kAcceptableTargetOvershoot), cfg.rc_target_bitrate); } TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) { vpx_codec_enc_cfg_t cfg = GetConfig(); - const int kTl0_kbps = 100; - const int kTl1_kbps = 100; - EXPECT_THAT(layers_->OnRatesUpdated(kTl0_kbps, kTl1_kbps, 5), - ElementsAre(kTl0_kbps)); - EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); + layers_->ConfigureBitrates(100, 100, 5, &cfg); - EXPECT_EQ(static_cast(kTl1_kbps), cfg.rc_target_bitrate); + EXPECT_EQ(100U, cfg.rc_target_bitrate); } TEST_F(ScreenshareLayerTest, EncoderDrop) { @@ -471,8 +453,7 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { const uint32_t kStartTimestamp = 1234; vpx_codec_enc_cfg_t cfg = GetConfig(); - layers_->OnRatesUpdated(kLowBitrateKbps, kLowBitrateKbps, 5); - layers_->UpdateConfiguration(&cfg); + layers_->ConfigureBitrates(kLowBitrateKbps, kLowBitrateKbps, 5, &cfg); EXPECT_EQ(ScreenshareLayers::kTl0Flags, layers_->EncodeFlags(kStartTimestamp)); @@ -570,9 +551,4 @@ TEST_F(ScreenshareLayerTest, UpdatesHistograms) { kDefaultTl1BitrateKbps)); } -TEST_F(ScreenshareLayerTest, AllowsUpdateConfigBeforeSetRates) { - vpx_codec_enc_cfg_t cfg = GetConfig(); - EXPECT_FALSE(layers_->UpdateConfiguration(&cfg)); -} - } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc index 32472750f5..880f45fa7e 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc @@ -83,6 +83,17 @@ int VerifyCodec(const webrtc::VideoCodec* inst) { return WEBRTC_VIDEO_CODEC_OK; } +struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory { + ScreenshareTemporalLayersFactory() {} + virtual ~ScreenshareTemporalLayersFactory() {} + + virtual webrtc::TemporalLayers* Create(int num_temporal_layers, + uint8_t initial_tl0_pic_idx) const { + return new webrtc::ScreenshareLayers(num_temporal_layers, rand(), + webrtc::Clock::GetRealTimeClock()); + } +}; + // An EncodedImageCallback implementation that forwards on calls to a // SimulcastEncoderAdapter, but with the stream index it's registered with as // the first parameter to Encoded. @@ -105,25 +116,6 @@ class AdapterEncodedImageCallback : public webrtc::EncodedImageCallback { const size_t stream_idx_; }; -// Utility class used to adapt the simulcast id as reported by the temporal -// layers factory, since each sub-encoder will report stream 0. -class TemporalLayersFactoryAdapter : public webrtc::TemporalLayersFactory { - public: - TemporalLayersFactoryAdapter(int adapted_simulcast_id, - const TemporalLayersFactory& tl_factory) - : adapted_simulcast_id_(adapted_simulcast_id), tl_factory_(tl_factory) {} - ~TemporalLayersFactoryAdapter() override {} - webrtc::TemporalLayers* Create(int simulcast_id, - int temporal_layers, - uint8_t initial_tl0_pic_idx) const override { - return tl_factory_.Create(adapted_simulcast_id_, temporal_layers, - initial_tl0_pic_idx); - } - - const int adapted_simulcast_id_; - const TemporalLayersFactory& tl_factory_; -}; - } // namespace namespace webrtc { @@ -133,6 +125,7 @@ SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory) encoded_complete_callback_(nullptr), implementation_name_("SimulcastEncoderAdapter") { memset(&codec_, 0, sizeof(webrtc::VideoCodec)); + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); } SimulcastEncoderAdapter::~SimulcastEncoderAdapter() { @@ -180,13 +173,14 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, } codec_ = *inst; - SimulcastRateAllocator rate_allocator(codec_, nullptr); - BitrateAllocation allocation = rate_allocator.GetAllocation( - codec_.startBitrate * 1000, codec_.maxFramerate); - std::vector start_bitrates; - for (int i = 0; i < kMaxSimulcastStreams; ++i) { - uint32_t stream_bitrate = allocation.GetSpatialLayerSum(i) / 1000; - start_bitrates.push_back(stream_bitrate); + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + std::vector start_bitrates = + rate_allocator_->GetAllocation(codec_.startBitrate); + + // Special mode when screensharing on a single stream. + if (number_of_streams == 1 && inst->mode == kScreensharing) { + screensharing_tl_factory_.reset(new ScreenshareTemporalLayersFactory()); + codec_.VP8()->tl_factory = screensharing_tl_factory_.get(); } std::string implementation_name; @@ -206,9 +200,6 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, PopulateStreamCodec(&codec_, i, start_bitrate_kbps, highest_resolution_stream, &stream_codec); } - TemporalLayersFactoryAdapter tl_factory_adapter(i, - *codec_.VP8()->tl_factory); - stream_codec.VP8()->tl_factory = &tl_factory_adapter; // TODO(ronghuawu): Remove once this is handled in VP8EncoderImpl. if (stream_codec.qpMax < kDefaultMinQp) { @@ -349,48 +340,61 @@ int SimulcastEncoderAdapter::SetChannelParameters(uint32_t packet_loss, return WEBRTC_VIDEO_CODEC_OK; } -int SimulcastEncoderAdapter::SetRateAllocation(const BitrateAllocation& bitrate, - uint32_t new_framerate) { - if (!Initialized()) +int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit, + uint32_t new_framerate) { + if (!Initialized()) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - - if (new_framerate < 1) + } + if (new_framerate < 1) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - - if (codec_.maxBitrate > 0 && bitrate.get_sum_kbps() > codec_.maxBitrate) - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - - if (bitrate.get_sum_bps() > 0) { - // Make sure the bitrate fits the configured min bitrates. 0 is a special - // value that means paused, though, so leave it alone. - if (bitrate.get_sum_kbps() < codec_.minBitrate) - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - - if (codec_.numberOfSimulcastStreams > 0 && - bitrate.get_sum_kbps() < codec_.simulcastStream[0].minBitrate) { - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } + } + if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) { + new_bitrate_kbit = codec_.maxBitrate; } + std::vector stream_bitrates; + if (new_bitrate_kbit > 0) { + // Make sure the bitrate fits the configured min bitrates. 0 is a special + // value that means paused, though, so leave it alone. + if (new_bitrate_kbit < codec_.minBitrate) { + new_bitrate_kbit = codec_.minBitrate; + } + if (codec_.numberOfSimulcastStreams > 0 && + new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) { + new_bitrate_kbit = codec_.simulcastStream[0].minBitrate; + } + stream_bitrates = rate_allocator_->GetAllocation(new_bitrate_kbit); + } codec_.maxFramerate = new_framerate; - for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { - uint32_t stream_bitrate_kbps = - bitrate.GetSpatialLayerSum(stream_idx) / 1000; + // Disable any stream not in the current allocation. + stream_bitrates.resize(streaminfos_.size(), 0U); + for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { + uint32_t stream_bitrate_kbps = stream_bitrates[stream_idx]; // Need a key frame if we have not sent this stream before. if (stream_bitrate_kbps > 0 && !streaminfos_[stream_idx].send_stream) { streaminfos_[stream_idx].key_frame_request = true; } streaminfos_[stream_idx].send_stream = stream_bitrate_kbps > 0; - // Slice the temporal layers out of the full allocation and pass it on to - // the encoder handling the current simulcast stream. - BitrateAllocation stream_allocation; - for (int i = 0; i < kMaxTemporalStreams; ++i) - stream_allocation.SetBitrate(0, i, bitrate.GetBitrate(stream_idx, i)); - streaminfos_[stream_idx].encoder->SetRateAllocation(stream_allocation, - new_framerate); + // TODO(holmer): This is a temporary hack for screensharing, where we + // interpret the startBitrate as the encoder target bitrate. This is + // to allow for a different max bitrate, so if the codec can't meet + // the target we still allow it to overshoot up to the max before dropping + // frames. This hack should be improved. + if (codec_.targetBitrate > 0 && + (codec_.VP8()->numberOfTemporalLayers == 2 || + codec_.simulcastStream[0].numberOfTemporalLayers == 2)) { + stream_bitrate_kbps = std::min(codec_.maxBitrate, stream_bitrate_kbps); + // TODO(ronghuawu): Can't change max bitrate via the VideoEncoder + // interface. And VP8EncoderImpl doesn't take negative framerate. + // max_bitrate = std::min(codec_.maxBitrate, stream_bitrate_kbps); + // new_framerate = -1; + } + + streaminfos_[stream_idx].encoder->SetRates(stream_bitrate_kbps, + new_framerate); } return WEBRTC_VIDEO_CODEC_OK; diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h index b76e7f4ce9..30d5abb6cc 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h @@ -48,8 +48,7 @@ class SimulcastEncoderAdapter : public VP8Encoder { const std::vector* frame_types) override; int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; int SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; - int SetRateAllocation(const BitrateAllocation& bitrate, - uint32_t new_framerate) override; + int SetRates(uint32_t new_bitrate_kbit, uint32_t new_framerate) override; // Eventual handler for the contained encoders' EncodedImageCallbacks, but // called from an internal helper that also knows the correct stream @@ -104,10 +103,12 @@ class SimulcastEncoderAdapter : public VP8Encoder { bool Initialized() const; std::unique_ptr factory_; + std::unique_ptr screensharing_tl_factory_; VideoCodec codec_; std::vector streaminfos_; EncodedImageCallback* encoded_complete_callback_; std::string implementation_name_; + std::unique_ptr rate_allocator_; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter_unittest.cc index b3fe8e053d..9a06c0ac5f 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter_unittest.cc @@ -95,6 +95,12 @@ TEST_F(TestSimulcastEncoderAdapter, TestSpatioTemporalLayers321PatternEncoder) { TestVp8Simulcast::TestSpatioTemporalLayers321PatternEncoder(); } +// TODO(ronghuawu): Enable this test when SkipEncodingUnusedStreams option is +// implemented for SimulcastEncoderAdapter. +TEST_F(TestSimulcastEncoderAdapter, DISABLED_TestSkipEncodingUnusedStreams) { + TestVp8Simulcast::TestSkipEncodingUnusedStreams(); +} + TEST_F(TestSimulcastEncoderAdapter, DISABLED_TestRPSIEncoder) { TestVp8Simulcast::TestRPSIEncoder(); } @@ -126,9 +132,8 @@ class MockVideoEncoder : public VideoEncoder { int32_t Release() /* override */ { return 0; } - int32_t SetRateAllocation(const BitrateAllocation& bitrate_allocation, - uint32_t framerate) { - last_set_bitrate_ = bitrate_allocation; + int32_t SetRates(uint32_t newBitRate, uint32_t frameRate) /* override */ { + last_set_bitrate_ = static_cast(newBitRate); return 0; } @@ -155,13 +160,13 @@ class MockVideoEncoder : public VideoEncoder { void set_supports_native_handle(bool enabled) { supports_native_handle_ = enabled; } - BitrateAllocation last_set_bitrate() const { return last_set_bitrate_; } + int32_t last_set_bitrate() const { return last_set_bitrate_; } MOCK_CONST_METHOD0(ImplementationName, const char*()); private: bool supports_native_handle_ = false; - BitrateAllocation last_set_bitrate_; + int32_t last_set_bitrate_ = -1; VideoCodec codec_; EncodedImageCallback* callback_; @@ -268,9 +273,6 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test, void SetupCodec() { TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); - rate_allocator_.reset(new SimulcastRateAllocator(codec_, nullptr)); - tl_factory_.SetListener(rate_allocator_.get()); - codec_.VP8()->tl_factory = &tl_factory_; EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); adapter_->RegisterEncodeCompleteCallback(this); } @@ -299,6 +301,7 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test, EXPECT_EQ(ref.VP8().automaticResizeOn, target.VP8().automaticResizeOn); EXPECT_EQ(ref.VP8().frameDroppingOn, target.VP8().frameDroppingOn); EXPECT_EQ(ref.VP8().keyFrameInterval, target.VP8().keyFrameInterval); + EXPECT_EQ(ref.VP8().tl_factory, target.VP8().tl_factory); EXPECT_EQ(ref.qpMax, target.qpMax); EXPECT_EQ(0, target.numberOfSimulcastStreams); EXPECT_EQ(ref.mode, target.mode); @@ -311,7 +314,6 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test, *ref_codec = codec_; ref_codec->VP8()->numberOfTemporalLayers = kTestTemporalLayerProfile[stream_index]; - ref_codec->VP8()->tl_factory = &tl_factory_; ref_codec->width = codec_.simulcastStream[stream_index].width; ref_codec->height = codec_.simulcastStream[stream_index].height; ref_codec->maxBitrate = codec_.simulcastStream[stream_index].maxBitrate; @@ -355,8 +357,6 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test, int last_encoded_image_width_; int last_encoded_image_height_; int last_encoded_image_simulcast_index_; - TemporalLayersFactory tl_factory_; - std::unique_ptr rate_allocator_; }; TEST_F(TestSimulcastEncoderAdapterFake, InitEncode) { @@ -376,7 +376,7 @@ TEST_F(TestSimulcastEncoderAdapterFake, EncodedCallbackForDifferentEncoders) { SetupCodec(); // Set bitrates so that we send all layers. - adapter_->SetRateAllocation(rate_allocator_->GetAllocation(1200, 30), 30); + adapter_->SetRates(1200, 30); // At this point, the simulcast encoder adapter should have 3 streams: HD, // quarter HD, and quarter quarter HD. We're going to mostly ignore the exact @@ -409,7 +409,6 @@ TEST_F(TestSimulcastEncoderAdapterFake, EncodedCallbackForDifferentEncoders) { TEST_F(TestSimulcastEncoderAdapterFake, SupportsNativeHandleForSingleStreams) { TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); - codec_.VP8()->tl_factory = &tl_factory_; codec_.numberOfSimulcastStreams = 1; EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); adapter_->RegisterEncodeCompleteCallback(this); @@ -423,37 +422,27 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsNativeHandleForSingleStreams) { TEST_F(TestSimulcastEncoderAdapterFake, SetRatesUnderMinBitrate) { TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); - codec_.VP8()->tl_factory = &tl_factory_; codec_.minBitrate = 50; codec_.numberOfSimulcastStreams = 1; EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); - rate_allocator_.reset(new SimulcastRateAllocator(codec_, nullptr)); // Above min should be respected. - BitrateAllocation target_bitrate = - rate_allocator_->GetAllocation(codec_.minBitrate * 1000, 30); - adapter_->SetRateAllocation(target_bitrate, 30); - EXPECT_EQ(target_bitrate, - helper_->factory()->encoders()[0]->last_set_bitrate()); + adapter_->SetRates(100, 30); + EXPECT_EQ(100, helper_->factory()->encoders()[0]->last_set_bitrate()); // Below min but non-zero should be replaced with the min bitrate. - BitrateAllocation too_low_bitrate = - rate_allocator_->GetAllocation((codec_.minBitrate - 1) * 1000, 30); - adapter_->SetRateAllocation(too_low_bitrate, 30); - EXPECT_EQ(target_bitrate, - helper_->factory()->encoders()[0]->last_set_bitrate()); + adapter_->SetRates(15, 30); + EXPECT_EQ(50, helper_->factory()->encoders()[0]->last_set_bitrate()); // Zero should be passed on as is, since it means "pause". - adapter_->SetRateAllocation(BitrateAllocation(), 30); - EXPECT_EQ(BitrateAllocation(), - helper_->factory()->encoders()[0]->last_set_bitrate()); + adapter_->SetRates(0, 30); + EXPECT_EQ(0, helper_->factory()->encoders()[0]->last_set_bitrate()); } TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) { EXPECT_STREQ("SimulcastEncoderAdapter", adapter_->ImplementationName()); TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); - codec_.VP8()->tl_factory = &tl_factory_; std::vector encoder_names; encoder_names.push_back("codec1"); encoder_names.push_back("codec2"); @@ -476,7 +465,6 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsNativeHandleForMultipleStreams) { TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); - codec_.VP8()->tl_factory = &tl_factory_; codec_.numberOfSimulcastStreams = 3; EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); adapter_->RegisterEncodeCompleteCallback(this); @@ -505,7 +493,6 @@ TEST_F(TestSimulcastEncoderAdapterFake, NativeHandleForwardingForMultipleStreams) { TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); - codec_.VP8()->tl_factory = &tl_factory_; codec_.numberOfSimulcastStreams = 3; // High start bitrate, so all streams are enabled. codec_.startBitrate = 3000; @@ -530,7 +517,6 @@ TEST_F(TestSimulcastEncoderAdapterFake, TEST_F(TestSimulcastEncoderAdapterFake, TestFailureReturnCodesFromEncodeCalls) { TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); - codec_.VP8()->tl_factory = &tl_factory_; codec_.numberOfSimulcastStreams = 3; EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); adapter_->RegisterEncodeCompleteCallback(this); diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.cc index d604c64816..9d57dc0996 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.cc @@ -86,5 +86,10 @@ TEST_F(TestVp8Impl, TestSpatioTemporalLayers321PatternEncoder) { TEST_F(TestVp8Impl, TestStrideEncodeDecode) { TestVp8Simulcast::TestStrideEncodeDecode(); } + +TEST_F(TestVp8Impl, TestSkipEncodingUnusedStreams) { + TestVp8Simulcast::TestSkipEncodingUnusedStreams(); +} + } // namespace testing } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h index 1740153252..56ad3a710d 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h @@ -12,7 +12,6 @@ #define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_SIMULCAST_UNITTEST_H_ #include -#include #include #include @@ -21,7 +20,6 @@ #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h" -#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include "webrtc/test/gtest.h" #include "webrtc/video_frame.h" @@ -149,6 +147,83 @@ class Vp8TestDecodedImageCallback : public DecodedImageCallback { int decoded_frames_; }; +class SkipEncodingUnusedStreamsTest { + public: + std::vector RunTest(VP8Encoder* encoder, + VideoCodec* settings, + uint32_t target_bitrate) { + SpyingTemporalLayersFactory spy_factory; + settings->VP8()->tl_factory = &spy_factory; + EXPECT_EQ(0, encoder->InitEncode(settings, 1, 1200)); + + encoder->SetRates(target_bitrate, 30); + + std::vector configured_bitrates; + for (std::vector::const_iterator it = + spy_factory.spying_layers_.begin(); + it != spy_factory.spying_layers_.end(); ++it) { + configured_bitrates.push_back( + static_cast(*it)->configured_bitrate_); + } + return configured_bitrates; + } + + class SpyingTemporalLayers : public TemporalLayers { + public: + explicit SpyingTemporalLayers(TemporalLayers* layers) + : configured_bitrate_(0), layers_(layers) {} + + virtual ~SpyingTemporalLayers() { delete layers_; } + + int EncodeFlags(uint32_t timestamp) override { + return layers_->EncodeFlags(timestamp); + } + + bool ConfigureBitrates(int bitrate_kbit, + int max_bitrate_kbit, + int framerate, + vpx_codec_enc_cfg_t* cfg) override { + configured_bitrate_ = bitrate_kbit; + return layers_->ConfigureBitrates(bitrate_kbit, max_bitrate_kbit, + framerate, cfg); + } + + void PopulateCodecSpecific(bool base_layer_sync, + CodecSpecificInfoVP8* vp8_info, + uint32_t timestamp) override { + layers_->PopulateCodecSpecific(base_layer_sync, vp8_info, timestamp); + } + + void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override { + layers_->FrameEncoded(size, timestamp, qp); + } + + int CurrentLayerId() const override { return layers_->CurrentLayerId(); } + + bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override { + return false; + } + + int configured_bitrate_; + TemporalLayers* layers_; + }; + + class SpyingTemporalLayersFactory : public TemporalLayersFactory { + public: + virtual ~SpyingTemporalLayersFactory() {} + TemporalLayers* Create(int temporal_layers, + uint8_t initial_tl0_pic_idx) const override { + SpyingTemporalLayers* layers = + new SpyingTemporalLayers(TemporalLayersFactory::Create( + temporal_layers, initial_tl0_pic_idx)); + spying_layers_.push_back(layers); + return layers; + } + + mutable std::vector spying_layers_; + }; +}; + class TestVp8Simulcast : public ::testing::Test { public: TestVp8Simulcast(VP8Encoder* encoder, VP8Decoder* decoder) @@ -190,7 +265,7 @@ class TestVp8Simulcast : public ::testing::Test { static void DefaultSettings(VideoCodec* settings, const int* temporal_layer_profile) { - RTC_CHECK(settings); + assert(settings); memset(settings, 0, sizeof(VideoCodec)); strncpy(settings->plName, "VP8", 4); settings->codecType = kVideoCodecVP8; @@ -240,18 +315,12 @@ class TestVp8Simulcast : public ::testing::Test { } protected: - void SetUp() override { SetUpCodec(kDefaultTemporalLayerProfile); } + virtual void SetUp() { SetUpCodec(kDefaultTemporalLayerProfile); } - void TearDown() override { - encoder_->Release(); - decoder_->Release(); - } - - void SetUpCodec(const int* temporal_layer_profile) { + virtual void SetUpCodec(const int* temporal_layer_profile) { encoder_->RegisterEncodeCompleteCallback(&encoder_callback_); decoder_->RegisterDecodeCompleteCallback(&decoder_callback_); DefaultSettings(&settings_, temporal_layer_profile); - SetUpRateAllocator(); EXPECT_EQ(0, encoder_->InitEncode(&settings_, 1, 1200)); EXPECT_EQ(0, decoder_->InitDecode(&settings_, 1)); int half_width = (kDefaultWidth + 1) / 2; @@ -262,16 +331,9 @@ class TestVp8Simulcast : public ::testing::Test { new VideoFrame(input_buffer_, 0, 0, webrtc::kVideoRotation_0)); } - void SetUpRateAllocator() { - TemporalLayersFactory* tl_factory = new TemporalLayersFactory(); - rate_allocator_.reset(new SimulcastRateAllocator( - settings_, std::unique_ptr(tl_factory))); - settings_.VP8()->tl_factory = tl_factory; - } - - void SetRates(uint32_t bitrate_kbps, uint32_t fps) { - encoder_->SetRateAllocation( - rate_allocator_->GetAllocation(bitrate_kbps * 1000, fps), fps); + virtual void TearDown() { + encoder_->Release(); + decoder_->Release(); } void ExpectStreams(FrameType frame_type, int expected_video_streams) { @@ -334,7 +396,7 @@ class TestVp8Simulcast : public ::testing::Test { // We currently expect all active streams to generate a key frame even though // a key frame was only requested for some of them. void TestKeyFrameRequestsOnAllStreams() { - SetRates(kMaxBitrates[2], 30); // To get all three streams. + encoder_->SetRates(kMaxBitrates[2], 30); // To get all three streams. std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, kNumberOfSimulcastStreams); @@ -369,7 +431,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestPaddingAllStreams() { // We should always encode the base layer. - SetRates(kMinBitrates[0] - 1, 30); + encoder_->SetRates(kMinBitrates[0] - 1, 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 1); @@ -382,7 +444,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestPaddingTwoStreams() { // We have just enough to get only the first stream and padding for two. - SetRates(kMinBitrates[0], 30); + encoder_->SetRates(kMinBitrates[0], 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 1); @@ -396,7 +458,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestPaddingTwoStreamsOneMaxedOut() { // We are just below limit of sending second stream, so we should get // the first stream maxed out (at |maxBitrate|), and padding for two. - SetRates(kTargetBitrates[0] + kMinBitrates[1] - 1, 30); + encoder_->SetRates(kTargetBitrates[0] + kMinBitrates[1] - 1, 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 1); @@ -409,7 +471,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestPaddingOneStream() { // We have just enough to send two streams, so padding for one stream. - SetRates(kTargetBitrates[0] + kMinBitrates[1], 30); + encoder_->SetRates(kTargetBitrates[0] + kMinBitrates[1], 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 2); @@ -423,7 +485,8 @@ class TestVp8Simulcast : public ::testing::Test { void TestPaddingOneStreamTwoMaxedOut() { // We are just below limit of sending third stream, so we should get // first stream's rate maxed out at |targetBitrate|, second at |maxBitrate|. - SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] - 1, 30); + encoder_->SetRates( + kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] - 1, 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 2); @@ -436,7 +499,8 @@ class TestVp8Simulcast : public ::testing::Test { void TestSendAllStreams() { // We have just enough to send all streams. - SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2], 30); + encoder_->SetRates( + kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2], 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 3); @@ -449,7 +513,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestDisablingStreams() { // We should get three media streams. - SetRates(kMaxBitrates[0] + kMaxBitrates[1] + kMaxBitrates[2], 30); + encoder_->SetRates(kMaxBitrates[0] + kMaxBitrates[1] + kMaxBitrates[2], 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 3); @@ -460,33 +524,36 @@ class TestVp8Simulcast : public ::testing::Test { EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL, &frame_types)); // We should only get two streams and padding for one. - SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] / 2, 30); + encoder_->SetRates( + kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] / 2, 30); ExpectStreams(kVideoFrameDelta, 2); input_frame_->set_timestamp(input_frame_->timestamp() + 3000); EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL, &frame_types)); // We should only get the first stream and padding for two. - SetRates(kTargetBitrates[0] + kMinBitrates[1] / 2, 30); + encoder_->SetRates(kTargetBitrates[0] + kMinBitrates[1] / 2, 30); ExpectStreams(kVideoFrameDelta, 1); input_frame_->set_timestamp(input_frame_->timestamp() + 3000); EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL, &frame_types)); // We don't have enough bitrate for the thumbnail stream, but we should get // it anyway with current configuration. - SetRates(kTargetBitrates[0] - 1, 30); + encoder_->SetRates(kTargetBitrates[0] - 1, 30); ExpectStreams(kVideoFrameDelta, 1); input_frame_->set_timestamp(input_frame_->timestamp() + 3000); EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL, &frame_types)); // We should only get two streams and padding for one. - SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] / 2, 30); + encoder_->SetRates( + kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] / 2, 30); // We get a key frame because a new stream is being enabled. ExpectStreams(kVideoFrameKey, 2); input_frame_->set_timestamp(input_frame_->timestamp() + 3000); EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL, &frame_types)); // We should get all three streams. - SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kTargetBitrates[2], 30); + encoder_->SetRates( + kTargetBitrates[0] + kTargetBitrates[1] + kTargetBitrates[2], 30); // We get a key frame because a new stream is being enabled. ExpectStreams(kVideoFrameKey, 3); input_frame_->set_timestamp(input_frame_->timestamp() + 3000); @@ -523,11 +590,10 @@ class TestVp8Simulcast : public ::testing::Test { settings_.width; settings_.simulcastStream[settings_.numberOfSimulcastStreams - 1].height = settings_.height; - SetUpRateAllocator(); EXPECT_EQ(0, encoder_->InitEncode(&settings_, 1, 1200)); // Encode one frame and verify. - SetRates(kMaxBitrates[0] + kMaxBitrates[1], 30); + encoder_->SetRates(kMaxBitrates[0] + kMaxBitrates[1], 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); EXPECT_CALL( @@ -545,9 +611,8 @@ class TestVp8Simulcast : public ::testing::Test { DefaultSettings(&settings_, kDefaultTemporalLayerProfile); // Start at the lowest bitrate for enabling base stream. settings_.startBitrate = kMinBitrates[0]; - SetUpRateAllocator(); EXPECT_EQ(0, encoder_->InitEncode(&settings_, 1, 1200)); - SetRates(settings_.startBitrate, 30); + encoder_->SetRates(settings_.startBitrate, 30); ExpectStreams(kVideoFrameKey, 1); // Resize |input_frame_| to the new resolution. half_width = (settings_.width + 1) / 2; @@ -569,7 +634,7 @@ class TestVp8Simulcast : public ::testing::Test { Vp8TestEncodedImageCallback encoder_callback; encoder_->RegisterEncodeCompleteCallback(&encoder_callback); - SetRates(kMaxBitrates[2], 30); // To get all three streams. + encoder_->SetRates(kMaxBitrates[2], 30); // To get all three streams. EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL, NULL)); int picture_id = -1; @@ -638,7 +703,7 @@ class TestVp8Simulcast : public ::testing::Test { encoder_->RegisterEncodeCompleteCallback(&encoder_callback); decoder_->RegisterDecodeCompleteCallback(&decoder_callback); - SetRates(kMaxBitrates[2], 30); // To get all three streams. + encoder_->SetRates(kMaxBitrates[2], 30); // To get all three streams. // Set color. int plane_offset[kNumOfPlanes]; @@ -712,7 +777,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestSaptioTemporalLayers333PatternEncoder() { Vp8TestEncodedImageCallback encoder_callback; encoder_->RegisterEncodeCompleteCallback(&encoder_callback); - SetRates(kMaxBitrates[2], 30); // To get all three streams. + encoder_->SetRates(kMaxBitrates[2], 30); // To get all three streams. int expected_temporal_idx[3] = {-1, -1, -1}; bool expected_layer_sync[3] = {false, false, false}; @@ -781,7 +846,7 @@ class TestVp8Simulcast : public ::testing::Test { SetUpCodec(temporal_layer_profile); Vp8TestEncodedImageCallback encoder_callback; encoder_->RegisterEncodeCompleteCallback(&encoder_callback); - SetRates(kMaxBitrates[2], 30); // To get all three streams. + encoder_->SetRates(kMaxBitrates[2], 30); // To get all three streams. int expected_temporal_idx[3] = {-1, -1, -1}; bool expected_layer_sync[3] = {false, false, false}; @@ -840,7 +905,7 @@ class TestVp8Simulcast : public ::testing::Test { encoder_->RegisterEncodeCompleteCallback(&encoder_callback); decoder_->RegisterDecodeCompleteCallback(&decoder_callback); - SetRates(kMaxBitrates[2], 30); // To get all three streams. + encoder_->SetRates(kMaxBitrates[2], 30); // To get all three streams. // Setting two (possibly) problematic use cases for stride: // 1. stride > width 2. stride_y != stride_uv/2 int stride_y = kDefaultWidth + 20; @@ -876,6 +941,30 @@ class TestVp8Simulcast : public ::testing::Test { EXPECT_EQ(2, decoder_callback.DecodedFrames()); } + void TestSkipEncodingUnusedStreams() { + SkipEncodingUnusedStreamsTest test; + std::vector configured_bitrate = + test.RunTest(encoder_.get(), &settings_, + 1); // Target bit rate 1, to force all streams but the + // base one to be exceeding bandwidth constraints. + EXPECT_EQ(static_cast(kNumberOfSimulcastStreams), + configured_bitrate.size()); + + unsigned int min_bitrate = + std::max(settings_.simulcastStream[0].minBitrate, settings_.minBitrate); + int stream = 0; + for (std::vector::const_iterator it = + configured_bitrate.begin(); + it != configured_bitrate.end(); ++it) { + if (stream == 0) { + EXPECT_EQ(min_bitrate, *it); + } else { + EXPECT_EQ(0u, *it); + } + ++stream; + } + } + std::unique_ptr encoder_; MockEncodedImageCallback encoder_callback_; std::unique_ptr decoder_; @@ -883,7 +972,6 @@ class TestVp8Simulcast : public ::testing::Test { VideoCodec settings_; rtc::scoped_refptr input_buffer_; std::unique_ptr input_frame_; - std::unique_ptr rate_allocator_; }; } // namespace testing diff --git a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h index e1f1e0abae..0d09839a8c 100644 --- a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h @@ -12,21 +12,18 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ #define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ -#include +#include "vpx/vpx_encoder.h" #include "webrtc/common_video/include/video_image.h" #include "webrtc/typedefs.h" -struct vpx_codec_enc_cfg; -typedef struct vpx_codec_enc_cfg vpx_codec_enc_cfg_t; - namespace webrtc { struct CodecSpecificInfoVP8; class TemporalLayers { public: - // Factory for TemporalLayer strategy. Default behavior is a fixed pattern + // Factory for TemporalLayer strategy. Default behaviour is a fixed pattern // of temporal layers. See default_temporal_layers.cc virtual ~TemporalLayers() {} @@ -34,15 +31,10 @@ class TemporalLayers { // and/or update the reference buffers. virtual int EncodeFlags(uint32_t timestamp) = 0; - // Update state based on new bitrate target and incoming framerate. - // Returns the bitrate allocation for the active temporal layers. - virtual std::vector OnRatesUpdated(int bitrate_kbps, - int max_bitrate_kbps, - int framerate) = 0; - - // Update the encoder configuration with target bitrates or other parameters. - // Returns true iff the configuration was actually modified. - virtual bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) = 0; + virtual bool ConfigureBitrates(int bitrate_kbit, + int max_bitrate_kbit, + int framerate, + vpx_codec_enc_cfg_t* cfg) = 0; virtual void PopulateCodecSpecific(bool base_layer_sync, CodecSpecificInfoVP8* vp8_info, @@ -51,42 +43,26 @@ class TemporalLayers { virtual void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) = 0; virtual int CurrentLayerId() const = 0; + + virtual bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) = 0; }; -class TemporalLayersListener; class TemporalLayersFactory { public: - TemporalLayersFactory() : listener_(nullptr) {} virtual ~TemporalLayersFactory() {} - virtual TemporalLayers* Create(int simulcast_id, - int temporal_layers, + virtual TemporalLayers* Create(int temporal_layers, uint8_t initial_tl0_pic_idx) const; - void SetListener(TemporalLayersListener* listener); - - protected: - TemporalLayersListener* listener_; }; // Factory for a temporal layers strategy that adaptively changes the number of -// layers based on input frame rate so that the base layer has an acceptable -// frame rate. See realtime_temporal_layers.cc +// layers based on input framerate so that the base layer has an acceptable +// framerate. See realtime_temporal_layers.cc class RealTimeTemporalLayersFactory : public TemporalLayersFactory { public: - RealTimeTemporalLayersFactory() {} ~RealTimeTemporalLayersFactory() override {} - TemporalLayers* Create(int simulcast_id, - int num_temporal_layers, + TemporalLayers* Create(int num_temporal_layers, uint8_t initial_tl0_pic_idx) const override; }; -class TemporalLayersListener { - public: - TemporalLayersListener() {} - virtual ~TemporalLayersListener() {} - - virtual void OnTemporalLayersCreated(int simulcast_id, - TemporalLayers* layers) = 0; -}; - } // namespace webrtc #endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ diff --git a/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc index d95b6d0adc..d636b14f50 100644 --- a/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc @@ -16,7 +16,6 @@ #include "webrtc/base/timeutils.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" -#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/test/frame_utils.h" #include "webrtc/test/gtest.h" #include "webrtc/test/testsupport/fileutils.h" @@ -162,8 +161,6 @@ class TestVp8Impl : public ::testing::Test { codec_inst_.maxBitrate = 4000; codec_inst_.qpMax = 56; codec_inst_.VP8()->denoisingOn = true; - codec_inst_.VP8()->tl_factory = &tl_factory_; - codec_inst_.VP8()->numberOfTemporalLayers = 1; EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->InitEncode(&codec_inst_, 1, 1440)); @@ -204,7 +201,6 @@ class TestVp8Impl : public ::testing::Test { EncodedImage encoded_frame_; VideoFrame decoded_frame_; VideoCodec codec_inst_; - TemporalLayersFactory tl_factory_; }; TEST_F(TestVp8Impl, EncoderParameterTest) { @@ -219,15 +215,11 @@ TEST_F(TestVp8Impl, EncoderParameterTest) { codec_inst_.qpMax = 56; codec_inst_.VP8()->complexity = kComplexityNormal; codec_inst_.VP8()->numberOfTemporalLayers = 1; - codec_inst_.VP8()->tl_factory = &tl_factory_; // Calls before InitEncode(). EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); int bit_rate = 300; - BitrateAllocation bitrate_allocation; - bitrate_allocation.SetBitrate(0, 0, bit_rate * 1000); EXPECT_EQ(WEBRTC_VIDEO_CODEC_UNINITIALIZED, - encoder_->SetRateAllocation(bitrate_allocation, - codec_inst_.maxFramerate)); + encoder_->SetRates(bit_rate, codec_inst_.maxFramerate)); EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->InitEncode(&codec_inst_, 1, 1440)); diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index eb5d2bd3a1..2e510ec897 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -113,6 +113,7 @@ VP8Decoder* VP8Decoder::Create() { VP8EncoderImpl::VP8EncoderImpl() : encoded_complete_callback_(nullptr), + rate_allocator_(new SimulcastRateAllocator(codec_)), inited_(false), timestamp_(0), feedback_mode_(false), @@ -174,32 +175,27 @@ int VP8EncoderImpl::Release() { return ret_val; } -int VP8EncoderImpl::SetRateAllocation(const BitrateAllocation& bitrate, - uint32_t new_framerate) { - if (!inited_) +int VP8EncoderImpl::SetRates(uint32_t new_bitrate_kbit, + uint32_t new_framerate) { + if (!inited_) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - - if (encoders_[0].err) - return WEBRTC_VIDEO_CODEC_ERROR; - - if (new_framerate < 1) - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - - if (bitrate.get_sum_bps() == 0) { - // Encoder paused, turn off all encoding. - const int num_streams = static_cast(encoders_.size()); - for (int i = 0; i < num_streams; ++i) - SetStreamState(false, i); - return WEBRTC_VIDEO_CODEC_OK; } - - // At this point, bitrate allocation should already match codec settings. - if (codec_.maxBitrate > 0) - RTC_DCHECK_LE(bitrate.get_sum_kbps(), codec_.maxBitrate); - RTC_DCHECK_GE(bitrate.get_sum_kbps(), codec_.minBitrate); - if (codec_.numberOfSimulcastStreams > 0) - RTC_DCHECK_GE(bitrate.get_sum_kbps(), codec_.simulcastStream[0].minBitrate); - + if (encoders_[0].err) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + if (new_framerate < 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) { + new_bitrate_kbit = codec_.maxBitrate; + } + if (new_bitrate_kbit < codec_.minBitrate) { + new_bitrate_kbit = codec_.minBitrate; + } + if (codec_.numberOfSimulcastStreams > 0 && + new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) { + new_bitrate_kbit = codec_.simulcastStream[0].minBitrate; + } codec_.maxFramerate = new_framerate; if (encoders_.size() == 1) { @@ -211,14 +207,14 @@ int VP8EncoderImpl::SetRateAllocation(const BitrateAllocation& bitrate, // Only trigger keyframes if we are allowed to scale down. if (configurations_[0].rc_resize_allowed) { if (!down_scale_requested_) { - if (k_pixels_per_frame > bitrate.get_sum_kbps()) { + if (k_pixels_per_frame > new_bitrate_kbit) { down_scale_requested_ = true; - down_scale_bitrate_ = bitrate.get_sum_kbps(); + down_scale_bitrate_ = new_bitrate_kbit; key_frame_request_[0] = true; } } else { - if (bitrate.get_sum_kbps() > (2 * down_scale_bitrate_) || - bitrate.get_sum_kbps() < (down_scale_bitrate_ / 2)) { + if (new_bitrate_kbit > (2 * down_scale_bitrate_) || + new_bitrate_kbit < (down_scale_bitrate_ / 2)) { down_scale_requested_ = false; } } @@ -237,18 +233,31 @@ int VP8EncoderImpl::SetRateAllocation(const BitrateAllocation& bitrate, } } + std::vector stream_bitrates = + rate_allocator_->GetAllocation(new_bitrate_kbit); size_t stream_idx = encoders_.size() - 1; for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) { - unsigned int target_bitrate_kbps = - bitrate.GetSpatialLayerSum(stream_idx) / 1000; - - bool send_stream = target_bitrate_kbps > 0; - if (send_stream || encoders_.size() > 1) - SetStreamState(send_stream, stream_idx); - - configurations_[i].rc_target_bitrate = target_bitrate_kbps; - temporal_layers_[stream_idx]->UpdateConfiguration(&configurations_[i]); + if (encoders_.size() > 1) + SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx); + unsigned int target_bitrate = stream_bitrates[stream_idx]; + unsigned int max_bitrate = codec_.maxBitrate; + int framerate = new_framerate; + // TODO(holmer): This is a temporary hack for screensharing, where we + // interpret the startBitrate as the encoder target bitrate. This is + // to allow for a different max bitrate, so if the codec can't meet + // the target we still allow it to overshoot up to the max before dropping + // frames. This hack should be improved. + if (codec_.targetBitrate > 0 && + (codec_.VP8()->numberOfTemporalLayers == 2 || + codec_.simulcastStream[0].numberOfTemporalLayers == 2)) { + int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate); + max_bitrate = std::min(codec_.maxBitrate, target_bitrate); + target_bitrate = tl0_bitrate; + } + configurations_[i].rc_target_bitrate = target_bitrate; + temporal_layers_[stream_idx]->ConfigureBitrates( + target_bitrate, max_bitrate, framerate, &configurations_[i]); if (vpx_codec_enc_config_set(&encoders_[i], &configurations_[i])) { return WEBRTC_VIDEO_CODEC_ERROR; } @@ -278,17 +287,26 @@ void VP8EncoderImpl::SetStreamState(bool send_stream, void VP8EncoderImpl::SetupTemporalLayers(int num_streams, int num_temporal_layers, const VideoCodec& codec) { - RTC_DCHECK(codec.VP8().tl_factory != nullptr); + TemporalLayersFactory default_factory; const TemporalLayersFactory* tl_factory = codec.VP8().tl_factory; + if (!tl_factory) + tl_factory = &default_factory; if (num_streams == 1) { - temporal_layers_.push_back( - tl_factory->Create(0, num_temporal_layers, rand())); + if (codec.mode == kScreensharing) { + // Special mode when screensharing on a single stream. + temporal_layers_.push_back(new ScreenshareLayers( + num_temporal_layers, rand(), webrtc::Clock::GetRealTimeClock())); + } else { + temporal_layers_.push_back( + tl_factory->Create(num_temporal_layers, rand())); + } } else { for (int i = 0; i < num_streams; ++i) { - RTC_CHECK_GT(num_temporal_layers, 0); - int layers = std::max(static_cast(1), - codec.simulcastStream[i].numberOfTemporalLayers); - temporal_layers_.push_back(tl_factory->Create(i, layers, rand())); + // TODO(andresp): crash if layers is invalid. + int layers = codec.simulcastStream[i].numberOfTemporalLayers; + if (layers < 1) + layers = 1; + temporal_layers_.push_back(tl_factory->Create(layers, rand())); } } } @@ -333,8 +351,10 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, int num_temporal_layers = doing_simulcast ? inst->simulcastStream[0].numberOfTemporalLayers : inst->VP8().numberOfTemporalLayers; - RTC_DCHECK_GT(num_temporal_layers, 0); + // TODO(andresp): crash if num temporal layers is bananas. + if (num_temporal_layers < 1) + num_temporal_layers = 1; SetupTemporalLayers(number_of_streams, num_temporal_layers, *inst); feedback_mode_ = inst->VP8().feedbackModeOn; @@ -342,6 +362,7 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, number_of_cores_ = number_of_cores; timestamp_ = 0; codec_ = *inst; + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); // Code expects simulcastStream resolutions to be correct, make sure they are // filled even when there are no simulcast layers. @@ -493,44 +514,45 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, vpx_img_wrap(&raw_images_[0], VPX_IMG_FMT_I420, inst->width, inst->height, 1, NULL); - // Note the order we use is different from webm, we have lowest resolution - // at position 0 and they have highest resolution at position 0. - int stream_idx = encoders_.size() - 1; - SimulcastRateAllocator init_allocator(codec_, nullptr); - BitrateAllocation allocation = init_allocator.GetAllocation( - inst->startBitrate * 1000, inst->maxFramerate); - std::vector stream_bitrates; - for (int i = 0; i == 0 || i < inst->numberOfSimulcastStreams; ++i) { - uint32_t bitrate = allocation.GetSpatialLayerSum(i) / 1000; - stream_bitrates.push_back(bitrate); - } - - configurations_[0].rc_target_bitrate = stream_bitrates[stream_idx]; - temporal_layers_[stream_idx]->OnRatesUpdated( - stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate); - temporal_layers_[stream_idx]->UpdateConfiguration(&configurations_[0]); - --stream_idx; - for (size_t i = 1; i < encoders_.size(); ++i, --stream_idx) { - memcpy(&configurations_[i], &configurations_[0], - sizeof(configurations_[0])); - - configurations_[i].g_w = inst->simulcastStream[stream_idx].width; - configurations_[i].g_h = inst->simulcastStream[stream_idx].height; - - // Use 1 thread for lower resolutions. - configurations_[i].g_threads = 1; - - // Setting alignment to 32 - as that ensures at least 16 for all - // planes (32 for Y, 16 for U,V). Libvpx sets the requested stride for - // the y plane, but only half of it to the u and v planes. - vpx_img_alloc(&raw_images_[i], VPX_IMG_FMT_I420, - inst->simulcastStream[stream_idx].width, - inst->simulcastStream[stream_idx].height, kVp832ByteAlign); + if (encoders_.size() == 1) { + configurations_[0].rc_target_bitrate = inst->startBitrate; + temporal_layers_[0]->ConfigureBitrates(inst->startBitrate, inst->maxBitrate, + inst->maxFramerate, + &configurations_[0]); + } else { + // Note the order we use is different from webm, we have lowest resolution + // at position 0 and they have highest resolution at position 0. + int stream_idx = encoders_.size() - 1; + std::vector stream_bitrates = + rate_allocator_->GetAllocation(inst->startBitrate); SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx); - configurations_[i].rc_target_bitrate = stream_bitrates[stream_idx]; - temporal_layers_[stream_idx]->OnRatesUpdated( - stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate); - temporal_layers_[stream_idx]->UpdateConfiguration(&configurations_[i]); + configurations_[0].rc_target_bitrate = stream_bitrates[stream_idx]; + temporal_layers_[stream_idx]->ConfigureBitrates( + stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate, + &configurations_[0]); + --stream_idx; + for (size_t i = 1; i < encoders_.size(); ++i, --stream_idx) { + memcpy(&configurations_[i], &configurations_[0], + sizeof(configurations_[0])); + + configurations_[i].g_w = inst->simulcastStream[stream_idx].width; + configurations_[i].g_h = inst->simulcastStream[stream_idx].height; + + // Use 1 thread for lower resolutions. + configurations_[i].g_threads = 1; + + // Setting alignment to 32 - as that ensures at least 16 for all + // planes (32 for Y, 16 for U,V). Libvpx sets the requested stride for + // the y plane, but only half of it to the u and v planes. + vpx_img_alloc(&raw_images_[i], VPX_IMG_FMT_I420, + inst->simulcastStream[stream_idx].width, + inst->simulcastStream[stream_idx].height, kVp832ByteAlign); + SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx); + configurations_[i].rc_target_bitrate = stream_bitrates[stream_idx]; + temporal_layers_[stream_idx]->ConfigureBitrates( + stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate, + &configurations_[i]); + } } rps_.Init(); diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h index 3b3a78a7e7..f2f208813c 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h @@ -32,6 +32,7 @@ namespace webrtc { +class SimulcastRateAllocator; class TemporalLayers; class VP8EncoderImpl : public VP8Encoder { @@ -54,8 +55,7 @@ class VP8EncoderImpl : public VP8Encoder { int SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; - int SetRateAllocation(const BitrateAllocation& bitrate, - uint32_t new_framerate) override; + int SetRates(uint32_t new_bitrate_kbit, uint32_t frame_rate) override; void OnDroppedFrame() override; @@ -94,6 +94,7 @@ class VP8EncoderImpl : public VP8Encoder { EncodedImageCallback* encoded_complete_callback_; VideoCodec codec_; + std::unique_ptr rate_allocator_; bool inited_; int64_t timestamp_; bool feedback_mode_; diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc index 7ab4a79bd2..8c798db665 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -77,7 +77,6 @@ VP9EncoderImpl::VP9EncoderImpl() frames_since_kf_(0), num_temporal_layers_(0), num_spatial_layers_(0), - is_flexible_mode_(false), frames_encoded_(0), // Use two spatial when screensharing with flexible mode. spatial_layer_(new ScreenshareLayersVP9(2)) { @@ -194,28 +193,24 @@ bool VP9EncoderImpl::SetSvcRates() { return true; } -int VP9EncoderImpl::SetRateAllocation( - const BitrateAllocation& bitrate_allocation, - uint32_t frame_rate) { +int VP9EncoderImpl::SetRates(uint32_t new_bitrate_kbit, + uint32_t new_framerate) { if (!inited_) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } if (encoder_->err) { return WEBRTC_VIDEO_CODEC_ERROR; } - if (frame_rate < 1) { + if (new_framerate < 1) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } // Update bit rate - if (codec_.maxBitrate > 0 && - bitrate_allocation.get_sum_kbps() > codec_.maxBitrate) { - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) { + new_bitrate_kbit = codec_.maxBitrate; } - - // TODO(sprang): Actually use BitrateAllocation layer info. - config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps(); - codec_.maxFramerate = frame_rate; - spatial_layer_->ConfigureBitrate(bitrate_allocation.get_sum_kbps(), 0); + config_->rc_target_bitrate = new_bitrate_kbit; + codec_.maxFramerate = new_framerate; + spatial_layer_->ConfigureBitrate(new_bitrate_kbit, 0); if (!SetSvcRates()) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h index 4eb31a4256..7e65f3b211 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h @@ -46,8 +46,7 @@ class VP9EncoderImpl : public VP9Encoder { int SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; - int SetRateAllocation(const BitrateAllocation& bitrate_allocation, - uint32_t frame_rate) override; + int SetRates(uint32_t new_bitrate_kbit, uint32_t frame_rate) override; void OnDroppedFrame() override {} diff --git a/webrtc/modules/video_coding/generic_encoder.cc b/webrtc/modules/video_coding/generic_encoder.cc index 80e48d1911..acef556b24 100644 --- a/webrtc/modules/video_coding/generic_encoder.cc +++ b/webrtc/modules/video_coding/generic_encoder.cc @@ -28,7 +28,7 @@ VCMGenericEncoder::VCMGenericEncoder( : encoder_(encoder), vcm_encoded_frame_callback_(encoded_frame_callback), internal_source_(internal_source), - encoder_params_({BitrateAllocation(), 0, 0, 0}), + encoder_params_({0, 0, 0, 0}), is_screenshare_(false) {} VCMGenericEncoder::~VCMGenericEncoder() {} @@ -90,23 +90,11 @@ void VCMGenericEncoder::SetEncoderParameters(const EncoderParameters& params) { params.input_frame_rate != encoder_params_.input_frame_rate; encoder_params_ = params; } - if (channel_parameters_have_changed) { - int res = encoder_->SetChannelParameters(params.loss_rate, params.rtt); - if (res != 0) { - LOG(LS_WARNING) << "Error set encoder parameters (loss = " - << params.loss_rate << ", rtt = " << params.rtt - << "): " << res; - } - } + if (channel_parameters_have_changed) + encoder_->SetChannelParameters(params.loss_rate, params.rtt); if (rates_have_changed) { - int res = encoder_->SetRateAllocation(params.target_bitrate, - params.input_frame_rate); - if (res != 0) { - LOG(LS_WARNING) << "Error set encoder rate (total bitrate bps = " - << params.target_bitrate.get_sum_bps() - << ", framerate = " << params.input_frame_rate - << "): " << res; - } + uint32_t target_bitrate_kbps = (params.target_bitrate + 500) / 1000; + encoder_->SetRates(target_bitrate_kbps, params.input_frame_rate); } } diff --git a/webrtc/modules/video_coding/generic_encoder.h b/webrtc/modules/video_coding/generic_encoder.h index 4d2ea2f364..bc1afae55f 100644 --- a/webrtc/modules/video_coding/generic_encoder.h +++ b/webrtc/modules/video_coding/generic_encoder.h @@ -28,7 +28,7 @@ class MediaOptimization; } // namespace media_optimization struct EncoderParameters { - BitrateAllocation target_bitrate; + uint32_t target_bitrate; uint8_t loss_rate; int64_t rtt; uint32_t input_frame_rate; diff --git a/webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h b/webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h index 7e2076ccdc..ce024973cf 100644 --- a/webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h +++ b/webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h @@ -45,9 +45,6 @@ class MockVideoEncoder : public VideoEncoder { MOCK_METHOD0(Reset, int32_t()); MOCK_METHOD2(SetChannelParameters, int32_t(uint32_t packetLoss, int64_t rtt)); MOCK_METHOD2(SetRates, int32_t(uint32_t newBitRate, uint32_t frameRate)); - MOCK_METHOD2(SetRateAllocation, - int32_t(const BitrateAllocation& newBitRate, - uint32_t frameRate)); MOCK_METHOD1(SetPeriodicKeyFrames, int32_t(bool enable)); }; diff --git a/webrtc/modules/video_coding/include/video_codec_initializer.h b/webrtc/modules/video_coding/include/video_codec_initializer.h deleted file mode 100644 index 8514bbbce0..0000000000 --- a/webrtc/modules/video_coding/include/video_codec_initializer.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODEC_INITIALIZER_H_ -#define WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODEC_INITIALIZER_H_ - -#include -#include -#include - -#include "webrtc/video_send_stream.h" - -namespace webrtc { - -class TemporalLayersFactory; -class VideoBitrateAllocator; -class VideoCodec; -class VideoEncoderConfig; - -class VideoCodecInitializer { - public: - // Takes an EncoderSettings, a VideoEncoderConfig and the VideoStream - // configuration and translated them into the old school VideoCodec type. - // It also creates a VideoBitrateAllocator instance, suitable for the codec - // type used. For instance, VP8 will create an allocator than can handle - // simulcast and temporal layering. - // GetBitrateAllocator is called implicitly from here, no need to call again. - static bool SetupCodec( - const VideoEncoderConfig& config, - const VideoSendStream::Config::EncoderSettings settings, - const std::vector& streams, - VideoCodec* codec, - std::unique_ptr* bitrate_allocator); - - // Create a bitrate allocator for the specified codec. |tl_factory| is - // optional, if it is populated, ownership of that instance will be - // transferred to the VideoBitrateAllocator instance. - static std::unique_ptr CreateBitrateAllocator( - const VideoCodec& codec, - std::unique_ptr tl_factory); - - private: - static VideoCodec VideoEncoderConfigToVideoCodec( - const VideoEncoderConfig& config, - const std::vector& streams, - const std::string& payload_name, - int payload_type); -}; - -} // namespace webrtc - -#endif // WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODEC_INITIALIZER_H_ diff --git a/webrtc/modules/video_coding/include/video_coding.h b/webrtc/modules/video_coding/include/video_coding.h index 7b6bd32416..eeda8a2e34 100644 --- a/webrtc/modules/video_coding/include/video_coding.h +++ b/webrtc/modules/video_coding/include/video_coding.h @@ -35,7 +35,6 @@ class EncodedImageCallback; // removing the VCM and use VideoSender/VideoReceiver as a public interface // directly. class VCMQMSettingsCallback; -class VideoBitrateAllocator; class VideoEncoder; class VideoDecoder; struct CodecSpecificInfo; diff --git a/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc b/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc deleted file mode 100644 index cb701f20b6..0000000000 --- a/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h" -#include "webrtc/base/basictypes.h" -#include "webrtc/base/constructormagic.h" - -namespace webrtc { - -DefaultVideoBitrateAllocator::DefaultVideoBitrateAllocator( - const VideoCodec& codec) - : codec_(codec) {} - -DefaultVideoBitrateAllocator::~DefaultVideoBitrateAllocator() {} - -BitrateAllocation DefaultVideoBitrateAllocator::GetAllocation( - uint32_t total_bitrate_bps, - uint32_t framerate) { - BitrateAllocation allocation; - if (total_bitrate_bps == 0) - return allocation; - - if (total_bitrate_bps < codec_.minBitrate * 1000) { - allocation.SetBitrate(0, 0, codec_.minBitrate * 1000); - } else if (codec_.maxBitrate > 0 && - total_bitrate_bps > codec_.maxBitrate * 1000) { - allocation.SetBitrate(0, 0, codec_.maxBitrate * 1000); - } else { - allocation.SetBitrate(0, 0, total_bitrate_bps); - } - - return allocation; -} - -uint32_t DefaultVideoBitrateAllocator::GetPreferredBitrateBps( - uint32_t framerate) { - return GetAllocation(codec_.maxBitrate * 1000, framerate).get_sum_bps(); -} - -} // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h b/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h deleted file mode 100644 index f254e8d4b1..0000000000 --- a/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_DEFAULT_VIDEO_BITRATE_ALLOCATOR_H_ -#define WEBRTC_MODULES_VIDEO_CODING_UTILITY_DEFAULT_VIDEO_BITRATE_ALLOCATOR_H_ - -#include "webrtc/common_video/include/video_bitrate_allocator.h" - -namespace webrtc { - -class DefaultVideoBitrateAllocator : public VideoBitrateAllocator { - public: - explicit DefaultVideoBitrateAllocator(const VideoCodec& codec); - ~DefaultVideoBitrateAllocator() override; - - BitrateAllocation GetAllocation(uint32_t total_bitrate, - uint32_t framerate) override; - uint32_t GetPreferredBitrateBps(uint32_t framerate) override; - - private: - const VideoCodec codec_; -}; - -} // namespace webrtc - -#endif // WEBRTC_MODULES_VIDEO_CODING_UTILITY_DEFAULT_VIDEO_BITRATE_ALLOCATOR_H_ diff --git a/webrtc/modules/video_coding/utility/default_video_bitrate_allocator_unittest.cc b/webrtc/modules/video_coding/utility/default_video_bitrate_allocator_unittest.cc deleted file mode 100644 index df5643353e..0000000000 --- a/webrtc/modules/video_coding/utility/default_video_bitrate_allocator_unittest.cc +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include -#include - -#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h" -#include "webrtc/test/gtest.h" - -namespace webrtc { -namespace { -uint32_t kMaxBitrateBps = 1000000; -uint32_t kMinBitrateBps = 50000; -uint32_t kMaxFramerate = 30; -} // namespace - -class DefaultVideoBitrateAllocatorTest : public ::testing::Test { - public: - DefaultVideoBitrateAllocatorTest() {} - virtual ~DefaultVideoBitrateAllocatorTest() {} - - void SetUp() override { - codec_.codecType = kVideoCodecVP8; - codec_.minBitrate = kMinBitrateBps / 1000; - codec_.maxBitrate = kMaxBitrateBps / 1000; - codec_.targetBitrate = (kMinBitrateBps + kMaxBitrateBps) / 2000; - codec_.maxFramerate = kMaxFramerate; - allocator_.reset(new DefaultVideoBitrateAllocator(codec_)); - } - - protected: - VideoCodec codec_; - std::unique_ptr allocator_; -}; - -TEST_F(DefaultVideoBitrateAllocatorTest, ZeroIsOff) { - BitrateAllocation allocation = allocator_->GetAllocation(0, kMaxFramerate); - EXPECT_EQ(0u, allocation.get_sum_bps()); -} - -TEST_F(DefaultVideoBitrateAllocatorTest, CapsToMin) { - BitrateAllocation allocation = allocator_->GetAllocation(1, kMaxFramerate); - EXPECT_EQ(kMinBitrateBps, allocation.get_sum_bps()); - - allocation = allocator_->GetAllocation(kMinBitrateBps - 1, kMaxFramerate); - EXPECT_EQ(kMinBitrateBps, allocation.get_sum_bps()); - - allocation = allocator_->GetAllocation(kMinBitrateBps, kMaxFramerate); - EXPECT_EQ(kMinBitrateBps, allocation.get_sum_bps()); -} - -TEST_F(DefaultVideoBitrateAllocatorTest, CapsToMax) { - BitrateAllocation allocation = - allocator_->GetAllocation(kMaxBitrateBps, kMaxFramerate); - EXPECT_EQ(kMaxBitrateBps, allocation.get_sum_bps()); - - allocation = allocator_->GetAllocation(kMaxBitrateBps + 1, kMaxFramerate); - EXPECT_EQ(kMaxBitrateBps, allocation.get_sum_bps()); - - allocation = allocator_->GetAllocation(std::numeric_limits::max(), - kMaxFramerate); - EXPECT_EQ(kMaxBitrateBps, allocation.get_sum_bps()); -} - -TEST_F(DefaultVideoBitrateAllocatorTest, GoodInBetween) { - BitrateAllocation allocation = - allocator_->GetAllocation(kMinBitrateBps + 1, kMaxFramerate); - EXPECT_EQ(kMinBitrateBps + 1, allocation.get_sum_bps()); - - allocation = allocator_->GetAllocation(kMaxBitrateBps - 1, kMaxFramerate); - EXPECT_EQ(kMaxBitrateBps - 1, allocation.get_sum_bps()); -} -} // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc index d0afc3757d..f6dccf0dda 100644 --- a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc +++ b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc @@ -11,133 +11,67 @@ #include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include -#include -#include -#include - -#include "webrtc/base/checks.h" namespace webrtc { -SimulcastRateAllocator::SimulcastRateAllocator( - const VideoCodec& codec, - std::unique_ptr tl_factory) - : codec_(codec), tl_factory_(std::move(tl_factory)) { - if (tl_factory_.get()) - tl_factory_->SetListener(this); -} +webrtc::SimulcastRateAllocator::SimulcastRateAllocator(const VideoCodec& codec) + : codec_(codec) {} -void SimulcastRateAllocator::OnTemporalLayersCreated(int simulcast_id, - TemporalLayers* layers) { - RTC_DCHECK(temporal_layers_.find(simulcast_id) == temporal_layers_.end()); - temporal_layers_[simulcast_id] = layers; -} +std::vector webrtc::SimulcastRateAllocator::GetAllocation( + uint32_t bitrate_kbps) const { + // Always allocate enough bitrate for the minimum bitrate of the first layer. + // Suspending below min bitrate is controlled outside the codec implementation + // and is not overridden by this. + const uint32_t min_bitrate_bps = codec_.numberOfSimulcastStreams == 0 + ? codec_.minBitrate + : codec_.simulcastStream[0].minBitrate; + uint32_t left_to_allocate = std::max(min_bitrate_bps, bitrate_kbps); + if (codec_.maxBitrate) + left_to_allocate = std::min(left_to_allocate, codec_.maxBitrate); -BitrateAllocation SimulcastRateAllocator::GetAllocation( - uint32_t total_bitrate_bps, - uint32_t framerate) { - uint32_t left_to_allocate = total_bitrate_bps; - if (codec_.maxBitrate && codec_.maxBitrate * 1000 < left_to_allocate) - left_to_allocate = codec_.maxBitrate * 1000; - - BitrateAllocation allocated_bitrates_bps; - if (codec_.numberOfSimulcastStreams == 0) { + if (codec_.numberOfSimulcastStreams < 2) { // No simulcast, just set the target as this has been capped already. - allocated_bitrates_bps.SetBitrate( - 0, 0, std::max(codec_.minBitrate * 1000, left_to_allocate)); - } else { - // Always allocate enough bitrate for the minimum bitrate of the first - // layer. Suspending below min bitrate is controlled outside the codec - // implementation and is not overridden by this. - left_to_allocate = - std::max(codec_.simulcastStream[0].minBitrate * 1000, left_to_allocate); - - // Begin by allocating bitrate to simulcast streams, putting all bitrate in - // temporal layer 0. We'll then distribute this bitrate, across potential - // temporal layers, when stream allocation is done. - - // Allocate up to the target bitrate for each simulcast layer. - size_t layer = 0; - for (; layer < codec_.numberOfSimulcastStreams; ++layer) { - const SimulcastStream& stream = codec_.simulcastStream[layer]; - if (left_to_allocate < stream.minBitrate * 1000) - break; - uint32_t allocation = - std::min(left_to_allocate, stream.targetBitrate * 1000); - allocated_bitrates_bps.SetBitrate(layer, 0, allocation); - RTC_DCHECK_LE(allocation, left_to_allocate); - left_to_allocate -= allocation; - } - - // Next, try allocate remaining bitrate, up to max bitrate, in top stream. - // TODO(sprang): Allocate up to max bitrate for all layers once we have a - // better idea of possible performance implications. - if (left_to_allocate > 0) { - size_t active_layer = layer - 1; - const SimulcastStream& stream = codec_.simulcastStream[active_layer]; - uint32_t bitrate_bps = - allocated_bitrates_bps.GetSpatialLayerSum(active_layer); - uint32_t allocation = - std::min(left_to_allocate, stream.maxBitrate * 1000 - bitrate_bps); - bitrate_bps += allocation; - RTC_DCHECK_LE(allocation, left_to_allocate); - left_to_allocate -= allocation; - allocated_bitrates_bps.SetBitrate(active_layer, 0, bitrate_bps); - } + return std::vector(1, left_to_allocate); } - const int num_spatial_streams = - std::max(1, static_cast(codec_.numberOfSimulcastStreams)); + // Initialize bitrates with zeroes. + std::vector allocated_bitrates_bps(codec_.numberOfSimulcastStreams, + 0); - // Finally, distribute the bitrate for the simulcast streams across the - // available temporal layers. - for (int simulcast_id = 0; simulcast_id < num_spatial_streams; - ++simulcast_id) { - auto tl_it = temporal_layers_.find(simulcast_id); - if (tl_it == temporal_layers_.end()) - continue; // TODO(sprang): If > 1 SS, assume default TL alloc? + // First try to allocate up to the target bitrate for each substream. + size_t layer = 0; + for (; layer < codec_.numberOfSimulcastStreams; ++layer) { + const SimulcastStream& stream = codec_.simulcastStream[layer]; + if (left_to_allocate < stream.minBitrate) + break; + uint32_t allocation = std::min(left_to_allocate, stream.targetBitrate); + allocated_bitrates_bps[layer] = allocation; + left_to_allocate -= allocation; + } - uint32_t target_bitrate_kbps = - allocated_bitrates_bps.GetBitrate(simulcast_id, 0) / 1000; - RTC_DCHECK_EQ( - target_bitrate_kbps, - allocated_bitrates_bps.GetSpatialLayerSum(simulcast_id) / 1000); - uint32_t max_bitrate_kbps; - if (codec_.numberOfSimulcastStreams == 0) { - max_bitrate_kbps = codec_.maxBitrate; - - // TODO(holmer): This is a temporary hack for screensharing, where we - // interpret the startBitrate as the encoder target bitrate. This is - // to allow for a different max bitrate, so if the codec can't meet - // the target we still allow it to overshoot up to the max before dropping - // frames. This hack should be improved. - if (codec_.mode == kScreensharing && codec_.targetBitrate > 0 && - (codec_.VP8().numberOfTemporalLayers == 2 || - codec_.simulcastStream[0].numberOfTemporalLayers == 2)) { - int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate_kbps); - max_bitrate_kbps = std::min(codec_.maxBitrate, target_bitrate_kbps); - target_bitrate_kbps = tl0_bitrate; - } - } else { - max_bitrate_kbps = codec_.simulcastStream[simulcast_id].maxBitrate; - } - - std::vector tl_allocation = tl_it->second->OnRatesUpdated( - target_bitrate_kbps, max_bitrate_kbps, framerate); - - for (size_t tl_index = 0; tl_index < tl_allocation.size(); ++tl_index) { - allocated_bitrates_bps.SetBitrate(simulcast_id, tl_index, - tl_allocation[tl_index] * 1000); - } + // Next, try allocate remaining bitrate, up to max bitrate, in top layer. + // TODO(sprang): Allocate up to max bitrate for all layers once we have a + // better idea of possible performance implications. + if (left_to_allocate > 0) { + size_t active_layer = layer - 1; + const SimulcastStream& stream = codec_.simulcastStream[active_layer]; + uint32_t allocation = + std::min(left_to_allocate, + stream.maxBitrate - allocated_bitrates_bps[active_layer]); + left_to_allocate -= allocation; + allocated_bitrates_bps[active_layer] += allocation; } return allocated_bitrates_bps; } -uint32_t SimulcastRateAllocator::GetPreferredBitrateBps(uint32_t framerate) { - BitrateAllocation allocation = - GetAllocation(codec_.maxBitrate * 1000, framerate); - return allocation.get_sum_bps(); +uint32_t SimulcastRateAllocator::GetPreferedBitrate() const { + std::vector rates = GetAllocation(codec_.maxBitrate); + uint32_t preferred_bitrate = 0; + for (const uint32_t& rate : rates) { + preferred_bitrate += rate; + } + return preferred_bitrate; } const VideoCodec& webrtc::SimulcastRateAllocator::GetCodec() const { diff --git a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.h b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.h index 149e866588..e7ee772b3c 100644 --- a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.h +++ b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.h @@ -11,36 +11,24 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_SIMULCAST_RATE_ALLOCATOR_H_ #define WEBRTC_MODULES_VIDEO_CODING_UTILITY_SIMULCAST_RATE_ALLOCATOR_H_ -#include -#include +#include #include "webrtc/base/basictypes.h" #include "webrtc/base/constructormagic.h" -#include "webrtc/common_video/include/video_bitrate_allocator.h" -#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/video_encoder.h" namespace webrtc { -class SimulcastRateAllocator : public VideoBitrateAllocator, - public TemporalLayersListener { +class SimulcastRateAllocator { public: - explicit SimulcastRateAllocator( - const VideoCodec& codec, - std::unique_ptr tl_factory); + explicit SimulcastRateAllocator(const VideoCodec& codec); - void OnTemporalLayersCreated(int simulcast_id, - TemporalLayers* layers) override; - - BitrateAllocation GetAllocation(uint32_t total_bitrate_bps, - uint32_t framerate) override; - uint32_t GetPreferredBitrateBps(uint32_t framerate) override; + std::vector GetAllocation(uint32_t bitrate_kbps) const; + uint32_t GetPreferedBitrate() const; const VideoCodec& GetCodec() const; private: const VideoCodec codec_; - std::map temporal_layers_; - std::unique_ptr tl_factory_; RTC_DISALLOW_COPY_AND_ASSIGN(SimulcastRateAllocator); }; diff --git a/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc index ef95cb113f..57fd5d22bd 100644 --- a/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc +++ b/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -12,7 +12,6 @@ #include #include -#include #include "webrtc/test/gtest.h" @@ -42,59 +41,43 @@ class SimulcastRateAllocatorTest : public ::testing::Test { EXPECT_EQ(expected[i], actual[i]) << "Mismatch at index " << i; } - template - void ExpectEqual(uint32_t (&expected)[S], const BitrateAllocation& actual) { - // EXPECT_EQ(S, actual.size()); - uint32_t sum = 0; - for (size_t i = 0; i < S; ++i) { - uint32_t layer_bitrate = actual.GetSpatialLayerSum(i); - EXPECT_EQ(expected[i] * 1000, layer_bitrate) << "Mismatch at index " << i; - sum += layer_bitrate; - } - EXPECT_EQ(sum, actual.get_sum_bps()); - } - void CreateAllocator() { - allocator_.reset(new SimulcastRateAllocator(codec_, nullptr)); - } - - BitrateAllocation GetAllocation(uint32_t target_bitrate) { - return allocator_->GetAllocation(target_bitrate * 1000, kDefaultFrameRate); + allocator_.reset(new SimulcastRateAllocator(codec_)); } protected: - static const int kDefaultFrameRate = 30; VideoCodec codec_; std::unique_ptr allocator_; }; TEST_F(SimulcastRateAllocatorTest, NoSimulcastBelowMin) { uint32_t expected[] = {codec_.minBitrate}; - ExpectEqual(expected, GetAllocation(codec_.minBitrate - 1)); - ExpectEqual(expected, GetAllocation(1)); - ExpectEqual(expected, GetAllocation(0)); + ExpectEqual(expected, allocator_->GetAllocation(codec_.minBitrate - 1)); + ExpectEqual(expected, allocator_->GetAllocation(1)); + ExpectEqual(expected, allocator_->GetAllocation(0)); } TEST_F(SimulcastRateAllocatorTest, NoSimulcastAboveMax) { uint32_t expected[] = {codec_.maxBitrate}; - ExpectEqual(expected, GetAllocation(codec_.maxBitrate + 1)); - ExpectEqual(expected, GetAllocation(std::numeric_limits::max())); + ExpectEqual(expected, allocator_->GetAllocation(codec_.maxBitrate + 1)); + ExpectEqual(expected, + allocator_->GetAllocation(std::numeric_limits::max())); } TEST_F(SimulcastRateAllocatorTest, NoSimulcastNoMax) { - const uint32_t kMax = BitrateAllocation::kMaxBitrateBps / 1000; + constexpr uint32_t kMax = std::numeric_limits::max(); codec_.maxBitrate = 0; CreateAllocator(); uint32_t expected[] = {kMax}; - ExpectEqual(expected, GetAllocation(kMax)); + ExpectEqual(expected, allocator_->GetAllocation(kMax)); } TEST_F(SimulcastRateAllocatorTest, NoSimulcastWithinLimits) { for (uint32_t bitrate = codec_.minBitrate; bitrate <= codec_.maxBitrate; ++bitrate) { uint32_t expected[] = {bitrate}; - ExpectEqual(expected, GetAllocation(bitrate)); + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); } } @@ -107,9 +90,9 @@ TEST_F(SimulcastRateAllocatorTest, SingleSimulcastBelowMin) { CreateAllocator(); uint32_t expected[] = {kMin}; - ExpectEqual(expected, GetAllocation(kMin - 1)); - ExpectEqual(expected, GetAllocation(1)); - ExpectEqual(expected, GetAllocation(0)); + ExpectEqual(expected, allocator_->GetAllocation(kMin - 1)); + ExpectEqual(expected, allocator_->GetAllocation(1)); + ExpectEqual(expected, allocator_->GetAllocation(0)); } TEST_F(SimulcastRateAllocatorTest, SingleSimulcastAboveMax) { @@ -120,9 +103,9 @@ TEST_F(SimulcastRateAllocatorTest, SingleSimulcastAboveMax) { CreateAllocator(); uint32_t expected[] = {kMax}; - ExpectEqual(expected, GetAllocation(kMax)); - ExpectEqual(expected, GetAllocation(kMax + 1)); - ExpectEqual(expected, GetAllocation(std::numeric_limits::max())); + ExpectEqual(expected, allocator_->GetAllocation(kMax + 1)); + ExpectEqual(expected, + allocator_->GetAllocation(std::numeric_limits::max())); } TEST_F(SimulcastRateAllocatorTest, SingleSimulcastWithinLimits) { @@ -134,7 +117,7 @@ TEST_F(SimulcastRateAllocatorTest, SingleSimulcastWithinLimits) { for (uint32_t bitrate = kMinBitrate; bitrate <= kMaxBitrate; ++bitrate) { uint32_t expected[] = {bitrate}; - ExpectEqual(expected, GetAllocation(bitrate)); + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); } } @@ -156,14 +139,14 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { // Single stream, min bitrate. const uint32_t bitrate = codec_.simulcastStream[0].minBitrate; uint32_t expected[] = {bitrate, 0, 0}; - ExpectEqual(expected, GetAllocation(bitrate)); + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); } { // Single stream at target bitrate. const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate; uint32_t expected[] = {bitrate, 0, 0}; - ExpectEqual(expected, GetAllocation(bitrate)); + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); } { @@ -171,7 +154,7 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate + codec_.simulcastStream[1].minBitrate - 1; uint32_t expected[] = {bitrate, 0, 0}; - ExpectEqual(expected, GetAllocation(bitrate)); + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); } { @@ -180,7 +163,7 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { codec_.simulcastStream[1].minBitrate; uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, codec_.simulcastStream[1].minBitrate, 0}; - ExpectEqual(expected, GetAllocation(bitrate)); + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); } { @@ -189,7 +172,7 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { codec_.simulcastStream[1].maxBitrate; uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, codec_.simulcastStream[1].maxBitrate, 0}; - ExpectEqual(expected, GetAllocation(bitrate)); + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); } { @@ -199,7 +182,7 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { codec_.simulcastStream[1].maxBitrate + 499; uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, codec_.simulcastStream[1].maxBitrate, 0}; - ExpectEqual(expected, GetAllocation(bitrate)); + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); } { @@ -210,7 +193,7 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, codec_.simulcastStream[1].targetBitrate, codec_.simulcastStream[2].minBitrate}; - ExpectEqual(expected, GetAllocation(bitrate)); + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); } { @@ -221,13 +204,12 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, codec_.simulcastStream[1].targetBitrate, codec_.simulcastStream[2].maxBitrate}; - ExpectEqual(expected, GetAllocation(bitrate)); + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); } } -TEST_F(SimulcastRateAllocatorTest, GetPreferredBitrateBps) { - EXPECT_EQ(codec_.maxBitrate * 1000, - allocator_->GetPreferredBitrateBps(codec_.maxFramerate)); +TEST_F(SimulcastRateAllocatorTest, GetPreferredBitrate) { + EXPECT_EQ(codec_.maxBitrate, allocator_->GetPreferedBitrate()); } TEST_F(SimulcastRateAllocatorTest, GetPreferredBitrateSimulcast) { @@ -246,13 +228,12 @@ TEST_F(SimulcastRateAllocatorTest, GetPreferredBitrateSimulcast) { codec_.simulcastStream[2].maxBitrate = 4000; CreateAllocator(); - uint32_t preferred_bitrate_kbps; - preferred_bitrate_kbps = codec_.simulcastStream[0].targetBitrate; - preferred_bitrate_kbps += codec_.simulcastStream[1].targetBitrate; - preferred_bitrate_kbps += codec_.simulcastStream[2].maxBitrate; + uint32_t preferred_bitrate; + preferred_bitrate = codec_.simulcastStream[0].targetBitrate; + preferred_bitrate += codec_.simulcastStream[1].targetBitrate; + preferred_bitrate += codec_.simulcastStream[2].maxBitrate; - EXPECT_EQ(preferred_bitrate_kbps * 1000, - allocator_->GetPreferredBitrateBps(codec_.maxFramerate)); + EXPECT_EQ(preferred_bitrate, allocator_->GetPreferedBitrate()); } } // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/video_coding_utility.gyp b/webrtc/modules/video_coding/utility/video_coding_utility.gyp index 91d1ac18a3..b1158a2298 100644 --- a/webrtc/modules/video_coding/utility/video_coding_utility.gyp +++ b/webrtc/modules/video_coding/utility/video_coding_utility.gyp @@ -19,8 +19,6 @@ '<(webrtc_root)/system_wrappers/system_wrappers.gyp:system_wrappers', ], 'sources': [ - 'default_video_bitrate_allocator.cc', - 'default_video_bitrate_allocator.h', 'frame_dropper.cc', 'frame_dropper.h', 'ivf_file_writer.cc', diff --git a/webrtc/modules/video_coding/video_codec_initializer.cc b/webrtc/modules/video_coding/video_codec_initializer.cc deleted file mode 100644 index e72a1692c3..0000000000 --- a/webrtc/modules/video_coding/video_codec_initializer.cc +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "webrtc/modules/video_coding/include/video_codec_initializer.h" - -#include "webrtc/base/basictypes.h" -#include "webrtc/common_video/include/video_bitrate_allocator.h" -#include "webrtc/common_types.h" -#include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" -#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" -#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" -#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h" -#include "webrtc/system_wrappers/include/clock.h" - -namespace webrtc { - -struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory { - ScreenshareTemporalLayersFactory() {} - virtual ~ScreenshareTemporalLayersFactory() {} - - virtual webrtc::TemporalLayers* Create(int simulcast_id, - int num_temporal_layers, - uint8_t initial_tl0_pic_idx) const { - webrtc::TemporalLayers* tl = new webrtc::ScreenshareLayers( - num_temporal_layers, rand(), webrtc::Clock::GetRealTimeClock()); - if (listener_) - listener_->OnTemporalLayersCreated(simulcast_id, tl); - return tl; - } -}; - -bool VideoCodecInitializer::SetupCodec( - const VideoEncoderConfig& config, - const VideoSendStream::Config::EncoderSettings settings, - const std::vector& streams, - VideoCodec* codec, - std::unique_ptr* bitrate_allocator) { - *codec = VideoEncoderConfigToVideoCodec( - config, streams, settings.payload_name, settings.payload_type); - - std::unique_ptr tl_factory; - switch (codec->codecType) { - case kVideoCodecVP8: { - if (!codec->VP8()->tl_factory) { - if (codec->mode == kScreensharing && - codec->numberOfSimulcastStreams == 1 && - codec->VP8()->numberOfTemporalLayers == 2) { - // Conference mode temporal layering for screen content. - tl_factory.reset(new ScreenshareTemporalLayersFactory()); - } else { - // Standard video temporal layers. - tl_factory.reset(new TemporalLayersFactory()); - } - codec->VP8()->tl_factory = tl_factory.get(); - } - break; - } - default: { - // TODO(sprang): Warn, once we have specific allocators for all supported - // codec types. - break; - } - } - *bitrate_allocator = CreateBitrateAllocator(*codec, std::move(tl_factory)); - - return true; -} - -std::unique_ptr -VideoCodecInitializer::CreateBitrateAllocator( - const VideoCodec& codec, - std::unique_ptr tl_factory) { - std::unique_ptr rate_allocator; - - switch (codec.codecType) { - case kVideoCodecVP8: { - // Set up default VP8 temporal layer factory, if not provided. - rate_allocator.reset( - new SimulcastRateAllocator(codec, std::move(tl_factory))); - } break; - default: - rate_allocator.reset(new DefaultVideoBitrateAllocator(codec)); - } - - return rate_allocator; -} - -// TODO(sprang): Split this up and separate the codec specific parts. -VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec( - const VideoEncoderConfig& config, - const std::vector& streams, - const std::string& payload_name, - int payload_type) { - static const int kEncoderMinBitrateKbps = 30; - RTC_DCHECK(!streams.empty()); - RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0); - - VideoCodec video_codec; - memset(&video_codec, 0, sizeof(video_codec)); - video_codec.codecType = PayloadNameToCodecType(payload_name) - .value_or(VideoCodecType::kVideoCodecGeneric); - - switch (config.content_type) { - case VideoEncoderConfig::ContentType::kRealtimeVideo: - video_codec.mode = kRealtimeVideo; - break; - case VideoEncoderConfig::ContentType::kScreen: - video_codec.mode = kScreensharing; - if (streams.size() == 1 && - streams[0].temporal_layer_thresholds_bps.size() == 1) { - video_codec.targetBitrate = - streams[0].temporal_layer_thresholds_bps[0] / 1000; - } - break; - } - - if (config.encoder_specific_settings) - config.encoder_specific_settings->FillEncoderSpecificSettings(&video_codec); - - switch (video_codec.codecType) { - case kVideoCodecVP8: { - if (!config.encoder_specific_settings) - *video_codec.VP8() = VideoEncoder::GetDefaultVp8Settings(); - video_codec.VP8()->numberOfTemporalLayers = static_cast( - streams.back().temporal_layer_thresholds_bps.size() + 1); - break; - } - case kVideoCodecVP9: { - if (!config.encoder_specific_settings) - *video_codec.VP9() = VideoEncoder::GetDefaultVp9Settings(); - if (video_codec.mode == kScreensharing && - config.encoder_specific_settings) { - video_codec.VP9()->flexibleMode = true; - // For now VP9 screensharing use 1 temporal and 2 spatial layers. - RTC_DCHECK_EQ(1, video_codec.VP9()->numberOfTemporalLayers); - RTC_DCHECK_EQ(2, video_codec.VP9()->numberOfSpatialLayers); - } - video_codec.VP9()->numberOfTemporalLayers = static_cast( - streams.back().temporal_layer_thresholds_bps.size() + 1); - break; - } - case kVideoCodecH264: { - if (!config.encoder_specific_settings) - *video_codec.H264() = VideoEncoder::GetDefaultH264Settings(); - break; - } - default: - // TODO(pbos): Support encoder_settings codec-agnostically. - RTC_DCHECK(!config.encoder_specific_settings) - << "Encoder-specific settings for codec type not wired up."; - break; - } - - strncpy(video_codec.plName, payload_name.c_str(), kPayloadNameSize - 1); - video_codec.plName[kPayloadNameSize - 1] = '\0'; - video_codec.plType = payload_type; - video_codec.numberOfSimulcastStreams = - static_cast(streams.size()); - video_codec.minBitrate = streams[0].min_bitrate_bps / 1000; - if (video_codec.minBitrate < kEncoderMinBitrateKbps) - video_codec.minBitrate = kEncoderMinBitrateKbps; - RTC_DCHECK_LE(streams.size(), static_cast(kMaxSimulcastStreams)); - if (video_codec.codecType == kVideoCodecVP9) { - // If the vector is empty, bitrates will be configured automatically. - RTC_DCHECK(config.spatial_layers.empty() || - config.spatial_layers.size() == - video_codec.VP9()->numberOfSpatialLayers); - RTC_DCHECK_LE(video_codec.VP9()->numberOfSpatialLayers, - kMaxSimulcastStreams); - for (size_t i = 0; i < config.spatial_layers.size(); ++i) - video_codec.spatialLayers[i] = config.spatial_layers[i]; - } - for (size_t i = 0; i < streams.size(); ++i) { - SimulcastStream* sim_stream = &video_codec.simulcastStream[i]; - RTC_DCHECK_GT(streams[i].width, 0u); - RTC_DCHECK_GT(streams[i].height, 0u); - RTC_DCHECK_GT(streams[i].max_framerate, 0); - // Different framerates not supported per stream at the moment. - RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate); - RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0); - RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps); - RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps); - RTC_DCHECK_GE(streams[i].max_qp, 0); - - sim_stream->width = static_cast(streams[i].width); - sim_stream->height = static_cast(streams[i].height); - sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000; - sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000; - sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000; - sim_stream->qpMax = streams[i].max_qp; - sim_stream->numberOfTemporalLayers = static_cast( - streams[i].temporal_layer_thresholds_bps.size() + 1); - - video_codec.width = - std::max(video_codec.width, static_cast(streams[i].width)); - video_codec.height = - std::max(video_codec.height, static_cast(streams[i].height)); - video_codec.minBitrate = - std::min(static_cast(video_codec.minBitrate), - static_cast(streams[i].min_bitrate_bps / 1000)); - video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000; - video_codec.qpMax = std::max(video_codec.qpMax, - static_cast(streams[i].max_qp)); - } - - if (video_codec.maxBitrate == 0) { - // Unset max bitrate -> cap to one bit per pixel. - video_codec.maxBitrate = - (video_codec.width * video_codec.height * video_codec.maxFramerate) / - 1000; - } - if (video_codec.maxBitrate < kEncoderMinBitrateKbps) - video_codec.maxBitrate = kEncoderMinBitrateKbps; - - RTC_DCHECK_GT(streams[0].max_framerate, 0); - video_codec.maxFramerate = streams[0].max_framerate; - return video_codec; -} - -} // namespace webrtc diff --git a/webrtc/modules/video_coding/video_coding.gypi b/webrtc/modules/video_coding/video_coding.gypi index f9b1cf512b..0b682c7d99 100644 --- a/webrtc/modules/video_coding/video_coding.gypi +++ b/webrtc/modules/video_coding/video_coding.gypi @@ -23,7 +23,6 @@ ], 'sources': [ # interfaces - 'include/video_codec_initializer.h', 'include/video_coding.h', 'include/video_coding_defines.h', @@ -87,7 +86,6 @@ 'session_info.cc', 'timestamp_map.cc', 'timing.cc', - 'video_codec_initializer.cc', 'video_coding_impl.cc', 'video_sender.cc', 'video_receiver.cc', diff --git a/webrtc/modules/video_coding/video_coding_impl.cc b/webrtc/modules/video_coding/video_coding_impl.cc index 021281bffa..2f709b610c 100644 --- a/webrtc/modules/video_coding/video_coding_impl.cc +++ b/webrtc/modules/video_coding/video_coding_impl.cc @@ -11,14 +11,10 @@ #include "webrtc/modules/video_coding/video_coding_impl.h" #include -#include #include "webrtc/common_types.h" -#include "webrtc/common_video/include/video_bitrate_allocator.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/base/criticalsection.h" -#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" -#include "webrtc/modules/video_coding/include/video_codec_initializer.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/encoded_frame.h" #include "webrtc/modules/video_coding/jitter_buffer.h" @@ -108,21 +104,6 @@ class VideoCodingModuleImpl : public VideoCodingModule { int32_t RegisterSendCodec(const VideoCodec* sendCodec, uint32_t numberOfCores, uint32_t maxPayloadSize) override { - if (sendCodec != nullptr && sendCodec->codecType == kVideoCodecVP8) { - // Set up a rate allocator and temporal layers factory for this vp8 - // instance. The codec impl will have a raw pointer to the TL factory, - // and will call it when initializing. Since this can happen - // asynchronously keep the instance alive until destruction or until a - // new send codec is registered. - VideoCodec vp8_codec = *sendCodec; - std::unique_ptr tl_factory( - new TemporalLayersFactory()); - vp8_codec.VP8()->tl_factory = tl_factory.get(); - rate_allocator_ = VideoCodecInitializer::CreateBitrateAllocator( - vp8_codec, std::move(tl_factory)); - return sender_.RegisterSendCodec(&vp8_codec, numberOfCores, - maxPayloadSize); - } return sender_.RegisterSendCodec(sendCodec, numberOfCores, maxPayloadSize); } @@ -145,8 +126,7 @@ class VideoCodingModuleImpl : public VideoCodingModule { int32_t SetChannelParameters(uint32_t target_bitrate, // bits/s. uint8_t lossRate, int64_t rtt) override { - return sender_.SetChannelParameters(target_bitrate, lossRate, rtt, - rate_allocator_.get()); + return sender_.SetChannelParameters(target_bitrate, lossRate, rtt); } int32_t RegisterProtectionCallback( @@ -276,7 +256,6 @@ class VideoCodingModuleImpl : public VideoCodingModule { private: EncodedImageCallbackWrapper post_encode_callback_; vcm::VideoSender sender_; - std::unique_ptr rate_allocator_; vcm::VideoReceiver receiver_; }; } // namespace diff --git a/webrtc/modules/video_coding/video_coding_impl.h b/webrtc/modules/video_coding/video_coding_impl.h index 0533634546..cb24654041 100644 --- a/webrtc/modules/video_coding/video_coding_impl.h +++ b/webrtc/modules/video_coding/video_coding_impl.h @@ -34,8 +34,6 @@ namespace webrtc { -class VideoBitrateAllocator; - namespace vcm { class VCMProcessTimer { @@ -77,15 +75,9 @@ class VideoSender : public Module { int Bitrate(unsigned int* bitrate) const; int FrameRate(unsigned int* framerate) const; - int32_t SetChannelParameters(uint32_t target_bitrate_bps, + int32_t SetChannelParameters(uint32_t target_bitrate, // bits/s. uint8_t lossRate, - int64_t rtt, - VideoBitrateAllocator* bitrate_allocator); - // Calls SetChannelParameters(), with the previous target bitrate, loss rate - // and RTT, but reallocates the bitrate allocation based on a presumably - // updated codec configuration. - int32_t UpdateChannelParemeters(VideoBitrateAllocator* bitrate_allocator); - + int64_t rtt); // Deprecated: // TODO(perkj): Remove once no projects use it. int32_t RegisterProtectionCallback(VCMProtectionCallback* protection); diff --git a/webrtc/modules/video_coding/video_sender.cc b/webrtc/modules/video_coding/video_sender.cc index 35fd39cd05..b7664a719b 100644 --- a/webrtc/modules/video_coding/video_sender.cc +++ b/webrtc/modules/video_coding/video_sender.cc @@ -14,12 +14,9 @@ #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/common_types.h" -#include "webrtc/common_video/include/video_bitrate_allocator.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" -#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/encoded_frame.h" -#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h" #include "webrtc/modules/video_coding/utility/quality_scaler.h" #include "webrtc/modules/video_coding/video_coding_impl.h" #include "webrtc/system_wrappers/include/clock.h" @@ -39,7 +36,7 @@ VideoSender::VideoSender(Clock* clock, frame_dropper_enabled_(true), _sendStatsTimer(1000, clock_), current_codec_(), - encoder_params_({BitrateAllocation(), 0, 0, 0}), + encoder_params_({0, 0, 0, 0}), encoder_has_internal_source_(false), next_frame_types_(1, kVideoFrameDelta) { _mediaOpt.Reset(); @@ -175,7 +172,7 @@ int VideoSender::Bitrate(unsigned int* bitrate) const { if (!_encoder) return VCM_UNINITIALIZED; - *bitrate = _encoder->GetEncoderParameters().target_bitrate.get_sum_bps(); + *bitrate = _encoder->GetEncoderParameters().target_bitrate; return 0; } @@ -192,38 +189,15 @@ int VideoSender::FrameRate(unsigned int* framerate) const { return 0; } -int32_t VideoSender::UpdateChannelParemeters( - VideoBitrateAllocator* bitrate_allocator) { - EncoderParameters encoder_params; - { - rtc::CritScope cs(¶ms_crit_); - encoder_params = encoder_params_; - } +int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate, + uint8_t lossRate, + int64_t rtt) { + uint32_t target_rate = + _mediaOpt.SetTargetRates(target_bitrate, lossRate, rtt); - return SetChannelParameters(encoder_params.target_bitrate.get_sum_bps(), - encoder_params.loss_rate, encoder_params.rtt, - bitrate_allocator); -} - -int32_t VideoSender::SetChannelParameters( - uint32_t target_bitrate_bps, - uint8_t lossRate, - int64_t rtt, - VideoBitrateAllocator* bitrate_allocator) { - uint32_t video_target_rate_bps = - _mediaOpt.SetTargetRates(target_bitrate_bps, lossRate, rtt); uint32_t input_frame_rate = _mediaOpt.InputFrameRate(); - BitrateAllocation bitrate_allocation; - if (bitrate_allocator) { - bitrate_allocation = bitrate_allocator->GetAllocation(video_target_rate_bps, - input_frame_rate); - } else { - DefaultVideoBitrateAllocator default_allocator(current_codec_); - bitrate_allocation = default_allocator.GetAllocation(video_target_rate_bps, - input_frame_rate); - } - EncoderParameters encoder_params = {bitrate_allocation, lossRate, rtt, + EncoderParameters encoder_params = {target_rate, lossRate, rtt, input_frame_rate}; bool encoder_has_internal_source; { @@ -254,7 +228,7 @@ void VideoSender::SetEncoderParameters(EncoderParameters params, // encoder implementations behave when given a zero target bitrate. // TODO(perkj): Make sure all known encoder implementations handle zero // target bitrate and remove this check. - if (!has_internal_source && params.target_bitrate.get_sum_bps() == 0) + if (!has_internal_source && params.target_bitrate == 0) return; if (params.input_frame_rate == 0) { @@ -294,8 +268,7 @@ int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame, SetEncoderParameters(encoder_params, encoder_has_internal_source); if (_mediaOpt.DropFrame()) { LOG(LS_VERBOSE) << "Drop Frame " - << "target bitrate " - << encoder_params.target_bitrate.get_sum_bps() + << "target bitrate " << encoder_params.target_bitrate << " loss rate " << encoder_params.loss_rate << " rtt " << encoder_params.rtt << " input frame rate " << encoder_params.input_frame_rate; diff --git a/webrtc/modules/video_coding/video_sender_unittest.cc b/webrtc/modules/video_coding/video_sender_unittest.cc index bda45521f6..71602f0ea1 100644 --- a/webrtc/modules/video_coding/video_sender_unittest.cc +++ b/webrtc/modules/video_coding/video_sender_unittest.cc @@ -19,8 +19,6 @@ #include "webrtc/modules/video_coding/include/video_coding.h" #include "webrtc/modules/video_coding/test/test_util.h" #include "webrtc/modules/video_coding/video_coding_impl.h" -#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h" -#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include "webrtc/system_wrappers/include/clock.h" #include "webrtc/test/frame_generator.h" #include "webrtc/test/gtest.h" @@ -198,10 +196,6 @@ class TestVideoSender : public ::testing::Test { }; class TestVideoSenderWithMockEncoder : public TestVideoSender { - public: - TestVideoSenderWithMockEncoder() {} - ~TestVideoSenderWithMockEncoder() override {} - protected: void SetUp() override { TestVideoSender::SetUp(); @@ -218,7 +212,6 @@ class TestVideoSenderWithMockEncoder : public TestVideoSender { generator_.reset( new EmptyFrameGenerator(settings_.width, settings_.height)); EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200)); - rate_allocator_.reset(new DefaultVideoBitrateAllocator(settings_)); } void TearDown() override { sender_.reset(); } @@ -268,7 +261,6 @@ class TestVideoSenderWithMockEncoder : public TestVideoSender { VideoCodec settings_; NiceMock encoder_; - std::unique_ptr rate_allocator_; }; TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequests) { @@ -299,19 +291,14 @@ TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequests) { } TEST_F(TestVideoSenderWithMockEncoder, TestSetRate) { - const uint32_t new_bitrate_kbps = settings_.startBitrate + 300; - BitrateAllocation new_rate_allocation = rate_allocator_->GetAllocation( - new_bitrate_kbps * 1000, settings_.maxFramerate); - EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, _)) - .Times(1) - .WillOnce(Return(0)); - sender_->SetChannelParameters(new_bitrate_kbps * 1000, 0, 200, - rate_allocator_.get()); + const uint32_t new_bitrate = settings_.startBitrate + 300; + EXPECT_CALL(encoder_, SetRates(new_bitrate, _)).Times(1).WillOnce(Return(0)); + sender_->SetChannelParameters(new_bitrate * 1000, 0, 200); AddFrame(); // Expect no call to encoder_.SetRates if the new bitrate is zero. - EXPECT_CALL(encoder_, SetRateAllocation(_, _)).Times(0); - sender_->SetChannelParameters(0, 0, 200, rate_allocator_.get()); + EXPECT_CALL(encoder_, SetRates(new_bitrate, _)).Times(0); + sender_->SetChannelParameters(0, 0, 200); AddFrame(); } @@ -342,19 +329,13 @@ TEST_F(TestVideoSenderWithMockEncoder, TestEncoderParametersForInternalSource) { EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200)); // Update encoder bitrate parameters. We expect that to immediately call // SetRates on the encoder without waiting for AddFrame processing. - const uint32_t new_bitrate_kbps = settings_.startBitrate + 300; - BitrateAllocation new_rate_allocation = rate_allocator_->GetAllocation( - new_bitrate_kbps * 1000, settings_.maxFramerate); - EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, _)) - .Times(1) - .WillOnce(Return(0)); - sender_->SetChannelParameters(new_bitrate_kbps * 1000, 0, 200, - rate_allocator_.get()); + const uint32_t new_bitrate = settings_.startBitrate + 300; + EXPECT_CALL(encoder_, SetRates(new_bitrate, _)).Times(1).WillOnce(Return(0)); + sender_->SetChannelParameters(new_bitrate * 1000, 0, 200); } TEST_F(TestVideoSenderWithMockEncoder, EncoderFramerateUpdatedViaProcess) { - sender_->SetChannelParameters(settings_.startBitrate * 1000, 0, 200, - rate_allocator_.get()); + sender_->SetChannelParameters(settings_.startBitrate * 1000, 0, 200); const int64_t kRateStatsWindowMs = 2000; const uint32_t kInputFps = 20; int64_t start_time = clock_.TimeInMilliseconds(); @@ -362,9 +343,7 @@ TEST_F(TestVideoSenderWithMockEncoder, EncoderFramerateUpdatedViaProcess) { AddFrame(); clock_.AdvanceTimeMilliseconds(1000 / kInputFps); } - EXPECT_CALL(encoder_, SetRateAllocation(_, kInputFps)) - .Times(1) - .WillOnce(Return(0)); + EXPECT_CALL(encoder_, SetRates(_, kInputFps)).Times(1).WillOnce(Return(0)); sender_->Process(); AddFrame(); } @@ -382,29 +361,23 @@ TEST_F(TestVideoSenderWithMockEncoder, EXPECT_CALL(encoder_, SetChannelParameters(kLossRate, kRtt)) .Times(1) .WillOnce(Return(0)); - sender_->SetChannelParameters(settings_.startBitrate * 1000, kLossRate, kRtt, - rate_allocator_.get()); + sender_->SetChannelParameters(settings_.startBitrate * 1000, kLossRate, kRtt); while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) { AddFrame(); clock_.AdvanceTimeMilliseconds(1000 / kInputFps); } // After process, input framerate should be updated but not ChannelParameters // as they are the same as before. - EXPECT_CALL(encoder_, SetRateAllocation(_, kInputFps)) - .Times(1) - .WillOnce(Return(0)); + EXPECT_CALL(encoder_, SetRates(_, kInputFps)).Times(1).WillOnce(Return(0)); sender_->Process(); AddFrame(); // Call to SetChannelParameters with changed bitrate should call encoder // SetRates but not encoder SetChannelParameters (that are unchanged). - uint32_t new_bitrate_bps = 2 * settings_.startBitrate * 1000; - BitrateAllocation new_rate_allocation = - rate_allocator_->GetAllocation(new_bitrate_bps, kInputFps); - EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, kInputFps)) + EXPECT_CALL(encoder_, SetRates(2 * settings_.startBitrate, kInputFps)) .Times(1) .WillOnce(Return(0)); - sender_->SetChannelParameters(new_bitrate_bps, kLossRate, kRtt, - rate_allocator_.get()); + sender_->SetChannelParameters(2 * settings_.startBitrate * 1000, kLossRate, + kRtt); AddFrame(); } @@ -427,12 +400,6 @@ class TestVideoSenderWithVp8 : public TestVideoSender { codec_.minBitrate = 10; codec_.startBitrate = codec_bitrate_kbps_; codec_.maxBitrate = codec_bitrate_kbps_; - - TemporalLayersFactory* tl_factory = new TemporalLayersFactory(); - rate_allocator_.reset(new SimulcastRateAllocator( - codec_, std::unique_ptr(tl_factory))); - codec_.VP8()->tl_factory = tl_factory; - encoder_.reset(VP8Encoder::Create()); sender_->RegisterExternalEncoder(encoder_.get(), codec_.plType, false); EXPECT_EQ(0, sender_->RegisterSendCodec(&codec_, 1, 1200)); @@ -458,9 +425,8 @@ class TestVideoSenderWithVp8 : public TestVideoSender { // Note: SetChannelParameters fails if less than 2 frames are in the // buffer since it will fail to calculate the framerate. if (i != 0) { - EXPECT_EQ(VCM_OK, - sender_->SetChannelParameters(available_bitrate_kbps_ * 1000, - 0, 200, rate_allocator_.get())); + EXPECT_EQ(VCM_OK, sender_->SetChannelParameters( + available_bitrate_kbps_ * 1000, 0, 200)); } } } @@ -481,7 +447,6 @@ class TestVideoSenderWithVp8 : public TestVideoSender { VideoCodec codec_; int codec_bitrate_kbps_; int available_bitrate_kbps_; - std::unique_ptr rate_allocator_; }; #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) @@ -511,15 +476,11 @@ TEST_F(TestVideoSenderWithVp8, MAYBE_FixedTemporalLayersStrategy) { #endif TEST_F(TestVideoSenderWithVp8, MAYBE_RealTimeTemporalLayersStrategy) { VideoCodec codec = MakeVp8VideoCodec(352, 288, 3); + RealTimeTemporalLayersFactory realtime_tl_factory; + codec.VP8()->tl_factory = &realtime_tl_factory; codec.minBitrate = 10; codec.startBitrate = codec_bitrate_kbps_; codec.maxBitrate = codec_bitrate_kbps_; - - TemporalLayersFactory* tl_factory = new RealTimeTemporalLayersFactory(); - rate_allocator_.reset(new SimulcastRateAllocator( - codec, std::unique_ptr(tl_factory))); - codec.VP8()->tl_factory = tl_factory; - EXPECT_EQ(0, sender_->RegisterSendCodec(&codec, 1, 1200)); const int low_b = codec_bitrate_kbps_ * 0.4; diff --git a/webrtc/test/configurable_frame_size_encoder.cc b/webrtc/test/configurable_frame_size_encoder.cc index bef62405dd..905b69aae8 100644 --- a/webrtc/test/configurable_frame_size_encoder.cc +++ b/webrtc/test/configurable_frame_size_encoder.cc @@ -12,7 +12,6 @@ #include -#include "webrtc/base/checks.h" #include "webrtc/common_video/include/video_image.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/test/gtest.h" @@ -73,9 +72,8 @@ int32_t ConfigurableFrameSizeEncoder::SetChannelParameters(uint32_t packet_loss, return WEBRTC_VIDEO_CODEC_OK; } -int32_t ConfigurableFrameSizeEncoder::SetRateAllocation( - const BitrateAllocation& allocation, - uint32_t framerate) { +int32_t ConfigurableFrameSizeEncoder::SetRates(uint32_t new_bit_rate, + uint32_t frame_rate) { return WEBRTC_VIDEO_CODEC_OK; } @@ -84,7 +82,7 @@ int32_t ConfigurableFrameSizeEncoder::SetPeriodicKeyFrames(bool enable) { } int32_t ConfigurableFrameSizeEncoder::SetFrameSize(size_t size) { - RTC_DCHECK_LE(size, max_frame_size_); + assert(size <= max_frame_size_); current_frame_size_ = size; return WEBRTC_VIDEO_CODEC_OK; } diff --git a/webrtc/test/configurable_frame_size_encoder.h b/webrtc/test/configurable_frame_size_encoder.h index a3820baf7e..d269441667 100644 --- a/webrtc/test/configurable_frame_size_encoder.h +++ b/webrtc/test/configurable_frame_size_encoder.h @@ -39,8 +39,7 @@ class ConfigurableFrameSizeEncoder : public VideoEncoder { int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; - int32_t SetRateAllocation(const BitrateAllocation& allocation, - uint32_t framerate) override; + int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override; int32_t SetPeriodicKeyFrames(bool enable) override; diff --git a/webrtc/test/fake_encoder.cc b/webrtc/test/fake_encoder.cc index 26ce0c6608..f518ce3919 100644 --- a/webrtc/test/fake_encoder.cc +++ b/webrtc/test/fake_encoder.cc @@ -23,6 +23,7 @@ namespace test { FakeEncoder::FakeEncoder(Clock* clock) : clock_(clock), callback_(NULL), + target_bitrate_kbps_(0), max_target_bitrate_kbps_(-1), last_encode_time_ms_(0) { // Generate some arbitrary not-all-zero data @@ -42,7 +43,7 @@ int32_t FakeEncoder::InitEncode(const VideoCodec* config, int32_t number_of_cores, size_t max_payload_size) { config_ = *config; - target_bitrate_.SetBitrate(0, 0, config_.startBitrate * 1000); + target_bitrate_kbps_ = config_.startBitrate; return 0; } @@ -64,8 +65,8 @@ int32_t FakeEncoder::Encode(const VideoFrame& input_image, time_since_last_encode_ms = 3 * 1000 / config_.maxFramerate; } - size_t bits_available = static_cast(target_bitrate_.get_sum_kbps() * - time_since_last_encode_ms); + size_t bits_available = + static_cast(target_bitrate_kbps_ * time_since_last_encode_ms); size_t min_bits = static_cast( config_.simulcastStream[0].minBitrate * time_since_last_encode_ms); if (bits_available < min_bits) @@ -132,9 +133,8 @@ int32_t FakeEncoder::SetChannelParameters(uint32_t packet_loss, int64_t rtt) { return 0; } -int32_t FakeEncoder::SetRateAllocation(const BitrateAllocation& rate_allocation, - uint32_t framerate) { - target_bitrate_ = rate_allocation; +int32_t FakeEncoder::SetRates(uint32_t new_target_bitrate, uint32_t framerate) { + target_bitrate_kbps_ = new_target_bitrate; return 0; } diff --git a/webrtc/test/fake_encoder.h b/webrtc/test/fake_encoder.h index 57b39a4404..d5e7b480df 100644 --- a/webrtc/test/fake_encoder.h +++ b/webrtc/test/fake_encoder.h @@ -39,8 +39,7 @@ class FakeEncoder : public VideoEncoder { EncodedImageCallback* callback) override; int32_t Release() override; int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; - int32_t SetRateAllocation(const BitrateAllocation& rate_allocation, - uint32_t framerate) override; + int32_t SetRates(uint32_t new_target_bitrate, uint32_t framerate) override; const char* ImplementationName() const override; static const char* kImplementationName; @@ -49,7 +48,7 @@ class FakeEncoder : public VideoEncoder { Clock* const clock_; VideoCodec config_; EncodedImageCallback* callback_; - BitrateAllocation target_bitrate_; + int target_bitrate_kbps_; int max_target_bitrate_kbps_; int64_t last_encode_time_ms_; uint8_t encoded_buffer_[100000]; diff --git a/webrtc/video/end_to_end_tests.cc b/webrtc/video/end_to_end_tests.cc index fde2ce29aa..492f55d6ec 100644 --- a/webrtc/video/end_to_end_tests.cc +++ b/webrtc/video/end_to_end_tests.cc @@ -2550,13 +2550,12 @@ TEST_F(EndToEndTest, ReportsSetEncoderRates) { RTC_DCHECK_EQ(1u, encoder_config->number_of_streams); } - int32_t SetRateAllocation(const BitrateAllocation& rate_allocation, - uint32_t framerate) override { + int32_t SetRates(uint32_t new_target_bitrate, uint32_t framerate) override { // Make sure not to trigger on any default zero bitrates. - if (rate_allocation.get_sum_bps() == 0) + if (new_target_bitrate == 0) return 0; rtc::CritScope lock(&crit_); - bitrate_kbps_ = rate_allocation.get_sum_kbps(); + bitrate_kbps_ = new_target_bitrate; observation_complete_.Set(); return 0; } diff --git a/webrtc/video/video_encoder.cc b/webrtc/video/video_encoder.cc index dbfd0eb8db..bb6e9e2308 100644 --- a/webrtc/video/video_encoder.cc +++ b/webrtc/video/video_encoder.cc @@ -67,13 +67,8 @@ VideoEncoder::EncoderType VideoEncoder::CodecToEncoderType( VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper( VideoCodecType codec_type, webrtc::VideoEncoder* encoder) - : number_of_cores_(0), - max_payload_size_(0), - rates_set_(false), - framerate_(0), + : rates_set_(false), channel_parameters_set_(false), - packet_loss_(0), - rtt_(0), encoder_type_(CodecToEncoderType(codec_type)), encoder_(encoder), callback_(nullptr) {} @@ -97,7 +92,7 @@ bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder() { if (callback_) fallback_encoder_->RegisterEncodeCompleteCallback(callback_); if (rates_set_) - fallback_encoder_->SetRateAllocation(bitrate_allocation_, framerate_); + fallback_encoder_->SetRates(bitrate_, framerate_); if (channel_parameters_set_) fallback_encoder_->SetChannelParameters(packet_loss_, rtt_); @@ -194,15 +189,14 @@ int32_t VideoEncoderSoftwareFallbackWrapper::SetChannelParameters( return ret; } -int32_t VideoEncoderSoftwareFallbackWrapper::SetRateAllocation( - const BitrateAllocation& bitrate_allocation, - uint32_t framerate) { +int32_t VideoEncoderSoftwareFallbackWrapper::SetRates(uint32_t bitrate, + uint32_t framerate) { rates_set_ = true; - bitrate_allocation_ = bitrate_allocation; + bitrate_ = bitrate; framerate_ = framerate; - int32_t ret = encoder_->SetRateAllocation(bitrate_allocation_, framerate); + int32_t ret = encoder_->SetRates(bitrate, framerate); if (fallback_encoder_) - return fallback_encoder_->SetRateAllocation(bitrate_allocation_, framerate); + return fallback_encoder_->SetRates(bitrate, framerate); return ret; } diff --git a/webrtc/video/video_encoder_unittest.cc b/webrtc/video/video_encoder_unittest.cc index 899304f886..5f6f17e7b4 100644 --- a/webrtc/video/video_encoder_unittest.cc +++ b/webrtc/video/video_encoder_unittest.cc @@ -10,11 +10,8 @@ #include "webrtc/video_encoder.h" -#include "webrtc/base/checks.h" -#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/include/video_error_codes.h" -#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include "webrtc/test/gtest.h" namespace webrtc { @@ -66,8 +63,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { return WEBRTC_VIDEO_CODEC_OK; } - int32_t SetRateAllocation(const BitrateAllocation& bitrate_allocation, - uint32_t framerate) override { + int32_t SetRates(uint32_t bitrate, uint32_t framerate) override { ++set_rates_count_; return WEBRTC_VIDEO_CODEC_OK; } @@ -121,7 +117,6 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { VideoEncoderSoftwareFallbackWrapper fallback_wrapper_; VideoCodec codec_ = {}; std::unique_ptr frame_; - std::unique_ptr rate_allocator_; }; void VideoEncoderSoftwareFallbackWrapperTest::EncodeFrame() { @@ -144,19 +139,10 @@ void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() { codec_.maxFramerate = 30; codec_.width = kWidth; codec_.height = kHeight; - codec_.VP8()->numberOfTemporalLayers = 1; - std::unique_ptr tl_factory( - new TemporalLayersFactory()); - codec_.VP8()->tl_factory = tl_factory.get(); - rate_allocator_.reset( - new SimulcastRateAllocator(codec_, std::move(tl_factory))); - fake_encoder_.init_encode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR; EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.InitEncode(&codec_, 2, kMaxPayloadSize)); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - fallback_wrapper_.SetRateAllocation( - rate_allocator_->GetAllocation(300000, 30), 30)); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.SetRates(300, 30)); int callback_count = callback_.callback_count_; int encode_count = fake_encoder_.encode_count_; @@ -171,16 +157,8 @@ void VideoEncoderSoftwareFallbackWrapperTest::FallbackFromEncodeRequest() { codec_.maxFramerate = 30; codec_.width = kWidth; codec_.height = kHeight; - codec_.VP8()->numberOfTemporalLayers = 1; - std::unique_ptr tl_factory( - new TemporalLayersFactory()); - codec_.VP8()->tl_factory = tl_factory.get(); - rate_allocator_.reset( - new SimulcastRateAllocator(codec_, std::move(tl_factory))); fallback_wrapper_.InitEncode(&codec_, 2, kMaxPayloadSize); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - fallback_wrapper_.SetRateAllocation( - rate_allocator_->GetAllocation(300000, 30), 30)); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.SetRates(300, 30)); EXPECT_EQ(1, fake_encoder_.init_encode_count_); // Have the non-fallback encoder request a software fallback. @@ -263,7 +241,7 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest, SetRatesForwardedDuringFallback) { UtilizeFallbackEncoder(); EXPECT_EQ(1, fake_encoder_.set_rates_count_); - fallback_wrapper_.SetRateAllocation(BitrateAllocation(), 1); + fallback_wrapper_.SetRates(1, 1); EXPECT_EQ(2, fake_encoder_.set_rates_count_); EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); } diff --git a/webrtc/video/video_send_stream_tests.cc b/webrtc/video/video_send_stream_tests.cc index 267487a1e5..2b5c6e6909 100644 --- a/webrtc/video/video_send_stream_tests.cc +++ b/webrtc/video/video_send_stream_tests.cc @@ -1478,8 +1478,7 @@ TEST_F(VideoSendStreamTest, return; } } - EXPECT_TRUE( - init_encode_called_.Wait(VideoSendStreamTest::kDefaultTimeoutMs)); + init_encode_called_.Wait(VideoSendStreamTest::kDefaultTimeoutMs); { rtc::CritScope lock(&crit_); EXPECT_EQ(width, last_initialized_frame_width_); @@ -1495,7 +1494,9 @@ TEST_F(VideoSendStreamTest, last_initialized_frame_width_ = config->width; last_initialized_frame_height_ = config->height; ++number_of_initializations_; - init_encode_called_.Set(); + // First time InitEncode is called, the frame size is unknown. + if (number_of_initializations_ > 1) + init_encode_called_.Set(); return FakeEncoder::InitEncode(config, number_of_cores, max_payload_size); } @@ -1625,12 +1626,11 @@ TEST_F(VideoSendStreamTest, VideoSendStreamStopSetEncoderRateToZero) { return FakeEncoder::InitEncode(config, number_of_cores, max_payload_size); } - int32_t SetRateAllocation(const BitrateAllocation& bitrate, - uint32_t framerate) override { + int32_t SetRates(uint32_t new_target_bitrate, uint32_t framerate) override { rtc::CritScope lock(&crit_); - bitrate_kbps_ = bitrate.get_sum_kbps(); + bitrate_kbps_ = new_target_bitrate; bitrate_changed_.Set(); - return FakeEncoder::SetRateAllocation(bitrate, framerate); + return FakeEncoder::SetRates(new_target_bitrate, framerate); } int GetBitrateKbps() const { @@ -1664,17 +1664,13 @@ TEST_F(VideoSendStreamTest, VideoSendStreamStopSetEncoderRateToZero) { CreateVideoStreams(); EXPECT_TRUE(encoder.WaitForEncoderInit()); - EXPECT_TRUE(encoder.WaitBitrateChanged()); - EXPECT_EQ(0, encoder.GetBitrateKbps()); - + EXPECT_GT(encoder.GetBitrateKbps(), 0); video_send_stream_->Start(); EXPECT_TRUE(encoder.WaitBitrateChanged()); EXPECT_GT(encoder.GetBitrateKbps(), 0); - video_send_stream_->Stop(); EXPECT_TRUE(encoder.WaitBitrateChanged()); EXPECT_EQ(0, encoder.GetBitrateKbps()); - video_send_stream_->Start(); EXPECT_TRUE(encoder.WaitBitrateChanged()); EXPECT_GT(encoder.GetBitrateKbps(), 0); @@ -2089,12 +2085,10 @@ void VideoCodecConfigObserver::VerifyCodecSpecifics( } // Set expected temporal layers as they should have been set when - // reconfiguring the encoder and not match the set config. Also copy the - // TemporalLayersFactory pointer that has been injected by ViEEncoder. + // reconfiguring the encoder and not match the set config. VideoCodecVP8 encoder_settings = encoder_settings_; encoder_settings.numberOfTemporalLayers = kVideoCodecConfigObserverNumberOfTemporalLayers; - encoder_settings.tl_factory = config.VP8().tl_factory; EXPECT_EQ( 0, memcmp(&config.VP8(), &encoder_settings, sizeof(encoder_settings_))); } @@ -2316,17 +2310,16 @@ TEST_F(VideoSendStreamTest, ReconfigureBitratesSetsEncoderBitratesCorrectly) { maxPayloadSize); } - int32_t SetRateAllocation(const BitrateAllocation& bitrate, - uint32_t frameRate) override { + int32_t SetRates(uint32_t newBitRate, uint32_t frameRate) override { { rtc::CritScope lock(&crit_); - if (target_bitrate_ == bitrate.get_sum_kbps()) { - return FakeEncoder::SetRateAllocation(bitrate, frameRate); + if (target_bitrate_ == newBitRate) { + return FakeEncoder::SetRates(newBitRate, frameRate); } - target_bitrate_ = bitrate.get_sum_kbps(); + target_bitrate_ = newBitRate; } bitrate_changed_event_.Set(); - return FakeEncoder::SetRateAllocation(bitrate, frameRate); + return FakeEncoder::SetRates(newBitRate, frameRate); } void WaitForSetRates(uint32_t expected_bitrate) { diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index 70aed4f752..a1709bfe63 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -14,13 +14,11 @@ #include #include -#include "webrtc/modules/video_coding/include/video_codec_initializer.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/base/trace_event.h" #include "webrtc/base/timeutils.h" #include "webrtc/modules/pacing/paced_sender.h" -#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/include/video_coding.h" #include "webrtc/modules/video_coding/include/video_coding_defines.h" #include "webrtc/video/overuse_frame_detector.h" @@ -33,6 +31,146 @@ namespace { // Time interval for logging frame counts. const int64_t kFrameLogIntervalMs = 60000; +VideoCodecType PayloadNameToCodecType(const std::string& payload_name) { + if (payload_name == "VP8") + return kVideoCodecVP8; + if (payload_name == "VP9") + return kVideoCodecVP9; + if (payload_name == "H264") + return kVideoCodecH264; + return kVideoCodecGeneric; +} + +VideoCodec VideoEncoderConfigToVideoCodec( + const VideoEncoderConfig& config, + const std::vector& streams, + const std::string& payload_name, + int payload_type) { + static const int kEncoderMinBitrateKbps = 30; + RTC_DCHECK(!streams.empty()); + RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0); + + VideoCodec video_codec; + memset(&video_codec, 0, sizeof(video_codec)); + video_codec.codecType = PayloadNameToCodecType(payload_name); + + switch (config.content_type) { + case VideoEncoderConfig::ContentType::kRealtimeVideo: + video_codec.mode = kRealtimeVideo; + break; + case VideoEncoderConfig::ContentType::kScreen: + video_codec.mode = kScreensharing; + if (streams.size() == 1 && + streams[0].temporal_layer_thresholds_bps.size() == 1) { + video_codec.targetBitrate = + streams[0].temporal_layer_thresholds_bps[0] / 1000; + } + break; + } + + if (config.encoder_specific_settings) + config.encoder_specific_settings->FillEncoderSpecificSettings(&video_codec); + + switch (video_codec.codecType) { + case kVideoCodecVP8: { + if (!config.encoder_specific_settings) + *video_codec.VP8() = VideoEncoder::GetDefaultVp8Settings(); + video_codec.VP8()->numberOfTemporalLayers = static_cast( + streams.back().temporal_layer_thresholds_bps.size() + 1); + break; + } + case kVideoCodecVP9: { + if (!config.encoder_specific_settings) + *video_codec.VP9() = VideoEncoder::GetDefaultVp9Settings(); + if (video_codec.mode == kScreensharing && + config.encoder_specific_settings) { + video_codec.VP9()->flexibleMode = true; + // For now VP9 screensharing use 1 temporal and 2 spatial layers. + RTC_DCHECK_EQ(1, video_codec.VP9()->numberOfTemporalLayers); + RTC_DCHECK_EQ(2, video_codec.VP9()->numberOfSpatialLayers); + } + video_codec.VP9()->numberOfTemporalLayers = static_cast( + streams.back().temporal_layer_thresholds_bps.size() + 1); + break; + } + case kVideoCodecH264: { + if (!config.encoder_specific_settings) + *video_codec.H264() = VideoEncoder::GetDefaultH264Settings(); + break; + } + default: + // TODO(pbos): Support encoder_settings codec-agnostically. + RTC_DCHECK(!config.encoder_specific_settings) + << "Encoder-specific settings for codec type not wired up."; + break; + } + + strncpy(video_codec.plName, payload_name.c_str(), kPayloadNameSize - 1); + video_codec.plName[kPayloadNameSize - 1] = '\0'; + video_codec.plType = payload_type; + video_codec.numberOfSimulcastStreams = + static_cast(streams.size()); + video_codec.minBitrate = streams[0].min_bitrate_bps / 1000; + if (video_codec.minBitrate < kEncoderMinBitrateKbps) + video_codec.minBitrate = kEncoderMinBitrateKbps; + RTC_DCHECK_LE(streams.size(), static_cast(kMaxSimulcastStreams)); + if (video_codec.codecType == kVideoCodecVP9) { + // If the vector is empty, bitrates will be configured automatically. + RTC_DCHECK(config.spatial_layers.empty() || + config.spatial_layers.size() == + video_codec.VP9()->numberOfSpatialLayers); + RTC_DCHECK_LE(video_codec.VP9()->numberOfSpatialLayers, + kMaxSimulcastStreams); + for (size_t i = 0; i < config.spatial_layers.size(); ++i) + video_codec.spatialLayers[i] = config.spatial_layers[i]; + } + for (size_t i = 0; i < streams.size(); ++i) { + SimulcastStream* sim_stream = &video_codec.simulcastStream[i]; + RTC_DCHECK_GT(streams[i].width, 0u); + RTC_DCHECK_GT(streams[i].height, 0u); + RTC_DCHECK_GT(streams[i].max_framerate, 0); + // Different framerates not supported per stream at the moment. + RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate); + RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0); + RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps); + RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps); + RTC_DCHECK_GE(streams[i].max_qp, 0); + + sim_stream->width = static_cast(streams[i].width); + sim_stream->height = static_cast(streams[i].height); + sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000; + sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000; + sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000; + sim_stream->qpMax = streams[i].max_qp; + sim_stream->numberOfTemporalLayers = static_cast( + streams[i].temporal_layer_thresholds_bps.size() + 1); + + video_codec.width = std::max(video_codec.width, + static_cast(streams[i].width)); + video_codec.height = std::max( + video_codec.height, static_cast(streams[i].height)); + video_codec.minBitrate = + std::min(static_cast(video_codec.minBitrate), + static_cast(streams[i].min_bitrate_bps / 1000)); + video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000; + video_codec.qpMax = std::max(video_codec.qpMax, + static_cast(streams[i].max_qp)); + } + + if (video_codec.maxBitrate == 0) { + // Unset max bitrate -> cap to one bit per pixel. + video_codec.maxBitrate = + (video_codec.width * video_codec.height * video_codec.maxFramerate) / + 1000; + } + if (video_codec.maxBitrate < kEncoderMinBitrateKbps) + video_codec.maxBitrate = kEncoderMinBitrateKbps; + + RTC_DCHECK_GT(streams[0].max_framerate, 0); + video_codec.maxFramerate = streams[0].max_framerate; + return video_codec; +} + // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle // pipelining encoders better (multiple input frames before something comes // out). This should effectively turn off CPU adaptations for systems that @@ -232,8 +370,7 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, source_proxy_(new VideoSourceProxy(this)), sink_(nullptr), settings_(settings), - codec_type_(PayloadNameToCodecType(settings.payload_name) - .value_or(VideoCodecType::kVideoCodecUnknown)), + codec_type_(PayloadNameToCodecType(settings.payload_name)), video_sender_(Clock::GetRealTimeClock(), this, this), overuse_detector_(Clock::GetRealTimeClock(), GetCpuOveruseOptions(settings.full_overuse_time), @@ -285,7 +422,6 @@ void ViEEncoder::Stop() { encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); overuse_detector_.StopCheckForOveruse(); - rate_allocator_.reset(); video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, false); shutdown_event_.Set(); @@ -380,11 +516,8 @@ void ViEEncoder::ReconfigureEncoder() { encoder_config_.video_stream_factory->CreateEncoderStreams( last_frame_info_->width, last_frame_info_->height, encoder_config_); - VideoCodec codec; - if (!VideoCodecInitializer::SetupCodec(encoder_config_, settings_, streams, - &codec, &rate_allocator_)) { - LOG(LS_ERROR) << "Failed to create encoder configuration."; - } + VideoCodec codec = VideoEncoderConfigToVideoCodec( + encoder_config_, streams, settings_.payload_name, settings_.payload_type); codec.startBitrate = std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate); @@ -399,18 +532,17 @@ void ViEEncoder::ReconfigureEncoder() { RTC_DCHECK(success); } - video_sender_.UpdateChannelParemeters(rate_allocator_.get()); - + rate_allocator_.reset(new SimulcastRateAllocator(codec)); if (stats_proxy_) { - int framerate = stats_proxy_->GetSendFrameRate(); - if (framerate == 0) - framerate = codec.maxFramerate; - stats_proxy_->OnEncoderReconfigured( - encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate)); + stats_proxy_->OnEncoderReconfigured(encoder_config_, + rate_allocator_->GetPreferedBitrate()); } pending_encoder_reconfiguration_ = false; - + if (stats_proxy_) { + stats_proxy_->OnEncoderReconfigured(encoder_config_, + rate_allocator_->GetPreferedBitrate()); + } sink_->OnEncoderConfigurationChanged( std::move(streams), encoder_config_.min_transmit_bitrate_bps); } @@ -632,12 +764,13 @@ void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, << " rtt " << round_trip_time_ms; video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, - round_trip_time_ms, rate_allocator_.get()); + round_trip_time_ms); encoder_start_bitrate_bps_ = bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_; bool video_is_suspended = bitrate_bps == 0; - bool video_suspension_changed = video_is_suspended != EncoderPaused(); + bool video_suspension_changed = + video_is_suspended != (last_observed_bitrate_bps_ == 0); last_observed_bitrate_bps_ = bitrate_bps; if (stats_proxy_ && video_suspension_changed) { diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h index 433e9ceb5e..b89f66e9d9 100644 --- a/webrtc/video/vie_encoder.h +++ b/webrtc/video/vie_encoder.h @@ -21,10 +21,10 @@ #include "webrtc/base/task_queue.h" #include "webrtc/call.h" #include "webrtc/common_types.h" -#include "webrtc/common_video/include/video_bitrate_allocator.h" #include "webrtc/common_video/rotation.h" #include "webrtc/media/base/videosinkinterface.h" #include "webrtc/modules/video_coding/include/video_coding_defines.h" +#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include "webrtc/modules/video_coding/video_coding_impl.h" #include "webrtc/modules/video_processing/include/video_processing.h" #include "webrtc/system_wrappers/include/atomic32.h" @@ -184,7 +184,10 @@ class ViEEncoder : public rtc::VideoSinkInterface, rtc::ThreadChecker thread_checker_; VideoEncoderConfig encoder_config_ ACCESS_ON(&encoder_queue_); - std::unique_ptr rate_allocator_ + // TODO(sprang): Change |rate_allocator_| to be a codec type + // agnostic interface. It is currently VP8 simulcast specific if more than + // one layer is specified. + std::unique_ptr rate_allocator_ ACCESS_ON(&encoder_queue_); // Set when ConfigureEncoder has been called in order to lazy reconfigure the diff --git a/webrtc/video/vie_encoder_unittest.cc b/webrtc/video/vie_encoder_unittest.cc index f6046be652..9f4fdc47e3 100644 --- a/webrtc/video/vie_encoder_unittest.cc +++ b/webrtc/video/vie_encoder_unittest.cc @@ -93,7 +93,6 @@ class ViEEncoderTest : public ::testing::Test { VideoEncoderConfig video_encoder_config; test::FillEncoderConfiguration(1, &video_encoder_config); - video_encoder_config_ = video_encoder_config.Copy(); vie_encoder_.reset(new ViEEncoderUnderTest( stats_proxy_.get(), video_send_config_.encoder_settings)); vie_encoder_->SetSink(&sink_, false /* rotation_applied */); @@ -237,7 +236,6 @@ class ViEEncoderTest : public ::testing::Test { }; VideoSendStream::Config video_send_config_; - VideoEncoderConfig video_encoder_config_; int codec_width_; int codec_height_; TestEncoder fake_encoder_; @@ -597,20 +595,6 @@ TEST_F(ViEEncoderTest, StatsTracksAdaptationStatsWhenSwitchingSource) { vie_encoder_->Stop(); } -TEST_F(ViEEncoderTest, StatsTracksPreferredBitrate) { - const int kTargetBitrateBps = 100000; - vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); - - video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); - sink_.WaitForEncodedFrame(1); - - VideoSendStream::Stats stats = stats_proxy_->GetStats(); - EXPECT_EQ(video_encoder_config_.max_bitrate_bps, - stats.preferred_media_bitrate_bps); - - vie_encoder_->Stop(); -} - TEST_F(ViEEncoderTest, UMACpuLimitedResolutionInPercent) { const int kTargetBitrateBps = 100000; vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); diff --git a/webrtc/video_encoder.h b/webrtc/video_encoder.h index 2dc8334d62..8bfa72fe1e 100644 --- a/webrtc/video_encoder.h +++ b/webrtc/video_encoder.h @@ -15,7 +15,6 @@ #include #include -#include "webrtc/base/checks.h" #include "webrtc/common_types.h" #include "webrtc/typedefs.h" #include "webrtc/video_frame.h" @@ -149,17 +148,7 @@ class VideoEncoder { // - framerate : The target frame rate // // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. - virtual int32_t SetRates(uint32_t bitrate, uint32_t framerate) { - RTC_NOTREACHED() << "SetRate(uint32_t, uint32_t) is deprecated."; - return -1; - } - - // Default fallback: Just use the sum of bitrates as the single target rate. - // TODO(sprang): Remove this default implementation when we remove SetRates(). - virtual int32_t SetRateAllocation(const BitrateAllocation& allocation, - uint32_t framerate) { - return SetRates(allocation.get_sum_kbps(), framerate); - } + virtual int32_t SetRates(uint32_t bitrate, uint32_t framerate) = 0; virtual int32_t SetPeriodicKeyFrames(bool enable) { return -1; } virtual void OnDroppedFrame() {} @@ -187,8 +176,8 @@ class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder { const CodecSpecificInfo* codec_specific_info, const std::vector* frame_types) override; int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; - int32_t SetRateAllocation(const BitrateAllocation& bitrate_allocation, - uint32_t framerate) override; + + int32_t SetRates(uint32_t bitrate, uint32_t framerate) override; void OnDroppedFrame() override; bool SupportsNativeHandle() const override; @@ -203,7 +192,7 @@ class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder { // The last bitrate/framerate set, and a flag for noting they are set. bool rates_set_; - BitrateAllocation bitrate_allocation_; + uint32_t bitrate_; uint32_t framerate_; // The last channel parameters set, and a flag for noting they are set.