From 1b761ca21ac76513d3abe3790fb4c2f73a81e127 Mon Sep 17 00:00:00 2001 From: Florent Castelli Date: Mon, 21 Jan 2019 14:33:02 +0100 Subject: [PATCH] Remove simulcast constraints in SimulcastEncoderAdapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lowest and highest resolution layers are also identified instead of assuming they are the first and last ones. Bug: webrtc:10069 Change-Id: If9c76d647415c5065b79dc71850709db6bf16f61 Reviewed-on: https://webrtc-review.googlesource.com/c/114429 Reviewed-by: Erik Språng Commit-Queue: Florent Castelli Cr-Commit-Position: refs/heads/master@{#26343} --- media/engine/simulcast_encoder_adapter.cc | 56 ++++++++++--------- media/engine/simulcast_encoder_adapter.h | 8 ++- .../simulcast_encoder_adapter_unittest.cc | 40 ++++++++++++- .../utility/simulcast_rate_allocator.cc | 34 ++++++++--- .../utility/simulcast_test_fixture_impl.cc | 35 +++++++----- .../utility/simulcast_test_fixture_impl.h | 3 +- 6 files changed, 124 insertions(+), 52 deletions(-) diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index d73c59a718..895489ca36 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -66,21 +66,6 @@ int NumberOfStreams(const webrtc::VideoCodec& codec) { return streams; } -bool ValidSimulcastResolutions(const webrtc::VideoCodec& codec, - int num_streams) { - if (codec.width != codec.simulcastStream[num_streams - 1].width || - codec.height != codec.simulcastStream[num_streams - 1].height) { - return false; - } - for (int i = 0; i < num_streams; ++i) { - if (codec.width * codec.simulcastStream[i].height != - codec.height * codec.simulcastStream[i].width) { - return false; - } - } - return true; -} - int VerifyCodec(const webrtc::VideoCodec* inst) { if (inst == nullptr) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; @@ -102,6 +87,12 @@ int VerifyCodec(const webrtc::VideoCodec* inst) { return WEBRTC_VIDEO_CODEC_OK; } +bool StreamResolutionCompare(const webrtc::SimulcastStream& a, + const webrtc::SimulcastStream& b) { + return std::tie(a.height, a.width, a.maxBitrate, a.maxFramerate) < + std::tie(b.height, b.width, b.maxBitrate, b.maxFramerate); +} + // An EncodedImageCallback implementation that forwards on calls to a // SimulcastEncoderAdapter, but with the stream index it's registered with as // the first parameter to Encoded. @@ -195,10 +186,6 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, RTC_DCHECK_LE(number_of_streams, kMaxSimulcastStreams); const bool doing_simulcast = (number_of_streams > 1); - if (doing_simulcast && !ValidSimulcastResolutions(*inst, number_of_streams)) { - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } - codec_ = *inst; SimulcastRateAllocator rate_allocator(codec_); VideoBitrateAllocation allocation = rate_allocator.GetAllocation( @@ -212,6 +199,19 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, encoder_info_.supports_native_handle = true; encoder_info_.scaling_settings.thresholds = absl::nullopt; // Create |number_of_streams| of encoder instances and init them. + + const auto minmax = std::minmax_element( + std::begin(codec_.simulcastStream), + std::begin(codec_.simulcastStream) + number_of_streams, + StreamResolutionCompare); + const auto lowest_resolution_stream_index = + std::distance(std::begin(codec_.simulcastStream), minmax.first); + const auto highest_resolution_stream_index = + std::distance(std::begin(codec_.simulcastStream), minmax.second); + + RTC_DCHECK_LT(lowest_resolution_stream_index, number_of_streams); + RTC_DCHECK_LT(highest_resolution_stream_index, number_of_streams); + for (int i = 0; i < number_of_streams; ++i) { VideoCodec stream_codec; uint32_t start_bitrate_kbps = start_bitrates[i]; @@ -223,11 +223,16 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, } else { // Cap start bitrate to the min bitrate in order to avoid strange codec // behavior. Since sending will be false, this should not matter. + StreamResolution stream_resolution = + i == highest_resolution_stream_index + ? StreamResolution::HIGHEST + : i == lowest_resolution_stream_index ? StreamResolution::LOWEST + : StreamResolution::OTHER; + start_bitrate_kbps = std::max(codec_.simulcastStream[i].minBitrate, start_bitrate_kbps); - bool highest_resolution_stream = (i == (number_of_streams - 1)); - PopulateStreamCodec(codec_, i, start_bitrate_kbps, - highest_resolution_stream, &stream_codec); + PopulateStreamCodec(codec_, i, start_bitrate_kbps, stream_resolution, + &stream_codec); } // TODO(ronghuawu): Remove once this is handled in LibvpxVp8Encoder. @@ -511,7 +516,7 @@ void SimulcastEncoderAdapter::PopulateStreamCodec( const webrtc::VideoCodec& inst, int stream_index, uint32_t start_bitrate_kbps, - bool highest_resolution_stream, + StreamResolution stream_resolution, webrtc::VideoCodec* stream_codec) { *stream_codec = inst; @@ -523,8 +528,7 @@ void SimulcastEncoderAdapter::PopulateStreamCodec( stream_codec->minBitrate = inst.simulcastStream[stream_index].minBitrate; stream_codec->qpMax = inst.simulcastStream[stream_index].qpMax; // Settings that are based on stream/resolution. - const bool lowest_resolution_stream = (stream_index == 0); - if (lowest_resolution_stream) { + if (stream_resolution == StreamResolution::LOWEST) { // Settings for lowest spatial resolutions. if (inst.mode == VideoCodecMode::kScreensharing) { if (experimental_boosted_screenshare_qp_) { @@ -537,7 +541,7 @@ void SimulcastEncoderAdapter::PopulateStreamCodec( if (inst.codecType == webrtc::kVideoCodecVP8) { stream_codec->VP8()->numberOfTemporalLayers = inst.simulcastStream[stream_index].numberOfTemporalLayers; - if (!highest_resolution_stream) { + if (stream_resolution != StreamResolution::HIGHEST) { // For resolutions below CIF, set the codec |complexity| parameter to // kComplexityHigher, which maps to cpu_used = -4. int pixels_per_frame = stream_codec->width * stream_codec->height; diff --git a/media/engine/simulcast_encoder_adapter.h b/media/engine/simulcast_encoder_adapter.h index 696503d16e..b7b6c7f20f 100644 --- a/media/engine/simulcast_encoder_adapter.h +++ b/media/engine/simulcast_encoder_adapter.h @@ -83,11 +83,17 @@ class SimulcastEncoderAdapter : public VideoEncoder { bool send_stream; }; + enum class StreamResolution { + OTHER, + HIGHEST, + LOWEST, + }; + // Populate the codec settings for each simulcast stream. void PopulateStreamCodec(const webrtc::VideoCodec& inst, int stream_index, uint32_t start_bitrate_kbps, - bool highest_resolution_stream, + StreamResolution stream_resolution, webrtc::VideoCodec* stream_codec); bool Initialized() const; diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc index 4cbaf56db2..c02a585a28 100644 --- a/media/engine/simulcast_encoder_adapter_unittest.cc +++ b/media/engine/simulcast_encoder_adapter_unittest.cc @@ -428,10 +428,13 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test, // always be 0. } - void InitRefCodec(int stream_index, VideoCodec* ref_codec) { + void InitRefCodec(int stream_index, + VideoCodec* ref_codec, + bool reverse_layer_order = false) { *ref_codec = codec_; ref_codec->VP8()->numberOfTemporalLayers = - kTestTemporalLayerProfile[stream_index]; + kTestTemporalLayerProfile[reverse_layer_order ? 2 - stream_index + : stream_index]; ref_codec->width = codec_.simulcastStream[stream_index].width; ref_codec->height = codec_.simulcastStream[stream_index].height; ref_codec->maxBitrate = codec_.simulcastStream[stream_index].maxBitrate; @@ -963,6 +966,39 @@ TEST_F(TestSimulcastEncoderAdapterFake, DoesNotAlterMaxQpForScreenshare) { VerifyCodec(ref_codec, 0); } +TEST_F(TestSimulcastEncoderAdapterFake, + DoesNotAlterMaxQpForScreenshareReversedLayer) { + const int kHighMaxQp = 56; + const int kLowMaxQp = 46; + + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast(kTestTemporalLayerProfile), + kVideoCodecVP8, true /* reverse_layer_order */); + codec_.numberOfSimulcastStreams = 3; + codec_.simulcastStream[2].qpMax = kHighMaxQp; + codec_.mode = VideoCodecMode::kScreensharing; + + EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); + EXPECT_EQ(3u, helper_->factory()->encoders().size()); + + // Just check the lowest stream, which is the one that where the adapter + // might alter the max qp setting. + VideoCodec ref_codec; + InitRefCodec(2, &ref_codec, true /* reverse_layer_order */); + ref_codec.qpMax = kHighMaxQp; + ref_codec.VP8()->complexity = webrtc::VideoCodecComplexity::kComplexityHigher; + ref_codec.VP8()->denoisingOn = false; + ref_codec.startBitrate = 100; // Should equal to the target bitrate. + VerifyCodec(ref_codec, 2); + + // Change the max qp and try again. + codec_.simulcastStream[2].qpMax = kLowMaxQp; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); + EXPECT_EQ(3u, helper_->factory()->encoders().size()); + ref_codec.qpMax = kLowMaxQp; + VerifyCodec(ref_codec, 2); +} + TEST_F(TestSimulcastEncoderAdapterFake, ActivatesCorrectStreamsInInitEncode) { // Set up common settings for three streams. SimulcastTestFixtureImpl::DefaultSettings( diff --git a/modules/video_coding/utility/simulcast_rate_allocator.cc b/modules/video_coding/utility/simulcast_rate_allocator.cc index 4e8b70815c..873f94f3d8 100644 --- a/modules/video_coding/utility/simulcast_rate_allocator.cc +++ b/modules/video_coding/utility/simulcast_rate_allocator.cc @@ -13,7 +13,9 @@ #include #include #include +#include #include +#include #include #include "common_types.h" // NOLINT(build/include) @@ -110,10 +112,21 @@ void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers( } return; } + + // Sort the layers by maxFramerate, they might not always be from smallest + // to biggest + std::vector layer_index(codec_.numberOfSimulcastStreams); + std::iota(layer_index.begin(), layer_index.end(), 0); + std::stable_sort(layer_index.begin(), layer_index.end(), + [this](size_t a, size_t b) { + return std::tie(codec_.simulcastStream[a].maxBitrate) < + std::tie(codec_.simulcastStream[b].maxBitrate); + }); + // Find the first active layer. We don't allocate to inactive layers. size_t active_layer = 0; for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) { - if (codec_.simulcastStream[active_layer].active) { + if (codec_.simulcastStream[layer_index[active_layer]].active) { // Found the first active layer. break; } @@ -127,7 +140,8 @@ void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers( // active layer. Suspending below min bitrate is controlled outside the // codec implementation and is not overridden by this. left_to_allocate = std::max( - codec_.simulcastStream[active_layer].minBitrate * 1000, left_to_allocate); + codec_.simulcastStream[layer_index[active_layer]].minBitrate * 1000, + left_to_allocate); // Begin by allocating bitrate to simulcast streams, putting all bitrate in // temporal layer 0. We'll then distribute this bitrate, across potential @@ -144,15 +158,16 @@ void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers( size_t top_active_layer = active_layer; // Allocate up to the target bitrate for each active simulcast layer. for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) { - const SimulcastStream& stream = codec_.simulcastStream[active_layer]; + const SimulcastStream& stream = + codec_.simulcastStream[layer_index[active_layer]]; if (!stream.active) { - stream_enabled_[active_layer] = false; + stream_enabled_[layer_index[active_layer]] = false; continue; } // If we can't allocate to the current layer we can't allocate to higher // layers because they require a higher minimum bitrate. uint32_t min_bitrate = stream.minBitrate * 1000; - if (!first_allocation && !stream_enabled_[active_layer]) { + if (!first_allocation && !stream_enabled_[layer_index[active_layer]]) { min_bitrate = std::min( static_cast(hysteresis_factor_ * min_bitrate + 0.5), stream.targetBitrate * 1000); @@ -162,18 +177,19 @@ void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers( } // We are allocating to this layer so it is the current active allocation. - top_active_layer = active_layer; - stream_enabled_[active_layer] = true; + top_active_layer = layer_index[active_layer]; + stream_enabled_[layer_index[active_layer]] = true; uint32_t allocation = std::min(left_to_allocate, stream.targetBitrate * 1000); - allocated_bitrates_bps->SetBitrate(active_layer, 0, allocation); + allocated_bitrates_bps->SetBitrate(layer_index[active_layer], 0, + allocation); RTC_DCHECK_LE(allocation, left_to_allocate); left_to_allocate -= allocation; } // All layers above this one are not active. for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) { - stream_enabled_[active_layer] = false; + stream_enabled_[layer_index[active_layer]] = false; } // Next, try allocate remaining bitrate, up to max bitrate, in top active diff --git a/modules/video_coding/utility/simulcast_test_fixture_impl.cc b/modules/video_coding/utility/simulcast_test_fixture_impl.cc index be5028652a..781c7d0f71 100644 --- a/modules/video_coding/utility/simulcast_test_fixture_impl.cc +++ b/modules/video_coding/utility/simulcast_test_fixture_impl.cc @@ -212,7 +212,8 @@ void ConfigureStream(int width, void SimulcastTestFixtureImpl::DefaultSettings( VideoCodec* settings, const int* temporal_layer_profile, - VideoCodecType codec_type) { + VideoCodecType codec_type, + bool reverse_layer_order) { RTC_CHECK(settings); memset(settings, 0, sizeof(VideoCodec)); settings->codecType = codec_type; @@ -227,26 +228,34 @@ void SimulcastTestFixtureImpl::DefaultSettings( settings->numberOfSimulcastStreams = kNumberOfSimulcastStreams; settings->active = true; ASSERT_EQ(3, kNumberOfSimulcastStreams); + int layer_order[3] = {0, 1, 2}; + if (reverse_layer_order) { + layer_order[0] = 2; + layer_order[2] = 0; + } settings->timing_frame_thresholds = {kDefaultTimingFramesDelayMs, kDefaultOutlierFrameSizePercent}; ConfigureStream(kDefaultWidth / 4, kDefaultHeight / 4, kMaxBitrates[0], kMinBitrates[0], kTargetBitrates[0], - &settings->simulcastStream[0], temporal_layer_profile[0]); + &settings->simulcastStream[layer_order[0]], + temporal_layer_profile[0]); ConfigureStream(kDefaultWidth / 2, kDefaultHeight / 2, kMaxBitrates[1], kMinBitrates[1], kTargetBitrates[1], - &settings->simulcastStream[1], temporal_layer_profile[1]); + &settings->simulcastStream[layer_order[1]], + temporal_layer_profile[1]); ConfigureStream(kDefaultWidth, kDefaultHeight, kMaxBitrates[2], kMinBitrates[2], kTargetBitrates[2], - &settings->simulcastStream[2], temporal_layer_profile[2]); - if (codec_type == kVideoCodecVP8) { - settings->VP8()->denoisingOn = true; - settings->VP8()->automaticResizeOn = false; - settings->VP8()->frameDroppingOn = true; - settings->VP8()->keyFrameInterval = 3000; - } else { - settings->H264()->frameDroppingOn = true; - settings->H264()->keyFrameInterval = 3000; - } + &settings->simulcastStream[layer_order[2]], + temporal_layer_profile[2]); + if (codec_type == kVideoCodecVP8) { + settings->VP8()->denoisingOn = true; + settings->VP8()->automaticResizeOn = false; + settings->VP8()->frameDroppingOn = true; + settings->VP8()->keyFrameInterval = 3000; + } else { + settings->H264()->frameDroppingOn = true; + settings->H264()->keyFrameInterval = 3000; + } } SimulcastTestFixtureImpl::SimulcastTestFixtureImpl( diff --git a/modules/video_coding/utility/simulcast_test_fixture_impl.h b/modules/video_coding/utility/simulcast_test_fixture_impl.h index 2f834bdfb5..8881e0671d 100644 --- a/modules/video_coding/utility/simulcast_test_fixture_impl.h +++ b/modules/video_coding/utility/simulcast_test_fixture_impl.h @@ -55,7 +55,8 @@ class SimulcastTestFixtureImpl final : public SimulcastTestFixture { static void DefaultSettings(VideoCodec* settings, const int* temporal_layer_profile, - VideoCodecType codec_type); + VideoCodecType codec_type, + bool reverse_layer_order = false); private: class TestEncodedImageCallback;