admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove simulcast constraints in SimulcastEncoderAdapter

Browse Source 
The lowest and highest resolution layers are also identified instead
of assuming they are the first and last ones.

Bug: webrtc:10069
Change-Id: If9c76d647415c5065b79dc71850709db6bf16f61
Reviewed-on: https://webrtc-review.googlesource.com/c/114429
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Florent Castelli <orphis@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26343}
This commit is contained in:
Florent Castelli 2019-01-21 14:33:02 +01:00 committed by Commit Bot
parent e6a4793b16
commit 1b761ca21a
6 changed files with 124 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
media/engine/simulcast_encoder_adapter.cc
View File

@ -66,21 +66,6 @@ int NumberOfStreams(const webrtc::VideoCodec& codec) {
return streams;
}
bool ValidSimulcastResolutions(const webrtc::VideoCodec& codec,
int num_streams) {
if (codec.width != codec.simulcastStream[num_streams - 1].width ||
codec.height != codec.simulcastStream[num_streams - 1].height) {
return false;
}
for (int i = 0; i < num_streams; ++i) {
if (codec.width * codec.simulcastStream[i].height !=
codec.height * codec.simulcastStream[i].width) {
return false;
}
}
return true;
}
int VerifyCodec(const webrtc::VideoCodec* inst) {
if (inst == nullptr) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
@ -102,6 +87,12 @@ int VerifyCodec(const webrtc::VideoCodec* inst) {
return WEBRTC_VIDEO_CODEC_OK;
}
bool StreamResolutionCompare(const webrtc::SimulcastStream& a,
const webrtc::SimulcastStream& b) {
return std::tie(a.height, a.width, a.maxBitrate, a.maxFramerate) <
std::tie(b.height, b.width, b.maxBitrate, b.maxFramerate);
}
// An EncodedImageCallback implementation that forwards on calls to a
// SimulcastEncoderAdapter, but with the stream index it's registered with as
// the first parameter to Encoded.
@ -195,10 +186,6 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst,
RTC_DCHECK_LE(number_of_streams, kMaxSimulcastStreams);
const bool doing_simulcast = (number_of_streams > 1);
if (doing_simulcast && !ValidSimulcastResolutions(*inst, number_of_streams)) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
codec_ = *inst;
SimulcastRateAllocator rate_allocator(codec_);
VideoBitrateAllocation allocation = rate_allocator.GetAllocation(
@ -212,6 +199,19 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst,
encoder_info_.supports_native_handle = true;
encoder_info_.scaling_settings.thresholds = absl::nullopt;
// Create |number_of_streams| of encoder instances and init them.
const auto minmax = std::minmax_element(
std::begin(codec_.simulcastStream),
std::begin(codec_.simulcastStream) + number_of_streams,
StreamResolutionCompare);
const auto lowest_resolution_stream_index =
std::distance(std::begin(codec_.simulcastStream), minmax.first);
const auto highest_resolution_stream_index =
std::distance(std::begin(codec_.simulcastStream), minmax.second);
RTC_DCHECK_LT(lowest_resolution_stream_index, number_of_streams);
RTC_DCHECK_LT(highest_resolution_stream_index, number_of_streams);
for (int i = 0; i < number_of_streams; ++i) {
VideoCodec stream_codec;
uint32_t start_bitrate_kbps = start_bitrates[i];
@ -223,11 +223,16 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst,
} else {
// Cap start bitrate to the min bitrate in order to avoid strange codec
// behavior. Since sending will be false, this should not matter.
StreamResolution stream_resolution =
i == highest_resolution_stream_index
? StreamResolution::HIGHEST
: i == lowest_resolution_stream_index ? StreamResolution::LOWEST
: StreamResolution::OTHER;
start_bitrate_kbps =
std::max(codec_.simulcastStream[i].minBitrate, start_bitrate_kbps);
bool highest_resolution_stream = (i == (number_of_streams - 1));
PopulateStreamCodec(codec_, i, start_bitrate_kbps,
highest_resolution_stream, &stream_codec);
PopulateStreamCodec(codec_, i, start_bitrate_kbps, stream_resolution,
&stream_codec);
}
// TODO(ronghuawu): Remove once this is handled in LibvpxVp8Encoder.
@ -511,7 +516,7 @@ void SimulcastEncoderAdapter::PopulateStreamCodec(
const webrtc::VideoCodec& inst,
int stream_index,
uint32_t start_bitrate_kbps,
bool highest_resolution_stream,
StreamResolution stream_resolution,
webrtc::VideoCodec* stream_codec) {
*stream_codec = inst;
@ -523,8 +528,7 @@ void SimulcastEncoderAdapter::PopulateStreamCodec(
stream_codec->minBitrate = inst.simulcastStream[stream_index].minBitrate;
stream_codec->qpMax = inst.simulcastStream[stream_index].qpMax;
// Settings that are based on stream/resolution.
const bool lowest_resolution_stream = (stream_index == 0);
if (lowest_resolution_stream) {
if (stream_resolution == StreamResolution::LOWEST) {
// Settings for lowest spatial resolutions.
if (inst.mode == VideoCodecMode::kScreensharing) {
if (experimental_boosted_screenshare_qp_) {
@ -537,7 +541,7 @@ void SimulcastEncoderAdapter::PopulateStreamCodec(
if (inst.codecType == webrtc::kVideoCodecVP8) {
stream_codec->VP8()->numberOfTemporalLayers =
inst.simulcastStream[stream_index].numberOfTemporalLayers;
if (!highest_resolution_stream) {
if (stream_resolution != StreamResolution::HIGHEST) {
// For resolutions below CIF, set the codec |complexity| parameter to
// kComplexityHigher, which maps to cpu_used = -4.
int pixels_per_frame = stream_codec->width * stream_codec->height;

8
media/engine/simulcast_encoder_adapter.h
View File

@ -83,11 +83,17 @@ class SimulcastEncoderAdapter : public VideoEncoder {
bool send_stream;
};
enum class StreamResolution {
OTHER,
HIGHEST,
LOWEST,
};
// Populate the codec settings for each simulcast stream.
void PopulateStreamCodec(const webrtc::VideoCodec& inst,
int stream_index,
uint32_t start_bitrate_kbps,
bool highest_resolution_stream,
StreamResolution stream_resolution,
webrtc::VideoCodec* stream_codec);
bool Initialized() const;

40
media/engine/simulcast_encoder_adapter_unittest.cc
View File

@ -428,10 +428,13 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test,
// always be 0.
}
void InitRefCodec(int stream_index, VideoCodec* ref_codec) {
void InitRefCodec(int stream_index,
VideoCodec* ref_codec,
bool reverse_layer_order = false) {
*ref_codec = codec_;
ref_codec->VP8()->numberOfTemporalLayers =
kTestTemporalLayerProfile[stream_index];
kTestTemporalLayerProfile[reverse_layer_order ? 2 - stream_index
: stream_index];
ref_codec->width = codec_.simulcastStream[stream_index].width;
ref_codec->height = codec_.simulcastStream[stream_index].height;
ref_codec->maxBitrate = codec_.simulcastStream[stream_index].maxBitrate;
@ -963,6 +966,39 @@ TEST_F(TestSimulcastEncoderAdapterFake, DoesNotAlterMaxQpForScreenshare) {
VerifyCodec(ref_codec, 0);
}
TEST_F(TestSimulcastEncoderAdapterFake,
DoesNotAlterMaxQpForScreenshareReversedLayer) {
const int kHighMaxQp = 56;
const int kLowMaxQp = 46;
SimulcastTestFixtureImpl::DefaultSettings(
&codec_, static_cast<const int*>(kTestTemporalLayerProfile),
kVideoCodecVP8, true /* reverse_layer_order */);
codec_.numberOfSimulcastStreams = 3;
codec_.simulcastStream[2].qpMax = kHighMaxQp;
codec_.mode = VideoCodecMode::kScreensharing;
EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200));
EXPECT_EQ(3u, helper_->factory()->encoders().size());
// Just check the lowest stream, which is the one that where the adapter
// might alter the max qp setting.
VideoCodec ref_codec;
InitRefCodec(2, &ref_codec, true /* reverse_layer_order */);
ref_codec.qpMax = kHighMaxQp;
ref_codec.VP8()->complexity = webrtc::VideoCodecComplexity::kComplexityHigher;
ref_codec.VP8()->denoisingOn = false;
ref_codec.startBitrate = 100; // Should equal to the target bitrate.
VerifyCodec(ref_codec, 2);
// Change the max qp and try again.
codec_.simulcastStream[2].qpMax = kLowMaxQp;
EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200));
EXPECT_EQ(3u, helper_->factory()->encoders().size());
ref_codec.qpMax = kLowMaxQp;
VerifyCodec(ref_codec, 2);
}
TEST_F(TestSimulcastEncoderAdapterFake, ActivatesCorrectStreamsInInitEncode) {
// Set up common settings for three streams.
SimulcastTestFixtureImpl::DefaultSettings(

34
modules/video_coding/utility/simulcast_rate_allocator.cc
View File

@ -13,7 +13,9 @@
#include <stdio.h>
#include <algorithm>
#include <cstdint>
#include <numeric>
#include <string>
#include <tuple>
#include <vector>
#include "common_types.h" // NOLINT(build/include)
@ -110,10 +112,21 @@ void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers(
}
return;
}
// Sort the layers by maxFramerate, they might not always be from smallest
// to biggest
std::vector<size_t> layer_index(codec_.numberOfSimulcastStreams);
std::iota(layer_index.begin(), layer_index.end(), 0);
std::stable_sort(layer_index.begin(), layer_index.end(),
[this](size_t a, size_t b) {
return std::tie(codec_.simulcastStream[a].maxBitrate) <
std::tie(codec_.simulcastStream[b].maxBitrate);
});
// Find the first active layer. We don't allocate to inactive layers.
size_t active_layer = 0;
for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) {
if (codec_.simulcastStream[active_layer].active) {
if (codec_.simulcastStream[layer_index[active_layer]].active) {
// Found the first active layer.
break;
}
@ -127,7 +140,8 @@ void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers(
// active layer. Suspending below min bitrate is controlled outside the
// codec implementation and is not overridden by this.
left_to_allocate = std::max(
codec_.simulcastStream[active_layer].minBitrate * 1000, left_to_allocate);
codec_.simulcastStream[layer_index[active_layer]].minBitrate * 1000,
left_to_allocate);
// Begin by allocating bitrate to simulcast streams, putting all bitrate in
// temporal layer 0. We'll then distribute this bitrate, across potential
@ -144,15 +158,16 @@ void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers(
size_t top_active_layer = active_layer;
// Allocate up to the target bitrate for each active simulcast layer.
for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) {
const SimulcastStream& stream = codec_.simulcastStream[active_layer];
const SimulcastStream& stream =
codec_.simulcastStream[layer_index[active_layer]];
if (!stream.active) {
stream_enabled_[active_layer] = false;
stream_enabled_[layer_index[active_layer]] = false;
continue;
}
// If we can't allocate to the current layer we can't allocate to higher
// layers because they require a higher minimum bitrate.
uint32_t min_bitrate = stream.minBitrate * 1000;
if (!first_allocation && !stream_enabled_[active_layer]) {
if (!first_allocation && !stream_enabled_[layer_index[active_layer]]) {
min_bitrate = std::min(
static_cast<uint32_t>(hysteresis_factor_ * min_bitrate + 0.5),
stream.targetBitrate * 1000);
@ -162,18 +177,19 @@ void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers(
}
// We are allocating to this layer so it is the current active allocation.
top_active_layer = active_layer;
stream_enabled_[active_layer] = true;
top_active_layer = layer_index[active_layer];
stream_enabled_[layer_index[active_layer]] = true;
uint32_t allocation =
std::min(left_to_allocate, stream.targetBitrate * 1000);
allocated_bitrates_bps->SetBitrate(active_layer, 0, allocation);
allocated_bitrates_bps->SetBitrate(layer_index[active_layer], 0,
allocation);
RTC_DCHECK_LE(allocation, left_to_allocate);
left_to_allocate -= allocation;
}
// All layers above this one are not active.
for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) {
stream_enabled_[active_layer] = false;
stream_enabled_[layer_index[active_layer]] = false;
}
// Next, try allocate remaining bitrate, up to max bitrate, in top active

35
modules/video_coding/utility/simulcast_test_fixture_impl.cc
View File

@ -212,7 +212,8 @@ void ConfigureStream(int width,
void SimulcastTestFixtureImpl::DefaultSettings(
VideoCodec* settings,
const int* temporal_layer_profile,
VideoCodecType codec_type) {
VideoCodecType codec_type,
bool reverse_layer_order) {
RTC_CHECK(settings);
memset(settings, 0, sizeof(VideoCodec));
settings->codecType = codec_type;
@ -227,26 +228,34 @@ void SimulcastTestFixtureImpl::DefaultSettings(
settings->numberOfSimulcastStreams = kNumberOfSimulcastStreams;
settings->active = true;
ASSERT_EQ(3, kNumberOfSimulcastStreams);
int layer_order[3] = {0, 1, 2};
if (reverse_layer_order) {
layer_order[0] = 2;
layer_order[2] = 0;
}
settings->timing_frame_thresholds = {kDefaultTimingFramesDelayMs,
kDefaultOutlierFrameSizePercent};
ConfigureStream(kDefaultWidth / 4, kDefaultHeight / 4, kMaxBitrates[0],
kMinBitrates[0], kTargetBitrates[0],
&settings->simulcastStream[0], temporal_layer_profile[0]);
&settings->simulcastStream[layer_order[0]],
temporal_layer_profile[0]);
ConfigureStream(kDefaultWidth / 2, kDefaultHeight / 2, kMaxBitrates[1],
kMinBitrates[1], kTargetBitrates[1],
&settings->simulcastStream[1], temporal_layer_profile[1]);
&settings->simulcastStream[layer_order[1]],
temporal_layer_profile[1]);
ConfigureStream(kDefaultWidth, kDefaultHeight, kMaxBitrates[2],
kMinBitrates[2], kTargetBitrates[2],
&settings->simulcastStream[2], temporal_layer_profile[2]);
if (codec_type == kVideoCodecVP8) {
settings->VP8()->denoisingOn = true;
settings->VP8()->automaticResizeOn = false;
settings->VP8()->frameDroppingOn = true;
settings->VP8()->keyFrameInterval = 3000;
} else {
settings->H264()->frameDroppingOn = true;
settings->H264()->keyFrameInterval = 3000;
}
&settings->simulcastStream[layer_order[2]],
temporal_layer_profile[2]);
if (codec_type == kVideoCodecVP8) {
settings->VP8()->denoisingOn = true;
settings->VP8()->automaticResizeOn = false;
settings->VP8()->frameDroppingOn = true;
settings->VP8()->keyFrameInterval = 3000;
} else {
settings->H264()->frameDroppingOn = true;
settings->H264()->keyFrameInterval = 3000;
}
}
SimulcastTestFixtureImpl::SimulcastTestFixtureImpl(

3
modules/video_coding/utility/simulcast_test_fixture_impl.h
View File

@ -55,7 +55,8 @@ class SimulcastTestFixtureImpl final : public SimulcastTestFixture {
static void DefaultSettings(VideoCodec* settings,
const int* temporal_layer_profile,
VideoCodecType codec_type);
VideoCodecType codec_type,
bool reverse_layer_order = false);
private:
class TestEncodedImageCallback;