admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use pixels from single active stream if set in CanDecreaseResolutionTo

Browse Source 
Simulcast with one active stream:
Use pixels from single active stream if set (instead of input stream which could be larger) to avoid going below the min_pixel_per_frame limit when downgrading resolution.

Bug: none
Change-Id: I65acb12cc53e46f726ccb5bfab8ce08ff0c4cf78
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/208101
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Åsa Persson <asapersson@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33309}
This commit is contained in:
Åsa Persson 2021-02-18 14:20:03 +01:00 committed by Commit Bot
parent 42dd9bc077
commit 1dd94a023a
11 changed files with 108 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
call/adaptation/video_stream_adapter.cc
View File

@ -62,13 +62,14 @@ int GetIncreasedMaxPixelsWanted(int target_pixels) {
}
bool CanDecreaseResolutionTo(int target_pixels,
int target_pixels_min,
const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions) {
int max_pixels_per_frame =
rtc::dchecked_cast<int>(restrictions.max_pixels_per_frame().value_or(
std::numeric_limits<int>::max()));
return target_pixels < max_pixels_per_frame &&
target_pixels >= input_state.min_pixels_per_frame();
target_pixels_min >= input_state.min_pixels_per_frame();
}
bool CanIncreaseResolutionTo(int target_pixels,
@ -96,6 +97,11 @@ bool CanIncreaseFrameRateTo(int max_frame_rate,
}
bool MinPixelLimitReached(const VideoStreamInputState& input_state) {
if (input_state.single_active_stream_pixels().has_value()) {
return GetLowerResolutionThan(
input_state.single_active_stream_pixels().value()) <
input_state.min_pixels_per_frame();
}
return input_state.frame_size_pixels().has_value() &&
GetLowerResolutionThan(input_state.frame_size_pixels().value()) <
input_state.min_pixels_per_frame();
@ -470,7 +476,11 @@ VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::DecreaseResolution(
const RestrictionsWithCounters& current_restrictions) {
int target_pixels =
GetLowerResolutionThan(input_state.frame_size_pixels().value());
if (!CanDecreaseResolutionTo(target_pixels, input_state,
// Use single active stream if set, this stream could be lower than the input.
int target_pixels_min =
GetLowerResolutionThan(input_state.single_active_stream_pixels().value_or(
input_state.frame_size_pixels().value()));
if (!CanDecreaseResolutionTo(target_pixels, target_pixels_min, input_state,
current_restrictions.restrictions)) {
return Adaptation::Status::kLimitReached;
}
@ -693,4 +703,27 @@ VideoStreamAdapter::AwaitingFrameSizeChange::AwaitingFrameSizeChange(
: pixels_increased(pixels_increased),
frame_size_pixels(frame_size_pixels) {}
absl::optional<uint32_t> VideoStreamAdapter::GetSingleActiveLayerPixels(
const VideoCodec& codec) {
int num_active = 0;
absl::optional<uint32_t> pixels;
if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) {
if (codec.spatialLayers[i].active) {
++num_active;
pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height;
}
}
} else {
for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
if (codec.simulcastStream[i].active) {
++num_active;
pixels =
codec.simulcastStream[i].width * codec.simulcastStream[i].height;
}
}
}
return (num_active > 1) ? absl::nullopt : pixels;
}
} // namespace webrtc

3
call/adaptation/video_stream_adapter.h
View File

@ -163,6 +163,9 @@ class VideoStreamAdapter {
VideoAdaptationCounters counters;
};
static absl::optional<uint32_t> GetSingleActiveLayerPixels(
const VideoCodec& codec);
private:
void BroadcastVideoRestrictionsUpdate(
const VideoStreamInputState& input_state,

12
call/adaptation/video_stream_input_state.cc
View File

@ -19,7 +19,8 @@ VideoStreamInputState::VideoStreamInputState()
frame_size_pixels_(absl::nullopt),
frames_per_second_(0),
video_codec_type_(VideoCodecType::kVideoCodecGeneric),
min_pixels_per_frame_(kDefaultMinPixelsPerFrame) {}
min_pixels_per_frame_(kDefaultMinPixelsPerFrame),
single_active_stream_pixels_(absl::nullopt) {}
void VideoStreamInputState::set_has_input(bool has_input) {
has_input_ = has_input;
@ -43,6 +44,11 @@ void VideoStreamInputState::set_min_pixels_per_frame(int min_pixels_per_frame) {
min_pixels_per_frame_ = min_pixels_per_frame;
}
void VideoStreamInputState::set_single_active_stream_pixels(
absl::optional<int> single_active_stream_pixels) {
single_active_stream_pixels_ = single_active_stream_pixels;
}
bool VideoStreamInputState::has_input() const {
return has_input_;
}
@ -63,6 +69,10 @@ int VideoStreamInputState::min_pixels_per_frame() const {
return min_pixels_per_frame_;
}
absl::optional<int> VideoStreamInputState::single_active_stream_pixels() const {
return single_active_stream_pixels_;
}
bool VideoStreamInputState::HasInputFrameSizeAndFramesPerSecond() const {
return has_input_ && frame_size_pixels_.has_value();
}

4
call/adaptation/video_stream_input_state.h
View File

@ -27,12 +27,15 @@ class VideoStreamInputState {
void set_frames_per_second(int frames_per_second);
void set_video_codec_type(VideoCodecType video_codec_type);
void set_min_pixels_per_frame(int min_pixels_per_frame);
void set_single_active_stream_pixels(
absl::optional<int> single_active_stream_pixels);
bool has_input() const;
absl::optional<int> frame_size_pixels() const;
int frames_per_second() const;
VideoCodecType video_codec_type() const;
int min_pixels_per_frame() const;
absl::optional<int> single_active_stream_pixels() const;
bool HasInputFrameSizeAndFramesPerSecond() const;
@ -42,6 +45,7 @@ class VideoStreamInputState {
int frames_per_second_;
VideoCodecType video_codec_type_;
int min_pixels_per_frame_;
absl::optional<int> single_active_stream_pixels_;
};
} // namespace webrtc

5
call/adaptation/video_stream_input_state_provider.cc
View File

@ -10,6 +10,8 @@
#include "call/adaptation/video_stream_input_state_provider.h"
#include "call/adaptation/video_stream_adapter.h"
namespace webrtc {
VideoStreamInputStateProvider::VideoStreamInputStateProvider(
@ -36,6 +38,9 @@ void VideoStreamInputStateProvider::OnEncoderSettingsChanged(
encoder_settings.encoder_config().codec_type);
input_state_.set_min_pixels_per_frame(
encoder_settings.encoder_info().scaling_settings.min_pixels_per_frame);
input_state_.set_single_active_stream_pixels(
VideoStreamAdapter::GetSingleActiveLayerPixels(
encoder_settings.video_codec()));
}
VideoStreamInputState VideoStreamInputStateProvider::InputState() {

11
call/adaptation/video_stream_input_state_provider_unittest.cc
View File

@ -28,6 +28,7 @@ TEST(VideoStreamInputStateProviderTest, DefaultValues) {
EXPECT_EQ(0, input_state.frames_per_second());
EXPECT_EQ(VideoCodecType::kVideoCodecGeneric, input_state.video_codec_type());
EXPECT_EQ(kDefaultMinPixelsPerFrame, input_state.min_pixels_per_frame());
EXPECT_EQ(absl::nullopt, input_state.single_active_stream_pixels());
}
TEST(VideoStreamInputStateProviderTest, ValuesSet) {
@ -40,14 +41,22 @@ TEST(VideoStreamInputStateProviderTest, ValuesSet) {
encoder_info.scaling_settings.min_pixels_per_frame = 1337;
VideoEncoderConfig encoder_config;
encoder_config.codec_type = VideoCodecType::kVideoCodecVP9;
VideoCodec video_codec;
video_codec.codecType = VideoCodecType::kVideoCodecVP8;
video_codec.numberOfSimulcastStreams = 2;
video_codec.simulcastStream[0].active = false;
video_codec.simulcastStream[1].active = true;
video_codec.simulcastStream[1].width = 111;
video_codec.simulcastStream[1].height = 222;
input_state_provider.OnEncoderSettingsChanged(EncoderSettings(
std::move(encoder_info), std::move(encoder_config), VideoCodec()));
std::move(encoder_info), std::move(encoder_config), video_codec));
VideoStreamInputState input_state = input_state_provider.InputState();
EXPECT_EQ(true, input_state.has_input());
EXPECT_EQ(42, input_state.frame_size_pixels());
EXPECT_EQ(123, input_state.frames_per_second());
EXPECT_EQ(VideoCodecType::kVideoCodecVP9, input_state.video_codec_type());
EXPECT_EQ(1337, input_state.min_pixels_per_frame());
EXPECT_EQ(111 * 222, input_state.single_active_stream_pixels());
}
} // namespace webrtc

2
video/adaptation/bitrate_constraint.cc
View File

@ -61,7 +61,7 @@ bool BitrateConstraint::IsAdaptationUpAllowed(
}
absl::optional<uint32_t> current_frame_size_px =
VideoStreamEncoderResourceManager::GetSingleActiveLayerPixels(
VideoStreamAdapter::GetSingleActiveLayerPixels(
encoder_settings_->video_codec());
if (!current_frame_size_px.has_value()) {
return true;

32
video/adaptation/video_stream_encoder_resource_manager.cc
View File

@ -173,7 +173,8 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper {
"stream parameters";
initial_framedrop_ = 0;
if (single_active_stream_pixels_ &&
GetSingleActiveLayerPixels(codec) > *single_active_stream_pixels_) {
VideoStreamAdapter::GetSingleActiveLayerPixels(codec) >
*single_active_stream_pixels_) {
// Resolution increased.
use_bandwidth_allocation_ = true;
}
@ -183,7 +184,8 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper {
last_active_flags_ = active_flags;
last_input_width_ = codec.width;
last_input_height_ = codec.height;
single_active_stream_pixels_ = GetSingleActiveLayerPixels(codec);
single_active_stream_pixels_ =
VideoStreamAdapter::GetSingleActiveLayerPixels(codec);
}
void OnFrameDroppedDueToSize() { ++initial_framedrop_; }
@ -712,32 +714,6 @@ void VideoStreamEncoderResourceManager::OnQualityRampUp() {
quality_rampup_experiment_.reset();
}
absl::optional<uint32_t>
VideoStreamEncoderResourceManager::GetSingleActiveLayerPixels(
const VideoCodec& codec) {
int num_active = 0;
absl::optional<uint32_t> pixels;
if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) {
if (codec.spatialLayers[i].active) {
++num_active;
pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height;
}
}
} else {
for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
if (codec.simulcastStream[i].active) {
++num_active;
pixels =
codec.simulcastStream[i].width * codec.simulcastStream[i].height;
}
}
}
if (num_active > 1)
return absl::nullopt;
return pixels;
}
bool VideoStreamEncoderResourceManager::IsSimulcast(
const VideoEncoderConfig& encoder_config) {
const std::vector<VideoStream>& simulcast_layers =

2
video/adaptation/video_stream_encoder_resource_manager.h
View File

@ -147,8 +147,6 @@ class VideoStreamEncoderResourceManager
// QualityRampUpExperimentListener implementation.
void OnQualityRampUp() override;
static absl::optional<uint32_t> GetSingleActiveLayerPixels(
const VideoCodec& codec);
static bool IsSimulcast(const VideoEncoderConfig& encoder_config);
private:

34
video/quality_scaling_tests.cc
View File

@ -197,6 +197,14 @@ TEST_F(QualityScalingTest, NoAdaptDownForLowStartBitrate_Simulcast) {
/*automatic_resize=*/false, /*expect_adaptation=*/false);
}
TEST_F(QualityScalingTest, AdaptsDownForHighQp_HighestStreamActive_Vp8) {
// qp_low:1, qp_high:1 -> kHighQp
test::ScopedFieldTrials field_trials(kPrefix + "1,1,0,0,0,0" + kEnd);
RunTest("VP8", {false, false, true}, kHighStartBps,
/*automatic_resize=*/true, /*expect_adaptation=*/true);
}
TEST_F(QualityScalingTest,
AdaptsDownForLowStartBitrate_HighestStreamActive_Vp8) {
// qp_low:1, qp_high:127 -> kNormalQp
@ -206,6 +214,14 @@ TEST_F(QualityScalingTest,
/*automatic_resize=*/true, /*expect_adaptation=*/true);
}
TEST_F(QualityScalingTest, NoAdaptDownForHighQp_LowestStreamActive_Vp8) {
// qp_low:1, qp_high:1 -> kHighQp
test::ScopedFieldTrials field_trials(kPrefix + "1,1,0,0,0,0" + kEnd);
RunTest("VP8", {true, false, false}, kHighStartBps,
/*automatic_resize=*/true, /*expect_adaptation=*/false);
}
TEST_F(QualityScalingTest,
NoAdaptDownForLowStartBitrate_LowestStreamActive_Vp8) {
// qp_low:1, qp_high:127 -> kNormalQp
@ -250,6 +266,15 @@ TEST_F(QualityScalingTest, AdaptsDownForLowStartBitrate_Vp9) {
/*automatic_resize=*/true, /*expect_adaptation=*/true);
}
TEST_F(QualityScalingTest, NoAdaptDownForHighQp_LowestStreamActive_Vp9) {
// qp_low:1, qp_high:1 -> kHighQp
test::ScopedFieldTrials field_trials(kPrefix + "0,0,1,1,0,0" + kEnd +
"WebRTC-VP9QualityScaler/Enabled/");
RunTest("VP9", {true, false, false}, kHighStartBps,
/*automatic_resize=*/true, /*expect_adaptation=*/false);
}
TEST_F(QualityScalingTest,
NoAdaptDownForLowStartBitrate_LowestStreamActive_Vp9) {
// qp_low:1, qp_high:255 -> kNormalQp
@ -260,6 +285,15 @@ TEST_F(QualityScalingTest,
/*automatic_resize=*/true, /*expect_adaptation=*/false);
}
TEST_F(QualityScalingTest, AdaptsDownForHighQp_MiddleStreamActive_Vp9) {
// qp_low:1, qp_high:1 -> kHighQp
test::ScopedFieldTrials field_trials(kPrefix + "0,0,1,1,0,0" + kEnd +
"WebRTC-VP9QualityScaler/Enabled/");
RunTest("VP9", {false, true, false}, kHighStartBps,
/*automatic_resize=*/true, /*expect_adaptation=*/true);
}
TEST_F(QualityScalingTest,
AdaptsDownForLowStartBitrate_MiddleStreamActive_Vp9) {
// qp_low:1, qp_high:255 -> kNormalQp

2
video/video_stream_encoder.cc
View File

@ -359,7 +359,7 @@ void ApplyVp9BitrateLimits(const VideoEncoder::EncoderInfo& encoder_info,
// Get bitrate limits for active stream.
absl::optional<uint32_t> pixels =
VideoStreamEncoderResourceManager::GetSingleActiveLayerPixels(*codec);
VideoStreamAdapter::GetSingleActiveLayerPixels(*codec);
if (!pixels.has_value()) {
return;
}