admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable initial frame drop for one active simulcast layer.

Browse Source 
Bug: webrtc:12216
Change-Id: Ib2ac2fab45e560ba3eae30a926ce72667a257b07
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/193840
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Åsa Persson <asapersson@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32652}
This commit is contained in:
Åsa Persson 2020-11-20 15:45:44 +01:00 committed by Commit Bot
parent 766a32c28d
commit f46723c8aa
4 changed files with 119 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
video/adaptation/video_stream_encoder_resource_manager.cc
View File

@ -58,6 +58,20 @@ std::string ToString(VideoAdaptationReason reason) {
RTC_CHECK_NOTREACHED();
}
absl::optional<uint32_t> GetSingleActiveStreamPixels(const VideoCodec& codec) {
int num_active = 0;
absl::optional<uint32_t> pixels;
for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
if (codec.simulcastStream[i].active) {
++num_active;
pixels = codec.simulcastStream[i].width * codec.simulcastStream[i].height;
}
if (num_active > 1)
return absl::nullopt;
}
return pixels;
}
} // namespace
class VideoStreamEncoderResourceManager::InitialFrameDropper {
@ -78,6 +92,10 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper {
return initial_framedrop_ < kMaxInitialFramedrop;
}
absl::optional<uint32_t> single_active_stream_pixels() const {
return single_active_stream_pixels_;
}
// Input signals.
void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) {
set_start_bitrate_ = start_bitrate;
@ -104,6 +122,10 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper {
}
}
void OnEncoderSettingsUpdated(const VideoCodec& codec) {
single_active_stream_pixels_ = GetSingleActiveStreamPixels(codec);
}
void OnFrameDroppedDueToSize() { ++initial_framedrop_; }
void OnMaybeEncodeFrame() { initial_framedrop_ = kMaxInitialFramedrop; }
@ -130,6 +152,7 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper {
int64_t set_start_bitrate_time_ms_;
// Counts how many frames we've dropped in the initial framedrop phase.
int initial_framedrop_;
absl::optional<uint32_t> single_active_stream_pixels_;
};
VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager(
@ -230,7 +253,7 @@ void VideoStreamEncoderResourceManager::AddResource(
RTC_DCHECK(resource);
bool inserted;
std::tie(std::ignore, inserted) = resources_.emplace(resource, reason);
RTC_DCHECK(inserted) << "Resurce " << resource->Name()
RTC_DCHECK(inserted) << "Resource " << resource->Name()
<< " already was inserted";
adaptation_processor_->AddResource(resource);
}
@ -259,6 +282,8 @@ void VideoStreamEncoderResourceManager::SetEncoderSettings(
RTC_DCHECK_RUN_ON(encoder_queue_);
encoder_settings_ = std::move(encoder_settings);
bitrate_constraint_->OnEncoderSettingsUpdated(encoder_settings_);
initial_frame_dropper_->OnEncoderSettingsUpdated(
encoder_settings_->video_codec());
MaybeUpdateTargetFrameRate();
}
@ -339,6 +364,12 @@ bool VideoStreamEncoderResourceManager::DropInitialFrames() const {
return initial_frame_dropper_->DropInitialFrames();
}
absl::optional<uint32_t>
VideoStreamEncoderResourceManager::SingleActiveStreamPixels() const {
RTC_DCHECK_RUN_ON(encoder_queue_);
return initial_frame_dropper_->single_active_stream_pixels();
}
void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() {
RTC_DCHECK_RUN_ON(encoder_queue_);
initial_frame_dropper_->OnMaybeEncodeFrame();

5
video/adaptation/video_stream_encoder_resource_manager.h
View File

@ -121,9 +121,10 @@ class VideoStreamEncoderResourceManager
VideoAdaptationReason reason);
void RemoveResource(rtc::scoped_refptr<Resource> resource);
std::vector<AdaptationConstraint*> AdaptationConstraints() const;
// If true, the VideoStreamEncoder should eexecute its logic to maybe drop
// frames baseed on size and bitrate.
// If true, the VideoStreamEncoder should execute its logic to maybe drop
// frames based on size and bitrate.
bool DropInitialFrames() const;
absl::optional<uint32_t> SingleActiveStreamPixels() const;
// VideoSourceRestrictionsListener implementation.
// Updates |video_source_restrictions_|.

86
video/quality_scaling_tests.cc
View File

@ -56,6 +56,7 @@ class QualityScalingTest : public test::CallTest {
protected:
void RunTest(VideoEncoderFactory* encoder_factory,
const std::string& payload_name,
const std::vector<bool>& streams_active,
int start_bps,
bool automatic_resize,
bool frame_dropping,
@ -67,6 +68,7 @@ class QualityScalingTest : public test::CallTest {
void QualityScalingTest::RunTest(VideoEncoderFactory* encoder_factory,
const std::string& payload_name,
const std::vector<bool>& streams_active,
int start_bps,
bool automatic_resize,
bool frame_dropping,
@ -77,6 +79,7 @@ void QualityScalingTest::RunTest(VideoEncoderFactory* encoder_factory,
public:
ScalingObserver(VideoEncoderFactory* encoder_factory,
const std::string& payload_name,
const std::vector<bool>& streams_active,
int start_bps,
bool automatic_resize,
bool frame_dropping,
@ -84,6 +87,7 @@ void QualityScalingTest::RunTest(VideoEncoderFactory* encoder_factory,
: SendTest(expect_adaptation ? kDefaultTimeoutMs : kTimeoutMs),
encoder_factory_(encoder_factory),
payload_name_(payload_name),
streams_active_(streams_active),
start_bps_(start_bps),
automatic_resize_(automatic_resize),
frame_dropping_(frame_dropping),
@ -108,6 +112,10 @@ void QualityScalingTest::RunTest(VideoEncoderFactory* encoder_factory,
bitrate_config->start_bitrate_bps = start_bps_;
}
size_t GetNumVideoStreams() const override {
return streams_active_.size();
}
void ModifyVideoConfigs(
VideoSendStream::Config* send_config,
std::vector<VideoReceiveStream::Config>* receive_configs,
@ -117,7 +125,15 @@ void QualityScalingTest::RunTest(VideoEncoderFactory* encoder_factory,
send_config->rtp.payload_type = kVideoSendPayloadType;
const VideoCodecType codec_type = PayloadStringToCodecType(payload_name_);
encoder_config->codec_type = codec_type;
encoder_config->max_bitrate_bps = start_bps_;
encoder_config->max_bitrate_bps =
std::max(start_bps_, encoder_config->max_bitrate_bps);
double scale_factor = 1.0;
for (int i = streams_active_.size() - 1; i >= 0; --i) {
VideoStream& stream = encoder_config->simulcast_layers[i];
stream.active = streams_active_[i];
stream.scale_resolution_down_by = scale_factor;
scale_factor *= 2.0;
}
SetEncoderSpecific(encoder_config, codec_type, automatic_resize_,
frame_dropping_);
}
@ -129,12 +145,13 @@ void QualityScalingTest::RunTest(VideoEncoderFactory* encoder_factory,
VideoEncoderFactory* const encoder_factory_;
const std::string payload_name_;
const std::vector<bool> streams_active_;
const int start_bps_;
const bool automatic_resize_;
const bool frame_dropping_;
const bool expect_adaptation_;
} test(encoder_factory, payload_name, start_bps, automatic_resize,
frame_dropping, expect_adaptation);
} test(encoder_factory, payload_name, streams_active, start_bps,
automatic_resize, frame_dropping, expect_adaptation);
RunBaseTest(&test);
}
@ -150,7 +167,7 @@ TEST_F(QualityScalingTest, AdaptsDownForHighQp_Vp8) {
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
RunTest(&encoder_factory, "VP8", kHighStartBps, kAutomaticResize,
RunTest(&encoder_factory, "VP8", {true}, kHighStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
@ -165,7 +182,7 @@ TEST_F(QualityScalingTest, NoAdaptDownForHighQpWithResizeOff_Vp8) {
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
RunTest(&encoder_factory, "VP8", kHighStartBps, kAutomaticResize,
RunTest(&encoder_factory, "VP8", {true}, kHighStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
@ -182,7 +199,7 @@ TEST_F(QualityScalingTest,
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
RunTest(&encoder_factory, "VP8", kHighStartBps, kAutomaticResize,
RunTest(&encoder_factory, "VP8", {true}, kHighStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
@ -197,7 +214,7 @@ TEST_F(QualityScalingTest, NoAdaptDownForNormalQp_Vp8) {
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
RunTest(&encoder_factory, "VP8", kHighStartBps, kAutomaticResize,
RunTest(&encoder_factory, "VP8", {true}, kHighStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
@ -212,10 +229,57 @@ TEST_F(QualityScalingTest, AdaptsDownForLowStartBitrate) {
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
RunTest(&encoder_factory, "VP8", kLowStartBps, kAutomaticResize,
RunTest(&encoder_factory, "VP8", {true}, kLowStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
TEST_F(QualityScalingTest, NoAdaptDownForLowStartBitrate_Simulcast) {
// VP8 QP thresholds, low:1, high:127 -> normal QP.
test::ScopedFieldTrials field_trials(kPrefix + "1,127,0,0,0,0" + kEnd);
// QualityScaler disabled.
const bool kAutomaticResize = false;
const bool kFrameDropping = true;
const bool kExpectAdapt = false;
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
RunTest(&encoder_factory, "VP8", {true, true}, kLowStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
TEST_F(QualityScalingTest,
AdaptsDownForLowStartBitrate_SimulcastOneActiveHighRes) {
// VP8 QP thresholds, low:1, high:127 -> normal QP.
test::ScopedFieldTrials field_trials(kPrefix + "1,127,0,0,0,0" + kEnd);
// QualityScaler enabled.
const bool kAutomaticResize = true;
const bool kFrameDropping = true;
const bool kExpectAdapt = true;
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
RunTest(&encoder_factory, "VP8", {false, false, true}, kLowStartBps,
kAutomaticResize, kFrameDropping, kExpectAdapt);
}
TEST_F(QualityScalingTest,
NoAdaptDownForLowStartBitrate_SimulcastOneActiveLowRes) {
// VP8 QP thresholds, low:1, high:127 -> normal QP.
test::ScopedFieldTrials field_trials(kPrefix + "1,127,0,0,0,0" + kEnd);
// QualityScaler enabled.
const bool kAutomaticResize = true;
const bool kFrameDropping = true;
const bool kExpectAdapt = false;
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
RunTest(&encoder_factory, "VP8", {true, false, false}, kLowStartBps,
kAutomaticResize, kFrameDropping, kExpectAdapt);
}
TEST_F(QualityScalingTest, NoAdaptDownForLowStartBitrateWithScalingOff) {
// VP8 QP thresholds, low:1, high:127 -> normal QP.
test::ScopedFieldTrials field_trials(kPrefix + "1,127,0,0,0,0" + kEnd);
@ -227,7 +291,7 @@ TEST_F(QualityScalingTest, NoAdaptDownForLowStartBitrateWithScalingOff) {
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
RunTest(&encoder_factory, "VP8", kLowStartBps, kAutomaticResize,
RunTest(&encoder_factory, "VP8", {true}, kLowStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
@ -243,7 +307,7 @@ TEST_F(QualityScalingTest, NoAdaptDownForHighQp_Vp9) {
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP9Encoder::Create(); });
RunTest(&encoder_factory, "VP9", kHighStartBps, kAutomaticResize,
RunTest(&encoder_factory, "VP9", {true}, kHighStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
@ -259,7 +323,7 @@ TEST_F(QualityScalingTest, AdaptsDownForHighQp_H264) {
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return H264Encoder::Create(cricket::VideoCodec("H264")); });
RunTest(&encoder_factory, "H264", kHighStartBps, kAutomaticResize,
RunTest(&encoder_factory, "H264", {true}, kHighStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
#endif // defined(WEBRTC_USE_H264)

11
video/video_stream_encoder.cc
View File

@ -565,6 +565,7 @@ void VideoStreamEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) {
encoder_queue_.PostTask([this, start_bitrate_bps] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
RTC_LOG(LS_INFO) << "SetStartBitrate " << start_bitrate_bps;
encoder_target_bitrate_bps_ =
start_bitrate_bps != 0 ? absl::optional<uint32_t>(start_bitrate_bps)
: absl::nullopt;
@ -1848,14 +1849,20 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const {
bool simulcast_or_svc =
(send_codec_.codecType == VideoCodecType::kVideoCodecVP9 &&
send_codec_.VP9().numberOfSpatialLayers > 1) ||
send_codec_.numberOfSimulcastStreams > 1 ||
encoder_config_.simulcast_layers.size() > 1;
((send_codec_.numberOfSimulcastStreams > 1 ||
encoder_config_.simulcast_layers.size() > 1) &&
!stream_resource_manager_.SingleActiveStreamPixels());
if (simulcast_or_svc || !stream_resource_manager_.DropInitialFrames() ||
!encoder_target_bitrate_bps_.has_value()) {
return false;
}
if (send_codec_.numberOfSimulcastStreams > 1 &&
stream_resource_manager_.SingleActiveStreamPixels()) {
pixel_count = stream_resource_manager_.SingleActiveStreamPixels().value();
}
absl::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits =
encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
pixel_count);