admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adjust max consecutive drops depending on target frame rate

Browse Source 
Current thresholds were tuned to guarantee no buffer overshoot in an extreme scenario (encoding a high complexity video in a low bitrate).

Bug: b/337757868, webrtc:351644568
Change-Id: I832b2564af6f18f06550338cc9b3618f8acdf831
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/356580
Reviewed-by: Dan Tan <dwtan@google.com>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42620}
This commit is contained in:
Sergey Silkin 2024-07-09 15:03:10 +02:00 committed by WebRTC LUCI CQ
parent 756f58963f
commit f7a1506703
4 changed files with 69 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
experiments/field_trials.py
View File

@ -92,9 +92,9 @@ ACTIVE_FIELD_TRIALS: FrozenSet[FieldTrial] = frozenset([
FieldTrial('WebRTC-JitterEstimatorConfig',
42224404,
date(2024, 4, 1)),
FieldTrial('WebRTC-LibaomAv1Encoder-MaxConsecFrameDrop',
42226184,
date(2024, 4, 1)),
FieldTrial('WebRTC-LibaomAv1Encoder-AdaptiveMaxConsecDrops',
351644568,
date(2025, 7, 1)),
FieldTrial('WebRTC-LibvpxVp9Encoder-SvcFrameDropConfig',
42226190,
date(2024, 9, 1)),

2
modules/video_coding/codecs/av1/BUILD.gn
View File

@ -60,6 +60,7 @@ rtc_library("libaom_av1_encoder") {
"../../../../api/video_codecs:scalability_mode",
"../../../../api/video_codecs:video_codecs_api",
"../../../../common_video",
"../../../../modules/rtp_rtcp:rtp_rtcp_format",
"../../../../rtc_base:checks",
"../../../../rtc_base:logging",
"../../../../rtc_base:rtc_numerics",
@ -104,6 +105,7 @@ if (rtc_include_tests) {
"../../../../api/units:data_size",
"../../../../api/units:time_delta",
"../../../../api/video:video_frame",
"../../../../modules/rtp_rtcp:rtp_rtcp_format",
"../../../../test:scoped_key_value_config",
"../../svc:scalability_mode_util",
"../../svc:scalability_structures",

42
modules/video_coding/codecs/av1/libaom_av1_encoder.cc
View File

@ -30,6 +30,7 @@
#include "api/video_codecs/scalability_mode.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "modules/video_coding/svc/create_scalability_structure.h"
@ -65,7 +66,6 @@ constexpr int kLowQindex = 145; // Low qindex threshold for QP scaling.
constexpr int kHighQindex = 205; // High qindex threshold for QP scaling.
constexpr int kBitDepth = 8;
constexpr int kLagInFrames = 0; // No look ahead.
constexpr int kRtpTicksPerSecond = 90000;
constexpr double kMinFrameRateFps = 1.0;
aom_superblock_size_t GetSuperblockSize(int width, int height, int threads) {
@ -133,7 +133,9 @@ class LibaomAv1Encoder final : public VideoEncoder {
double framerate_fps_; // Current target frame rate.
int64_t timestamp_;
const LibaomAv1EncoderInfoSettings encoder_info_override_;
int max_consec_frame_drop_;
// TODO(webrtc:351644568): Remove this kill-switch after the feature is fully
// deployed.
bool adaptive_max_consec_drops_;
};
int32_t VerifyCodecSettings(const VideoCodec& codec_settings) {
@ -164,12 +166,12 @@ int32_t VerifyCodecSettings(const VideoCodec& codec_settings) {
return WEBRTC_VIDEO_CODEC_OK;
}
int GetMaxConsecutiveFrameDrop(const FieldTrialsView& field_trials) {
webrtc::FieldTrialParameter<int> maxdrop("maxdrop", 0);
webrtc::ParseFieldTrial(
{&maxdrop},
field_trials.Lookup("WebRTC-LibaomAv1Encoder-MaxConsecFrameDrop"));
return maxdrop;
int GetMaxConsecDrops(double framerate_fps) {
// Consecutive frame drops result in a video freeze. We want to minimize the
// max number of consecutive drops and, at the same time, keep the value high
// enough to let encoder drain the buffer at overshoot.
constexpr double kMaxFreezeSeconds = 0.25;
return std::ceil(kMaxFreezeSeconds * framerate_fps);
}
LibaomAv1Encoder::LibaomAv1Encoder(const Environment& env,
@ -182,7 +184,8 @@ LibaomAv1Encoder::LibaomAv1Encoder(const Environment& env,
framerate_fps_(0),
timestamp_(0),
encoder_info_override_(env.field_trials()),
max_consec_frame_drop_(GetMaxConsecutiveFrameDrop(env.field_trials())) {}
adaptive_max_consec_drops_(!env.field_trials().IsDisabled(
"WebRTC-LibaomAv1Encoder-AdaptiveMaxConsecDrops")) {}
LibaomAv1Encoder::~LibaomAv1Encoder() {
Release();
@ -242,7 +245,7 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings,
cfg_.g_threads =
NumberOfThreads(cfg_.g_w, cfg_.g_h, settings.number_of_cores);
cfg_.g_timebase.num = 1;
cfg_.g_timebase.den = kRtpTicksPerSecond;
cfg_.g_timebase.den = kVideoPayloadTypeFrequency;
cfg_.rc_target_bitrate = encoder_settings_.startBitrate; // kilobits/sec.
cfg_.rc_dropframe_thresh = encoder_settings_.GetFrameDropEnabled() ? 30 : 0;
cfg_.g_input_bit_depth = kBitDepth;
@ -304,12 +307,6 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings,
SET_ENCODER_PARAM_OR_RETURN_ERROR(AV1E_SET_ENABLE_PALETTE, 0);
}
if (codec_settings->mode == VideoCodecMode::kRealtimeVideo &&
encoder_settings_.GetFrameDropEnabled() && max_consec_frame_drop_ > 0) {
SET_ENCODER_PARAM_OR_RETURN_ERROR(AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR,
max_consec_frame_drop_);
}
if (cfg_.g_threads == 8) {
// Values passed to AV1E_SET_TILE_ROWS and AV1E_SET_TILE_COLUMNS are log2()
// based.
@ -659,7 +656,7 @@ int32_t LibaomAv1Encoder::Encode(
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
const uint32_t duration = kRtpTicksPerSecond / framerate_fps_;
const uint32_t duration = kVideoPayloadTypeFrequency / framerate_fps_;
timestamp_ += duration;
const size_t num_spatial_layers =
@ -836,6 +833,17 @@ void LibaomAv1Encoder::SetRates(const RateControlParameters& parameters) {
SetEncoderControlParameters(AV1E_SET_SVC_PARAMS, &*svc_params_);
}
if (adaptive_max_consec_drops_ &&
(!rates_configured_ || framerate_fps_ != parameters.framerate_fps)) {
int max_consec_drops = GetMaxConsecDrops(parameters.framerate_fps);
if (!SetEncoderControlParameters(AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR,
max_consec_drops)) {
RTC_LOG(LS_WARNING)
<< "Failed to set AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR to "
<< max_consec_drops;
}
}
framerate_fps_ = parameters.framerate_fps;
rates_configured_ = true;

53
modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc
View File

@ -10,6 +10,7 @@
#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
#include <algorithm>
#include <limits>
#include <memory>
#include <utility>
@ -22,6 +23,7 @@
#include "api/test/frame_generator_interface.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/video_coding/codecs/test/encoded_video_frame_producer.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "test/gmock.h"
@ -37,6 +39,7 @@ using ::testing::Eq;
using ::testing::Field;
using ::testing::IsEmpty;
using ::testing::SizeIs;
using ::testing::Values;
VideoCodec DefaultCodecSettings() {
VideoCodec codec_settings;
@ -199,32 +202,54 @@ TEST(LibaomAv1EncoderTest, CheckOddDimensionsWithSpatialLayers) {
ASSERT_THAT(encoded_frames, SizeIs(6));
}
TEST(LibaomAv1EncoderTest, WithMaximumConsecutiveFrameDrop) {
auto field_trials = std::make_unique<ScopedKeyValueConfig>(
"WebRTC-LibaomAv1Encoder-MaxConsecFrameDrop/maxdrop:2/");
const Environment env = CreateEnvironment(std::move(field_trials));
class LibaomAv1EncoderMaxConsecDropTest
: public ::testing::TestWithParam</*framerate_fps=*/int> {};
TEST_P(LibaomAv1EncoderMaxConsecDropTest, MaxConsecDrops) {
VideoBitrateAllocation allocation;
allocation.SetBitrate(0, 0, 1000); // some very low bitrate
std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder(env);
allocation.SetBitrate(0, 0,
1000); // Very low bitrate to provoke frame drops.
std::unique_ptr<VideoEncoder> encoder =
CreateLibaomAv1Encoder(CreateEnvironment());
VideoCodec codec_settings = DefaultCodecSettings();
codec_settings.SetFrameDropEnabled(true);
codec_settings.SetScalabilityMode(ScalabilityMode::kL1T1);
codec_settings.startBitrate = allocation.get_sum_kbps();
codec_settings.maxFramerate = GetParam();
ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
WEBRTC_VIDEO_CODEC_OK);
encoder->SetRates(VideoEncoder::RateControlParameters(
allocation, codec_settings.maxFramerate));
EncodedVideoFrameProducer evfp(*encoder);
evfp.SetResolution(
RenderResolution{codec_settings.width, codec_settings.height});
// We should code the first frame, skip two, then code another frame.
std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames =
evfp.SetNumInputFrames(4).Encode();
ASSERT_THAT(encoded_frames, SizeIs(2));
// The 4 frames have default Rtp-timestamps of 1000, 4000, 7000, 10000.
ASSERT_THAT(encoded_frames[1].encoded_image.RtpTimestamp(), 10000);
EncodedVideoFrameProducer(*encoder)
.SetNumInputFrames(60)
.SetFramerateFps(codec_settings.maxFramerate)
.SetResolution(RenderResolution{320, 180})
.Encode();
ASSERT_GE(encoded_frames.size(), 2u);
int max_consec_drops = 0;
for (size_t i = 1; i < encoded_frames.size(); ++i) {
uint32_t frame_duration_rtp =
encoded_frames[i].encoded_image.RtpTimestamp() -
encoded_frames[i - 1].encoded_image.RtpTimestamp();
// X consecutive drops result in a freeze of (X + 1) frame duration.
// Subtract 1 to get pure number of drops.
int num_drops = frame_duration_rtp * codec_settings.maxFramerate /
kVideoPayloadTypeFrequency -
1;
max_consec_drops = std::max(max_consec_drops, num_drops);
}
const int expected_max_consec_drops =
std::ceil(0.25 * codec_settings.maxFramerate);
EXPECT_EQ(max_consec_drops, expected_max_consec_drops);
}
INSTANTIATE_TEST_SUITE_P(LibaomAv1EncoderMaxConsecDropTests,
LibaomAv1EncoderMaxConsecDropTest,
Values(1, 2, 5, 15, 30, 60));
TEST(LibaomAv1EncoderTest, EncoderInfoWithoutResolutionBitrateLimits) {
std::unique_ptr<VideoEncoder> encoder =
CreateLibaomAv1Encoder(CreateEnvironment());