diff --git a/experiments/field_trials.py b/experiments/field_trials.py index 26328e6834..3f94963648 100755 --- a/experiments/field_trials.py +++ b/experiments/field_trials.py @@ -92,9 +92,9 @@ ACTIVE_FIELD_TRIALS: FrozenSet[FieldTrial] = frozenset([ FieldTrial('WebRTC-JitterEstimatorConfig', 42224404, date(2024, 4, 1)), - FieldTrial('WebRTC-LibaomAv1Encoder-MaxConsecFrameDrop', - 42226184, - date(2024, 4, 1)), + FieldTrial('WebRTC-LibaomAv1Encoder-AdaptiveMaxConsecDrops', + 351644568, + date(2025, 7, 1)), FieldTrial('WebRTC-LibvpxVp9Encoder-SvcFrameDropConfig', 42226190, date(2024, 9, 1)), diff --git a/modules/video_coding/codecs/av1/BUILD.gn b/modules/video_coding/codecs/av1/BUILD.gn index 7d93de7a3a..197e1f3e19 100644 --- a/modules/video_coding/codecs/av1/BUILD.gn +++ b/modules/video_coding/codecs/av1/BUILD.gn @@ -60,6 +60,7 @@ rtc_library("libaom_av1_encoder") { "../../../../api/video_codecs:scalability_mode", "../../../../api/video_codecs:video_codecs_api", "../../../../common_video", + "../../../../modules/rtp_rtcp:rtp_rtcp_format", "../../../../rtc_base:checks", "../../../../rtc_base:logging", "../../../../rtc_base:rtc_numerics", @@ -104,6 +105,7 @@ if (rtc_include_tests) { "../../../../api/units:data_size", "../../../../api/units:time_delta", "../../../../api/video:video_frame", + "../../../../modules/rtp_rtcp:rtp_rtcp_format", "../../../../test:scoped_key_value_config", "../../svc:scalability_mode_util", "../../svc:scalability_structures", diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc index 3a8830252d..258d3d3b88 100644 --- a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc +++ b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc @@ -30,6 +30,7 @@ #include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/include/video_error_codes.h" #include "modules/video_coding/svc/create_scalability_structure.h" @@ -65,7 +66,6 @@ constexpr int kLowQindex = 145; // Low qindex threshold for QP scaling. constexpr int kHighQindex = 205; // High qindex threshold for QP scaling. constexpr int kBitDepth = 8; constexpr int kLagInFrames = 0; // No look ahead. -constexpr int kRtpTicksPerSecond = 90000; constexpr double kMinFrameRateFps = 1.0; aom_superblock_size_t GetSuperblockSize(int width, int height, int threads) { @@ -133,7 +133,9 @@ class LibaomAv1Encoder final : public VideoEncoder { double framerate_fps_; // Current target frame rate. int64_t timestamp_; const LibaomAv1EncoderInfoSettings encoder_info_override_; - int max_consec_frame_drop_; + // TODO(webrtc:351644568): Remove this kill-switch after the feature is fully + // deployed. + bool adaptive_max_consec_drops_; }; int32_t VerifyCodecSettings(const VideoCodec& codec_settings) { @@ -164,12 +166,12 @@ int32_t VerifyCodecSettings(const VideoCodec& codec_settings) { return WEBRTC_VIDEO_CODEC_OK; } -int GetMaxConsecutiveFrameDrop(const FieldTrialsView& field_trials) { - webrtc::FieldTrialParameter maxdrop("maxdrop", 0); - webrtc::ParseFieldTrial( - {&maxdrop}, - field_trials.Lookup("WebRTC-LibaomAv1Encoder-MaxConsecFrameDrop")); - return maxdrop; +int GetMaxConsecDrops(double framerate_fps) { + // Consecutive frame drops result in a video freeze. We want to minimize the + // max number of consecutive drops and, at the same time, keep the value high + // enough to let encoder drain the buffer at overshoot. + constexpr double kMaxFreezeSeconds = 0.25; + return std::ceil(kMaxFreezeSeconds * framerate_fps); } LibaomAv1Encoder::LibaomAv1Encoder(const Environment& env, @@ -182,7 +184,8 @@ LibaomAv1Encoder::LibaomAv1Encoder(const Environment& env, framerate_fps_(0), timestamp_(0), encoder_info_override_(env.field_trials()), - max_consec_frame_drop_(GetMaxConsecutiveFrameDrop(env.field_trials())) {} + adaptive_max_consec_drops_(!env.field_trials().IsDisabled( + "WebRTC-LibaomAv1Encoder-AdaptiveMaxConsecDrops")) {} LibaomAv1Encoder::~LibaomAv1Encoder() { Release(); @@ -242,7 +245,7 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings, cfg_.g_threads = NumberOfThreads(cfg_.g_w, cfg_.g_h, settings.number_of_cores); cfg_.g_timebase.num = 1; - cfg_.g_timebase.den = kRtpTicksPerSecond; + cfg_.g_timebase.den = kVideoPayloadTypeFrequency; cfg_.rc_target_bitrate = encoder_settings_.startBitrate; // kilobits/sec. cfg_.rc_dropframe_thresh = encoder_settings_.GetFrameDropEnabled() ? 30 : 0; cfg_.g_input_bit_depth = kBitDepth; @@ -304,12 +307,6 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings, SET_ENCODER_PARAM_OR_RETURN_ERROR(AV1E_SET_ENABLE_PALETTE, 0); } - if (codec_settings->mode == VideoCodecMode::kRealtimeVideo && - encoder_settings_.GetFrameDropEnabled() && max_consec_frame_drop_ > 0) { - SET_ENCODER_PARAM_OR_RETURN_ERROR(AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR, - max_consec_frame_drop_); - } - if (cfg_.g_threads == 8) { // Values passed to AV1E_SET_TILE_ROWS and AV1E_SET_TILE_COLUMNS are log2() // based. @@ -659,7 +656,7 @@ int32_t LibaomAv1Encoder::Encode( return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; } - const uint32_t duration = kRtpTicksPerSecond / framerate_fps_; + const uint32_t duration = kVideoPayloadTypeFrequency / framerate_fps_; timestamp_ += duration; const size_t num_spatial_layers = @@ -836,6 +833,17 @@ void LibaomAv1Encoder::SetRates(const RateControlParameters& parameters) { SetEncoderControlParameters(AV1E_SET_SVC_PARAMS, &*svc_params_); } + if (adaptive_max_consec_drops_ && + (!rates_configured_ || framerate_fps_ != parameters.framerate_fps)) { + int max_consec_drops = GetMaxConsecDrops(parameters.framerate_fps); + if (!SetEncoderControlParameters(AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR, + max_consec_drops)) { + RTC_LOG(LS_WARNING) + << "Failed to set AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR to " + << max_consec_drops; + } + } + framerate_fps_ = parameters.framerate_fps; rates_configured_ = true; diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc b/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc index abb6fce0cf..a00b03aeda 100644 --- a/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc +++ b/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc @@ -10,6 +10,7 @@ #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" +#include #include #include #include @@ -22,6 +23,7 @@ #include "api/test/frame_generator_interface.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/video_coding/codecs/test/encoded_video_frame_producer.h" #include "modules/video_coding/include/video_error_codes.h" #include "test/gmock.h" @@ -37,6 +39,7 @@ using ::testing::Eq; using ::testing::Field; using ::testing::IsEmpty; using ::testing::SizeIs; +using ::testing::Values; VideoCodec DefaultCodecSettings() { VideoCodec codec_settings; @@ -199,32 +202,54 @@ TEST(LibaomAv1EncoderTest, CheckOddDimensionsWithSpatialLayers) { ASSERT_THAT(encoded_frames, SizeIs(6)); } -TEST(LibaomAv1EncoderTest, WithMaximumConsecutiveFrameDrop) { - auto field_trials = std::make_unique( - "WebRTC-LibaomAv1Encoder-MaxConsecFrameDrop/maxdrop:2/"); - const Environment env = CreateEnvironment(std::move(field_trials)); +class LibaomAv1EncoderMaxConsecDropTest + : public ::testing::TestWithParam {}; + +TEST_P(LibaomAv1EncoderMaxConsecDropTest, MaxConsecDrops) { VideoBitrateAllocation allocation; - allocation.SetBitrate(0, 0, 1000); // some very low bitrate - std::unique_ptr encoder = CreateLibaomAv1Encoder(env); + allocation.SetBitrate(0, 0, + 1000); // Very low bitrate to provoke frame drops. + std::unique_ptr encoder = + CreateLibaomAv1Encoder(CreateEnvironment()); VideoCodec codec_settings = DefaultCodecSettings(); codec_settings.SetFrameDropEnabled(true); codec_settings.SetScalabilityMode(ScalabilityMode::kL1T1); codec_settings.startBitrate = allocation.get_sum_kbps(); + codec_settings.maxFramerate = GetParam(); ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()), WEBRTC_VIDEO_CODEC_OK); encoder->SetRates(VideoEncoder::RateControlParameters( allocation, codec_settings.maxFramerate)); - EncodedVideoFrameProducer evfp(*encoder); - evfp.SetResolution( - RenderResolution{codec_settings.width, codec_settings.height}); - // We should code the first frame, skip two, then code another frame. std::vector encoded_frames = - evfp.SetNumInputFrames(4).Encode(); - ASSERT_THAT(encoded_frames, SizeIs(2)); - // The 4 frames have default Rtp-timestamps of 1000, 4000, 7000, 10000. - ASSERT_THAT(encoded_frames[1].encoded_image.RtpTimestamp(), 10000); + EncodedVideoFrameProducer(*encoder) + .SetNumInputFrames(60) + .SetFramerateFps(codec_settings.maxFramerate) + .SetResolution(RenderResolution{320, 180}) + .Encode(); + ASSERT_GE(encoded_frames.size(), 2u); + + int max_consec_drops = 0; + for (size_t i = 1; i < encoded_frames.size(); ++i) { + uint32_t frame_duration_rtp = + encoded_frames[i].encoded_image.RtpTimestamp() - + encoded_frames[i - 1].encoded_image.RtpTimestamp(); + // X consecutive drops result in a freeze of (X + 1) frame duration. + // Subtract 1 to get pure number of drops. + int num_drops = frame_duration_rtp * codec_settings.maxFramerate / + kVideoPayloadTypeFrequency - + 1; + max_consec_drops = std::max(max_consec_drops, num_drops); + } + + const int expected_max_consec_drops = + std::ceil(0.25 * codec_settings.maxFramerate); + EXPECT_EQ(max_consec_drops, expected_max_consec_drops); } +INSTANTIATE_TEST_SUITE_P(LibaomAv1EncoderMaxConsecDropTests, + LibaomAv1EncoderMaxConsecDropTest, + Values(1, 2, 5, 15, 30, 60)); + TEST(LibaomAv1EncoderTest, EncoderInfoWithoutResolutionBitrateLimits) { std::unique_ptr encoder = CreateLibaomAv1Encoder(CreateEnvironment());