From 7404f07ad91552ea2d66aebae5e82ad2a3956feb Mon Sep 17 00:00:00 2001 From: Sergey Silkin Date: Thu, 17 Nov 2022 11:00:38 +0100 Subject: [PATCH] Sync target rates Cache target bit- and framerate in a frame_num -> rates map and fetch the rates accociated with the current frame when needed. This solves the issue when wrong target rates may be used due to frames buffering in encoder. Bug: b/254447893 Change-Id: I369c8d8e71234c957dc2362b055061d12cec818f Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/283841 Reviewed-by: Rasmus Brandt Commit-Queue: Sergey Silkin Cr-Commit-Position: refs/heads/main@{#38673} --- .../codecs/test/videoprocessor.cc | 29 ++++++++++++++----- .../video_coding/codecs/test/videoprocessor.h | 5 ++-- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/modules/video_coding/codecs/test/videoprocessor.cc b/modules/video_coding/codecs/test/videoprocessor.cc index 353a00df79..bf3c72ace4 100644 --- a/modules/video_coding/codecs/test/videoprocessor.cc +++ b/modules/video_coding/codecs/test/videoprocessor.cc @@ -153,7 +153,6 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder, bitrate_allocator_( CreateBuiltinVideoBitrateAllocatorFactory() ->CreateVideoBitrateAllocator(config_.codec_settings)), - framerate_fps_(0), encode_callback_(this), input_frame_reader_(input_frame_reader), merged_encoded_frames_(num_simulcast_or_spatial_layers_), @@ -231,6 +230,11 @@ void VideoProcessor::ProcessFrame() { RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK(!is_finalized_); + RTC_DCHECK_GT(target_rates_.size(), 0u); + RTC_DCHECK_EQ(target_rates_.begin()->first, 0u); + RateProfile target_rate = + std::prev(target_rates_.upper_bound(last_inputed_frame_num_))->second; + const size_t frame_number = last_inputed_frame_num_++; // Get input frame and store for future quality calculation. @@ -239,7 +243,7 @@ void VideoProcessor::ProcessFrame() { RTC_CHECK(buffer) << "Tried to read too many frames from the file."; const size_t timestamp = last_inputed_timestamp_ + - static_cast(kVideoPayloadTypeFrequency / framerate_fps_); + static_cast(kVideoPayloadTypeFrequency / target_rate.input_fps); VideoFrame input_frame = VideoFrame::Builder() .set_video_frame_buffer(buffer) @@ -316,12 +320,14 @@ void VideoProcessor::SetRates(size_t bitrate_kbps, double framerate_fps) { RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK(!is_finalized_); - framerate_fps_ = framerate_fps; - bitrate_allocation_ = + target_rates_[last_inputed_frame_num_] = + RateProfile({.target_kbps = bitrate_kbps, .input_fps = framerate_fps}); + + auto bitrate_allocation = bitrate_allocator_->Allocate(VideoBitrateAllocationParameters( - static_cast(bitrate_kbps * 1000), framerate_fps_)); + static_cast(bitrate_kbps * 1000), framerate_fps)); encoder_->SetRates( - VideoEncoder::RateControlParameters(bitrate_allocation_, framerate_fps_)); + VideoEncoder::RateControlParameters(bitrate_allocation, framerate_fps)); } int32_t VideoProcessor::VideoProcessorDecodeCompleteCallback::Decoded( @@ -389,13 +395,20 @@ void VideoProcessor::FrameEncoded( first_encoded_frame_[spatial_idx] = false; last_encoded_frame_num_[spatial_idx] = frame_number; + RateProfile target_rate = + std::prev(target_rates_.upper_bound(frame_number))->second; + auto bitrate_allocation = + bitrate_allocator_->Allocate(VideoBitrateAllocationParameters( + static_cast(target_rate.target_kbps * 1000), + target_rate.input_fps)); + // Update frame statistics. frame_stat->encoding_successful = true; frame_stat->encode_time_us = GetElapsedTimeMicroseconds( frame_stat->encode_start_ns, encode_stop_ns - post_encode_time_ns_); frame_stat->target_bitrate_kbps = - bitrate_allocation_.GetTemporalLayerSum(spatial_idx, temporal_idx) / 1000; - frame_stat->target_framerate_fps = framerate_fps_; + bitrate_allocation.GetTemporalLayerSum(spatial_idx, temporal_idx) / 1000; + frame_stat->target_framerate_fps = target_rate.input_fps; frame_stat->length_bytes = encoded_image.size(); frame_stat->frame_type = encoded_image._frameType; frame_stat->temporal_idx = temporal_idx; diff --git a/modules/video_coding/codecs/test/videoprocessor.h b/modules/video_coding/codecs/test/videoprocessor.h index 4c89c790a9..79bf582a20 100644 --- a/modules/video_coding/codecs/test/videoprocessor.h +++ b/modules/video_coding/codecs/test/videoprocessor.h @@ -191,8 +191,9 @@ class VideoProcessor { webrtc::VideoEncoder* const encoder_; VideoDecoderList* const decoders_; const std::unique_ptr bitrate_allocator_; - VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(sequence_checker_); - double framerate_fps_ RTC_GUARDED_BY(sequence_checker_); + + // Target bitrate and framerate per frame. + std::map target_rates_ RTC_GUARDED_BY(sequence_checker_); // Adapters for the codec callbacks. VideoProcessorEncodeCompleteCallback encode_callback_;