From ebe5acb27a4797b29c41af53270a30bcede1e64f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Spr=C3=A5ng?= Date: Thu, 3 Dec 2020 16:18:44 +0100 Subject: [PATCH] VideoCodecTextFixture and YuvFrameReader improvements. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds ability to specify desired frame size separate from actual clip resolution, as well as clip and desired fps. This allows e.g. reading an HD clip but running benchmarks in VGA, and to specify e.g. 60fps for the clip but 30for encoding where frame dropping kicks in so that motion is actually correct rather than just plaing the clip slowly. Bug: webrtc:12229 Change-Id: I4ad4fcc335611a449dc2723ffafbec6731e89f55 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195324 Commit-Queue: Erik Språng Reviewed-by: Philip Eliasson Cr-Commit-Position: refs/heads/master@{#32839} --- api/test/videocodec_test_fixture.h | 11 ++ .../test/videocodec_test_fixture_impl.cc | 16 ++- .../codecs/test/videoprocessor.cc | 30 ++++- test/testsupport/frame_reader.h | 32 ++++- test/testsupport/y4m_frame_reader.cc | 6 +- test/testsupport/yuv_frame_reader.cc | 111 ++++++++++++++++-- 6 files changed, 182 insertions(+), 24 deletions(-) diff --git a/api/test/videocodec_test_fixture.h b/api/test/videocodec_test_fixture.h index 395c5cb800..379d46d096 100644 --- a/api/test/videocodec_test_fixture.h +++ b/api/test/videocodec_test_fixture.h @@ -88,6 +88,17 @@ class VideoCodecTestFixture { // Plain name of YUV file to process without file extension. std::string filename; + // Dimensions of test clip. Falls back to (codec_settings.width/height) if + // not set. + absl::optional clip_width; + absl::optional clip_height; + // Framerate of input clip. Defaults to 30fps if not set. + absl::optional clip_fps; + + // The resolution at which psnr/ssim comparisons should be made. Frames + // will be scaled to this size if different. + absl::optional reference_width; + absl::optional reference_height; // File to process. This must be a video file in the YUV format. std::string filepath; diff --git a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc index ebe90b826f..f6770d631e 100644 --- a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc +++ b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc @@ -58,7 +58,7 @@ using VideoStatistics = VideoCodecTestStats::VideoStatistics; namespace { const int kBaseKeyFrameInterval = 3000; const double kBitratePriority = 1.0; -const int kMaxFramerateFps = 30; +const int kDefaultMaxFramerateFps = 30; const int kMaxQp = 56; void ConfigureSimulcast(VideoCodec* codec_settings) { @@ -86,7 +86,7 @@ void ConfigureSvc(VideoCodec* codec_settings) { RTC_CHECK_EQ(kVideoCodecVP9, codec_settings->codecType); const std::vector layers = GetSvcConfig( - codec_settings->width, codec_settings->height, kMaxFramerateFps, + codec_settings->width, codec_settings->height, kDefaultMaxFramerateFps, /*first_active_layer=*/0, codec_settings->VP9()->numberOfSpatialLayers, codec_settings->VP9()->numberOfTemporalLayers, /* is_screen_sharing = */ false); @@ -646,10 +646,16 @@ void VideoCodecTestFixtureImpl::SetUpAndInitObjects( config_.codec_settings.startBitrate = static_cast(initial_bitrate_kbps); config_.codec_settings.maxFramerate = std::ceil(initial_framerate_fps); + int clip_width = config_.clip_width.value_or(config_.codec_settings.width); + int clip_height = config_.clip_height.value_or(config_.codec_settings.height); + // Create file objects for quality analysis. - source_frame_reader_.reset( - new YuvFrameReaderImpl(config_.filepath, config_.codec_settings.width, - config_.codec_settings.height)); + source_frame_reader_.reset(new YuvFrameReaderImpl( + config_.filepath, clip_width, clip_height, + config_.reference_width.value_or(clip_width), + config_.reference_height.value_or(clip_height), + YuvFrameReaderImpl::RepeatMode::kPingPong, config_.clip_fps, + config_.codec_settings.maxFramerate)); EXPECT_TRUE(source_frame_reader_->Init()); RTC_DCHECK(encoded_frame_writers_.empty()); diff --git a/modules/video_coding/codecs/test/videoprocessor.cc b/modules/video_coding/codecs/test/videoprocessor.cc index 1532695b23..a4918ae73d 100644 --- a/modules/video_coding/codecs/test/videoprocessor.cc +++ b/modules/video_coding/codecs/test/videoprocessor.cc @@ -251,7 +251,27 @@ void VideoProcessor::ProcessFrame() { if (input_frames_.size() == kMaxBufferedInputFrames) { input_frames_.erase(input_frames_.begin()); } - input_frames_.emplace(frame_number, input_frame); + + if (config_.reference_width != -1 && config_.reference_height != -1 && + (input_frame.width() != config_.reference_width || + input_frame.height() != config_.reference_height)) { + rtc::scoped_refptr scaled_buffer = I420Buffer::Create( + config_.codec_settings.width, config_.codec_settings.height); + scaled_buffer->ScaleFrom(*input_frame.video_frame_buffer()->ToI420()); + + VideoFrame scaled_reference_frame = input_frame; + scaled_reference_frame.set_video_frame_buffer(scaled_buffer); + input_frames_.emplace(frame_number, scaled_reference_frame); + + if (config_.reference_width == config_.codec_settings.width && + config_.reference_height == config_.codec_settings.height) { + // Both encoding and comparison uses the same down-scale factor, reuse + // it for encoder below. + input_frame = scaled_reference_frame; + } + } else { + input_frames_.emplace(frame_number, input_frame); + } } last_inputed_timestamp_ = timestamp; @@ -271,6 +291,14 @@ void VideoProcessor::ProcessFrame() { frame_stat->encode_start_ns = encode_start_ns; } + if (input_frame.width() != config_.codec_settings.width || + input_frame.height() != config_.codec_settings.height) { + rtc::scoped_refptr scaled_buffer = I420Buffer::Create( + config_.codec_settings.width, config_.codec_settings.height); + scaled_buffer->ScaleFrom(*input_frame.video_frame_buffer()->ToI420()); + input_frame.set_video_frame_buffer(scaled_buffer); + } + // Encode. const std::vector frame_types = (frame_number == 0) diff --git a/test/testsupport/frame_reader.h b/test/testsupport/frame_reader.h index 7f313e8b82..ac399655df 100644 --- a/test/testsupport/frame_reader.h +++ b/test/testsupport/frame_reader.h @@ -15,6 +15,7 @@ #include +#include "absl/types/optional.h" #include "api/scoped_refptr.h" namespace webrtc { @@ -47,11 +48,32 @@ class FrameReader { class YuvFrameReaderImpl : public FrameReader { public: + enum class RepeatMode { kSingle, kRepeat, kPingPong }; + class DropperUtil { + public: + DropperUtil(int source_fps, int target_fps); + + enum class DropDecision { kDropframe, kKeepFrame }; + DropDecision UpdateLevel(); + + private: + const double frame_size_buckets_; + double bucket_level_; + }; + // Creates a file handler. The input file is assumed to exist and be readable. // Parameters: // input_filename The file to read from. // width, height Size of each frame to read. YuvFrameReaderImpl(std::string input_filename, int width, int height); + YuvFrameReaderImpl(std::string input_filename, + int input_width, + int input_height, + int desired_width, + int desired_height, + RepeatMode repeat_mode, + absl::optional clip_fps, + int target_fps); ~YuvFrameReaderImpl() override; bool Init() override; rtc::scoped_refptr ReadFrame() override; @@ -63,9 +85,15 @@ class YuvFrameReaderImpl : public FrameReader { const std::string input_filename_; // It is not const, so subclasses will be able to add frame header size. size_t frame_length_in_bytes_; - const int width_; - const int height_; + const int input_width_; + const int input_height_; + const int desired_width_; + const int desired_height_; + const size_t frame_size_bytes_; + const RepeatMode repeat_mode_; int number_of_frames_; + int current_frame_index_; + std::unique_ptr dropper_; FILE* input_file_; }; diff --git a/test/testsupport/y4m_frame_reader.cc b/test/testsupport/y4m_frame_reader.cc index 6008d1ef16..3f037a3b4b 100644 --- a/test/testsupport/y4m_frame_reader.cc +++ b/test/testsupport/y4m_frame_reader.cc @@ -40,9 +40,9 @@ Y4mFrameReaderImpl::~Y4mFrameReaderImpl() { } bool Y4mFrameReaderImpl::Init() { - if (width_ <= 0 || height_ <= 0) { - fprintf(stderr, "Frame width and height must be >0, was %d x %d\n", width_, - height_); + if (input_width_ <= 0 || input_height_ <= 0) { + fprintf(stderr, "Frame width and height must be >0, was %d x %d\n", + input_width_, input_height_); return false; } input_file_ = fopen(input_filename_.c_str(), "rb"); diff --git a/test/testsupport/yuv_frame_reader.cc b/test/testsupport/yuv_frame_reader.cc index 91b31a6e72..fca982bf34 100644 --- a/test/testsupport/yuv_frame_reader.cc +++ b/test/testsupport/yuv_frame_reader.cc @@ -20,16 +20,64 @@ namespace webrtc { namespace test { +size_t FrameSizeBytes(int width, int height) { + int half_width = (width + 1) / 2; + size_t size_y = static_cast(width) * height; + size_t size_uv = static_cast(half_width) * ((height + 1) / 2); + return size_y + 2 * size_uv; +} + +YuvFrameReaderImpl::DropperUtil::DropperUtil(int source_fps, int target_fps) + : frame_size_buckets_( + std::max(1.0, static_cast(source_fps) / target_fps)), + bucket_level_(0.0) {} + +YuvFrameReaderImpl::DropperUtil::DropDecision +YuvFrameReaderImpl::DropperUtil::UpdateLevel() { + DropDecision decision; + if (bucket_level_ <= 0.0) { + decision = DropDecision::kKeepFrame; + bucket_level_ += frame_size_buckets_; + } else { + decision = DropDecision::kDropframe; + } + bucket_level_ -= 1.0; + return decision; +} YuvFrameReaderImpl::YuvFrameReaderImpl(std::string input_filename, int width, int height) + : YuvFrameReaderImpl(input_filename, + width, + height, + width, + height, + RepeatMode::kSingle, + 30, + 30) {} +YuvFrameReaderImpl::YuvFrameReaderImpl(std::string input_filename, + int input_width, + int input_height, + int desired_width, + int desired_height, + RepeatMode repeat_mode, + absl::optional clip_fps, + int target_fps) : input_filename_(input_filename), - frame_length_in_bytes_(width * height + - 2 * ((width + 1) / 2) * ((height + 1) / 2)), - width_(width), - height_(height), + frame_length_in_bytes_(input_width * input_height + + 2 * ((input_width + 1) / 2) * + ((input_height + 1) / 2)), + input_width_(input_width), + input_height_(input_height), + desired_width_(desired_width), + desired_height_(desired_height), + frame_size_bytes_(FrameSizeBytes(input_width, input_height)), + repeat_mode_(repeat_mode), number_of_frames_(-1), + current_frame_index_(-1), + dropper_(clip_fps.has_value() ? new DropperUtil(*clip_fps, target_fps) + : nullptr), input_file_(nullptr) {} YuvFrameReaderImpl::~YuvFrameReaderImpl() { @@ -37,9 +85,9 @@ YuvFrameReaderImpl::~YuvFrameReaderImpl() { } bool YuvFrameReaderImpl::Init() { - if (width_ <= 0 || height_ <= 0) { - fprintf(stderr, "Frame width and height must be >0, was %d x %d\n", width_, - height_); + if (input_width_ <= 0 || input_height_ <= 0) { + fprintf(stderr, "Frame width and height must be >0, was %d x %d\n", + input_width_, input_height_); return false; } input_file_ = fopen(input_filename_.c_str(), "rb"); @@ -56,6 +104,7 @@ bool YuvFrameReaderImpl::Init() { } number_of_frames_ = static_cast(source_file_size / frame_length_in_bytes_); + current_frame_index_ = 0; return true; } @@ -65,13 +114,49 @@ rtc::scoped_refptr YuvFrameReaderImpl::ReadFrame() { "YuvFrameReaderImpl is not initialized (input file is NULL)\n"); return nullptr; } - rtc::scoped_refptr buffer( - ReadI420Buffer(width_, height_, input_file_)); - if (!buffer && ferror(input_file_)) { - fprintf(stderr, "Error reading from input file: %s\n", - input_filename_.c_str()); + + rtc::scoped_refptr buffer; + + do { + if (current_frame_index_ >= number_of_frames_) { + switch (repeat_mode_) { + case RepeatMode::kSingle: + return nullptr; + case RepeatMode::kRepeat: + fseek(input_file_, 0, SEEK_SET); + current_frame_index_ = 0; + break; + case RepeatMode::kPingPong: + if (current_frame_index_ == number_of_frames_ * 2) { + fseek(input_file_, 0, SEEK_SET); + current_frame_index_ = 0; + } else { + int reverse_frame_index = current_frame_index_ - number_of_frames_; + int seek_frame_pos = (number_of_frames_ - reverse_frame_index - 1); + fseek(input_file_, seek_frame_pos * frame_size_bytes_, SEEK_SET); + } + break; + } + } + ++current_frame_index_; + + buffer = ReadI420Buffer(input_width_, input_height_, input_file_); + if (!buffer && ferror(input_file_)) { + fprintf(stderr, "Error reading from input file: %s\n", + input_filename_.c_str()); + } + } while (dropper_ && + dropper_->UpdateLevel() == DropperUtil::DropDecision::kDropframe); + + if (input_width_ == desired_width_ && input_height_ == desired_height_) { + return buffer; } - return buffer; + + rtc::scoped_refptr rescaled_buffer( + I420Buffer::Create(desired_width_, desired_height_)); + rescaled_buffer->ScaleFrom(*buffer.get()); + + return rescaled_buffer; } void YuvFrameReaderImpl::Close() {