diff --git a/api/test/videocodec_test_fixture.h b/api/test/videocodec_test_fixture.h index 2ed40910b2..395c5cb800 100644 --- a/api/test/videocodec_test_fixture.h +++ b/api/test/videocodec_test_fixture.h @@ -138,6 +138,9 @@ class VideoCodecTestFixture { bool save_encoded_ivf = false; bool save_decoded_y4m = false; } visualization_params; + + // Enables quality analysis for dropped frames. + bool analyze_quality_of_dropped_frames = false; }; virtual ~VideoCodecTestFixture() = default; diff --git a/api/test/videocodec_test_stats.h b/api/test/videocodec_test_stats.h index 63e15768dc..df1aed73aa 100644 --- a/api/test/videocodec_test_stats.h +++ b/api/test/videocodec_test_stats.h @@ -67,6 +67,7 @@ class VideoCodecTestStats { int qp = -1; // Quality. + bool quality_analysis_successful = false; float psnr_y = 0.0f; float psnr_u = 0.0f; float psnr_v = 0.0f; diff --git a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc index aa3ffede52..ebe90b826f 100644 --- a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc +++ b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc @@ -450,6 +450,8 @@ void VideoCodecTestFixtureImpl::ProcessAllFrames( } } + task_queue->PostTask([this] { processor_->Finalize(); }); + // Wait until we know that the last frame has been sent for encode. task_queue->SendTask([] {}, RTC_FROM_HERE); diff --git a/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc b/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc index 7290d29bf6..aa0ff0b9c8 100644 --- a/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc +++ b/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc @@ -252,12 +252,6 @@ VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcVideoStatistic( video_stat.height = std::max(video_stat.height, frame_stat.decoded_height); - psnr_y.AddSample(frame_stat.psnr_y); - psnr_u.AddSample(frame_stat.psnr_u); - psnr_v.AddSample(frame_stat.psnr_v); - psnr.AddSample(frame_stat.psnr); - ssim.AddSample(frame_stat.ssim); - if (video_stat.num_decoded_frames > 1) { if (last_successfully_decoded_frame.decoded_width != frame_stat.decoded_width || @@ -271,6 +265,14 @@ VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcVideoStatistic( last_successfully_decoded_frame = frame_stat; } + if (frame_stat.quality_analysis_successful) { + psnr_y.AddSample(frame_stat.psnr_y); + psnr_u.AddSample(frame_stat.psnr_u); + psnr_v.AddSample(frame_stat.psnr_v); + psnr.AddSample(frame_stat.psnr); + ssim.AddSample(frame_stat.ssim); + } + if (video_stat.num_input_frames > 0) { if (video_stat.time_to_reach_target_bitrate_sec == 0.0f) { RTC_CHECK_GT(time_since_first_frame_sec, 0); diff --git a/modules/video_coding/codecs/test/videoprocessor.cc b/modules/video_coding/codecs/test/videoprocessor.cc index f495bdd6c7..f43326836d 100644 --- a/modules/video_coding/codecs/test/videoprocessor.cc +++ b/modules/video_coding/codecs/test/videoprocessor.cc @@ -41,8 +41,6 @@ namespace webrtc { namespace test { -using FrameStatistics = VideoCodecTestStats::FrameStatistics; - namespace { const int kMsToRtpTimestamp = kVideoPayloadTypeFrequency / 1000; const int kMaxBufferedInputFrames = 20; @@ -86,34 +84,9 @@ int GetElapsedTimeMicroseconds(int64_t start_ns, int64_t stop_ns) { return static_cast(diff_us); } -void ExtractI420BufferWithSize(const VideoFrame& image, - int width, - int height, - rtc::Buffer* buffer) { - if (image.width() != width || image.height() != height) { - EXPECT_DOUBLE_EQ(static_cast(width) / height, - static_cast(image.width()) / image.height()); - // Same aspect ratio, no cropping needed. - rtc::scoped_refptr scaled(I420Buffer::Create(width, height)); - scaled->ScaleFrom(*image.video_frame_buffer()->ToI420()); - - size_t length = - CalcBufferSize(VideoType::kI420, scaled->width(), scaled->height()); - buffer->SetSize(length); - RTC_CHECK_NE(ExtractBuffer(scaled, length, buffer->data()), -1); - return; - } - - // No resize. - size_t length = - CalcBufferSize(VideoType::kI420, image.width(), image.height()); - buffer->SetSize(length); - RTC_CHECK_NE(ExtractBuffer(image, length, buffer->data()), -1); -} - void CalculateFrameQuality(const I420BufferInterface& ref_buffer, const I420BufferInterface& dec_buffer, - FrameStatistics* frame_stat, + VideoCodecTestStats::FrameStatistics* frame_stat, bool calc_ssim) { if (ref_buffer.width() != dec_buffer.width() || ref_buffer.height() != dec_buffer.height()) { @@ -174,6 +147,7 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder, num_simulcast_or_spatial_layers_( std::max(config_.NumberOfSimulcastStreams(), config_.NumberOfSpatialLayers())), + analyze_frame_quality_(!config_.measure_cpu), stats_(stats), encoder_(encoder), decoders_(decoders), @@ -192,8 +166,9 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder, last_encoded_frame_num_(num_simulcast_or_spatial_layers_), first_decoded_frame_(num_simulcast_or_spatial_layers_, true), last_decoded_frame_num_(num_simulcast_or_spatial_layers_), - decoded_frame_buffer_(num_simulcast_or_spatial_layers_), - post_encode_time_ns_(0) { + last_decoded_frame_buffer_(num_simulcast_or_spatial_layers_), + post_encode_time_ns_(0), + is_finalized_(false) { // Sanity checks. RTC_CHECK(TaskQueueBase::Current()) << "VideoProcessor must be run on a task queue."; @@ -234,6 +209,10 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder, VideoProcessor::~VideoProcessor() { RTC_DCHECK_RUN_ON(&sequence_checker_); + if (!is_finalized_) { + Finalize(); + } + // Explicitly reset codecs, in case they don't do that themselves when they // go out of scope. RTC_CHECK_EQ(encoder_->Release(), WEBRTC_VIDEO_CODEC_OK); @@ -249,6 +228,8 @@ VideoProcessor::~VideoProcessor() { void VideoProcessor::ProcessFrame() { RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(!is_finalized_); + const size_t frame_number = last_inputed_frame_num_++; // Get input frame and store for future quality calculation. @@ -304,6 +285,8 @@ void VideoProcessor::ProcessFrame() { void VideoProcessor::SetRates(size_t bitrate_kbps, double framerate_fps) { RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(!is_finalized_); + framerate_fps_ = framerate_fps; bitrate_allocation_ = bitrate_allocator_->Allocate(VideoBitrateAllocationParameters( @@ -460,6 +443,56 @@ void VideoProcessor::FrameEncoded( } } +void VideoProcessor::CalcFrameQuality(const I420BufferInterface& decoded_frame, + FrameStatistics* frame_stat) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + + const auto reference_frame = input_frames_.find(frame_stat->frame_number); + RTC_CHECK(reference_frame != input_frames_.cend()) + << "The codecs are either buffering too much, dropping too much, or " + "being too slow relative to the input frame rate."; + + // SSIM calculation is not optimized. Skip it in real-time mode. + const bool calc_ssim = !config_.encode_in_real_time; + CalculateFrameQuality(*reference_frame->second.video_frame_buffer()->ToI420(), + decoded_frame, frame_stat, calc_ssim); + + frame_stat->quality_analysis_successful = true; +} + +void VideoProcessor::WriteDecodedFrame(const I420BufferInterface& decoded_frame, + FrameWriter& frame_writer) { + int input_video_width = config_.codec_settings.width; + int input_video_height = config_.codec_settings.height; + + rtc::scoped_refptr scaled_buffer; + const I420BufferInterface* scaled_frame; + + if (decoded_frame.width() == input_video_width && + decoded_frame.height() == input_video_height) { + scaled_frame = &decoded_frame; + } else { + EXPECT_DOUBLE_EQ( + static_cast(input_video_width) / input_video_height, + static_cast(decoded_frame.width()) / decoded_frame.height()); + + scaled_buffer = I420Buffer::Create(input_video_width, input_video_height); + scaled_buffer->ScaleFrom(decoded_frame); + + scaled_frame = scaled_buffer; + } + + // Ensure there is no padding. + RTC_CHECK_EQ(scaled_frame->StrideY(), input_video_width); + RTC_CHECK_EQ(scaled_frame->StrideU(), input_video_width / 2); + RTC_CHECK_EQ(scaled_frame->StrideV(), input_video_width / 2); + + RTC_CHECK_EQ(3 * input_video_width * input_video_height / 2, + frame_writer.FrameLength()); + + RTC_CHECK(frame_writer.WriteFrame(scaled_frame->DataY())); +} + void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame, size_t spatial_idx) { RTC_DCHECK_RUN_ON(&sequence_checker_); @@ -472,13 +505,24 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame, stats_->GetFrameWithTimestamp(decoded_frame.timestamp(), spatial_idx); const size_t frame_number = frame_stat->frame_number; - if (decoded_frame_writers_ && !first_decoded_frame_[spatial_idx]) { - // Fill drops with last decoded frame to make them look like freeze at - // playback and to keep decoded layers in sync. - for (size_t i = last_decoded_frame_num_[spatial_idx] + 1; i < frame_number; - ++i) { - RTC_CHECK(decoded_frame_writers_->at(spatial_idx) - ->WriteFrame(decoded_frame_buffer_[spatial_idx].data())); + if (!first_decoded_frame_[spatial_idx]) { + for (size_t dropped_frame_number = last_decoded_frame_num_[spatial_idx] + 1; + dropped_frame_number < frame_number; ++dropped_frame_number) { + FrameStatistics* dropped_frame_stat = + stats_->GetFrame(dropped_frame_number, spatial_idx); + + if (analyze_frame_quality_ && config_.analyze_quality_of_dropped_frames) { + // Calculate frame quality comparing input frame with last decoded one. + CalcFrameQuality(*last_decoded_frame_buffer_[spatial_idx], + dropped_frame_stat); + } + + if (decoded_frame_writers_ != nullptr) { + // Fill drops with last decoded frame to make them look like freeze at + // playback and to keep decoded layers in sync. + WriteDecodedFrame(*last_decoded_frame_buffer_[spatial_idx], + *decoded_frame_writers_->at(spatial_idx)); + } } } @@ -497,41 +541,40 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame, frame_stat->decoded_height = decoded_frame.height(); // Skip quality metrics calculation to not affect CPU usage. - if (!config_.measure_cpu) { - const auto reference_frame = input_frames_.find(frame_number); - RTC_CHECK(reference_frame != input_frames_.cend()) - << "The codecs are either buffering too much, dropping too much, or " - "being too slow relative the input frame rate."; + if (analyze_frame_quality_ || decoded_frame_writers_) { + // Save last decoded frame to handle possible future drops. + rtc::scoped_refptr i420buffer = + decoded_frame.video_frame_buffer()->ToI420(); - // SSIM calculation is not optimized. Skip it in real-time mode. - const bool calc_ssim = !config_.encode_in_real_time; - CalculateFrameQuality( - *reference_frame->second.video_frame_buffer()->ToI420(), - *decoded_frame.video_frame_buffer()->ToI420(), frame_stat, calc_ssim); - - // Erase all buffered input frames that we have moved past for all - // simulcast/spatial layers. Never buffer more than - // |kMaxBufferedInputFrames| frames, to protect against long runs of - // consecutive frame drops for a particular layer. - const auto min_last_decoded_frame_num = std::min_element( - last_decoded_frame_num_.cbegin(), last_decoded_frame_num_.cend()); - const size_t min_buffered_frame_num = std::max( - 0, static_cast(frame_number) - kMaxBufferedInputFrames + 1); - RTC_CHECK(min_last_decoded_frame_num != last_decoded_frame_num_.cend()); - const auto input_frames_erase_before = input_frames_.lower_bound( - std::max(*min_last_decoded_frame_num, min_buffered_frame_num)); - input_frames_.erase(input_frames_.cbegin(), input_frames_erase_before); + // Copy decoded frame to a buffer without padding/stride such that we can + // dump Y, U and V planes into a file in one shot. + last_decoded_frame_buffer_[spatial_idx] = I420Buffer::Copy( + i420buffer->width(), i420buffer->height(), i420buffer->DataY(), + i420buffer->StrideY(), i420buffer->DataU(), i420buffer->StrideU(), + i420buffer->DataV(), i420buffer->StrideV()); } - if (decoded_frame_writers_) { - ExtractI420BufferWithSize(decoded_frame, config_.codec_settings.width, - config_.codec_settings.height, - &decoded_frame_buffer_[spatial_idx]); - RTC_CHECK_EQ(decoded_frame_buffer_[spatial_idx].size(), - decoded_frame_writers_->at(spatial_idx)->FrameLength()); - RTC_CHECK(decoded_frame_writers_->at(spatial_idx) - ->WriteFrame(decoded_frame_buffer_[spatial_idx].data())); + if (analyze_frame_quality_) { + CalcFrameQuality(*decoded_frame.video_frame_buffer()->ToI420(), frame_stat); } + + if (decoded_frame_writers_ != nullptr) { + WriteDecodedFrame(*last_decoded_frame_buffer_[spatial_idx], + *decoded_frame_writers_->at(spatial_idx)); + } + + // Erase all buffered input frames that we have moved past for all + // simulcast/spatial layers. Never buffer more than + // |kMaxBufferedInputFrames| frames, to protect against long runs of + // consecutive frame drops for a particular layer. + const auto min_last_decoded_frame_num = std::min_element( + last_decoded_frame_num_.cbegin(), last_decoded_frame_num_.cend()); + const size_t min_buffered_frame_num = + std::max(0, static_cast(frame_number) - kMaxBufferedInputFrames + 1); + RTC_CHECK(min_last_decoded_frame_num != last_decoded_frame_num_.cend()); + const auto input_frames_erase_before = input_frames_.lower_bound( + std::max(*min_last_decoded_frame_num, min_buffered_frame_num)); + input_frames_.erase(input_frames_.cbegin(), input_frames_erase_before); } void VideoProcessor::DecodeFrame(const EncodedImage& encoded_image, @@ -588,5 +631,41 @@ const webrtc::EncodedImage* VideoProcessor::BuildAndStoreSuperframe( return &merged_encoded_frames_.at(spatial_idx); } +void VideoProcessor::Finalize() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(!is_finalized_); + is_finalized_ = true; + + if (!(analyze_frame_quality_ && config_.analyze_quality_of_dropped_frames) && + decoded_frame_writers_ == nullptr) { + return; + } + + for (size_t spatial_idx = 0; spatial_idx < num_simulcast_or_spatial_layers_; + ++spatial_idx) { + if (first_decoded_frame_[spatial_idx]) { + continue; // No decoded frames on this spatial layer. + } + + for (size_t dropped_frame_number = last_decoded_frame_num_[spatial_idx] + 1; + dropped_frame_number < last_inputed_frame_num_; + ++dropped_frame_number) { + FrameStatistics* frame_stat = + stats_->GetFrame(dropped_frame_number, spatial_idx); + + RTC_DCHECK(!frame_stat->decoding_successful); + + if (analyze_frame_quality_ && config_.analyze_quality_of_dropped_frames) { + CalcFrameQuality(*last_decoded_frame_buffer_[spatial_idx], frame_stat); + } + + if (decoded_frame_writers_ != nullptr) { + WriteDecodedFrame(*last_decoded_frame_buffer_[spatial_idx], + *decoded_frame_writers_->at(spatial_idx)); + } + } + } +} + } // namespace test } // namespace webrtc diff --git a/modules/video_coding/codecs/test/videoprocessor.h b/modules/video_coding/codecs/test/videoprocessor.h index a1c96c6da8..cd755ea0e0 100644 --- a/modules/video_coding/codecs/test/videoprocessor.h +++ b/modules/video_coding/codecs/test/videoprocessor.h @@ -24,6 +24,7 @@ #include "api/task_queue/task_queue_base.h" #include "api/test/videocodec_test_fixture.h" #include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" #include "api/video/video_bitrate_allocation.h" #include "api/video/video_bitrate_allocator.h" #include "api/video/video_frame.h" @@ -58,6 +59,7 @@ class VideoProcessor { // TODO(brandtr): Consider changing FrameWriterList to be a FrameWriterMap, // to be able to save different TLs separately. using FrameWriterList = std::vector>; + using FrameStatistics = VideoCodecTestStats::FrameStatistics; VideoProcessor(webrtc::VideoEncoder* encoder, VideoDecoderList* decoders, @@ -77,6 +79,11 @@ class VideoProcessor { // Updates the encoder with target rates. Must be called at least once. void SetRates(size_t bitrate_kbps, double framerate_fps); + // Signals processor to finalize frame processing and handle possible tail + // drops. If not called expelicitly, this will be called in dtor. It is + // unexpected to get ProcessFrame() or SetRates() calls after Finalize(). + void Finalize(); + private: class VideoProcessorEncodeCompleteCallback : public webrtc::EncodedImageCallback { @@ -182,9 +189,20 @@ class VideoProcessor { size_t simulcast_svc_idx, bool inter_layer_predicted) RTC_RUN_ON(sequence_checker_); - // Test input/output. - VideoCodecTestFixture::Config config_ RTC_GUARDED_BY(sequence_checker_); + void CalcFrameQuality(const I420BufferInterface& decoded_frame, + FrameStatistics* frame_stat); + + void WriteDecodedFrame(const I420BufferInterface& decoded_frame, + FrameWriter& frame_writer); + + void HandleTailDrops(); + + // Test config. + const VideoCodecTestFixture::Config config_; const size_t num_simulcast_or_spatial_layers_; + const bool analyze_frame_quality_; + + // Frame statistics. VideoCodecTestStatsImpl* const stats_; // Codecs. @@ -240,7 +258,7 @@ class VideoProcessor { // simulcast_svc_idx -> frame_number. std::vector last_decoded_frame_num_ RTC_GUARDED_BY(sequence_checker_); // simulcast_svc_idx -> buffer. - std::vector decoded_frame_buffer_ + std::vector> last_decoded_frame_buffer_ RTC_GUARDED_BY(sequence_checker_); // Time spent in frame encode callback. It is accumulated for layers and @@ -248,6 +266,9 @@ class VideoProcessor { // is substracted from measured encode time. Thus we get pure encode time. int64_t post_encode_time_ns_ RTC_GUARDED_BY(sequence_checker_); + // Indicates whether Finalize() was called or not. + bool is_finalized_ RTC_GUARDED_BY(sequence_checker_); + // This class must be operated on a TaskQueue. SequenceChecker sequence_checker_; diff --git a/test/testsupport/frame_writer.h b/test/testsupport/frame_writer.h index b91e57c963..5f85d8bcd4 100644 --- a/test/testsupport/frame_writer.h +++ b/test/testsupport/frame_writer.h @@ -32,7 +32,7 @@ class FrameWriter { // Writes a frame of the configured frame length to the output file. // Returns true if the write was successful, false otherwise. - virtual bool WriteFrame(uint8_t* frame_buffer) = 0; + virtual bool WriteFrame(const uint8_t* frame_buffer) = 0; // Closes the output file if open. Essentially makes this class impossible // to use anymore. Will also be invoked by the destructor. @@ -54,7 +54,7 @@ class YuvFrameWriterImpl : public FrameWriter { YuvFrameWriterImpl(std::string output_filename, int width, int height); ~YuvFrameWriterImpl() override; bool Init() override; - bool WriteFrame(uint8_t* frame_buffer) override; + bool WriteFrame(const uint8_t* frame_buffer) override; void Close() override; size_t FrameLength() override; @@ -76,7 +76,7 @@ class Y4mFrameWriterImpl : public YuvFrameWriterImpl { int frame_rate); ~Y4mFrameWriterImpl() override; bool Init() override; - bool WriteFrame(uint8_t* frame_buffer) override; + bool WriteFrame(const uint8_t* frame_buffer) override; private: const int frame_rate_; diff --git a/test/testsupport/y4m_frame_writer.cc b/test/testsupport/y4m_frame_writer.cc index 25106d886c..896524fa0c 100644 --- a/test/testsupport/y4m_frame_writer.cc +++ b/test/testsupport/y4m_frame_writer.cc @@ -41,7 +41,7 @@ bool Y4mFrameWriterImpl::Init() { return true; } -bool Y4mFrameWriterImpl::WriteFrame(uint8_t* frame_buffer) { +bool Y4mFrameWriterImpl::WriteFrame(const uint8_t* frame_buffer) { if (output_file_ == nullptr) { fprintf(stderr, "Y4mFrameWriterImpl is not initialized (output file is NULL)\n"); diff --git a/test/testsupport/yuv_frame_writer.cc b/test/testsupport/yuv_frame_writer.cc index 24f4f53133..59cb74ebeb 100644 --- a/test/testsupport/yuv_frame_writer.cc +++ b/test/testsupport/yuv_frame_writer.cc @@ -50,7 +50,7 @@ bool YuvFrameWriterImpl::Init() { return true; } -bool YuvFrameWriterImpl::WriteFrame(uint8_t* frame_buffer) { +bool YuvFrameWriterImpl::WriteFrame(const uint8_t* frame_buffer) { RTC_DCHECK(frame_buffer); if (output_file_ == nullptr) { fprintf(stderr,