diff --git a/api/BUILD.gn b/api/BUILD.gn index 7d76433959..38ba78feb4 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -985,22 +985,50 @@ if (rtc_include_tests) { ] } - rtc_library("videocodec_test_fixture_api") { + rtc_library("videocodec_test_stats_api") { visibility = [ "*" ] testonly = true sources = [ - "test/videocodec_test_fixture.h", "test/videocodec_test_stats.cc", "test/videocodec_test_stats.h", ] deps = [ - "../modules/video_coding:video_codec_interface", + "../api/units:data_rate", + "../api/units:frequency", "../rtc_base:stringutils", "video:video_frame_type", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("videocodec_test_fixture_api") { + visibility = [ "*" ] + testonly = true + sources = [ "test/videocodec_test_fixture.h" ] + deps = [ + ":videocodec_test_stats_api", + "../modules/video_coding:video_codec_interface", "video_codecs:video_codecs_api", ] } + rtc_library("video_codec_tester_api") { + visibility = [ "*" ] + testonly = true + sources = [ "test/video_codec_tester.h" ] + deps = [ + ":videocodec_test_stats_api", + "../modules/video_coding/svc:scalability_mode_util", + "video:encoded_image", + "video:resolution", + "video:video_frame", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/functional:any_invocable", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + rtc_library("create_videocodec_test_fixture_api") { visibility = [ "*" ] testonly = true @@ -1016,6 +1044,19 @@ if (rtc_include_tests) { ] } + rtc_library("create_video_codec_tester_api") { + visibility = [ "*" ] + testonly = true + sources = [ + "test/create_video_codec_tester.cc", + "test/create_video_codec_tester.h", + ] + deps = [ + ":video_codec_tester_api", + "../modules/video_coding:videocodec_test_impl", + ] + } + rtc_source_set("mock_audio_mixer") { visibility = [ "*" ] testonly = true diff --git a/api/test/create_video_codec_tester.cc b/api/test/create_video_codec_tester.cc new file mode 100644 index 0000000000..a1efefdb48 --- /dev/null +++ b/api/test/create_video_codec_tester.cc @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_video_codec_tester.h" + +#include +#include + +#include "api/test/video_codec_tester.h" +#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" + +namespace webrtc { +namespace test { + +std::unique_ptr CreateVideoCodecTester() { + return std::make_unique(); +} + +} // namespace test +} // namespace webrtc diff --git a/api/test/create_video_codec_tester.h b/api/test/create_video_codec_tester.h new file mode 100644 index 0000000000..c68864ce85 --- /dev/null +++ b/api/test/create_video_codec_tester.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ +#define API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ + +#include + +#include "api/test/video_codec_tester.h" + +namespace webrtc { +namespace test { + +std::unique_ptr CreateVideoCodecTester(); + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ diff --git a/api/test/video_codec_tester.h b/api/test/video_codec_tester.h new file mode 100644 index 0000000000..0eaaa1b895 --- /dev/null +++ b/api/test/video_codec_tester.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_CODEC_TESTER_H_ +#define API_TEST_VIDEO_CODEC_TESTER_H_ + +#include + +#include "absl/functional/any_invocable.h" +#include "api/test/videocodec_test_stats.h" +#include "api/video/encoded_image.h" +#include "api/video/resolution.h" +#include "api/video/video_frame.h" + +namespace webrtc { +namespace test { + +// Interface for a video codec tester. The interface provides minimalistic set +// of data structures that enables implementation of decode-only, encode-only +// and encode-decode tests. +class VideoCodecTester { + public: + // Pacing settings for codec input. + struct PacingSettings { + enum PacingMode { + // Pacing is not used. Frames are sent to codec back-to-back. + kNoPacing, + // Pace with the rate equal to the target video frame rate. Pacing time is + // derived from RTP timestamp. + kRealTime, + // Pace with the explicitly provided rate. + kConstantRate, + }; + PacingMode mode = PacingMode::kNoPacing; + // Pacing rate for `kConstantRate` mode. + Frequency constant_rate = Frequency::Zero(); + }; + + struct DecoderSettings { + PacingSettings pacing; + }; + + struct EncoderSettings { + PacingSettings pacing; + }; + + virtual ~VideoCodecTester() = default; + + // Interface for a raw video frames source. + class RawVideoSource { + public: + virtual ~RawVideoSource() = default; + + // Returns next frame. If no more frames to pull, returns `absl::nullopt`. + // For analysis and pacing purposes, frame must have RTP timestamp set. The + // timestamp must represent the target video frame rate and be unique. + virtual absl::optional PullFrame() = 0; + + // Returns early pulled frame with RTP timestamp equal to `timestamp_rtp`. + virtual VideoFrame GetFrame(uint32_t timestamp_rtp, + Resolution resolution) = 0; + }; + + // Interface for a coded video frames source. + class CodedVideoSource { + public: + virtual ~CodedVideoSource() = default; + + // Returns next frame. If no more frames to pull, returns `absl::nullopt`. + // For analysis and pacing purposes, frame must have RTP timestamp set. The + // timestamp must represent the target video frame rate and be unique. + virtual absl::optional PullFrame() = 0; + }; + + // Interface for a video encoder. + class Encoder { + public: + using EncodeCallback = + absl::AnyInvocable; + + virtual ~Encoder() = default; + + virtual void Encode(const VideoFrame& frame, EncodeCallback callback) = 0; + }; + + // Interface for a video decoder. + class Decoder { + public: + using DecodeCallback = + absl::AnyInvocable; + + virtual ~Decoder() = default; + + virtual void Decode(const EncodedImage& frame, DecodeCallback callback) = 0; + }; + + // Pulls coded video frames from `video_source` and passes them to `decoder`. + // Returns `VideoCodecTestStats` object that contains collected per-frame + // metrics. + virtual std::unique_ptr RunDecodeTest( + std::unique_ptr video_source, + std::unique_ptr decoder, + const DecoderSettings& decoder_settings) = 0; + + // Pulls raw video frames from `video_source` and passes them to `encoder`. + // Returns `VideoCodecTestStats` object that contains collected per-frame + // metrics. + virtual std::unique_ptr RunEncodeTest( + std::unique_ptr video_source, + std::unique_ptr encoder, + const EncoderSettings& encoder_settings) = 0; + + // Pulls raw video frames from `video_source`, passes them to `encoder` and + // then passes encoded frames to `decoder`. Returns `VideoCodecTestStats` + // object that contains collected per-frame metrics. + virtual std::unique_ptr RunEncodeDecodeTest( + std::unique_ptr video_source, + std::unique_ptr encoder, + std::unique_ptr decoder, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings) = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEO_CODEC_TESTER_H_ diff --git a/api/test/videocodec_test_stats.h b/api/test/videocodec_test_stats.h index a05985a665..12c60638db 100644 --- a/api/test/videocodec_test_stats.h +++ b/api/test/videocodec_test_stats.h @@ -18,6 +18,9 @@ #include #include +#include "absl/types/optional.h" +#include "api/units/data_rate.h" +#include "api/units/frequency.h" #include "api/video/video_frame_type.h" namespace webrtc { @@ -135,11 +138,16 @@ class VideoCodecTestStats { virtual ~VideoCodecTestStats() = default; - virtual std::vector GetFrameStatistics() = 0; + virtual std::vector GetFrameStatistics() const = 0; virtual std::vector SliceAndCalcLayerVideoStatistic( size_t first_frame_num, size_t last_frame_num) = 0; + + virtual VideoStatistics CalcVideoStatistic(size_t first_frame, + size_t last_frame, + DataRate target_bitrate, + Frequency target_framerate) = 0; }; } // namespace test diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 2686047bfd..b097daa922 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -877,6 +877,8 @@ if (rtc_include_tests) { rtc_library("video_codecs_test_framework") { testonly = true sources = [ + "codecs/test/video_codec_analyzer.cc", + "codecs/test/video_codec_analyzer.h", "codecs/test/video_codec_unittest.cc", "codecs/test/video_codec_unittest.h", "codecs/test/videoprocessor.cc", @@ -895,14 +897,17 @@ if (rtc_include_tests) { "../../api:frame_generator_api", "../../api:scoped_refptr", "../../api:sequence_checker", + "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", "../../api/task_queue", + "../../api/task_queue:default_task_queue_factory", "../../api/video:builtin_video_bitrate_allocator_factory", "../../api/video:encoded_image", "../../api/video:resolution", "../../api/video:video_bitrate_allocation", "../../api/video:video_bitrate_allocator", "../../api/video:video_bitrate_allocator_factory", + "../../api/video:video_codec_constants", "../../api/video:video_frame", "../../api/video:video_rtp_headers", "../../api/video_codecs:video_codecs_api", @@ -911,6 +916,7 @@ if (rtc_include_tests) { "../../rtc_base:checks", "../../rtc_base:macromagic", "../../rtc_base:rtc_event", + "../../rtc_base:task_queue_for_test", "../../rtc_base:timeutils", "../../rtc_base/synchronization:mutex", "../../rtc_base/system:no_unique_address", @@ -959,6 +965,8 @@ if (rtc_include_tests) { rtc_library("videocodec_test_impl") { testonly = true sources = [ + "codecs/test/video_codec_tester_impl.cc", + "codecs/test/video_codec_tester_impl.h", "codecs/test/videocodec_test_fixture_impl.cc", "codecs/test/videocodec_test_fixture_impl.h", ] @@ -970,12 +978,20 @@ if (rtc_include_tests) { ":videocodec_test_stats_impl", ":webrtc_vp9_helpers", "../../api:array_view", + "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", + "../../api/task_queue:default_task_queue_factory", + "../../api/task_queue:task_queue", "../../api/test/metrics:global_metrics_logger_and_exporter", "../../api/test/metrics:metric", "../../api/test/video:function_video_factory", "../../api/transport:field_trial_based_config", + "../../api/units:frequency", + "../../api/units:time_delta", + "../../api/units:timestamp", + "../../api/video:encoded_image", "../../api/video:video_bitrate_allocation", + "../../api/video:video_frame", "../../api/video_codecs:video_codecs_api", "../../api/video_codecs:video_decoder_factory_template", "../../api/video_codecs:video_decoder_factory_template_dav1d_adapter", @@ -994,6 +1010,7 @@ if (rtc_include_tests) { "../../rtc_base:checks", "../../rtc_base:logging", "../../rtc_base:rtc_base_tests_utils", + "../../rtc_base:rtc_event", "../../rtc_base:stringutils", "../../rtc_base:task_queue_for_test", "../../rtc_base:timeutils", @@ -1018,7 +1035,7 @@ if (rtc_include_tests) { "codecs/test/videocodec_test_stats_impl.h", ] deps = [ - "../../api:videocodec_test_fixture_api", + "../../api:videocodec_test_stats_api", "../../api/numerics", "../../rtc_base:checks", "../../rtc_base:rtc_numerics", @@ -1035,6 +1052,7 @@ if (rtc_include_tests) { sources = [ "codecs/h264/test/h264_impl_unittest.cc", "codecs/multiplex/test/multiplex_adapter_unittest.cc", + "codecs/test/video_codec_test.cc", "codecs/test/video_encoder_decoder_instantiation_tests.cc", "codecs/test/videocodec_test_av1.cc", "codecs/test/videocodec_test_libvpx.cc", @@ -1063,18 +1081,27 @@ if (rtc_include_tests) { ":webrtc_vp9", ":webrtc_vp9_helpers", "../../api:create_frame_generator", + "../../api:create_video_codec_tester_api", "../../api:create_videocodec_test_fixture_api", "../../api:frame_generator_api", "../../api:mock_video_codec_factory", "../../api:mock_video_decoder", "../../api:mock_video_encoder", "../../api:scoped_refptr", + "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", + "../../api:videocodec_test_stats_api", "../../api/test/video:function_video_factory", + "../../api/units:data_rate", + "../../api/units:frequency", "../../api/video:encoded_image", + "../../api/video:resolution", "../../api/video:video_frame", "../../api/video:video_rtp_headers", + "../../api/video_codecs:builtin_video_decoder_factory", + "../../api/video_codecs:builtin_video_encoder_factory", "../../api/video_codecs:rtc_software_fallback_wrappers", + "../../api/video_codecs:scalability_mode", "../../api/video_codecs:video_codecs_api", "../../common_video", "../../common_video/test:utilities", @@ -1090,11 +1117,14 @@ if (rtc_include_tests) { "../../test:fileutils", "../../test:test_support", "../../test:video_test_common", + "../../test:video_test_support", "../rtp_rtcp:rtp_rtcp_format", "codecs/av1:dav1d_decoder", + "svc:scalability_mode_util", "//third_party/libyuv", ] absl_deps = [ + "//third_party/abseil-cpp/absl/functional:any_invocable", "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/types:optional", ] @@ -1130,6 +1160,8 @@ if (rtc_include_tests) { sources = [ "chain_diff_calculator_unittest.cc", + "codecs/test/video_codec_analyzer_unittest.cc", + "codecs/test/video_codec_tester_impl_unittest.cc", "codecs/test/videocodec_test_fixture_config_unittest.cc", "codecs/test/videocodec_test_stats_impl_unittest.cc", "codecs/test/videoprocessor_unittest.cc", @@ -1213,9 +1245,11 @@ if (rtc_include_tests) { "../../api:rtp_packet_info", "../../api:scoped_refptr", "../../api:simulcast_test_fixture_api", + "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", "../../api/task_queue", "../../api/task_queue:default_task_queue_factory", + "../../api/task_queue/test:mock_task_queue_base", "../../api/test/video:function_video_factory", "../../api/units:data_size", "../../api/units:frequency", @@ -1223,6 +1257,7 @@ if (rtc_include_tests) { "../../api/units:timestamp", "../../api/video:builtin_video_bitrate_allocator_factory", "../../api/video:encoded_frame", + "../../api/video:encoded_image", "../../api/video:render_resolution", "../../api/video:video_adaptation", "../../api/video:video_bitrate_allocation", @@ -1239,6 +1274,7 @@ if (rtc_include_tests) { "../../media:rtc_media_base", "../../rtc_base", "../../rtc_base:checks", + "../../rtc_base:gunit_helpers", "../../rtc_base:histogram_percentile_counter", "../../rtc_base:platform_thread", "../../rtc_base:random", @@ -1265,6 +1301,7 @@ if (rtc_include_tests) { "../../test:video_test_common", "../../test:video_test_support", "../../test/time_controller:time_controller", + "../../third_party/libyuv:libyuv", "../rtp_rtcp:rtp_rtcp_format", "../rtp_rtcp:rtp_video_header", "codecs/av1:video_coding_codecs_av1_tests", diff --git a/modules/video_coding/codecs/test/video_codec_analyzer.cc b/modules/video_coding/codecs/test/video_codec_analyzer.cc new file mode 100644 index 0000000000..50af417bcf --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_analyzer.cc @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/video_codec_analyzer.h" + +#include + +#include "api/task_queue/default_task_queue_factory.h" +#include "api/test/video_codec_tester.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_codec_constants.h" +#include "api/video/video_frame.h" +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "rtc_base/time_utils.h" +#include "third_party/libyuv/include/libyuv/compare.h" + +namespace webrtc { +namespace test { + +namespace { + +struct Psnr { + double y; + double u; + double v; + double yuv; +}; + +Psnr CalcPsnr(const I420BufferInterface& ref_buffer, + const I420BufferInterface& dec_buffer) { + RTC_CHECK_EQ(ref_buffer.width(), dec_buffer.width()); + RTC_CHECK_EQ(ref_buffer.height(), dec_buffer.height()); + + uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataY(), dec_buffer.StrideY(), ref_buffer.DataY(), + ref_buffer.StrideY(), dec_buffer.width(), dec_buffer.height()); + + uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataU(), dec_buffer.StrideU(), ref_buffer.DataU(), + ref_buffer.StrideU(), dec_buffer.width() / 2, dec_buffer.height() / 2); + + uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataV(), dec_buffer.StrideV(), ref_buffer.DataV(), + ref_buffer.StrideV(), dec_buffer.width() / 2, dec_buffer.height() / 2); + + int num_y_samples = dec_buffer.width() * dec_buffer.height(); + Psnr psnr; + psnr.y = libyuv::SumSquareErrorToPsnr(sse_y, num_y_samples); + psnr.u = libyuv::SumSquareErrorToPsnr(sse_u, num_y_samples / 4); + psnr.v = libyuv::SumSquareErrorToPsnr(sse_v, num_y_samples / 4); + psnr.yuv = libyuv::SumSquareErrorToPsnr(sse_y + sse_u + sse_v, + num_y_samples + num_y_samples / 2); + return psnr; +} + +} // namespace + +VideoCodecAnalyzer::VideoCodecAnalyzer( + rtc::TaskQueue& task_queue, + ReferenceVideoSource* reference_video_source) + : task_queue_(task_queue), reference_video_source_(reference_video_source) { + sequence_checker_.Detach(); +} + +void VideoCodecAnalyzer::StartEncode(const VideoFrame& input_frame) { + int64_t encode_started_ns = rtc::TimeNanos(); + task_queue_.PostTask( + [this, timestamp_rtp = input_frame.timestamp(), encode_started_ns]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoCodecTestStats::FrameStatistics* fs = + stats_.GetOrAddFrame(timestamp_rtp, /*spatial_idx=*/0); + fs->encode_start_ns = encode_started_ns; + }); +} + +void VideoCodecAnalyzer::FinishEncode(const EncodedImage& frame) { + int64_t encode_finished_ns = rtc::TimeNanos(); + + task_queue_.PostTask([this, timestamp_rtp = frame.Timestamp(), + spatial_idx = frame.SpatialIndex().value_or(0), + temporal_idx = frame.TemporalIndex().value_or(0), + frame_type = frame._frameType, qp = frame.qp_, + frame_size_bytes = frame.size(), encode_finished_ns]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoCodecTestStats::FrameStatistics* fs = + stats_.GetOrAddFrame(timestamp_rtp, spatial_idx); + VideoCodecTestStats::FrameStatistics* fs_base = + stats_.GetOrAddFrame(timestamp_rtp, 0); + + fs->encode_start_ns = fs_base->encode_start_ns; + fs->spatial_idx = spatial_idx; + fs->temporal_idx = temporal_idx; + fs->frame_type = frame_type; + fs->qp = qp; + + fs->encode_time_us = (encode_finished_ns - fs->encode_start_ns) / + rtc::kNumNanosecsPerMicrosec; + fs->length_bytes = frame_size_bytes; + + fs->encoding_successful = true; + }); +} + +void VideoCodecAnalyzer::StartDecode(const EncodedImage& frame) { + int64_t decode_start_ns = rtc::TimeNanos(); + task_queue_.PostTask([this, timestamp_rtp = frame.Timestamp(), + spatial_idx = frame.SpatialIndex().value_or(0), + frame_size_bytes = frame.size(), decode_start_ns]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoCodecTestStats::FrameStatistics* fs = + stats_.GetOrAddFrame(timestamp_rtp, spatial_idx); + if (fs->length_bytes == 0) { + // In encode-decode test the frame size is set in EncodeFinished. In + // decode-only test set it here. + fs->length_bytes = frame_size_bytes; + } + fs->decode_start_ns = decode_start_ns; + }); +} + +void VideoCodecAnalyzer::FinishDecode(const VideoFrame& frame, + int spatial_idx) { + int64_t decode_finished_ns = rtc::TimeNanos(); + task_queue_.PostTask([this, timestamp_rtp = frame.timestamp(), spatial_idx, + width = frame.width(), height = frame.height(), + decode_finished_ns]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoCodecTestStats::FrameStatistics* fs = + stats_.GetFrameWithTimestamp(timestamp_rtp, spatial_idx); + fs->decode_time_us = (decode_finished_ns - fs->decode_start_ns) / + rtc::kNumNanosecsPerMicrosec; + fs->decoded_width = width; + fs->decoded_height = height; + fs->decoding_successful = true; + }); + + if (reference_video_source_ != nullptr) { + // Copy hardware-backed frame into main memory to release output buffers + // which number may be limited in hardware decoders. + rtc::scoped_refptr decoded_buffer = + frame.video_frame_buffer()->ToI420(); + + task_queue_.PostTask([this, decoded_buffer, + timestamp_rtp = frame.timestamp(), spatial_idx]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoFrame ref_frame = reference_video_source_->GetFrame( + timestamp_rtp, {.width = decoded_buffer->width(), + .height = decoded_buffer->height()}); + rtc::scoped_refptr ref_buffer = + ref_frame.video_frame_buffer()->ToI420(); + + Psnr psnr = CalcPsnr(*decoded_buffer, *ref_buffer); + VideoCodecTestStats::FrameStatistics* fs = + this->stats_.GetFrameWithTimestamp(timestamp_rtp, spatial_idx); + fs->psnr_y = static_cast(psnr.y); + fs->psnr_u = static_cast(psnr.u); + fs->psnr_v = static_cast(psnr.v); + fs->psnr = static_cast(psnr.yuv); + + fs->quality_analysis_successful = true; + }); + } +} + +std::unique_ptr VideoCodecAnalyzer::GetStats() { + std::unique_ptr stats; + rtc::Event ready; + task_queue_.PostTask([this, &stats, &ready]() mutable { + RTC_DCHECK_RUN_ON(&sequence_checker_); + stats.reset(new VideoCodecTestStatsImpl(stats_)); + ready.Set(); + }); + ready.Wait(rtc::Event::kForever); + return stats; +} + +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_analyzer.h b/modules/video_coding/codecs/test/video_codec_analyzer.h new file mode 100644 index 0000000000..63a864e810 --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_analyzer.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ +#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ + +#include + +#include "absl/types/optional.h" +#include "api/sequence_checker.h" +#include "api/video/encoded_image.h" +#include "api/video/resolution.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/codecs/test/videocodec_test_stats_impl.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system/no_unique_address.h" +#include "rtc_base/task_queue_for_test.h" + +namespace webrtc { +namespace test { + +// Analyzer measures and collects metrics necessary for evaluation of video +// codec quality and performance. This class is thread-safe. +class VideoCodecAnalyzer { + public: + // An interface that provides reference frames for spatial quality analysis. + class ReferenceVideoSource { + public: + virtual ~ReferenceVideoSource() = default; + + virtual VideoFrame GetFrame(uint32_t timestamp_rtp, + Resolution resolution) = 0; + }; + + VideoCodecAnalyzer(rtc::TaskQueue& task_queue, + ReferenceVideoSource* reference_video_source = nullptr); + + void StartEncode(const VideoFrame& frame); + + void FinishEncode(const EncodedImage& frame); + + void StartDecode(const EncodedImage& frame); + + void FinishDecode(const VideoFrame& frame, int spatial_idx); + + std::unique_ptr GetStats(); + + protected: + rtc::TaskQueue& task_queue_; + ReferenceVideoSource* const reference_video_source_; + VideoCodecTestStatsImpl stats_ RTC_GUARDED_BY(sequence_checker_); + RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ diff --git a/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc b/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc new file mode 100644 index 0000000000..3f9de6dac2 --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/video_codec_analyzer.h" + +#include "absl/types/optional.h" +#include "api/video/i420_buffer.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "third_party/libyuv/include/libyuv/planar_functions.h" + +namespace webrtc { +namespace test { + +namespace { +using ::testing::Return; +using ::testing::Values; + +const size_t kTimestamp = 3000; +const size_t kSpatialIdx = 2; + +class MockReferenceVideoSource + : public VideoCodecAnalyzer::ReferenceVideoSource { + public: + MOCK_METHOD(VideoFrame, GetFrame, (uint32_t, Resolution), (override)); +}; + +VideoFrame CreateVideoFrame(uint32_t timestamp_rtp, + uint8_t y = 0, + uint8_t u = 0, + uint8_t v = 0) { + rtc::scoped_refptr buffer(I420Buffer::Create(2, 2)); + + libyuv::I420Rect(buffer->MutableDataY(), buffer->StrideY(), + buffer->MutableDataU(), buffer->StrideU(), + buffer->MutableDataV(), buffer->StrideV(), 0, 0, + buffer->width(), buffer->height(), y, u, v); + + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .build(); +} + +EncodedImage CreateEncodedImage(uint32_t timestamp_rtp, int spatial_idx = 0) { + EncodedImage encoded_image; + encoded_image.SetTimestamp(timestamp_rtp); + encoded_image.SetSpatialIndex(spatial_idx); + return encoded_image; +} +} // namespace + +TEST(VideoCodecAnalyzerTest, EncodeStartedCreatesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartEncode(CreateVideoFrame(kTimestamp)); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); + EXPECT_EQ(fs[0].rtp_timestamp, kTimestamp); +} + +TEST(VideoCodecAnalyzerTest, EncodeFinishedUpdatesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartEncode(CreateVideoFrame(kTimestamp)); + + EncodedImage encoded_frame = CreateEncodedImage(kTimestamp, kSpatialIdx); + analyzer.FinishEncode(encoded_frame); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(2u, fs.size()); + EXPECT_TRUE(fs[1].encoding_successful); +} + +TEST(VideoCodecAnalyzerTest, DecodeStartedNoFrameStatsCreatesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); + EXPECT_EQ(fs[0].rtp_timestamp, kTimestamp); +} + +TEST(VideoCodecAnalyzerTest, DecodeStartedFrameStatsExistsReusesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartEncode(CreateVideoFrame(kTimestamp)); + analyzer.StartDecode(CreateEncodedImage(kTimestamp, /*spatial_idx=*/0)); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); +} + +TEST(VideoCodecAnalyzerTest, DecodeFinishedUpdatesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); + VideoFrame decoded_frame = CreateVideoFrame(kTimestamp); + analyzer.FinishDecode(decoded_frame, kSpatialIdx); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); + + EXPECT_TRUE(fs[0].decoding_successful); + EXPECT_EQ(static_cast(fs[0].decoded_width), decoded_frame.width()); + EXPECT_EQ(static_cast(fs[0].decoded_height), decoded_frame.height()); +} + +TEST(VideoCodecAnalyzerTest, DecodeFinishedComputesPsnr) { + TaskQueueForTest task_queue; + MockReferenceVideoSource reference_video_source; + VideoCodecAnalyzer analyzer(task_queue, &reference_video_source); + analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); + + EXPECT_CALL(reference_video_source, GetFrame) + .WillOnce(Return(CreateVideoFrame(kTimestamp, /*y=*/0, + /*u=*/0, /*v=*/0))); + + analyzer.FinishDecode( + CreateVideoFrame(kTimestamp, /*value_y=*/1, /*value_u=*/2, /*value_v=*/3), + kSpatialIdx); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); + + EXPECT_NEAR(fs[0].psnr_y, 48, 1); + EXPECT_NEAR(fs[0].psnr_u, 42, 1); + EXPECT_NEAR(fs[0].psnr_v, 38, 1); +} + +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_test.cc b/modules/video_coding/codecs/test/video_codec_test.cc new file mode 100644 index 0000000000..bd4c8e07f2 --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_test.cc @@ -0,0 +1,456 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/video_codec.h" + +#include +#include +#include +#include + +#include "absl/functional/any_invocable.h" +#include "api/test/create_video_codec_tester.h" +#include "api/test/videocodec_test_stats.h" +#include "api/units/data_rate.h" +#include "api/units/frequency.h" +#include "api/video/i420_buffer.h" +#include "api/video/resolution.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_encoder.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "media/base/media_constants.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "rtc_base/strings/string_builder.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/frame_reader.h" + +namespace webrtc { +namespace test { + +namespace { +using ::testing::Combine; +using ::testing::Values; +using Layer = std::pair; + +struct VideoInfo { + std::string name; + Resolution resolution; +}; + +struct CodecInfo { + std::string type; + std::string encoder; + std::string decoder; +}; + +struct EncodingSettings { + ScalabilityMode scalability_mode; + // Spatial layer resolution. + std::map resolution; + // Top temporal layer frame rate. + Frequency framerate; + // Bitrate of spatial and temporal layers. + std::map bitrate; +}; + +struct EncodingTestSettings { + std::string name; + int num_frames = 1; + std::map frame_settings; +}; + +struct DecodingTestSettings { + std::string name; +}; + +struct QualityExpectations { + double min_apsnr_y; +}; + +struct EncodeDecodeTestParams { + CodecInfo codec; + VideoInfo video; + VideoCodecTester::EncoderSettings encoder_settings; + VideoCodecTester::DecoderSettings decoder_settings; + EncodingTestSettings encoding_settings; + DecodingTestSettings decoding_settings; + QualityExpectations quality_expectations; +}; + +const EncodingSettings kQvga64Kbps30Fps = { + .scalability_mode = ScalabilityMode::kL1T1, + .resolution = {{0, {.width = 320, .height = 180}}}, + .framerate = Frequency::Hertz(30), + .bitrate = {{Layer(0, 0), DataRate::KilobitsPerSec(64)}}}; + +const EncodingTestSettings kConstantRateQvga64Kbps30Fps = { + .name = "ConstantRateQvga64Kbps30Fps", + .num_frames = 300, + .frame_settings = {{/*frame_num=*/0, kQvga64Kbps30Fps}}}; + +const QualityExpectations kLowQuality = {.min_apsnr_y = 30}; + +const VideoInfo kFourPeople_1280x720_30 = { + .name = "FourPeople_1280x720_30", + .resolution = {.width = 1280, .height = 720}}; + +const CodecInfo kLibvpxVp8 = {.type = "VP8", + .encoder = "libvpx", + .decoder = "libvpx"}; + +const CodecInfo kLibvpxVp9 = {.type = "VP9", + .encoder = "libvpx", + .decoder = "libvpx"}; + +const CodecInfo kOpenH264 = {.type = "H264", + .encoder = "openh264", + .decoder = "ffmpeg"}; + +class TestRawVideoSource : public VideoCodecTester::RawVideoSource { + public: + static constexpr Frequency k90kHz = Frequency::Hertz(90000); + + TestRawVideoSource(std::unique_ptr frame_reader, + const EncodingTestSettings& test_settings) + : frame_reader_(std::move(frame_reader)), + test_settings_(test_settings), + frame_num_(0), + timestamp_rtp_(0) { + // Ensure settings for the first frame are provided. + RTC_CHECK_GT(test_settings_.frame_settings.size(), 0u); + RTC_CHECK_EQ(test_settings_.frame_settings.begin()->first, 0); + } + + // Pulls next frame. Frame RTP timestamp is set accordingly to + // `EncodingSettings::framerate`. + absl::optional PullFrame() override { + if (frame_num_ >= test_settings_.num_frames) { + // End of stream. + return absl::nullopt; + } + + EncodingSettings frame_settings = + std::prev(test_settings_.frame_settings.upper_bound(frame_num_)) + ->second; + + int pulled_frame; + auto buffer = frame_reader_->PullFrame( + &pulled_frame, frame_settings.resolution.rbegin()->second, + {.num = 30, .den = static_cast(frame_settings.framerate.hertz())}); + RTC_CHECK(buffer) << "Cannot pull frame " << frame_num_; + + auto frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp_) + .build(); + + pulled_frames_[timestamp_rtp_] = pulled_frame; + timestamp_rtp_ += k90kHz / frame_settings.framerate; + ++frame_num_; + + return frame; + } + + // Reads frame specified by `timestamp_rtp`, scales it to `resolution` and + // returns. Frame with the given `timestamp_rtp` is expected to be pulled + // before. + VideoFrame GetFrame(uint32_t timestamp_rtp, Resolution resolution) override { + RTC_CHECK(pulled_frames_.find(timestamp_rtp) != pulled_frames_.end()) + << "Frame with RTP timestamp " << timestamp_rtp + << " was not pulled before"; + auto buffer = + frame_reader_->ReadFrame(pulled_frames_[timestamp_rtp], resolution); + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .build(); + } + + protected: + std::unique_ptr frame_reader_; + const EncodingTestSettings& test_settings_; + int frame_num_; + uint32_t timestamp_rtp_; + std::map pulled_frames_; +}; + +class TestEncoder : public VideoCodecTester::Encoder, + public EncodedImageCallback { + public: + TestEncoder(std::unique_ptr encoder, + const CodecInfo& codec_info, + const std::map& frame_settings) + : encoder_(std::move(encoder)), + codec_info_(codec_info), + frame_settings_(frame_settings), + frame_num_(0) { + // Ensure settings for the first frame is provided. + RTC_CHECK_GT(frame_settings_.size(), 0u); + RTC_CHECK_EQ(frame_settings_.begin()->first, 0); + + encoder_->RegisterEncodeCompleteCallback(this); + } + + void Encode(const VideoFrame& frame, EncodeCallback callback) override { + callbacks_[frame.timestamp()] = std::move(callback); + + if (auto fs = frame_settings_.find(frame_num_); + fs != frame_settings_.end()) { + if (fs == frame_settings_.begin() || + ConfigChanged(fs->second, std::prev(fs)->second)) { + Configure(fs->second); + } + if (fs == frame_settings_.begin() || + RateChanged(fs->second, std::prev(fs)->second)) { + SetRates(fs->second); + } + } + + int result = encoder_->Encode(frame, nullptr); + RTC_CHECK_EQ(result, WEBRTC_VIDEO_CODEC_OK); + ++frame_num_; + } + + protected: + Result OnEncodedImage(const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) override { + auto cb = callbacks_.find(encoded_image.Timestamp()); + RTC_CHECK(cb != callbacks_.end()); + cb->second(encoded_image); + + callbacks_.erase(callbacks_.begin(), cb); + return Result(Result::Error::OK); + } + + void Configure(const EncodingSettings& es) { + VideoCodec vc; + const Resolution& resolution = es.resolution.rbegin()->second; + vc.width = resolution.width; + vc.height = resolution.height; + const DataRate& bitrate = es.bitrate.rbegin()->second; + vc.startBitrate = bitrate.kbps(); + vc.maxBitrate = bitrate.kbps(); + vc.minBitrate = 0; + vc.maxFramerate = static_cast(es.framerate.hertz()); + vc.active = true; + vc.qpMax = 0; + vc.numberOfSimulcastStreams = 0; + vc.mode = webrtc::VideoCodecMode::kRealtimeVideo; + vc.SetFrameDropEnabled(true); + + vc.codecType = PayloadStringToCodecType(codec_info_.type); + if (vc.codecType == kVideoCodecVP8) { + *(vc.VP8()) = VideoEncoder::GetDefaultVp8Settings(); + } else if (vc.codecType == kVideoCodecVP9) { + *(vc.VP9()) = VideoEncoder::GetDefaultVp9Settings(); + } else if (vc.codecType == kVideoCodecH264) { + *(vc.H264()) = VideoEncoder::GetDefaultH264Settings(); + } + + VideoEncoder::Settings ves( + VideoEncoder::Capabilities(/*loss_notification=*/false), + /*number_of_cores=*/1, + /*max_payload_size=*/1440); + + int result = encoder_->InitEncode(&vc, ves); + RTC_CHECK_EQ(result, WEBRTC_VIDEO_CODEC_OK); + } + + void SetRates(const EncodingSettings& es) { + VideoEncoder::RateControlParameters rc; + int num_spatial_layers = + ScalabilityModeToNumSpatialLayers(es.scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumSpatialLayers(es.scalability_mode); + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + RTC_CHECK(es.bitrate.find(Layer(sidx, tidx)) != es.bitrate.end()) + << "Bitrate for layer S=" << sidx << " T=" << tidx << " is not set"; + rc.bitrate.SetBitrate(sidx, tidx, + es.bitrate.at(Layer(sidx, tidx)).bps()); + } + } + + rc.framerate_fps = es.framerate.millihertz() / 1000.0; + encoder_->SetRates(rc); + } + + bool ConfigChanged(const EncodingSettings& es, + const EncodingSettings& prev_es) const { + return es.scalability_mode != prev_es.scalability_mode || + es.resolution != prev_es.resolution; + } + + bool RateChanged(const EncodingSettings& es, + const EncodingSettings& prev_es) const { + return es.bitrate != prev_es.bitrate || es.framerate != prev_es.framerate; + } + + std::unique_ptr encoder_; + const CodecInfo& codec_info_; + const std::map& frame_settings_; + int frame_num_; + std::map callbacks_; +}; + +class TestDecoder : public VideoCodecTester::Decoder, + public DecodedImageCallback { + public: + TestDecoder(std::unique_ptr decoder, + const CodecInfo& codec_info) + : decoder_(std::move(decoder)), codec_info_(codec_info), frame_num_(0) { + decoder_->RegisterDecodeCompleteCallback(this); + } + void Decode(const EncodedImage& frame, DecodeCallback callback) override { + callbacks_[frame.Timestamp()] = std::move(callback); + + if (frame_num_ == 0) { + Configure(); + } + + decoder_->Decode(frame, /*missing_frames=*/false, + /*render_time_ms=*/0); + ++frame_num_; + } + + void Configure() { + VideoDecoder::Settings ds; + ds.set_codec_type(PayloadStringToCodecType(codec_info_.type)); + ds.set_number_of_cores(1); + + bool result = decoder_->Configure(ds); + RTC_CHECK(result); + } + + protected: + int Decoded(VideoFrame& decoded_frame) override { + auto cb = callbacks_.find(decoded_frame.timestamp()); + RTC_CHECK(cb != callbacks_.end()); + cb->second(decoded_frame); + + callbacks_.erase(callbacks_.begin(), cb); + return WEBRTC_VIDEO_CODEC_OK; + } + + std::unique_ptr decoder_; + const CodecInfo& codec_info_; + int frame_num_; + std::map callbacks_; +}; + +std::unique_ptr CreateEncoder( + const CodecInfo& codec_info, + const std::map& frame_settings) { + auto factory = CreateBuiltinVideoEncoderFactory(); + auto encoder = factory->CreateVideoEncoder(SdpVideoFormat(codec_info.type)); + return std::make_unique(std::move(encoder), codec_info, + frame_settings); +} + +std::unique_ptr CreateDecoder( + const CodecInfo& codec_info) { + auto factory = CreateBuiltinVideoDecoderFactory(); + auto decoder = factory->CreateVideoDecoder(SdpVideoFormat(codec_info.type)); + return std::make_unique(std::move(decoder), codec_info); +} + +} // namespace + +class EncodeDecodeTest + : public ::testing::TestWithParam { + public: + EncodeDecodeTest() : test_params_(GetParam()) {} + + void SetUp() override { + std::unique_ptr frame_reader = + CreateYuvFrameReader(ResourcePath(test_params_.video.name, "yuv"), + test_params_.video.resolution, + YuvFrameReaderImpl::RepeatMode::kPingPong); + video_source_ = std::make_unique( + std::move(frame_reader), test_params_.encoding_settings); + + encoder_ = CreateEncoder(test_params_.codec, + test_params_.encoding_settings.frame_settings); + decoder_ = CreateDecoder(test_params_.codec); + + tester_ = CreateVideoCodecTester(); + } + + static std::string TestParametersToStr( + const ::testing::TestParamInfo& info) { + return std::string(info.param.encoding_settings.name + + info.param.codec.type + info.param.codec.encoder + + info.param.codec.decoder); + } + + protected: + EncodeDecodeTestParams test_params_; + std::unique_ptr video_source_; + std::unique_ptr encoder_; + std::unique_ptr decoder_; + std::unique_ptr tester_; +}; + +TEST_P(EncodeDecodeTest, DISABLED_TestEncodeDecode) { + std::unique_ptr stats = tester_->RunEncodeDecodeTest( + std::move(video_source_), std::move(encoder_), std::move(decoder_), + test_params_.encoder_settings, test_params_.decoder_settings); + + const auto& frame_settings = test_params_.encoding_settings.frame_settings; + for (auto fs = frame_settings.begin(); fs != frame_settings.end(); ++fs) { + int first_frame = fs->first; + int last_frame = std::next(fs) != frame_settings.end() + ? std::next(fs)->first - 1 + : test_params_.encoding_settings.num_frames - 1; + + const EncodingSettings& encoding_settings = fs->second; + auto metrics = stats->CalcVideoStatistic( + first_frame, last_frame, encoding_settings.bitrate.rbegin()->second, + encoding_settings.framerate); + + EXPECT_GE(metrics.avg_psnr_y, + test_params_.quality_expectations.min_apsnr_y); + } +} + +std::list ConstantRateTestParameters() { + std::list test_params; + std::vector codecs = {kLibvpxVp8}; + std::vector videos = {kFourPeople_1280x720_30}; + std::vector> + encoding_settings = {{kConstantRateQvga64Kbps30Fps, kLowQuality}}; + for (const CodecInfo& codec : codecs) { + for (const VideoInfo& video : videos) { + for (const auto& es : encoding_settings) { + EncodeDecodeTestParams p; + p.codec = codec; + p.video = video; + p.encoding_settings = es.first; + p.quality_expectations = es.second; + test_params.push_back(p); + } + } + } + return test_params; +} + +INSTANTIATE_TEST_SUITE_P(ConstantRate, + EncodeDecodeTest, + ::testing::ValuesIn(ConstantRateTestParameters()), + EncodeDecodeTest::TestParametersToStr); +} // namespace test + +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl.cc b/modules/video_coding/codecs/test/video_codec_tester_impl.cc new file mode 100644 index 0000000000..3000c1adee --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_tester_impl.cc @@ -0,0 +1,325 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" + +#include +#include +#include + +#include "api/task_queue/default_task_queue_factory.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/codecs/test/video_codec_analyzer.h" +#include "rtc_base/event.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/sleep.h" + +namespace webrtc { +namespace test { + +namespace { +using RawVideoSource = VideoCodecTester::RawVideoSource; +using CodedVideoSource = VideoCodecTester::CodedVideoSource; +using Decoder = VideoCodecTester::Decoder; +using Encoder = VideoCodecTester::Encoder; +using EncoderSettings = VideoCodecTester::EncoderSettings; +using DecoderSettings = VideoCodecTester::DecoderSettings; +using PacingSettings = VideoCodecTester::PacingSettings; +using PacingMode = PacingSettings::PacingMode; + +constexpr Frequency k90kHz = Frequency::Hertz(90000); + +// A thread-safe wrapper for video source to be shared with the quality analyzer +// that reads reference frames from a separate thread. +class SyncRawVideoSource : public VideoCodecAnalyzer::ReferenceVideoSource { + public: + explicit SyncRawVideoSource(std::unique_ptr video_source) + : video_source_(std::move(video_source)) {} + + absl::optional PullFrame() { + MutexLock lock(&mutex_); + return video_source_->PullFrame(); + } + + VideoFrame GetFrame(uint32_t timestamp_rtp, Resolution resolution) override { + MutexLock lock(&mutex_); + return video_source_->GetFrame(timestamp_rtp, resolution); + } + + protected: + std::unique_ptr video_source_ RTC_GUARDED_BY(mutex_); + Mutex mutex_; +}; + +// Pacer calculates delay necessary to keep frame encode or decode call spaced +// from the previous calls by the pacing time. `Delay` is expected to be called +// as close as possible to posting frame encode or decode task. This class is +// not thread safe. +class Pacer { + public: + explicit Pacer(PacingSettings settings) + : settings_(settings), delay_(TimeDelta::Zero()) {} + TimeDelta Delay(Timestamp beat) { + if (settings_.mode == PacingMode::kNoPacing) { + return TimeDelta::Zero(); + } + + Timestamp now = Timestamp::Micros(rtc::TimeMicros()); + if (prev_time_.has_value()) { + delay_ += PacingTime(beat); + delay_ -= (now - *prev_time_); + if (delay_.ns() < 0) { + delay_ = TimeDelta::Zero(); + } + } + + prev_beat_ = beat; + prev_time_ = now; + return delay_; + } + + private: + TimeDelta PacingTime(Timestamp beat) { + if (settings_.mode == PacingMode::kRealTime) { + return beat - *prev_beat_; + } + RTC_CHECK_EQ(PacingMode::kConstantRate, settings_.mode); + return 1 / settings_.constant_rate; + } + + PacingSettings settings_; + absl::optional prev_beat_; + absl::optional prev_time_; + TimeDelta delay_; +}; + +// Task queue that keeps the number of queued tasks below a certain limit. If +// the limit is reached, posting of a next task is blocked until execution of a +// previously posted task starts. This class is not thread-safe. +class LimitedTaskQueue { + public: + // The codec tester reads frames from video source in the main thread. + // Encoding and decoding are done in separate threads. If encoding or + // decoding is slow, the reading may go far ahead and may buffer too many + // frames in memory. To prevent this we limit the encoding/decoding queue + // size. When the queue is full, the main thread and, hence, reading frames + // from video source is blocked until a previously posted encoding/decoding + // task starts. + static constexpr int kMaxTaskQueueSize = 3; + + explicit LimitedTaskQueue(rtc::TaskQueue& task_queue) + : task_queue_(task_queue), queue_size_(0) {} + + void PostDelayedTask(absl::AnyInvocable task, TimeDelta delay) { + ++queue_size_; + task_queue_.PostDelayedTask( + [this, task = std::move(task)]() mutable { + std::move(task)(); + --queue_size_; + task_executed_.Set(); + }, + delay); + + task_executed_.Reset(); + if (queue_size_ > kMaxTaskQueueSize) { + task_executed_.Wait(rtc::Event::kForever); + } + RTC_CHECK(queue_size_ <= kMaxTaskQueueSize); + } + + void WaitForPreviouslyPostedTasks() { + while (queue_size_ > 0) { + task_executed_.Wait(rtc::Event::kForever); + task_executed_.Reset(); + } + } + + rtc::TaskQueue& task_queue_; + std::atomic_int queue_size_; + rtc::Event task_executed_; +}; + +class TesterDecoder { + public: + TesterDecoder(std::unique_ptr decoder, + VideoCodecAnalyzer* analyzer, + const DecoderSettings& settings, + rtc::TaskQueue& task_queue) + : decoder_(std::move(decoder)), + analyzer_(analyzer), + settings_(settings), + pacer_(settings.pacing), + task_queue_(task_queue) { + RTC_CHECK(analyzer_) << "Analyzer must be provided"; + } + + void Decode(const EncodedImage& frame) { + Timestamp timestamp = Timestamp::Micros((frame.Timestamp() / k90kHz).us()); + + task_queue_.PostDelayedTask( + [this, frame] { + analyzer_->StartDecode(frame); + decoder_->Decode(frame, [this](const VideoFrame& decoded_frame) { + this->analyzer_->FinishDecode(decoded_frame, /*spatial_idx=*/0); + }); + }, + pacer_.Delay(timestamp)); + } + + void Flush() { task_queue_.WaitForPreviouslyPostedTasks(); } + + protected: + std::unique_ptr decoder_; + VideoCodecAnalyzer* const analyzer_; + const DecoderSettings& settings_; + Pacer pacer_; + LimitedTaskQueue task_queue_; +}; + +class TesterEncoder { + public: + TesterEncoder(std::unique_ptr encoder, + TesterDecoder* decoder, + VideoCodecAnalyzer* analyzer, + const EncoderSettings& settings, + rtc::TaskQueue& task_queue) + : encoder_(std::move(encoder)), + decoder_(decoder), + analyzer_(analyzer), + settings_(settings), + pacer_(settings.pacing), + task_queue_(task_queue) { + RTC_CHECK(analyzer_) << "Analyzer must be provided"; + } + + void Encode(const VideoFrame& frame) { + Timestamp timestamp = Timestamp::Micros((frame.timestamp() / k90kHz).us()); + + task_queue_.PostDelayedTask( + [this, frame] { + analyzer_->StartEncode(frame); + encoder_->Encode(frame, [this](const EncodedImage& encoded_frame) { + this->analyzer_->FinishEncode(encoded_frame); + if (decoder_ != nullptr) { + this->decoder_->Decode(encoded_frame); + } + }); + }, + pacer_.Delay(timestamp)); + } + + void Flush() { task_queue_.WaitForPreviouslyPostedTasks(); } + + protected: + std::unique_ptr encoder_; + TesterDecoder* const decoder_; + VideoCodecAnalyzer* const analyzer_; + const EncoderSettings& settings_; + Pacer pacer_; + LimitedTaskQueue task_queue_; +}; + +} // namespace + +VideoCodecTesterImpl::VideoCodecTesterImpl() + : VideoCodecTesterImpl(/*task_queue_factory=*/nullptr) {} + +VideoCodecTesterImpl::VideoCodecTesterImpl(TaskQueueFactory* task_queue_factory) + : task_queue_factory_(task_queue_factory) { + if (task_queue_factory_ == nullptr) { + owned_task_queue_factory_ = CreateDefaultTaskQueueFactory(); + task_queue_factory_ = owned_task_queue_factory_.get(); + } +} + +std::unique_ptr VideoCodecTesterImpl::RunDecodeTest( + std::unique_ptr video_source, + std::unique_ptr decoder, + const DecoderSettings& decoder_settings) { + rtc::TaskQueue analyser_task_queue(task_queue_factory_->CreateTaskQueue( + "Analyzer", TaskQueueFactory::Priority::NORMAL)); + rtc::TaskQueue decoder_task_queue(task_queue_factory_->CreateTaskQueue( + "Decoder", TaskQueueFactory::Priority::NORMAL)); + + VideoCodecAnalyzer perf_analyzer(analyser_task_queue); + TesterDecoder tester_decoder(std::move(decoder), &perf_analyzer, + decoder_settings, decoder_task_queue); + + while (auto frame = video_source->PullFrame()) { + tester_decoder.Decode(*frame); + } + + tester_decoder.Flush(); + + return perf_analyzer.GetStats(); +} + +std::unique_ptr VideoCodecTesterImpl::RunEncodeTest( + std::unique_ptr video_source, + std::unique_ptr encoder, + const EncoderSettings& encoder_settings) { + rtc::TaskQueue analyser_task_queue(task_queue_factory_->CreateTaskQueue( + "Analyzer", TaskQueueFactory::Priority::NORMAL)); + rtc::TaskQueue encoder_task_queue(task_queue_factory_->CreateTaskQueue( + "Encoder", TaskQueueFactory::Priority::NORMAL)); + + SyncRawVideoSource sync_source(std::move(video_source)); + VideoCodecAnalyzer perf_analyzer(analyser_task_queue); + TesterEncoder tester_encoder(std::move(encoder), /*decoder=*/nullptr, + &perf_analyzer, encoder_settings, + encoder_task_queue); + + while (auto frame = sync_source.PullFrame()) { + tester_encoder.Encode(*frame); + } + + tester_encoder.Flush(); + + return perf_analyzer.GetStats(); +} + +std::unique_ptr VideoCodecTesterImpl::RunEncodeDecodeTest( + std::unique_ptr video_source, + std::unique_ptr encoder, + std::unique_ptr decoder, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings) { + rtc::TaskQueue analyser_task_queue(task_queue_factory_->CreateTaskQueue( + "Analyzer", TaskQueueFactory::Priority::NORMAL)); + rtc::TaskQueue decoder_task_queue(task_queue_factory_->CreateTaskQueue( + "Decoder", TaskQueueFactory::Priority::NORMAL)); + rtc::TaskQueue encoder_task_queue(task_queue_factory_->CreateTaskQueue( + "Encoder", TaskQueueFactory::Priority::NORMAL)); + + SyncRawVideoSource sync_source(std::move(video_source)); + VideoCodecAnalyzer perf_analyzer(analyser_task_queue, &sync_source); + TesterDecoder tester_decoder(std::move(decoder), &perf_analyzer, + decoder_settings, decoder_task_queue); + TesterEncoder tester_encoder(std::move(encoder), &tester_decoder, + &perf_analyzer, encoder_settings, + encoder_task_queue); + + while (auto frame = sync_source.PullFrame()) { + tester_encoder.Encode(*frame); + } + + tester_encoder.Flush(); + tester_decoder.Flush(); + + return perf_analyzer.GetStats(); +} + +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl.h b/modules/video_coding/codecs/test/video_codec_tester_impl.h new file mode 100644 index 0000000000..b64adeb882 --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_tester_impl.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ +#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ + +#include + +#include "api/task_queue/task_queue_factory.h" +#include "api/test/video_codec_tester.h" + +namespace webrtc { +namespace test { + +// A stateless implementation of `VideoCodecTester`. This class is thread safe. +class VideoCodecTesterImpl : public VideoCodecTester { + public: + VideoCodecTesterImpl(); + explicit VideoCodecTesterImpl(TaskQueueFactory* task_queue_factory); + + std::unique_ptr RunDecodeTest( + std::unique_ptr video_source, + std::unique_ptr decoder, + const DecoderSettings& decoder_settings) override; + + std::unique_ptr RunEncodeTest( + std::unique_ptr video_source, + std::unique_ptr encoder, + const EncoderSettings& encoder_settings) override; + + std::unique_ptr RunEncodeDecodeTest( + std::unique_ptr video_source, + std::unique_ptr encoder, + std::unique_ptr decoder, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings) override; + + protected: + std::unique_ptr owned_task_queue_factory_; + TaskQueueFactory* task_queue_factory_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc b/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc new file mode 100644 index 0000000000..29fb006fb5 --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc @@ -0,0 +1,259 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" + +#include +#include +#include +#include + +#include "api/task_queue/task_queue_factory.h" +#include "api/task_queue/test/mock_task_queue_base.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "rtc_base/fake_clock.h" +#include "rtc_base/gunit.h" +#include "rtc_base/task_queue_for_test.h" +#include "rtc_base/time_utils.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +namespace { +using ::testing::_; +using ::testing::Invoke; +using ::testing::InvokeWithoutArgs; +using ::testing::Return; + +using Decoder = VideoCodecTester::Decoder; +using Encoder = VideoCodecTester::Encoder; +using CodedVideoSource = VideoCodecTester::CodedVideoSource; +using RawVideoSource = VideoCodecTester::RawVideoSource; +using DecoderSettings = VideoCodecTester::DecoderSettings; +using EncoderSettings = VideoCodecTester::EncoderSettings; +using PacingSettings = VideoCodecTester::PacingSettings; +using PacingMode = PacingSettings::PacingMode; + +constexpr Frequency k90kHz = Frequency::Hertz(90000); + +VideoFrame CreateVideoFrame(uint32_t timestamp_rtp) { + rtc::scoped_refptr buffer(I420Buffer::Create(2, 2)); + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .build(); +} + +EncodedImage CreateEncodedImage(uint32_t timestamp_rtp) { + EncodedImage encoded_image; + encoded_image.SetTimestamp(timestamp_rtp); + return encoded_image; +} + +class MockRawVideoSource : public RawVideoSource { + public: + MOCK_METHOD(absl::optional, PullFrame, (), (override)); + MOCK_METHOD(VideoFrame, + GetFrame, + (uint32_t timestamp_rtp, Resolution), + (override)); +}; + +class MockCodedVideoSource : public CodedVideoSource { + public: + MOCK_METHOD(absl::optional, PullFrame, (), (override)); +}; + +class MockDecoder : public Decoder { + public: + MOCK_METHOD(void, + Decode, + (const EncodedImage& frame, DecodeCallback callback), + (override)); +}; + +class MockEncoder : public Encoder { + public: + MOCK_METHOD(void, + Encode, + (const VideoFrame& frame, EncodeCallback callback), + (override)); +}; + +class MockTaskQueueFactory : public TaskQueueFactory { + public: + explicit MockTaskQueueFactory(TaskQueueBase& task_queue) + : task_queue_(task_queue) {} + + std::unique_ptr CreateTaskQueue( + absl::string_view name, + Priority priority) const override { + return std::unique_ptr(&task_queue_); + } + + protected: + TaskQueueBase& task_queue_; +}; +} // namespace + +class VideoCodecTesterImplPacingTest + : public ::testing::TestWithParam, + std::vector, + std::vector>> { + public: + VideoCodecTesterImplPacingTest() + : pacing_settings_(std::get<0>(GetParam())), + frame_timestamp_ms_(std::get<1>(GetParam())), + frame_capture_delay_ms_(std::get<2>(GetParam())), + expected_frame_start_ms_(std::get<3>(GetParam())), + num_frames_(frame_timestamp_ms_.size()), + task_queue_factory_(task_queue_) {} + + void SetUp() override { + ON_CALL(task_queue_, PostTask) + .WillByDefault(Invoke( + [](absl::AnyInvocable task) { std::move(task)(); })); + + ON_CALL(task_queue_, PostDelayedTask) + .WillByDefault( + Invoke([&](absl::AnyInvocable task, TimeDelta delay) { + clock_.AdvanceTime(delay); + std::move(task)(); + })); + } + + protected: + PacingSettings pacing_settings_; + std::vector frame_timestamp_ms_; + std::vector frame_capture_delay_ms_; + std::vector expected_frame_start_ms_; + size_t num_frames_; + + rtc::ScopedFakeClock clock_; + MockTaskQueueBase task_queue_; + MockTaskQueueFactory task_queue_factory_; +}; + +TEST_P(VideoCodecTesterImplPacingTest, PaceEncode) { + auto video_source = std::make_unique(); + + size_t frame_num = 0; + EXPECT_CALL(*video_source, PullFrame).WillRepeatedly(Invoke([&]() mutable { + if (frame_num >= num_frames_) { + return absl::optional(); + } + clock_.AdvanceTime(TimeDelta::Millis(frame_capture_delay_ms_[frame_num])); + + uint32_t timestamp_rtp = frame_timestamp_ms_[frame_num] * k90kHz.hertz() / + rtc::kNumMillisecsPerSec; + ++frame_num; + return absl::optional(CreateVideoFrame(timestamp_rtp)); + })); + + auto encoder = std::make_unique(); + EncoderSettings encoder_settings; + encoder_settings.pacing = pacing_settings_; + + VideoCodecTesterImpl tester(&task_queue_factory_); + auto fs = tester + .RunEncodeTest(std::move(video_source), std::move(encoder), + encoder_settings) + ->GetFrameStatistics(); + ASSERT_EQ(fs.size(), num_frames_); + + for (size_t i = 0; i < fs.size(); ++i) { + int encode_start_ms = (fs[i].encode_start_ns - fs[0].encode_start_ns) / + rtc::kNumNanosecsPerMillisec; + EXPECT_NEAR(encode_start_ms, expected_frame_start_ms_[i], 10); + } +} + +TEST_P(VideoCodecTesterImplPacingTest, PaceDecode) { + auto video_source = std::make_unique(); + + size_t frame_num = 0; + EXPECT_CALL(*video_source, PullFrame).WillRepeatedly(Invoke([&]() mutable { + if (frame_num >= num_frames_) { + return absl::optional(); + } + clock_.AdvanceTime(TimeDelta::Millis(frame_capture_delay_ms_[frame_num])); + + uint32_t timestamp_rtp = frame_timestamp_ms_[frame_num] * k90kHz.hertz() / + rtc::kNumMillisecsPerSec; + ++frame_num; + return absl::optional(CreateEncodedImage(timestamp_rtp)); + })); + + auto decoder = std::make_unique(); + DecoderSettings decoder_settings; + decoder_settings.pacing = pacing_settings_; + + VideoCodecTesterImpl tester(&task_queue_factory_); + auto fs = tester + .RunDecodeTest(std::move(video_source), std::move(decoder), + decoder_settings) + ->GetFrameStatistics(); + ASSERT_EQ(fs.size(), num_frames_); + + for (size_t i = 0; i < fs.size(); ++i) { + int decode_start_ms = (fs[i].decode_start_ns - fs[0].decode_start_ns) / + rtc::kNumNanosecsPerMillisec; + EXPECT_NEAR(decode_start_ms, expected_frame_start_ms_[i], 10); + } +} + +INSTANTIATE_TEST_SUITE_P( + All, + VideoCodecTesterImplPacingTest, + ::testing::ValuesIn( + {std::make_tuple(PacingSettings({.mode = PacingMode::kNoPacing}), + /*frame_timestamp_ms=*/std::vector{0, 100}, + /*frame_capture_delay_ms=*/std::vector{0, 0}, + /*expected_frame_start_ms=*/std::vector{0, 0}), + // Pace with rate equal to the source frame rate. Frames are captured + // instantly. Verify that frames are paced with the source frame rate. + std::make_tuple(PacingSettings({.mode = PacingMode::kRealTime}), + /*frame_timestamp_ms=*/std::vector{0, 100}, + /*frame_capture_delay_ms=*/std::vector{0, 0}, + /*expected_frame_start_ms=*/std::vector{0, 100}), + // Pace with rate equal to the source frame rate. Frame capture is + // delayed by more than pacing time. Verify that no extra delay is + // added. + std::make_tuple(PacingSettings({.mode = PacingMode::kRealTime}), + /*frame_timestamp_ms=*/std::vector{0, 100}, + /*frame_capture_delay_ms=*/std::vector{0, 200}, + /*expected_frame_start_ms=*/std::vector{0, 200}), + // Pace with constant rate less then source frame rate. Frames are + // captured instantly. Verify that frames are paced with the requested + // constant rate. + std::make_tuple( + PacingSettings({.mode = PacingMode::kConstantRate, + .constant_rate = Frequency::Hertz(20)}), + /*frame_timestamp_ms=*/std::vector{0, 100}, + /*frame_capture_delay_ms=*/std::vector{0, 0}, + /*expected_frame_start_ms=*/std::vector{0, 50}), + // Pace with constant rate less then source frame rate. Frame capture + // is delayed by more than the pacing time. Verify that no extra delay + // is added. + std::make_tuple( + PacingSettings({.mode = PacingMode::kConstantRate, + .constant_rate = Frequency::Hertz(20)}), + /*frame_timestamp_ms=*/std::vector{0, 100}, + /*frame_capture_delay_ms=*/std::vector{0, 200}, + /*expected_frame_start_ms=*/std::vector{0, 200})})); +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc b/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc index efb7502e5d..390348b97a 100644 --- a/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc +++ b/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc @@ -58,7 +58,20 @@ FrameStatistics* VideoCodecTestStatsImpl::GetFrameWithTimestamp( return GetFrame(rtp_timestamp_to_frame_num_[layer_idx][timestamp], layer_idx); } -std::vector VideoCodecTestStatsImpl::GetFrameStatistics() { +FrameStatistics* VideoCodecTestStatsImpl::GetOrAddFrame(size_t timestamp_rtp, + size_t spatial_idx) { + if (rtp_timestamp_to_frame_num_[spatial_idx].count(timestamp_rtp) > 0) { + return GetFrameWithTimestamp(timestamp_rtp, spatial_idx); + } + + size_t frame_num = layer_stats_[spatial_idx].size(); + AddFrame(FrameStatistics(frame_num, timestamp_rtp, spatial_idx)); + + return GetFrameWithTimestamp(timestamp_rtp, spatial_idx); +} + +std::vector VideoCodecTestStatsImpl::GetFrameStatistics() + const { size_t capacity = 0; for (const auto& layer_stat : layer_stats_) { capacity += layer_stat.second.size(); @@ -92,7 +105,8 @@ VideoCodecTestStatsImpl::SliceAndCalcLayerVideoStatistic( for (size_t temporal_idx = 0; temporal_idx < num_temporal_layers; ++temporal_idx) { VideoStatistics layer_stat = SliceAndCalcVideoStatistic( - first_frame_num, last_frame_num, spatial_idx, temporal_idx, false); + first_frame_num, last_frame_num, spatial_idx, temporal_idx, false, + /*target_bitrate=*/absl::nullopt, /*target_framerate=*/absl::nullopt); layer_stats.push_back(layer_stat); } } @@ -110,9 +124,24 @@ VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcAggregatedVideoStatistic( RTC_CHECK_GT(num_spatial_layers, 0); RTC_CHECK_GT(num_temporal_layers, 0); - return SliceAndCalcVideoStatistic(first_frame_num, last_frame_num, - num_spatial_layers - 1, - num_temporal_layers - 1, true); + return SliceAndCalcVideoStatistic( + first_frame_num, last_frame_num, num_spatial_layers - 1, + num_temporal_layers - 1, true, /*target_bitrate=*/absl::nullopt, + /*target_framerate=*/absl::nullopt); +} + +VideoStatistics VideoCodecTestStatsImpl::CalcVideoStatistic( + size_t first_frame_num, + size_t last_frame_num, + DataRate target_bitrate, + Frequency target_framerate) { + size_t num_spatial_layers = 0; + size_t num_temporal_layers = 0; + GetNumberOfEncodedLayers(first_frame_num, last_frame_num, &num_spatial_layers, + &num_temporal_layers); + return SliceAndCalcVideoStatistic( + first_frame_num, last_frame_num, num_spatial_layers - 1, + num_temporal_layers - 1, true, target_bitrate, target_framerate); } size_t VideoCodecTestStatsImpl::Size(size_t spatial_idx) { @@ -175,7 +204,9 @@ VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcVideoStatistic( size_t last_frame_num, size_t spatial_idx, size_t temporal_idx, - bool aggregate_independent_layers) { + bool aggregate_independent_layers, + absl::optional target_bitrate, + absl::optional target_framerate) { VideoStatistics video_stat; float buffer_level_bits = 0.0f; @@ -200,8 +231,11 @@ VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcVideoStatistic( FrameStatistics last_successfully_decoded_frame(0, 0, 0); const size_t target_bitrate_kbps = - CalcLayerTargetBitrateKbps(first_frame_num, last_frame_num, spatial_idx, - temporal_idx, aggregate_independent_layers); + target_bitrate.has_value() + ? target_bitrate->kbps() + : CalcLayerTargetBitrateKbps(first_frame_num, last_frame_num, + spatial_idx, temporal_idx, + aggregate_independent_layers); const size_t target_bitrate_bps = 1000 * target_bitrate_kbps; RTC_CHECK_GT(target_bitrate_kbps, 0); // We divide by `target_bitrate_kbps`. @@ -303,7 +337,9 @@ VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcVideoStatistic( GetFrame(first_frame_num, spatial_idx)->rtp_timestamp; RTC_CHECK_GT(timestamp_delta, 0); const float input_framerate_fps = - 1.0 * kVideoPayloadTypeFrequency / timestamp_delta; + target_framerate.has_value() + ? target_framerate->millihertz() / 1000.0 + : 1.0 * kVideoPayloadTypeFrequency / timestamp_delta; RTC_CHECK_GT(input_framerate_fps, 0); const float duration_sec = num_frames / input_framerate_fps; diff --git a/modules/video_coding/codecs/test/videocodec_test_stats_impl.h b/modules/video_coding/codecs/test/videocodec_test_stats_impl.h index 61850d3622..1a7980aa0a 100644 --- a/modules/video_coding/codecs/test/videocodec_test_stats_impl.h +++ b/modules/video_coding/codecs/test/videocodec_test_stats_impl.h @@ -35,8 +35,12 @@ class VideoCodecTestStatsImpl : public VideoCodecTestStats { FrameStatistics* GetFrame(size_t frame_number, size_t spatial_idx); FrameStatistics* GetFrameWithTimestamp(size_t timestamp, size_t spatial_idx); + // Creates FrameStatisticts if it doesn't exists and/or returns + // created/existing FrameStatisticts. + FrameStatistics* GetOrAddFrame(size_t timestamp_rtp, size_t spatial_idx); + // Implements VideoCodecTestStats. - std::vector GetFrameStatistics() override; + std::vector GetFrameStatistics() const override; std::vector SliceAndCalcLayerVideoStatistic( size_t first_frame_num, size_t last_frame_num) override; @@ -44,6 +48,11 @@ class VideoCodecTestStatsImpl : public VideoCodecTestStats { VideoStatistics SliceAndCalcAggregatedVideoStatistic(size_t first_frame_num, size_t last_frame_num); + VideoStatistics CalcVideoStatistic(size_t first_frame, + size_t last_frame, + DataRate target_bitrate, + Frequency target_framerate) override; + size_t Size(size_t spatial_idx); void Clear(); @@ -65,7 +74,9 @@ class VideoCodecTestStatsImpl : public VideoCodecTestStats { size_t last_frame_num, size_t spatial_idx, size_t temporal_idx, - bool aggregate_independent_layers); + bool aggregate_independent_layers, + absl::optional target_bitrate, + absl::optional target_framerate); void GetNumberOfEncodedLayers(size_t first_frame_num, size_t last_frame_num, diff --git a/modules/video_coding/codecs/test/videocodec_test_stats_impl_unittest.cc b/modules/video_coding/codecs/test/videocodec_test_stats_impl_unittest.cc index 6477b6ab8c..89e7d2e1c4 100644 --- a/modules/video_coding/codecs/test/videocodec_test_stats_impl_unittest.cc +++ b/modules/video_coding/codecs/test/videocodec_test_stats_impl_unittest.cc @@ -38,6 +38,21 @@ TEST(StatsTest, AddAndGetFrame) { EXPECT_EQ(kTimestamp, frame_stat->rtp_timestamp); } +TEST(StatsTest, GetOrAddFrame_noFrame_createsNewFrameStat) { + VideoCodecTestStatsImpl stats; + stats.GetOrAddFrame(kTimestamp, 0); + FrameStatistics* frame_stat = stats.GetFrameWithTimestamp(kTimestamp, 0); + EXPECT_EQ(kTimestamp, frame_stat->rtp_timestamp); +} + +TEST(StatsTest, GetOrAddFrame_frameExists_returnsExistingFrameStat) { + VideoCodecTestStatsImpl stats; + stats.AddFrame(FrameStatistics(0, kTimestamp, 0)); + FrameStatistics* frame_stat1 = stats.GetFrameWithTimestamp(kTimestamp, 0); + FrameStatistics* frame_stat2 = stats.GetOrAddFrame(kTimestamp, 0); + EXPECT_EQ(frame_stat1, frame_stat2); +} + TEST(StatsTest, AddAndGetFrames) { VideoCodecTestStatsImpl stats; const size_t kNumFrames = 1000;