From 2e1a9a4ae0234d4b1ea7a6fd4188afa1fb20379d Mon Sep 17 00:00:00 2001 From: Sergey Silkin Date: Thu, 15 Dec 2022 10:15:18 +0100 Subject: [PATCH] Add video codec tester. This tester is an improved version of VideoProcessor and VideoCodecTestFixture and will eventually replace them. The tester provides better separation between codecs and testing logic. Its knowledge about codecs is limited to frame encode/decode calls and frame ready callbacks. Instantiation and configuration of codecs are the test responsibilities. Other differences: - Run encoding and decoding in separate threads - Run quality analysis in a separate thread - Reference frame buffering is moved into video source (which re-read frames from the file). - Make it possible to run decode-only tests This CL is MVP implementation: it adds only 1 test (video_codec_test.cc, ConstantRate/EncodeDecodeTest) and the test is disabled for now. Bug: b/261160916 Change-Id: Ida24a2fca1b1496237fa695c812084877c76379f Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/283525 Commit-Queue: Sergey Silkin Reviewed-by: Rasmus Brandt Reviewed-by: Mirko Bonadei Cr-Commit-Position: refs/heads/main@{#38901} --- api/BUILD.gn | 47 +- api/test/create_video_codec_tester.cc | 27 ++ api/test/create_video_codec_tester.h | 26 + api/test/video_codec_tester.h | 134 +++++ api/test/videocodec_test_stats.h | 10 +- modules/video_coding/BUILD.gn | 39 +- .../codecs/test/video_codec_analyzer.cc | 186 +++++++ .../codecs/test/video_codec_analyzer.h | 65 +++ .../test/video_codec_analyzer_unittest.cc | 141 ++++++ .../codecs/test/video_codec_test.cc | 456 ++++++++++++++++++ .../codecs/test/video_codec_tester_impl.cc | 325 +++++++++++++ .../codecs/test/video_codec_tester_impl.h | 53 ++ .../test/video_codec_tester_impl_unittest.cc | 259 ++++++++++ .../codecs/test/videocodec_test_stats_impl.cc | 54 ++- .../codecs/test/videocodec_test_stats_impl.h | 15 +- .../videocodec_test_stats_impl_unittest.cc | 15 + 16 files changed, 1836 insertions(+), 16 deletions(-) create mode 100644 api/test/create_video_codec_tester.cc create mode 100644 api/test/create_video_codec_tester.h create mode 100644 api/test/video_codec_tester.h create mode 100644 modules/video_coding/codecs/test/video_codec_analyzer.cc create mode 100644 modules/video_coding/codecs/test/video_codec_analyzer.h create mode 100644 modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc create mode 100644 modules/video_coding/codecs/test/video_codec_test.cc create mode 100644 modules/video_coding/codecs/test/video_codec_tester_impl.cc create mode 100644 modules/video_coding/codecs/test/video_codec_tester_impl.h create mode 100644 modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc diff --git a/api/BUILD.gn b/api/BUILD.gn index 7d76433959..38ba78feb4 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -985,22 +985,50 @@ if (rtc_include_tests) { ] } - rtc_library("videocodec_test_fixture_api") { + rtc_library("videocodec_test_stats_api") { visibility = [ "*" ] testonly = true sources = [ - "test/videocodec_test_fixture.h", "test/videocodec_test_stats.cc", "test/videocodec_test_stats.h", ] deps = [ - "../modules/video_coding:video_codec_interface", + "../api/units:data_rate", + "../api/units:frequency", "../rtc_base:stringutils", "video:video_frame_type", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("videocodec_test_fixture_api") { + visibility = [ "*" ] + testonly = true + sources = [ "test/videocodec_test_fixture.h" ] + deps = [ + ":videocodec_test_stats_api", + "../modules/video_coding:video_codec_interface", "video_codecs:video_codecs_api", ] } + rtc_library("video_codec_tester_api") { + visibility = [ "*" ] + testonly = true + sources = [ "test/video_codec_tester.h" ] + deps = [ + ":videocodec_test_stats_api", + "../modules/video_coding/svc:scalability_mode_util", + "video:encoded_image", + "video:resolution", + "video:video_frame", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/functional:any_invocable", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + rtc_library("create_videocodec_test_fixture_api") { visibility = [ "*" ] testonly = true @@ -1016,6 +1044,19 @@ if (rtc_include_tests) { ] } + rtc_library("create_video_codec_tester_api") { + visibility = [ "*" ] + testonly = true + sources = [ + "test/create_video_codec_tester.cc", + "test/create_video_codec_tester.h", + ] + deps = [ + ":video_codec_tester_api", + "../modules/video_coding:videocodec_test_impl", + ] + } + rtc_source_set("mock_audio_mixer") { visibility = [ "*" ] testonly = true diff --git a/api/test/create_video_codec_tester.cc b/api/test/create_video_codec_tester.cc new file mode 100644 index 0000000000..a1efefdb48 --- /dev/null +++ b/api/test/create_video_codec_tester.cc @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_video_codec_tester.h" + +#include +#include + +#include "api/test/video_codec_tester.h" +#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" + +namespace webrtc { +namespace test { + +std::unique_ptr CreateVideoCodecTester() { + return std::make_unique(); +} + +} // namespace test +} // namespace webrtc diff --git a/api/test/create_video_codec_tester.h b/api/test/create_video_codec_tester.h new file mode 100644 index 0000000000..c68864ce85 --- /dev/null +++ b/api/test/create_video_codec_tester.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ +#define API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ + +#include + +#include "api/test/video_codec_tester.h" + +namespace webrtc { +namespace test { + +std::unique_ptr CreateVideoCodecTester(); + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ diff --git a/api/test/video_codec_tester.h b/api/test/video_codec_tester.h new file mode 100644 index 0000000000..0eaaa1b895 --- /dev/null +++ b/api/test/video_codec_tester.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_CODEC_TESTER_H_ +#define API_TEST_VIDEO_CODEC_TESTER_H_ + +#include + +#include "absl/functional/any_invocable.h" +#include "api/test/videocodec_test_stats.h" +#include "api/video/encoded_image.h" +#include "api/video/resolution.h" +#include "api/video/video_frame.h" + +namespace webrtc { +namespace test { + +// Interface for a video codec tester. The interface provides minimalistic set +// of data structures that enables implementation of decode-only, encode-only +// and encode-decode tests. +class VideoCodecTester { + public: + // Pacing settings for codec input. + struct PacingSettings { + enum PacingMode { + // Pacing is not used. Frames are sent to codec back-to-back. + kNoPacing, + // Pace with the rate equal to the target video frame rate. Pacing time is + // derived from RTP timestamp. + kRealTime, + // Pace with the explicitly provided rate. + kConstantRate, + }; + PacingMode mode = PacingMode::kNoPacing; + // Pacing rate for `kConstantRate` mode. + Frequency constant_rate = Frequency::Zero(); + }; + + struct DecoderSettings { + PacingSettings pacing; + }; + + struct EncoderSettings { + PacingSettings pacing; + }; + + virtual ~VideoCodecTester() = default; + + // Interface for a raw video frames source. + class RawVideoSource { + public: + virtual ~RawVideoSource() = default; + + // Returns next frame. If no more frames to pull, returns `absl::nullopt`. + // For analysis and pacing purposes, frame must have RTP timestamp set. The + // timestamp must represent the target video frame rate and be unique. + virtual absl::optional PullFrame() = 0; + + // Returns early pulled frame with RTP timestamp equal to `timestamp_rtp`. + virtual VideoFrame GetFrame(uint32_t timestamp_rtp, + Resolution resolution) = 0; + }; + + // Interface for a coded video frames source. + class CodedVideoSource { + public: + virtual ~CodedVideoSource() = default; + + // Returns next frame. If no more frames to pull, returns `absl::nullopt`. + // For analysis and pacing purposes, frame must have RTP timestamp set. The + // timestamp must represent the target video frame rate and be unique. + virtual absl::optional PullFrame() = 0; + }; + + // Interface for a video encoder. + class Encoder { + public: + using EncodeCallback = + absl::AnyInvocable; + + virtual ~Encoder() = default; + + virtual void Encode(const VideoFrame& frame, EncodeCallback callback) = 0; + }; + + // Interface for a video decoder. + class Decoder { + public: + using DecodeCallback = + absl::AnyInvocable; + + virtual ~Decoder() = default; + + virtual void Decode(const EncodedImage& frame, DecodeCallback callback) = 0; + }; + + // Pulls coded video frames from `video_source` and passes them to `decoder`. + // Returns `VideoCodecTestStats` object that contains collected per-frame + // metrics. + virtual std::unique_ptr RunDecodeTest( + std::unique_ptr video_source, + std::unique_ptr decoder, + const DecoderSettings& decoder_settings) = 0; + + // Pulls raw video frames from `video_source` and passes them to `encoder`. + // Returns `VideoCodecTestStats` object that contains collected per-frame + // metrics. + virtual std::unique_ptr RunEncodeTest( + std::unique_ptr video_source, + std::unique_ptr encoder, + const EncoderSettings& encoder_settings) = 0; + + // Pulls raw video frames from `video_source`, passes them to `encoder` and + // then passes encoded frames to `decoder`. Returns `VideoCodecTestStats` + // object that contains collected per-frame metrics. + virtual std::unique_ptr RunEncodeDecodeTest( + std::unique_ptr video_source, + std::unique_ptr encoder, + std::unique_ptr decoder, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings) = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEO_CODEC_TESTER_H_ diff --git a/api/test/videocodec_test_stats.h b/api/test/videocodec_test_stats.h index a05985a665..12c60638db 100644 --- a/api/test/videocodec_test_stats.h +++ b/api/test/videocodec_test_stats.h @@ -18,6 +18,9 @@ #include #include +#include "absl/types/optional.h" +#include "api/units/data_rate.h" +#include "api/units/frequency.h" #include "api/video/video_frame_type.h" namespace webrtc { @@ -135,11 +138,16 @@ class VideoCodecTestStats { virtual ~VideoCodecTestStats() = default; - virtual std::vector GetFrameStatistics() = 0; + virtual std::vector GetFrameStatistics() const = 0; virtual std::vector SliceAndCalcLayerVideoStatistic( size_t first_frame_num, size_t last_frame_num) = 0; + + virtual VideoStatistics CalcVideoStatistic(size_t first_frame, + size_t last_frame, + DataRate target_bitrate, + Frequency target_framerate) = 0; }; } // namespace test diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 2686047bfd..b097daa922 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -877,6 +877,8 @@ if (rtc_include_tests) { rtc_library("video_codecs_test_framework") { testonly = true sources = [ + "codecs/test/video_codec_analyzer.cc", + "codecs/test/video_codec_analyzer.h", "codecs/test/video_codec_unittest.cc", "codecs/test/video_codec_unittest.h", "codecs/test/videoprocessor.cc", @@ -895,14 +897,17 @@ if (rtc_include_tests) { "../../api:frame_generator_api", "../../api:scoped_refptr", "../../api:sequence_checker", + "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", "../../api/task_queue", + "../../api/task_queue:default_task_queue_factory", "../../api/video:builtin_video_bitrate_allocator_factory", "../../api/video:encoded_image", "../../api/video:resolution", "../../api/video:video_bitrate_allocation", "../../api/video:video_bitrate_allocator", "../../api/video:video_bitrate_allocator_factory", + "../../api/video:video_codec_constants", "../../api/video:video_frame", "../../api/video:video_rtp_headers", "../../api/video_codecs:video_codecs_api", @@ -911,6 +916,7 @@ if (rtc_include_tests) { "../../rtc_base:checks", "../../rtc_base:macromagic", "../../rtc_base:rtc_event", + "../../rtc_base:task_queue_for_test", "../../rtc_base:timeutils", "../../rtc_base/synchronization:mutex", "../../rtc_base/system:no_unique_address", @@ -959,6 +965,8 @@ if (rtc_include_tests) { rtc_library("videocodec_test_impl") { testonly = true sources = [ + "codecs/test/video_codec_tester_impl.cc", + "codecs/test/video_codec_tester_impl.h", "codecs/test/videocodec_test_fixture_impl.cc", "codecs/test/videocodec_test_fixture_impl.h", ] @@ -970,12 +978,20 @@ if (rtc_include_tests) { ":videocodec_test_stats_impl", ":webrtc_vp9_helpers", "../../api:array_view", + "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", + "../../api/task_queue:default_task_queue_factory", + "../../api/task_queue:task_queue", "../../api/test/metrics:global_metrics_logger_and_exporter", "../../api/test/metrics:metric", "../../api/test/video:function_video_factory", "../../api/transport:field_trial_based_config", + "../../api/units:frequency", + "../../api/units:time_delta", + "../../api/units:timestamp", + "../../api/video:encoded_image", "../../api/video:video_bitrate_allocation", + "../../api/video:video_frame", "../../api/video_codecs:video_codecs_api", "../../api/video_codecs:video_decoder_factory_template", "../../api/video_codecs:video_decoder_factory_template_dav1d_adapter", @@ -994,6 +1010,7 @@ if (rtc_include_tests) { "../../rtc_base:checks", "../../rtc_base:logging", "../../rtc_base:rtc_base_tests_utils", + "../../rtc_base:rtc_event", "../../rtc_base:stringutils", "../../rtc_base:task_queue_for_test", "../../rtc_base:timeutils", @@ -1018,7 +1035,7 @@ if (rtc_include_tests) { "codecs/test/videocodec_test_stats_impl.h", ] deps = [ - "../../api:videocodec_test_fixture_api", + "../../api:videocodec_test_stats_api", "../../api/numerics", "../../rtc_base:checks", "../../rtc_base:rtc_numerics", @@ -1035,6 +1052,7 @@ if (rtc_include_tests) { sources = [ "codecs/h264/test/h264_impl_unittest.cc", "codecs/multiplex/test/multiplex_adapter_unittest.cc", + "codecs/test/video_codec_test.cc", "codecs/test/video_encoder_decoder_instantiation_tests.cc", "codecs/test/videocodec_test_av1.cc", "codecs/test/videocodec_test_libvpx.cc", @@ -1063,18 +1081,27 @@ if (rtc_include_tests) { ":webrtc_vp9", ":webrtc_vp9_helpers", "../../api:create_frame_generator", + "../../api:create_video_codec_tester_api", "../../api:create_videocodec_test_fixture_api", "../../api:frame_generator_api", "../../api:mock_video_codec_factory", "../../api:mock_video_decoder", "../../api:mock_video_encoder", "../../api:scoped_refptr", + "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", + "../../api:videocodec_test_stats_api", "../../api/test/video:function_video_factory", + "../../api/units:data_rate", + "../../api/units:frequency", "../../api/video:encoded_image", + "../../api/video:resolution", "../../api/video:video_frame", "../../api/video:video_rtp_headers", + "../../api/video_codecs:builtin_video_decoder_factory", + "../../api/video_codecs:builtin_video_encoder_factory", "../../api/video_codecs:rtc_software_fallback_wrappers", + "../../api/video_codecs:scalability_mode", "../../api/video_codecs:video_codecs_api", "../../common_video", "../../common_video/test:utilities", @@ -1090,11 +1117,14 @@ if (rtc_include_tests) { "../../test:fileutils", "../../test:test_support", "../../test:video_test_common", + "../../test:video_test_support", "../rtp_rtcp:rtp_rtcp_format", "codecs/av1:dav1d_decoder", + "svc:scalability_mode_util", "//third_party/libyuv", ] absl_deps = [ + "//third_party/abseil-cpp/absl/functional:any_invocable", "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/types:optional", ] @@ -1130,6 +1160,8 @@ if (rtc_include_tests) { sources = [ "chain_diff_calculator_unittest.cc", + "codecs/test/video_codec_analyzer_unittest.cc", + "codecs/test/video_codec_tester_impl_unittest.cc", "codecs/test/videocodec_test_fixture_config_unittest.cc", "codecs/test/videocodec_test_stats_impl_unittest.cc", "codecs/test/videoprocessor_unittest.cc", @@ -1213,9 +1245,11 @@ if (rtc_include_tests) { "../../api:rtp_packet_info", "../../api:scoped_refptr", "../../api:simulcast_test_fixture_api", + "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", "../../api/task_queue", "../../api/task_queue:default_task_queue_factory", + "../../api/task_queue/test:mock_task_queue_base", "../../api/test/video:function_video_factory", "../../api/units:data_size", "../../api/units:frequency", @@ -1223,6 +1257,7 @@ if (rtc_include_tests) { "../../api/units:timestamp", "../../api/video:builtin_video_bitrate_allocator_factory", "../../api/video:encoded_frame", + "../../api/video:encoded_image", "../../api/video:render_resolution", "../../api/video:video_adaptation", "../../api/video:video_bitrate_allocation", @@ -1239,6 +1274,7 @@ if (rtc_include_tests) { "../../media:rtc_media_base", "../../rtc_base", "../../rtc_base:checks", + "../../rtc_base:gunit_helpers", "../../rtc_base:histogram_percentile_counter", "../../rtc_base:platform_thread", "../../rtc_base:random", @@ -1265,6 +1301,7 @@ if (rtc_include_tests) { "../../test:video_test_common", "../../test:video_test_support", "../../test/time_controller:time_controller", + "../../third_party/libyuv:libyuv", "../rtp_rtcp:rtp_rtcp_format", "../rtp_rtcp:rtp_video_header", "codecs/av1:video_coding_codecs_av1_tests", diff --git a/modules/video_coding/codecs/test/video_codec_analyzer.cc b/modules/video_coding/codecs/test/video_codec_analyzer.cc new file mode 100644 index 0000000000..50af417bcf --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_analyzer.cc @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/video_codec_analyzer.h" + +#include + +#include "api/task_queue/default_task_queue_factory.h" +#include "api/test/video_codec_tester.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_codec_constants.h" +#include "api/video/video_frame.h" +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "rtc_base/time_utils.h" +#include "third_party/libyuv/include/libyuv/compare.h" + +namespace webrtc { +namespace test { + +namespace { + +struct Psnr { + double y; + double u; + double v; + double yuv; +}; + +Psnr CalcPsnr(const I420BufferInterface& ref_buffer, + const I420BufferInterface& dec_buffer) { + RTC_CHECK_EQ(ref_buffer.width(), dec_buffer.width()); + RTC_CHECK_EQ(ref_buffer.height(), dec_buffer.height()); + + uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataY(), dec_buffer.StrideY(), ref_buffer.DataY(), + ref_buffer.StrideY(), dec_buffer.width(), dec_buffer.height()); + + uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataU(), dec_buffer.StrideU(), ref_buffer.DataU(), + ref_buffer.StrideU(), dec_buffer.width() / 2, dec_buffer.height() / 2); + + uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataV(), dec_buffer.StrideV(), ref_buffer.DataV(), + ref_buffer.StrideV(), dec_buffer.width() / 2, dec_buffer.height() / 2); + + int num_y_samples = dec_buffer.width() * dec_buffer.height(); + Psnr psnr; + psnr.y = libyuv::SumSquareErrorToPsnr(sse_y, num_y_samples); + psnr.u = libyuv::SumSquareErrorToPsnr(sse_u, num_y_samples / 4); + psnr.v = libyuv::SumSquareErrorToPsnr(sse_v, num_y_samples / 4); + psnr.yuv = libyuv::SumSquareErrorToPsnr(sse_y + sse_u + sse_v, + num_y_samples + num_y_samples / 2); + return psnr; +} + +} // namespace + +VideoCodecAnalyzer::VideoCodecAnalyzer( + rtc::TaskQueue& task_queue, + ReferenceVideoSource* reference_video_source) + : task_queue_(task_queue), reference_video_source_(reference_video_source) { + sequence_checker_.Detach(); +} + +void VideoCodecAnalyzer::StartEncode(const VideoFrame& input_frame) { + int64_t encode_started_ns = rtc::TimeNanos(); + task_queue_.PostTask( + [this, timestamp_rtp = input_frame.timestamp(), encode_started_ns]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoCodecTestStats::FrameStatistics* fs = + stats_.GetOrAddFrame(timestamp_rtp, /*spatial_idx=*/0); + fs->encode_start_ns = encode_started_ns; + }); +} + +void VideoCodecAnalyzer::FinishEncode(const EncodedImage& frame) { + int64_t encode_finished_ns = rtc::TimeNanos(); + + task_queue_.PostTask([this, timestamp_rtp = frame.Timestamp(), + spatial_idx = frame.SpatialIndex().value_or(0), + temporal_idx = frame.TemporalIndex().value_or(0), + frame_type = frame._frameType, qp = frame.qp_, + frame_size_bytes = frame.size(), encode_finished_ns]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoCodecTestStats::FrameStatistics* fs = + stats_.GetOrAddFrame(timestamp_rtp, spatial_idx); + VideoCodecTestStats::FrameStatistics* fs_base = + stats_.GetOrAddFrame(timestamp_rtp, 0); + + fs->encode_start_ns = fs_base->encode_start_ns; + fs->spatial_idx = spatial_idx; + fs->temporal_idx = temporal_idx; + fs->frame_type = frame_type; + fs->qp = qp; + + fs->encode_time_us = (encode_finished_ns - fs->encode_start_ns) / + rtc::kNumNanosecsPerMicrosec; + fs->length_bytes = frame_size_bytes; + + fs->encoding_successful = true; + }); +} + +void VideoCodecAnalyzer::StartDecode(const EncodedImage& frame) { + int64_t decode_start_ns = rtc::TimeNanos(); + task_queue_.PostTask([this, timestamp_rtp = frame.Timestamp(), + spatial_idx = frame.SpatialIndex().value_or(0), + frame_size_bytes = frame.size(), decode_start_ns]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoCodecTestStats::FrameStatistics* fs = + stats_.GetOrAddFrame(timestamp_rtp, spatial_idx); + if (fs->length_bytes == 0) { + // In encode-decode test the frame size is set in EncodeFinished. In + // decode-only test set it here. + fs->length_bytes = frame_size_bytes; + } + fs->decode_start_ns = decode_start_ns; + }); +} + +void VideoCodecAnalyzer::FinishDecode(const VideoFrame& frame, + int spatial_idx) { + int64_t decode_finished_ns = rtc::TimeNanos(); + task_queue_.PostTask([this, timestamp_rtp = frame.timestamp(), spatial_idx, + width = frame.width(), height = frame.height(), + decode_finished_ns]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoCodecTestStats::FrameStatistics* fs = + stats_.GetFrameWithTimestamp(timestamp_rtp, spatial_idx); + fs->decode_time_us = (decode_finished_ns - fs->decode_start_ns) / + rtc::kNumNanosecsPerMicrosec; + fs->decoded_width = width; + fs->decoded_height = height; + fs->decoding_successful = true; + }); + + if (reference_video_source_ != nullptr) { + // Copy hardware-backed frame into main memory to release output buffers + // which number may be limited in hardware decoders. + rtc::scoped_refptr decoded_buffer = + frame.video_frame_buffer()->ToI420(); + + task_queue_.PostTask([this, decoded_buffer, + timestamp_rtp = frame.timestamp(), spatial_idx]() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoFrame ref_frame = reference_video_source_->GetFrame( + timestamp_rtp, {.width = decoded_buffer->width(), + .height = decoded_buffer->height()}); + rtc::scoped_refptr ref_buffer = + ref_frame.video_frame_buffer()->ToI420(); + + Psnr psnr = CalcPsnr(*decoded_buffer, *ref_buffer); + VideoCodecTestStats::FrameStatistics* fs = + this->stats_.GetFrameWithTimestamp(timestamp_rtp, spatial_idx); + fs->psnr_y = static_cast(psnr.y); + fs->psnr_u = static_cast(psnr.u); + fs->psnr_v = static_cast(psnr.v); + fs->psnr = static_cast(psnr.yuv); + + fs->quality_analysis_successful = true; + }); + } +} + +std::unique_ptr VideoCodecAnalyzer::GetStats() { + std::unique_ptr stats; + rtc::Event ready; + task_queue_.PostTask([this, &stats, &ready]() mutable { + RTC_DCHECK_RUN_ON(&sequence_checker_); + stats.reset(new VideoCodecTestStatsImpl(stats_)); + ready.Set(); + }); + ready.Wait(rtc::Event::kForever); + return stats; +} + +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_analyzer.h b/modules/video_coding/codecs/test/video_codec_analyzer.h new file mode 100644 index 0000000000..63a864e810 --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_analyzer.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ +#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ + +#include + +#include "absl/types/optional.h" +#include "api/sequence_checker.h" +#include "api/video/encoded_image.h" +#include "api/video/resolution.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/codecs/test/videocodec_test_stats_impl.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system/no_unique_address.h" +#include "rtc_base/task_queue_for_test.h" + +namespace webrtc { +namespace test { + +// Analyzer measures and collects metrics necessary for evaluation of video +// codec quality and performance. This class is thread-safe. +class VideoCodecAnalyzer { + public: + // An interface that provides reference frames for spatial quality analysis. + class ReferenceVideoSource { + public: + virtual ~ReferenceVideoSource() = default; + + virtual VideoFrame GetFrame(uint32_t timestamp_rtp, + Resolution resolution) = 0; + }; + + VideoCodecAnalyzer(rtc::TaskQueue& task_queue, + ReferenceVideoSource* reference_video_source = nullptr); + + void StartEncode(const VideoFrame& frame); + + void FinishEncode(const EncodedImage& frame); + + void StartDecode(const EncodedImage& frame); + + void FinishDecode(const VideoFrame& frame, int spatial_idx); + + std::unique_ptr GetStats(); + + protected: + rtc::TaskQueue& task_queue_; + ReferenceVideoSource* const reference_video_source_; + VideoCodecTestStatsImpl stats_ RTC_GUARDED_BY(sequence_checker_); + RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ diff --git a/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc b/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc new file mode 100644 index 0000000000..3f9de6dac2 --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/video_codec_analyzer.h" + +#include "absl/types/optional.h" +#include "api/video/i420_buffer.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "third_party/libyuv/include/libyuv/planar_functions.h" + +namespace webrtc { +namespace test { + +namespace { +using ::testing::Return; +using ::testing::Values; + +const size_t kTimestamp = 3000; +const size_t kSpatialIdx = 2; + +class MockReferenceVideoSource + : public VideoCodecAnalyzer::ReferenceVideoSource { + public: + MOCK_METHOD(VideoFrame, GetFrame, (uint32_t, Resolution), (override)); +}; + +VideoFrame CreateVideoFrame(uint32_t timestamp_rtp, + uint8_t y = 0, + uint8_t u = 0, + uint8_t v = 0) { + rtc::scoped_refptr buffer(I420Buffer::Create(2, 2)); + + libyuv::I420Rect(buffer->MutableDataY(), buffer->StrideY(), + buffer->MutableDataU(), buffer->StrideU(), + buffer->MutableDataV(), buffer->StrideV(), 0, 0, + buffer->width(), buffer->height(), y, u, v); + + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .build(); +} + +EncodedImage CreateEncodedImage(uint32_t timestamp_rtp, int spatial_idx = 0) { + EncodedImage encoded_image; + encoded_image.SetTimestamp(timestamp_rtp); + encoded_image.SetSpatialIndex(spatial_idx); + return encoded_image; +} +} // namespace + +TEST(VideoCodecAnalyzerTest, EncodeStartedCreatesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartEncode(CreateVideoFrame(kTimestamp)); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); + EXPECT_EQ(fs[0].rtp_timestamp, kTimestamp); +} + +TEST(VideoCodecAnalyzerTest, EncodeFinishedUpdatesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartEncode(CreateVideoFrame(kTimestamp)); + + EncodedImage encoded_frame = CreateEncodedImage(kTimestamp, kSpatialIdx); + analyzer.FinishEncode(encoded_frame); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(2u, fs.size()); + EXPECT_TRUE(fs[1].encoding_successful); +} + +TEST(VideoCodecAnalyzerTest, DecodeStartedNoFrameStatsCreatesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); + EXPECT_EQ(fs[0].rtp_timestamp, kTimestamp); +} + +TEST(VideoCodecAnalyzerTest, DecodeStartedFrameStatsExistsReusesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartEncode(CreateVideoFrame(kTimestamp)); + analyzer.StartDecode(CreateEncodedImage(kTimestamp, /*spatial_idx=*/0)); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); +} + +TEST(VideoCodecAnalyzerTest, DecodeFinishedUpdatesFrameStats) { + TaskQueueForTest task_queue; + VideoCodecAnalyzer analyzer(task_queue); + analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); + VideoFrame decoded_frame = CreateVideoFrame(kTimestamp); + analyzer.FinishDecode(decoded_frame, kSpatialIdx); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); + + EXPECT_TRUE(fs[0].decoding_successful); + EXPECT_EQ(static_cast(fs[0].decoded_width), decoded_frame.width()); + EXPECT_EQ(static_cast(fs[0].decoded_height), decoded_frame.height()); +} + +TEST(VideoCodecAnalyzerTest, DecodeFinishedComputesPsnr) { + TaskQueueForTest task_queue; + MockReferenceVideoSource reference_video_source; + VideoCodecAnalyzer analyzer(task_queue, &reference_video_source); + analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); + + EXPECT_CALL(reference_video_source, GetFrame) + .WillOnce(Return(CreateVideoFrame(kTimestamp, /*y=*/0, + /*u=*/0, /*v=*/0))); + + analyzer.FinishDecode( + CreateVideoFrame(kTimestamp, /*value_y=*/1, /*value_u=*/2, /*value_v=*/3), + kSpatialIdx); + + auto fs = analyzer.GetStats()->GetFrameStatistics(); + EXPECT_EQ(1u, fs.size()); + + EXPECT_NEAR(fs[0].psnr_y, 48, 1); + EXPECT_NEAR(fs[0].psnr_u, 42, 1); + EXPECT_NEAR(fs[0].psnr_v, 38, 1); +} + +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_test.cc b/modules/video_coding/codecs/test/video_codec_test.cc new file mode 100644 index 0000000000..bd4c8e07f2 --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_test.cc @@ -0,0 +1,456 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/video_codec.h" + +#include +#include +#include +#include + +#include "absl/functional/any_invocable.h" +#include "api/test/create_video_codec_tester.h" +#include "api/test/videocodec_test_stats.h" +#include "api/units/data_rate.h" +#include "api/units/frequency.h" +#include "api/video/i420_buffer.h" +#include "api/video/resolution.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_encoder.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "media/base/media_constants.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "rtc_base/strings/string_builder.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/frame_reader.h" + +namespace webrtc { +namespace test { + +namespace { +using ::testing::Combine; +using ::testing::Values; +using Layer = std::pair; + +struct VideoInfo { + std::string name; + Resolution resolution; +}; + +struct CodecInfo { + std::string type; + std::string encoder; + std::string decoder; +}; + +struct EncodingSettings { + ScalabilityMode scalability_mode; + // Spatial layer resolution. + std::map resolution; + // Top temporal layer frame rate. + Frequency framerate; + // Bitrate of spatial and temporal layers. + std::map bitrate; +}; + +struct EncodingTestSettings { + std::string name; + int num_frames = 1; + std::map frame_settings; +}; + +struct DecodingTestSettings { + std::string name; +}; + +struct QualityExpectations { + double min_apsnr_y; +}; + +struct EncodeDecodeTestParams { + CodecInfo codec; + VideoInfo video; + VideoCodecTester::EncoderSettings encoder_settings; + VideoCodecTester::DecoderSettings decoder_settings; + EncodingTestSettings encoding_settings; + DecodingTestSettings decoding_settings; + QualityExpectations quality_expectations; +}; + +const EncodingSettings kQvga64Kbps30Fps = { + .scalability_mode = ScalabilityMode::kL1T1, + .resolution = {{0, {.width = 320, .height = 180}}}, + .framerate = Frequency::Hertz(30), + .bitrate = {{Layer(0, 0), DataRate::KilobitsPerSec(64)}}}; + +const EncodingTestSettings kConstantRateQvga64Kbps30Fps = { + .name = "ConstantRateQvga64Kbps30Fps", + .num_frames = 300, + .frame_settings = {{/*frame_num=*/0, kQvga64Kbps30Fps}}}; + +const QualityExpectations kLowQuality = {.min_apsnr_y = 30}; + +const VideoInfo kFourPeople_1280x720_30 = { + .name = "FourPeople_1280x720_30", + .resolution = {.width = 1280, .height = 720}}; + +const CodecInfo kLibvpxVp8 = {.type = "VP8", + .encoder = "libvpx", + .decoder = "libvpx"}; + +const CodecInfo kLibvpxVp9 = {.type = "VP9", + .encoder = "libvpx", + .decoder = "libvpx"}; + +const CodecInfo kOpenH264 = {.type = "H264", + .encoder = "openh264", + .decoder = "ffmpeg"}; + +class TestRawVideoSource : public VideoCodecTester::RawVideoSource { + public: + static constexpr Frequency k90kHz = Frequency::Hertz(90000); + + TestRawVideoSource(std::unique_ptr frame_reader, + const EncodingTestSettings& test_settings) + : frame_reader_(std::move(frame_reader)), + test_settings_(test_settings), + frame_num_(0), + timestamp_rtp_(0) { + // Ensure settings for the first frame are provided. + RTC_CHECK_GT(test_settings_.frame_settings.size(), 0u); + RTC_CHECK_EQ(test_settings_.frame_settings.begin()->first, 0); + } + + // Pulls next frame. Frame RTP timestamp is set accordingly to + // `EncodingSettings::framerate`. + absl::optional PullFrame() override { + if (frame_num_ >= test_settings_.num_frames) { + // End of stream. + return absl::nullopt; + } + + EncodingSettings frame_settings = + std::prev(test_settings_.frame_settings.upper_bound(frame_num_)) + ->second; + + int pulled_frame; + auto buffer = frame_reader_->PullFrame( + &pulled_frame, frame_settings.resolution.rbegin()->second, + {.num = 30, .den = static_cast(frame_settings.framerate.hertz())}); + RTC_CHECK(buffer) << "Cannot pull frame " << frame_num_; + + auto frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp_) + .build(); + + pulled_frames_[timestamp_rtp_] = pulled_frame; + timestamp_rtp_ += k90kHz / frame_settings.framerate; + ++frame_num_; + + return frame; + } + + // Reads frame specified by `timestamp_rtp`, scales it to `resolution` and + // returns. Frame with the given `timestamp_rtp` is expected to be pulled + // before. + VideoFrame GetFrame(uint32_t timestamp_rtp, Resolution resolution) override { + RTC_CHECK(pulled_frames_.find(timestamp_rtp) != pulled_frames_.end()) + << "Frame with RTP timestamp " << timestamp_rtp + << " was not pulled before"; + auto buffer = + frame_reader_->ReadFrame(pulled_frames_[timestamp_rtp], resolution); + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .build(); + } + + protected: + std::unique_ptr frame_reader_; + const EncodingTestSettings& test_settings_; + int frame_num_; + uint32_t timestamp_rtp_; + std::map pulled_frames_; +}; + +class TestEncoder : public VideoCodecTester::Encoder, + public EncodedImageCallback { + public: + TestEncoder(std::unique_ptr encoder, + const CodecInfo& codec_info, + const std::map& frame_settings) + : encoder_(std::move(encoder)), + codec_info_(codec_info), + frame_settings_(frame_settings), + frame_num_(0) { + // Ensure settings for the first frame is provided. + RTC_CHECK_GT(frame_settings_.size(), 0u); + RTC_CHECK_EQ(frame_settings_.begin()->first, 0); + + encoder_->RegisterEncodeCompleteCallback(this); + } + + void Encode(const VideoFrame& frame, EncodeCallback callback) override { + callbacks_[frame.timestamp()] = std::move(callback); + + if (auto fs = frame_settings_.find(frame_num_); + fs != frame_settings_.end()) { + if (fs == frame_settings_.begin() || + ConfigChanged(fs->second, std::prev(fs)->second)) { + Configure(fs->second); + } + if (fs == frame_settings_.begin() || + RateChanged(fs->second, std::prev(fs)->second)) { + SetRates(fs->second); + } + } + + int result = encoder_->Encode(frame, nullptr); + RTC_CHECK_EQ(result, WEBRTC_VIDEO_CODEC_OK); + ++frame_num_; + } + + protected: + Result OnEncodedImage(const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) override { + auto cb = callbacks_.find(encoded_image.Timestamp()); + RTC_CHECK(cb != callbacks_.end()); + cb->second(encoded_image); + + callbacks_.erase(callbacks_.begin(), cb); + return Result(Result::Error::OK); + } + + void Configure(const EncodingSettings& es) { + VideoCodec vc; + const Resolution& resolution = es.resolution.rbegin()->second; + vc.width = resolution.width; + vc.height = resolution.height; + const DataRate& bitrate = es.bitrate.rbegin()->second; + vc.startBitrate = bitrate.kbps(); + vc.maxBitrate = bitrate.kbps(); + vc.minBitrate = 0; + vc.maxFramerate = static_cast(es.framerate.hertz()); + vc.active = true; + vc.qpMax = 0; + vc.numberOfSimulcastStreams = 0; + vc.mode = webrtc::VideoCodecMode::kRealtimeVideo; + vc.SetFrameDropEnabled(true); + + vc.codecType = PayloadStringToCodecType(codec_info_.type); + if (vc.codecType == kVideoCodecVP8) { + *(vc.VP8()) = VideoEncoder::GetDefaultVp8Settings(); + } else if (vc.codecType == kVideoCodecVP9) { + *(vc.VP9()) = VideoEncoder::GetDefaultVp9Settings(); + } else if (vc.codecType == kVideoCodecH264) { + *(vc.H264()) = VideoEncoder::GetDefaultH264Settings(); + } + + VideoEncoder::Settings ves( + VideoEncoder::Capabilities(/*loss_notification=*/false), + /*number_of_cores=*/1, + /*max_payload_size=*/1440); + + int result = encoder_->InitEncode(&vc, ves); + RTC_CHECK_EQ(result, WEBRTC_VIDEO_CODEC_OK); + } + + void SetRates(const EncodingSettings& es) { + VideoEncoder::RateControlParameters rc; + int num_spatial_layers = + ScalabilityModeToNumSpatialLayers(es.scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumSpatialLayers(es.scalability_mode); + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + RTC_CHECK(es.bitrate.find(Layer(sidx, tidx)) != es.bitrate.end()) + << "Bitrate for layer S=" << sidx << " T=" << tidx << " is not set"; + rc.bitrate.SetBitrate(sidx, tidx, + es.bitrate.at(Layer(sidx, tidx)).bps()); + } + } + + rc.framerate_fps = es.framerate.millihertz() / 1000.0; + encoder_->SetRates(rc); + } + + bool ConfigChanged(const EncodingSettings& es, + const EncodingSettings& prev_es) const { + return es.scalability_mode != prev_es.scalability_mode || + es.resolution != prev_es.resolution; + } + + bool RateChanged(const EncodingSettings& es, + const EncodingSettings& prev_es) const { + return es.bitrate != prev_es.bitrate || es.framerate != prev_es.framerate; + } + + std::unique_ptr encoder_; + const CodecInfo& codec_info_; + const std::map& frame_settings_; + int frame_num_; + std::map callbacks_; +}; + +class TestDecoder : public VideoCodecTester::Decoder, + public DecodedImageCallback { + public: + TestDecoder(std::unique_ptr decoder, + const CodecInfo& codec_info) + : decoder_(std::move(decoder)), codec_info_(codec_info), frame_num_(0) { + decoder_->RegisterDecodeCompleteCallback(this); + } + void Decode(const EncodedImage& frame, DecodeCallback callback) override { + callbacks_[frame.Timestamp()] = std::move(callback); + + if (frame_num_ == 0) { + Configure(); + } + + decoder_->Decode(frame, /*missing_frames=*/false, + /*render_time_ms=*/0); + ++frame_num_; + } + + void Configure() { + VideoDecoder::Settings ds; + ds.set_codec_type(PayloadStringToCodecType(codec_info_.type)); + ds.set_number_of_cores(1); + + bool result = decoder_->Configure(ds); + RTC_CHECK(result); + } + + protected: + int Decoded(VideoFrame& decoded_frame) override { + auto cb = callbacks_.find(decoded_frame.timestamp()); + RTC_CHECK(cb != callbacks_.end()); + cb->second(decoded_frame); + + callbacks_.erase(callbacks_.begin(), cb); + return WEBRTC_VIDEO_CODEC_OK; + } + + std::unique_ptr decoder_; + const CodecInfo& codec_info_; + int frame_num_; + std::map callbacks_; +}; + +std::unique_ptr CreateEncoder( + const CodecInfo& codec_info, + const std::map& frame_settings) { + auto factory = CreateBuiltinVideoEncoderFactory(); + auto encoder = factory->CreateVideoEncoder(SdpVideoFormat(codec_info.type)); + return std::make_unique(std::move(encoder), codec_info, + frame_settings); +} + +std::unique_ptr CreateDecoder( + const CodecInfo& codec_info) { + auto factory = CreateBuiltinVideoDecoderFactory(); + auto decoder = factory->CreateVideoDecoder(SdpVideoFormat(codec_info.type)); + return std::make_unique(std::move(decoder), codec_info); +} + +} // namespace + +class EncodeDecodeTest + : public ::testing::TestWithParam { + public: + EncodeDecodeTest() : test_params_(GetParam()) {} + + void SetUp() override { + std::unique_ptr frame_reader = + CreateYuvFrameReader(ResourcePath(test_params_.video.name, "yuv"), + test_params_.video.resolution, + YuvFrameReaderImpl::RepeatMode::kPingPong); + video_source_ = std::make_unique( + std::move(frame_reader), test_params_.encoding_settings); + + encoder_ = CreateEncoder(test_params_.codec, + test_params_.encoding_settings.frame_settings); + decoder_ = CreateDecoder(test_params_.codec); + + tester_ = CreateVideoCodecTester(); + } + + static std::string TestParametersToStr( + const ::testing::TestParamInfo& info) { + return std::string(info.param.encoding_settings.name + + info.param.codec.type + info.param.codec.encoder + + info.param.codec.decoder); + } + + protected: + EncodeDecodeTestParams test_params_; + std::unique_ptr video_source_; + std::unique_ptr encoder_; + std::unique_ptr decoder_; + std::unique_ptr tester_; +}; + +TEST_P(EncodeDecodeTest, DISABLED_TestEncodeDecode) { + std::unique_ptr stats = tester_->RunEncodeDecodeTest( + std::move(video_source_), std::move(encoder_), std::move(decoder_), + test_params_.encoder_settings, test_params_.decoder_settings); + + const auto& frame_settings = test_params_.encoding_settings.frame_settings; + for (auto fs = frame_settings.begin(); fs != frame_settings.end(); ++fs) { + int first_frame = fs->first; + int last_frame = std::next(fs) != frame_settings.end() + ? std::next(fs)->first - 1 + : test_params_.encoding_settings.num_frames - 1; + + const EncodingSettings& encoding_settings = fs->second; + auto metrics = stats->CalcVideoStatistic( + first_frame, last_frame, encoding_settings.bitrate.rbegin()->second, + encoding_settings.framerate); + + EXPECT_GE(metrics.avg_psnr_y, + test_params_.quality_expectations.min_apsnr_y); + } +} + +std::list ConstantRateTestParameters() { + std::list test_params; + std::vector codecs = {kLibvpxVp8}; + std::vector videos = {kFourPeople_1280x720_30}; + std::vector> + encoding_settings = {{kConstantRateQvga64Kbps30Fps, kLowQuality}}; + for (const CodecInfo& codec : codecs) { + for (const VideoInfo& video : videos) { + for (const auto& es : encoding_settings) { + EncodeDecodeTestParams p; + p.codec = codec; + p.video = video; + p.encoding_settings = es.first; + p.quality_expectations = es.second; + test_params.push_back(p); + } + } + } + return test_params; +} + +INSTANTIATE_TEST_SUITE_P(ConstantRate, + EncodeDecodeTest, + ::testing::ValuesIn(ConstantRateTestParameters()), + EncodeDecodeTest::TestParametersToStr); +} // namespace test + +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl.cc b/modules/video_coding/codecs/test/video_codec_tester_impl.cc new file mode 100644 index 0000000000..3000c1adee --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_tester_impl.cc @@ -0,0 +1,325 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" + +#include +#include +#include + +#include "api/task_queue/default_task_queue_factory.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/codecs/test/video_codec_analyzer.h" +#include "rtc_base/event.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/sleep.h" + +namespace webrtc { +namespace test { + +namespace { +using RawVideoSource = VideoCodecTester::RawVideoSource; +using CodedVideoSource = VideoCodecTester::CodedVideoSource; +using Decoder = VideoCodecTester::Decoder; +using Encoder = VideoCodecTester::Encoder; +using EncoderSettings = VideoCodecTester::EncoderSettings; +using DecoderSettings = VideoCodecTester::DecoderSettings; +using PacingSettings = VideoCodecTester::PacingSettings; +using PacingMode = PacingSettings::PacingMode; + +constexpr Frequency k90kHz = Frequency::Hertz(90000); + +// A thread-safe wrapper for video source to be shared with the quality analyzer +// that reads reference frames from a separate thread. +class SyncRawVideoSource : public VideoCodecAnalyzer::ReferenceVideoSource { + public: + explicit SyncRawVideoSource(std::unique_ptr video_source) + : video_source_(std::move(video_source)) {} + + absl::optional PullFrame() { + MutexLock lock(&mutex_); + return video_source_->PullFrame(); + } + + VideoFrame GetFrame(uint32_t timestamp_rtp, Resolution resolution) override { + MutexLock lock(&mutex_); + return video_source_->GetFrame(timestamp_rtp, resolution); + } + + protected: + std::unique_ptr video_source_ RTC_GUARDED_BY(mutex_); + Mutex mutex_; +}; + +// Pacer calculates delay necessary to keep frame encode or decode call spaced +// from the previous calls by the pacing time. `Delay` is expected to be called +// as close as possible to posting frame encode or decode task. This class is +// not thread safe. +class Pacer { + public: + explicit Pacer(PacingSettings settings) + : settings_(settings), delay_(TimeDelta::Zero()) {} + TimeDelta Delay(Timestamp beat) { + if (settings_.mode == PacingMode::kNoPacing) { + return TimeDelta::Zero(); + } + + Timestamp now = Timestamp::Micros(rtc::TimeMicros()); + if (prev_time_.has_value()) { + delay_ += PacingTime(beat); + delay_ -= (now - *prev_time_); + if (delay_.ns() < 0) { + delay_ = TimeDelta::Zero(); + } + } + + prev_beat_ = beat; + prev_time_ = now; + return delay_; + } + + private: + TimeDelta PacingTime(Timestamp beat) { + if (settings_.mode == PacingMode::kRealTime) { + return beat - *prev_beat_; + } + RTC_CHECK_EQ(PacingMode::kConstantRate, settings_.mode); + return 1 / settings_.constant_rate; + } + + PacingSettings settings_; + absl::optional prev_beat_; + absl::optional prev_time_; + TimeDelta delay_; +}; + +// Task queue that keeps the number of queued tasks below a certain limit. If +// the limit is reached, posting of a next task is blocked until execution of a +// previously posted task starts. This class is not thread-safe. +class LimitedTaskQueue { + public: + // The codec tester reads frames from video source in the main thread. + // Encoding and decoding are done in separate threads. If encoding or + // decoding is slow, the reading may go far ahead and may buffer too many + // frames in memory. To prevent this we limit the encoding/decoding queue + // size. When the queue is full, the main thread and, hence, reading frames + // from video source is blocked until a previously posted encoding/decoding + // task starts. + static constexpr int kMaxTaskQueueSize = 3; + + explicit LimitedTaskQueue(rtc::TaskQueue& task_queue) + : task_queue_(task_queue), queue_size_(0) {} + + void PostDelayedTask(absl::AnyInvocable task, TimeDelta delay) { + ++queue_size_; + task_queue_.PostDelayedTask( + [this, task = std::move(task)]() mutable { + std::move(task)(); + --queue_size_; + task_executed_.Set(); + }, + delay); + + task_executed_.Reset(); + if (queue_size_ > kMaxTaskQueueSize) { + task_executed_.Wait(rtc::Event::kForever); + } + RTC_CHECK(queue_size_ <= kMaxTaskQueueSize); + } + + void WaitForPreviouslyPostedTasks() { + while (queue_size_ > 0) { + task_executed_.Wait(rtc::Event::kForever); + task_executed_.Reset(); + } + } + + rtc::TaskQueue& task_queue_; + std::atomic_int queue_size_; + rtc::Event task_executed_; +}; + +class TesterDecoder { + public: + TesterDecoder(std::unique_ptr decoder, + VideoCodecAnalyzer* analyzer, + const DecoderSettings& settings, + rtc::TaskQueue& task_queue) + : decoder_(std::move(decoder)), + analyzer_(analyzer), + settings_(settings), + pacer_(settings.pacing), + task_queue_(task_queue) { + RTC_CHECK(analyzer_) << "Analyzer must be provided"; + } + + void Decode(const EncodedImage& frame) { + Timestamp timestamp = Timestamp::Micros((frame.Timestamp() / k90kHz).us()); + + task_queue_.PostDelayedTask( + [this, frame] { + analyzer_->StartDecode(frame); + decoder_->Decode(frame, [this](const VideoFrame& decoded_frame) { + this->analyzer_->FinishDecode(decoded_frame, /*spatial_idx=*/0); + }); + }, + pacer_.Delay(timestamp)); + } + + void Flush() { task_queue_.WaitForPreviouslyPostedTasks(); } + + protected: + std::unique_ptr decoder_; + VideoCodecAnalyzer* const analyzer_; + const DecoderSettings& settings_; + Pacer pacer_; + LimitedTaskQueue task_queue_; +}; + +class TesterEncoder { + public: + TesterEncoder(std::unique_ptr encoder, + TesterDecoder* decoder, + VideoCodecAnalyzer* analyzer, + const EncoderSettings& settings, + rtc::TaskQueue& task_queue) + : encoder_(std::move(encoder)), + decoder_(decoder), + analyzer_(analyzer), + settings_(settings), + pacer_(settings.pacing), + task_queue_(task_queue) { + RTC_CHECK(analyzer_) << "Analyzer must be provided"; + } + + void Encode(const VideoFrame& frame) { + Timestamp timestamp = Timestamp::Micros((frame.timestamp() / k90kHz).us()); + + task_queue_.PostDelayedTask( + [this, frame] { + analyzer_->StartEncode(frame); + encoder_->Encode(frame, [this](const EncodedImage& encoded_frame) { + this->analyzer_->FinishEncode(encoded_frame); + if (decoder_ != nullptr) { + this->decoder_->Decode(encoded_frame); + } + }); + }, + pacer_.Delay(timestamp)); + } + + void Flush() { task_queue_.WaitForPreviouslyPostedTasks(); } + + protected: + std::unique_ptr encoder_; + TesterDecoder* const decoder_; + VideoCodecAnalyzer* const analyzer_; + const EncoderSettings& settings_; + Pacer pacer_; + LimitedTaskQueue task_queue_; +}; + +} // namespace + +VideoCodecTesterImpl::VideoCodecTesterImpl() + : VideoCodecTesterImpl(/*task_queue_factory=*/nullptr) {} + +VideoCodecTesterImpl::VideoCodecTesterImpl(TaskQueueFactory* task_queue_factory) + : task_queue_factory_(task_queue_factory) { + if (task_queue_factory_ == nullptr) { + owned_task_queue_factory_ = CreateDefaultTaskQueueFactory(); + task_queue_factory_ = owned_task_queue_factory_.get(); + } +} + +std::unique_ptr VideoCodecTesterImpl::RunDecodeTest( + std::unique_ptr video_source, + std::unique_ptr decoder, + const DecoderSettings& decoder_settings) { + rtc::TaskQueue analyser_task_queue(task_queue_factory_->CreateTaskQueue( + "Analyzer", TaskQueueFactory::Priority::NORMAL)); + rtc::TaskQueue decoder_task_queue(task_queue_factory_->CreateTaskQueue( + "Decoder", TaskQueueFactory::Priority::NORMAL)); + + VideoCodecAnalyzer perf_analyzer(analyser_task_queue); + TesterDecoder tester_decoder(std::move(decoder), &perf_analyzer, + decoder_settings, decoder_task_queue); + + while (auto frame = video_source->PullFrame()) { + tester_decoder.Decode(*frame); + } + + tester_decoder.Flush(); + + return perf_analyzer.GetStats(); +} + +std::unique_ptr VideoCodecTesterImpl::RunEncodeTest( + std::unique_ptr video_source, + std::unique_ptr encoder, + const EncoderSettings& encoder_settings) { + rtc::TaskQueue analyser_task_queue(task_queue_factory_->CreateTaskQueue( + "Analyzer", TaskQueueFactory::Priority::NORMAL)); + rtc::TaskQueue encoder_task_queue(task_queue_factory_->CreateTaskQueue( + "Encoder", TaskQueueFactory::Priority::NORMAL)); + + SyncRawVideoSource sync_source(std::move(video_source)); + VideoCodecAnalyzer perf_analyzer(analyser_task_queue); + TesterEncoder tester_encoder(std::move(encoder), /*decoder=*/nullptr, + &perf_analyzer, encoder_settings, + encoder_task_queue); + + while (auto frame = sync_source.PullFrame()) { + tester_encoder.Encode(*frame); + } + + tester_encoder.Flush(); + + return perf_analyzer.GetStats(); +} + +std::unique_ptr VideoCodecTesterImpl::RunEncodeDecodeTest( + std::unique_ptr video_source, + std::unique_ptr encoder, + std::unique_ptr decoder, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings) { + rtc::TaskQueue analyser_task_queue(task_queue_factory_->CreateTaskQueue( + "Analyzer", TaskQueueFactory::Priority::NORMAL)); + rtc::TaskQueue decoder_task_queue(task_queue_factory_->CreateTaskQueue( + "Decoder", TaskQueueFactory::Priority::NORMAL)); + rtc::TaskQueue encoder_task_queue(task_queue_factory_->CreateTaskQueue( + "Encoder", TaskQueueFactory::Priority::NORMAL)); + + SyncRawVideoSource sync_source(std::move(video_source)); + VideoCodecAnalyzer perf_analyzer(analyser_task_queue, &sync_source); + TesterDecoder tester_decoder(std::move(decoder), &perf_analyzer, + decoder_settings, decoder_task_queue); + TesterEncoder tester_encoder(std::move(encoder), &tester_decoder, + &perf_analyzer, encoder_settings, + encoder_task_queue); + + while (auto frame = sync_source.PullFrame()) { + tester_encoder.Encode(*frame); + } + + tester_encoder.Flush(); + tester_decoder.Flush(); + + return perf_analyzer.GetStats(); +} + +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl.h b/modules/video_coding/codecs/test/video_codec_tester_impl.h new file mode 100644 index 0000000000..b64adeb882 --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_tester_impl.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ +#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ + +#include + +#include "api/task_queue/task_queue_factory.h" +#include "api/test/video_codec_tester.h" + +namespace webrtc { +namespace test { + +// A stateless implementation of `VideoCodecTester`. This class is thread safe. +class VideoCodecTesterImpl : public VideoCodecTester { + public: + VideoCodecTesterImpl(); + explicit VideoCodecTesterImpl(TaskQueueFactory* task_queue_factory); + + std::unique_ptr RunDecodeTest( + std::unique_ptr video_source, + std::unique_ptr decoder, + const DecoderSettings& decoder_settings) override; + + std::unique_ptr RunEncodeTest( + std::unique_ptr video_source, + std::unique_ptr encoder, + const EncoderSettings& encoder_settings) override; + + std::unique_ptr RunEncodeDecodeTest( + std::unique_ptr video_source, + std::unique_ptr encoder, + std::unique_ptr decoder, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings) override; + + protected: + std::unique_ptr owned_task_queue_factory_; + TaskQueueFactory* task_queue_factory_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc b/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc new file mode 100644 index 0000000000..29fb006fb5 --- /dev/null +++ b/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc @@ -0,0 +1,259 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" + +#include +#include +#include +#include + +#include "api/task_queue/task_queue_factory.h" +#include "api/task_queue/test/mock_task_queue_base.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "rtc_base/fake_clock.h" +#include "rtc_base/gunit.h" +#include "rtc_base/task_queue_for_test.h" +#include "rtc_base/time_utils.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +namespace { +using ::testing::_; +using ::testing::Invoke; +using ::testing::InvokeWithoutArgs; +using ::testing::Return; + +using Decoder = VideoCodecTester::Decoder; +using Encoder = VideoCodecTester::Encoder; +using CodedVideoSource = VideoCodecTester::CodedVideoSource; +using RawVideoSource = VideoCodecTester::RawVideoSource; +using DecoderSettings = VideoCodecTester::DecoderSettings; +using EncoderSettings = VideoCodecTester::EncoderSettings; +using PacingSettings = VideoCodecTester::PacingSettings; +using PacingMode = PacingSettings::PacingMode; + +constexpr Frequency k90kHz = Frequency::Hertz(90000); + +VideoFrame CreateVideoFrame(uint32_t timestamp_rtp) { + rtc::scoped_refptr buffer(I420Buffer::Create(2, 2)); + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .build(); +} + +EncodedImage CreateEncodedImage(uint32_t timestamp_rtp) { + EncodedImage encoded_image; + encoded_image.SetTimestamp(timestamp_rtp); + return encoded_image; +} + +class MockRawVideoSource : public RawVideoSource { + public: + MOCK_METHOD(absl::optional, PullFrame, (), (override)); + MOCK_METHOD(VideoFrame, + GetFrame, + (uint32_t timestamp_rtp, Resolution), + (override)); +}; + +class MockCodedVideoSource : public CodedVideoSource { + public: + MOCK_METHOD(absl::optional, PullFrame, (), (override)); +}; + +class MockDecoder : public Decoder { + public: + MOCK_METHOD(void, + Decode, + (const EncodedImage& frame, DecodeCallback callback), + (override)); +}; + +class MockEncoder : public Encoder { + public: + MOCK_METHOD(void, + Encode, + (const VideoFrame& frame, EncodeCallback callback), + (override)); +}; + +class MockTaskQueueFactory : public TaskQueueFactory { + public: + explicit MockTaskQueueFactory(TaskQueueBase& task_queue) + : task_queue_(task_queue) {} + + std::unique_ptr CreateTaskQueue( + absl::string_view name, + Priority priority) const override { + return std::unique_ptr(&task_queue_); + } + + protected: + TaskQueueBase& task_queue_; +}; +} // namespace + +class VideoCodecTesterImplPacingTest + : public ::testing::TestWithParam, + std::vector, + std::vector>> { + public: + VideoCodecTesterImplPacingTest() + : pacing_settings_(std::get<0>(GetParam())), + frame_timestamp_ms_(std::get<1>(GetParam())), + frame_capture_delay_ms_(std::get<2>(GetParam())), + expected_frame_start_ms_(std::get<3>(GetParam())), + num_frames_(frame_timestamp_ms_.size()), + task_queue_factory_(task_queue_) {} + + void SetUp() override { + ON_CALL(task_queue_, PostTask) + .WillByDefault(Invoke( + [](absl::AnyInvocable task) { std::move(task)(); })); + + ON_CALL(task_queue_, PostDelayedTask) + .WillByDefault( + Invoke([&](absl::AnyInvocable task, TimeDelta delay) { + clock_.AdvanceTime(delay); + std::move(task)(); + })); + } + + protected: + PacingSettings pacing_settings_; + std::vector frame_timestamp_ms_; + std::vector frame_capture_delay_ms_; + std::vector expected_frame_start_ms_; + size_t num_frames_; + + rtc::ScopedFakeClock clock_; + MockTaskQueueBase task_queue_; + MockTaskQueueFactory task_queue_factory_; +}; + +TEST_P(VideoCodecTesterImplPacingTest, PaceEncode) { + auto video_source = std::make_unique(); + + size_t frame_num = 0; + EXPECT_CALL(*video_source, PullFrame).WillRepeatedly(Invoke([&]() mutable { + if (frame_num >= num_frames_) { + return absl::optional(); + } + clock_.AdvanceTime(TimeDelta::Millis(frame_capture_delay_ms_[frame_num])); + + uint32_t timestamp_rtp = frame_timestamp_ms_[frame_num] * k90kHz.hertz() / + rtc::kNumMillisecsPerSec; + ++frame_num; + return absl::optional(CreateVideoFrame(timestamp_rtp)); + })); + + auto encoder = std::make_unique(); + EncoderSettings encoder_settings; + encoder_settings.pacing = pacing_settings_; + + VideoCodecTesterImpl tester(&task_queue_factory_); + auto fs = tester + .RunEncodeTest(std::move(video_source), std::move(encoder), + encoder_settings) + ->GetFrameStatistics(); + ASSERT_EQ(fs.size(), num_frames_); + + for (size_t i = 0; i < fs.size(); ++i) { + int encode_start_ms = (fs[i].encode_start_ns - fs[0].encode_start_ns) / + rtc::kNumNanosecsPerMillisec; + EXPECT_NEAR(encode_start_ms, expected_frame_start_ms_[i], 10); + } +} + +TEST_P(VideoCodecTesterImplPacingTest, PaceDecode) { + auto video_source = std::make_unique(); + + size_t frame_num = 0; + EXPECT_CALL(*video_source, PullFrame).WillRepeatedly(Invoke([&]() mutable { + if (frame_num >= num_frames_) { + return absl::optional(); + } + clock_.AdvanceTime(TimeDelta::Millis(frame_capture_delay_ms_[frame_num])); + + uint32_t timestamp_rtp = frame_timestamp_ms_[frame_num] * k90kHz.hertz() / + rtc::kNumMillisecsPerSec; + ++frame_num; + return absl::optional(CreateEncodedImage(timestamp_rtp)); + })); + + auto decoder = std::make_unique(); + DecoderSettings decoder_settings; + decoder_settings.pacing = pacing_settings_; + + VideoCodecTesterImpl tester(&task_queue_factory_); + auto fs = tester + .RunDecodeTest(std::move(video_source), std::move(decoder), + decoder_settings) + ->GetFrameStatistics(); + ASSERT_EQ(fs.size(), num_frames_); + + for (size_t i = 0; i < fs.size(); ++i) { + int decode_start_ms = (fs[i].decode_start_ns - fs[0].decode_start_ns) / + rtc::kNumNanosecsPerMillisec; + EXPECT_NEAR(decode_start_ms, expected_frame_start_ms_[i], 10); + } +} + +INSTANTIATE_TEST_SUITE_P( + All, + VideoCodecTesterImplPacingTest, + ::testing::ValuesIn( + {std::make_tuple(PacingSettings({.mode = PacingMode::kNoPacing}), + /*frame_timestamp_ms=*/std::vector{0, 100}, + /*frame_capture_delay_ms=*/std::vector{0, 0}, + /*expected_frame_start_ms=*/std::vector{0, 0}), + // Pace with rate equal to the source frame rate. Frames are captured + // instantly. Verify that frames are paced with the source frame rate. + std::make_tuple(PacingSettings({.mode = PacingMode::kRealTime}), + /*frame_timestamp_ms=*/std::vector{0, 100}, + /*frame_capture_delay_ms=*/std::vector{0, 0}, + /*expected_frame_start_ms=*/std::vector{0, 100}), + // Pace with rate equal to the source frame rate. Frame capture is + // delayed by more than pacing time. Verify that no extra delay is + // added. + std::make_tuple(PacingSettings({.mode = PacingMode::kRealTime}), + /*frame_timestamp_ms=*/std::vector{0, 100}, + /*frame_capture_delay_ms=*/std::vector{0, 200}, + /*expected_frame_start_ms=*/std::vector{0, 200}), + // Pace with constant rate less then source frame rate. Frames are + // captured instantly. Verify that frames are paced with the requested + // constant rate. + std::make_tuple( + PacingSettings({.mode = PacingMode::kConstantRate, + .constant_rate = Frequency::Hertz(20)}), + /*frame_timestamp_ms=*/std::vector{0, 100}, + /*frame_capture_delay_ms=*/std::vector{0, 0}, + /*expected_frame_start_ms=*/std::vector{0, 50}), + // Pace with constant rate less then source frame rate. Frame capture + // is delayed by more than the pacing time. Verify that no extra delay + // is added. + std::make_tuple( + PacingSettings({.mode = PacingMode::kConstantRate, + .constant_rate = Frequency::Hertz(20)}), + /*frame_timestamp_ms=*/std::vector{0, 100}, + /*frame_capture_delay_ms=*/std::vector{0, 200}, + /*expected_frame_start_ms=*/std::vector{0, 200})})); +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc b/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc index efb7502e5d..390348b97a 100644 --- a/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc +++ b/modules/video_coding/codecs/test/videocodec_test_stats_impl.cc @@ -58,7 +58,20 @@ FrameStatistics* VideoCodecTestStatsImpl::GetFrameWithTimestamp( return GetFrame(rtp_timestamp_to_frame_num_[layer_idx][timestamp], layer_idx); } -std::vector VideoCodecTestStatsImpl::GetFrameStatistics() { +FrameStatistics* VideoCodecTestStatsImpl::GetOrAddFrame(size_t timestamp_rtp, + size_t spatial_idx) { + if (rtp_timestamp_to_frame_num_[spatial_idx].count(timestamp_rtp) > 0) { + return GetFrameWithTimestamp(timestamp_rtp, spatial_idx); + } + + size_t frame_num = layer_stats_[spatial_idx].size(); + AddFrame(FrameStatistics(frame_num, timestamp_rtp, spatial_idx)); + + return GetFrameWithTimestamp(timestamp_rtp, spatial_idx); +} + +std::vector VideoCodecTestStatsImpl::GetFrameStatistics() + const { size_t capacity = 0; for (const auto& layer_stat : layer_stats_) { capacity += layer_stat.second.size(); @@ -92,7 +105,8 @@ VideoCodecTestStatsImpl::SliceAndCalcLayerVideoStatistic( for (size_t temporal_idx = 0; temporal_idx < num_temporal_layers; ++temporal_idx) { VideoStatistics layer_stat = SliceAndCalcVideoStatistic( - first_frame_num, last_frame_num, spatial_idx, temporal_idx, false); + first_frame_num, last_frame_num, spatial_idx, temporal_idx, false, + /*target_bitrate=*/absl::nullopt, /*target_framerate=*/absl::nullopt); layer_stats.push_back(layer_stat); } } @@ -110,9 +124,24 @@ VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcAggregatedVideoStatistic( RTC_CHECK_GT(num_spatial_layers, 0); RTC_CHECK_GT(num_temporal_layers, 0); - return SliceAndCalcVideoStatistic(first_frame_num, last_frame_num, - num_spatial_layers - 1, - num_temporal_layers - 1, true); + return SliceAndCalcVideoStatistic( + first_frame_num, last_frame_num, num_spatial_layers - 1, + num_temporal_layers - 1, true, /*target_bitrate=*/absl::nullopt, + /*target_framerate=*/absl::nullopt); +} + +VideoStatistics VideoCodecTestStatsImpl::CalcVideoStatistic( + size_t first_frame_num, + size_t last_frame_num, + DataRate target_bitrate, + Frequency target_framerate) { + size_t num_spatial_layers = 0; + size_t num_temporal_layers = 0; + GetNumberOfEncodedLayers(first_frame_num, last_frame_num, &num_spatial_layers, + &num_temporal_layers); + return SliceAndCalcVideoStatistic( + first_frame_num, last_frame_num, num_spatial_layers - 1, + num_temporal_layers - 1, true, target_bitrate, target_framerate); } size_t VideoCodecTestStatsImpl::Size(size_t spatial_idx) { @@ -175,7 +204,9 @@ VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcVideoStatistic( size_t last_frame_num, size_t spatial_idx, size_t temporal_idx, - bool aggregate_independent_layers) { + bool aggregate_independent_layers, + absl::optional target_bitrate, + absl::optional target_framerate) { VideoStatistics video_stat; float buffer_level_bits = 0.0f; @@ -200,8 +231,11 @@ VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcVideoStatistic( FrameStatistics last_successfully_decoded_frame(0, 0, 0); const size_t target_bitrate_kbps = - CalcLayerTargetBitrateKbps(first_frame_num, last_frame_num, spatial_idx, - temporal_idx, aggregate_independent_layers); + target_bitrate.has_value() + ? target_bitrate->kbps() + : CalcLayerTargetBitrateKbps(first_frame_num, last_frame_num, + spatial_idx, temporal_idx, + aggregate_independent_layers); const size_t target_bitrate_bps = 1000 * target_bitrate_kbps; RTC_CHECK_GT(target_bitrate_kbps, 0); // We divide by `target_bitrate_kbps`. @@ -303,7 +337,9 @@ VideoStatistics VideoCodecTestStatsImpl::SliceAndCalcVideoStatistic( GetFrame(first_frame_num, spatial_idx)->rtp_timestamp; RTC_CHECK_GT(timestamp_delta, 0); const float input_framerate_fps = - 1.0 * kVideoPayloadTypeFrequency / timestamp_delta; + target_framerate.has_value() + ? target_framerate->millihertz() / 1000.0 + : 1.0 * kVideoPayloadTypeFrequency / timestamp_delta; RTC_CHECK_GT(input_framerate_fps, 0); const float duration_sec = num_frames / input_framerate_fps; diff --git a/modules/video_coding/codecs/test/videocodec_test_stats_impl.h b/modules/video_coding/codecs/test/videocodec_test_stats_impl.h index 61850d3622..1a7980aa0a 100644 --- a/modules/video_coding/codecs/test/videocodec_test_stats_impl.h +++ b/modules/video_coding/codecs/test/videocodec_test_stats_impl.h @@ -35,8 +35,12 @@ class VideoCodecTestStatsImpl : public VideoCodecTestStats { FrameStatistics* GetFrame(size_t frame_number, size_t spatial_idx); FrameStatistics* GetFrameWithTimestamp(size_t timestamp, size_t spatial_idx); + // Creates FrameStatisticts if it doesn't exists and/or returns + // created/existing FrameStatisticts. + FrameStatistics* GetOrAddFrame(size_t timestamp_rtp, size_t spatial_idx); + // Implements VideoCodecTestStats. - std::vector GetFrameStatistics() override; + std::vector GetFrameStatistics() const override; std::vector SliceAndCalcLayerVideoStatistic( size_t first_frame_num, size_t last_frame_num) override; @@ -44,6 +48,11 @@ class VideoCodecTestStatsImpl : public VideoCodecTestStats { VideoStatistics SliceAndCalcAggregatedVideoStatistic(size_t first_frame_num, size_t last_frame_num); + VideoStatistics CalcVideoStatistic(size_t first_frame, + size_t last_frame, + DataRate target_bitrate, + Frequency target_framerate) override; + size_t Size(size_t spatial_idx); void Clear(); @@ -65,7 +74,9 @@ class VideoCodecTestStatsImpl : public VideoCodecTestStats { size_t last_frame_num, size_t spatial_idx, size_t temporal_idx, - bool aggregate_independent_layers); + bool aggregate_independent_layers, + absl::optional target_bitrate, + absl::optional target_framerate); void GetNumberOfEncodedLayers(size_t first_frame_num, size_t last_frame_num, diff --git a/modules/video_coding/codecs/test/videocodec_test_stats_impl_unittest.cc b/modules/video_coding/codecs/test/videocodec_test_stats_impl_unittest.cc index 6477b6ab8c..89e7d2e1c4 100644 --- a/modules/video_coding/codecs/test/videocodec_test_stats_impl_unittest.cc +++ b/modules/video_coding/codecs/test/videocodec_test_stats_impl_unittest.cc @@ -38,6 +38,21 @@ TEST(StatsTest, AddAndGetFrame) { EXPECT_EQ(kTimestamp, frame_stat->rtp_timestamp); } +TEST(StatsTest, GetOrAddFrame_noFrame_createsNewFrameStat) { + VideoCodecTestStatsImpl stats; + stats.GetOrAddFrame(kTimestamp, 0); + FrameStatistics* frame_stat = stats.GetFrameWithTimestamp(kTimestamp, 0); + EXPECT_EQ(kTimestamp, frame_stat->rtp_timestamp); +} + +TEST(StatsTest, GetOrAddFrame_frameExists_returnsExistingFrameStat) { + VideoCodecTestStatsImpl stats; + stats.AddFrame(FrameStatistics(0, kTimestamp, 0)); + FrameStatistics* frame_stat1 = stats.GetFrameWithTimestamp(kTimestamp, 0); + FrameStatistics* frame_stat2 = stats.GetOrAddFrame(kTimestamp, 0); + EXPECT_EQ(frame_stat1, frame_stat2); +} + TEST(StatsTest, AddAndGetFrames) { VideoCodecTestStatsImpl stats; const size_t kNumFrames = 1000;