From d431156c0e227313c4c76d9e06ae052989b22c86 Mon Sep 17 00:00:00 2001 From: Sergey Silkin Date: Mon, 13 Nov 2023 15:48:25 +0100 Subject: [PATCH] Move codecs handling from test to tester * Pass codec factories to the video codec tester instead of creating and wrapping codecs into a tester-specific wrappers in video_codec_test.cc. The motivation for this change is to simplify the tests by moving complexity to the tester. * Merge codec stats and analysis into the tester and move the tester. The merge fixes circular deps issues. Modularization is not strictly needed for testing framework like the video codec tester. It is still possible to unit test underlaying modules with rather small overhead. * Move the video codec tester from api/ to test/. test/ is accessible from outside of WebRTC which enables reusing the tester in downstream projects. Test output ~matches before and after this refactoring. There is a small difference that is caused by changes in qpMax: 63 -> 56 (kDefaultVideoMaxQpVpx). 56 is what WebRTC uses by default for VPx/AV1 encoders. Bug: webrtc:14852 Change-Id: I762707b7144fcff870119ad741ebe7091ea109ba Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/327260 Reviewed-by: Rasmus Brandt Commit-Queue: Sergey Silkin Reviewed-by: Mirko Bonadei Cr-Commit-Position: refs/heads/main@{#41144} --- api/BUILD.gn | 48 - api/test/create_video_codec_tester.cc | 27 - api/test/create_video_codec_tester.h | 26 - api/test/video_codec_stats.cc | 97 -- api/test/video_codec_stats.h | 120 -- api/test/video_codec_tester.h | 148 -- modules/video_coding/BUILD.gn | 64 +- .../codecs/test/video_codec_analyzer.cc | 193 --- .../codecs/test/video_codec_analyzer.h | 75 - .../test/video_codec_analyzer_unittest.cc | 127 -- .../codecs/test/video_codec_stats_impl.cc | 278 ---- .../codecs/test/video_codec_stats_impl.h | 62 - .../test/video_codec_stats_impl_unittest.cc | 148 -- .../codecs/test/video_codec_test.cc | 705 +++------ .../codecs/test/video_codec_tester_impl.cc | 437 ------ .../codecs/test/video_codec_tester_impl.h | 45 - .../test/video_codec_tester_impl_unittest.cc | 205 --- test/BUILD.gn | 51 + test/video_codec_tester.cc | 1260 +++++++++++++++++ test/video_codec_tester.h | 221 +++ test/video_codec_tester_unittest.cc | 513 +++++++ 21 files changed, 2216 insertions(+), 2634 deletions(-) delete mode 100644 api/test/create_video_codec_tester.cc delete mode 100644 api/test/create_video_codec_tester.h delete mode 100644 api/test/video_codec_stats.cc delete mode 100644 api/test/video_codec_stats.h delete mode 100644 api/test/video_codec_tester.h delete mode 100644 modules/video_coding/codecs/test/video_codec_analyzer.cc delete mode 100644 modules/video_coding/codecs/test/video_codec_analyzer.h delete mode 100644 modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc delete mode 100644 modules/video_coding/codecs/test/video_codec_stats_impl.cc delete mode 100644 modules/video_coding/codecs/test/video_codec_stats_impl.h delete mode 100644 modules/video_coding/codecs/test/video_codec_stats_impl_unittest.cc delete mode 100644 modules/video_coding/codecs/test/video_codec_tester_impl.cc delete mode 100644 modules/video_coding/codecs/test/video_codec_tester_impl.h delete mode 100644 modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc create mode 100644 test/video_codec_tester.cc create mode 100644 test/video_codec_tester.h create mode 100644 test/video_codec_tester_unittest.cc diff --git a/api/BUILD.gn b/api/BUILD.gn index a163315440..ff461b8e73 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -1071,24 +1071,6 @@ if (rtc_include_tests) { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } - rtc_library("video_codec_stats_api") { - visibility = [ "*" ] - testonly = true - sources = [ - "test/video_codec_stats.cc", - "test/video_codec_stats.h", - ] - deps = [ - "../api/numerics:numerics", - "../api/units:data_rate", - "../api/units:data_size", - "../api/units:frequency", - "test/metrics:metric", - "test/metrics:metrics_logger", - ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] - } - rtc_library("videocodec_test_fixture_api") { visibility = [ "*" ] testonly = true @@ -1100,23 +1082,6 @@ if (rtc_include_tests) { ] } - rtc_library("video_codec_tester_api") { - visibility = [ "*" ] - testonly = true - sources = [ "test/video_codec_tester.h" ] - deps = [ - ":video_codec_stats_api", - "../modules/video_coding/svc:scalability_mode_util", - "video:encoded_image", - "video:resolution", - "video:video_frame", - ] - absl_deps = [ - "//third_party/abseil-cpp/absl/functional:any_invocable", - "//third_party/abseil-cpp/absl/types:optional", - ] - } - rtc_library("create_videocodec_test_fixture_api") { visibility = [ "*" ] testonly = true @@ -1132,19 +1097,6 @@ if (rtc_include_tests) { ] } - rtc_library("create_video_codec_tester_api") { - visibility = [ "*" ] - testonly = true - sources = [ - "test/create_video_codec_tester.cc", - "test/create_video_codec_tester.h", - ] - deps = [ - ":video_codec_tester_api", - "../modules/video_coding:video_codec_tester", - ] - } - rtc_source_set("mock_audio_mixer") { visibility = [ "*" ] testonly = true diff --git a/api/test/create_video_codec_tester.cc b/api/test/create_video_codec_tester.cc deleted file mode 100644 index a1efefdb48..0000000000 --- a/api/test/create_video_codec_tester.cc +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "api/test/create_video_codec_tester.h" - -#include -#include - -#include "api/test/video_codec_tester.h" -#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" - -namespace webrtc { -namespace test { - -std::unique_ptr CreateVideoCodecTester() { - return std::make_unique(); -} - -} // namespace test -} // namespace webrtc diff --git a/api/test/create_video_codec_tester.h b/api/test/create_video_codec_tester.h deleted file mode 100644 index c68864ce85..0000000000 --- a/api/test/create_video_codec_tester.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ -#define API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ - -#include - -#include "api/test/video_codec_tester.h" - -namespace webrtc { -namespace test { - -std::unique_ptr CreateVideoCodecTester(); - -} // namespace test -} // namespace webrtc - -#endif // API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ diff --git a/api/test/video_codec_stats.cc b/api/test/video_codec_stats.cc deleted file mode 100644 index fb7226701e..0000000000 --- a/api/test/video_codec_stats.cc +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "api/test/video_codec_stats.h" - -namespace webrtc { -namespace test { - -void VideoCodecStats::Stream::LogMetrics( - MetricsLogger* logger, - std::string test_case_name, - std::map metadata) const { - logger->LogMetric("width", test_case_name, width, Unit::kCount, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("height", test_case_name, height, Unit::kCount, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric( - "frame_size_bytes", test_case_name, frame_size_bytes, Unit::kBytes, - webrtc::test::ImprovementDirection::kNeitherIsBetter, metadata); - - logger->LogMetric("keyframe", test_case_name, keyframe, Unit::kCount, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("qp", test_case_name, qp, Unit::kUnitless, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric( - "encode_time_ms", test_case_name, encode_time_ms, Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter, metadata); - - logger->LogMetric( - "decode_time_ms", test_case_name, decode_time_ms, Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter, metadata); - - logger->LogMetric("target_bitrate_kbps", test_case_name, target_bitrate_kbps, - Unit::kKilobitsPerSecond, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("target_framerate_fps", test_case_name, - target_framerate_fps, Unit::kHertz, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("encoded_bitrate_kbps", test_case_name, - encoded_bitrate_kbps, Unit::kKilobitsPerSecond, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("encoded_framerate_fps", test_case_name, - encoded_framerate_fps, Unit::kHertz, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("bitrate_mismatch_pct", test_case_name, - bitrate_mismatch_pct, Unit::kPercent, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("framerate_mismatch_pct", test_case_name, - framerate_mismatch_pct, Unit::kPercent, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("transmission_time_ms", test_case_name, - transmission_time_ms, Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("psnr_y_db", test_case_name, psnr.y, Unit::kUnitless, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("psnr_u_db", test_case_name, psnr.u, Unit::kUnitless, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("psnr_v_db", test_case_name, psnr.v, Unit::kUnitless, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); -} - -} // namespace test -} // namespace webrtc diff --git a/api/test/video_codec_stats.h b/api/test/video_codec_stats.h deleted file mode 100644 index 80f8287848..0000000000 --- a/api/test/video_codec_stats.h +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef API_TEST_VIDEO_CODEC_STATS_H_ -#define API_TEST_VIDEO_CODEC_STATS_H_ - -#include -#include -#include - -#include "absl/types/optional.h" -#include "api/numerics/samples_stats_counter.h" -#include "api/test/metrics/metric.h" -#include "api/test/metrics/metrics_logger.h" -#include "api/units/data_rate.h" -#include "api/units/data_size.h" -#include "api/units/frequency.h" - -namespace webrtc { -namespace test { - -// Interface for encoded and/or decoded video frame and stream statistics. -class VideoCodecStats { - public: - // Filter for slicing frames. - struct Filter { - absl::optional first_frame; - absl::optional last_frame; - absl::optional spatial_idx; - absl::optional temporal_idx; - }; - - struct Frame { - int frame_num = 0; - uint32_t timestamp_rtp = 0; - - int spatial_idx = 0; - int temporal_idx = 0; - - int width = 0; - int height = 0; - DataSize frame_size = DataSize::Zero(); - bool keyframe = false; - absl::optional qp; - absl::optional base_spatial_idx; - - Timestamp encode_start = Timestamp::Zero(); - TimeDelta encode_time = TimeDelta::Zero(); - Timestamp decode_start = Timestamp::Zero(); - TimeDelta decode_time = TimeDelta::Zero(); - - struct Psnr { - double y = 0.0; - double u = 0.0; - double v = 0.0; - }; - absl::optional psnr; - - absl::optional target_bitrate; - absl::optional target_framerate; - - bool encoded = false; - bool decoded = false; - }; - - struct Stream { - SamplesStatsCounter width; - SamplesStatsCounter height; - SamplesStatsCounter frame_size_bytes; - SamplesStatsCounter keyframe; - SamplesStatsCounter qp; - - SamplesStatsCounter encode_time_ms; - SamplesStatsCounter decode_time_ms; - - SamplesStatsCounter target_bitrate_kbps; - SamplesStatsCounter target_framerate_fps; - - SamplesStatsCounter encoded_bitrate_kbps; - SamplesStatsCounter encoded_framerate_fps; - - SamplesStatsCounter bitrate_mismatch_pct; - SamplesStatsCounter framerate_mismatch_pct; - - SamplesStatsCounter transmission_time_ms; - - struct Psnr { - SamplesStatsCounter y; - SamplesStatsCounter u; - SamplesStatsCounter v; - } psnr; - - // Logs `Stream` metrics to provided `MetricsLogger`. - void LogMetrics(MetricsLogger* logger, - std::string test_case_name, - std::map metadata = {}) const; - }; - - virtual ~VideoCodecStats() = default; - - // Returns frames from interval, spatial and temporal layer specified by given - // `filter`. - virtual std::vector Slice( - absl::optional filter = absl::nullopt) const = 0; - - // Returns video statistics aggregated for given `frames`. - virtual Stream Aggregate(const std::vector& frames) const = 0; -}; - -} // namespace test -} // namespace webrtc - -#endif // API_TEST_VIDEO_CODEC_STATS_H_ diff --git a/api/test/video_codec_tester.h b/api/test/video_codec_tester.h deleted file mode 100644 index c2fb89e2cb..0000000000 --- a/api/test/video_codec_tester.h +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef API_TEST_VIDEO_CODEC_TESTER_H_ -#define API_TEST_VIDEO_CODEC_TESTER_H_ - -#include -#include - -#include "absl/functional/any_invocable.h" -#include "absl/types/optional.h" -#include "api/test/video_codec_stats.h" -#include "api/video/encoded_image.h" -#include "api/video/resolution.h" -#include "api/video/video_frame.h" - -namespace webrtc { -namespace test { - -// Interface for a video codec tester. The interface provides minimalistic set -// of data structures that enables implementation of decode-only, encode-only -// and encode-decode tests. -class VideoCodecTester { - public: - // Pacing settings for codec input. - struct PacingSettings { - enum PacingMode { - // Pacing is not used. Frames are sent to codec back-to-back. - kNoPacing, - // Pace with the rate equal to the target video frame rate. Pacing time is - // derived from RTP timestamp. - kRealTime, - // Pace with the explicitly provided rate. - kConstantRate, - }; - PacingMode mode = PacingMode::kNoPacing; - // Pacing rate for `kConstantRate` mode. - Frequency constant_rate = Frequency::Zero(); - }; - - struct DecoderSettings { - PacingSettings pacing; - absl::optional decoder_input_base_path; - absl::optional decoder_output_base_path; - }; - - struct EncoderSettings { - PacingSettings pacing; - absl::optional encoder_input_base_path; - absl::optional encoder_output_base_path; - }; - - virtual ~VideoCodecTester() = default; - - // Interface for a raw video frames source. - class RawVideoSource { - public: - virtual ~RawVideoSource() = default; - - // Returns next frame. If no more frames to pull, returns `absl::nullopt`. - // For analysis and pacing purposes, frame must have RTP timestamp set. The - // timestamp must represent the target video frame rate and be unique. - virtual absl::optional PullFrame() = 0; - - // Returns early pulled frame with RTP timestamp equal to `timestamp_rtp`. - virtual VideoFrame GetFrame(uint32_t timestamp_rtp, - Resolution resolution) = 0; - }; - - // Interface for a coded video frames source. - class CodedVideoSource { - public: - virtual ~CodedVideoSource() = default; - - // Returns next frame. If no more frames to pull, returns `absl::nullopt`. - // For analysis and pacing purposes, frame must have RTP timestamp set. The - // timestamp must represent the target video frame rate and be unique. - virtual absl::optional PullFrame() = 0; - }; - - // Interface for a video encoder. - class Encoder { - public: - using EncodeCallback = - absl::AnyInvocable; - - virtual ~Encoder() = default; - - virtual void Initialize() = 0; - - virtual void Encode(const VideoFrame& frame, EncodeCallback callback) = 0; - - virtual void Flush() = 0; - }; - - // Interface for a video decoder. - class Decoder { - public: - using DecodeCallback = - absl::AnyInvocable; - - virtual ~Decoder() = default; - - virtual void Initialize() = 0; - - virtual void Decode(const EncodedImage& frame, DecodeCallback callback) = 0; - - virtual void Flush() = 0; - }; - - // Pulls coded video frames from `video_source` and passes them to `decoder`. - // Returns `VideoCodecTestStats` object that contains collected per-frame - // metrics. - virtual std::unique_ptr RunDecodeTest( - CodedVideoSource* video_source, - Decoder* decoder, - const DecoderSettings& decoder_settings) = 0; - - // Pulls raw video frames from `video_source` and passes them to `encoder`. - // Returns `VideoCodecTestStats` object that contains collected per-frame - // metrics. - virtual std::unique_ptr RunEncodeTest( - RawVideoSource* video_source, - Encoder* encoder, - const EncoderSettings& encoder_settings) = 0; - - // Pulls raw video frames from `video_source`, passes them to `encoder` and - // then passes encoded frames to `decoder`. Returns `VideoCodecTestStats` - // object that contains collected per-frame metrics. - virtual std::unique_ptr RunEncodeDecodeTest( - RawVideoSource* video_source, - Encoder* encoder, - Decoder* decoder, - const EncoderSettings& encoder_settings, - const DecoderSettings& decoder_settings) = 0; -}; - -} // namespace test -} // namespace webrtc - -#endif // API_TEST_VIDEO_CODEC_TESTER_H_ diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index a3f2befae9..c75b433cd4 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -851,8 +851,6 @@ if (rtc_include_tests) { "../../api:frame_generator_api", "../../api:scoped_refptr", "../../api:sequence_checker", - "../../api:video_codec_stats_api", - "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", "../../api/numerics:numerics", "../../api/task_queue", @@ -994,46 +992,6 @@ if (rtc_include_tests) { ] } - rtc_library("video_codec_tester") { - testonly = true - sources = [ - "codecs/test/video_codec_analyzer.cc", - "codecs/test/video_codec_analyzer.h", - "codecs/test/video_codec_stats_impl.cc", - "codecs/test/video_codec_stats_impl.h", - "codecs/test/video_codec_tester_impl.cc", - "codecs/test/video_codec_tester_impl.h", - ] - - deps = [ - ":video_coding_utility", - "../../api:sequence_checker", - "../../api:video_codec_stats_api", - "../../api:video_codec_tester_api", - "../../api/numerics:numerics", - "../../api/task_queue:default_task_queue_factory", - "../../api/test/metrics:metrics_logger", - "../../api/units:data_rate", - "../../api/units:frequency", - "../../api/units:time_delta", - "../../api/units:timestamp", - "../../api/video:encoded_image", - "../../api/video:resolution", - "../../api/video:video_codec_constants", - "../../api/video:video_frame", - "../../rtc_base:checks", - "../../rtc_base:rtc_event", - "../../rtc_base:task_queue_for_test", - "../../rtc_base:timeutils", - "../../rtc_base/system:no_unique_address", - "../../system_wrappers", - "../../test:video_test_support", - "//third_party/libyuv", - ] - - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] - } - rtc_test("video_codec_perf_tests") { testonly = true @@ -1041,28 +999,18 @@ if (rtc_include_tests) { deps = [ ":video_codec_interface", - ":video_codec_tester", - "../../api:create_video_codec_tester_api", - "../../api:video_codec_tester_api", - "../../api:videocodec_test_stats_api", "../../api/test/metrics:global_metrics_logger_and_exporter", "../../api/units:data_rate", "../../api/units:frequency", - "../../api/video:encoded_image", "../../api/video:resolution", - "../../api/video:video_frame", - "../../api/video_codecs:scalability_mode", - "../../api/video_codecs:video_codecs_api", - "../../media:rtc_internal_video_codecs", + "../../api/video_codecs:builtin_video_decoder_factory", + "../../api/video_codecs:builtin_video_encoder_factory", "../../rtc_base:logging", "../../test:fileutils", "../../test:test_flags", "../../test:test_main", "../../test:test_support", - "../../test:video_test_support", - "../rtp_rtcp:rtp_rtcp_format", - "svc:scalability_mode_util", - "//third_party/libyuv", + "../../test:video_codec_tester", ] if (is_android) { @@ -1190,9 +1138,6 @@ if (rtc_include_tests) { sources = [ "chain_diff_calculator_unittest.cc", - "codecs/test/video_codec_analyzer_unittest.cc", - "codecs/test/video_codec_stats_impl_unittest.cc", - "codecs/test/video_codec_tester_impl_unittest.cc", "codecs/test/videocodec_test_fixture_config_unittest.cc", "codecs/test/videocodec_test_stats_impl_unittest.cc", "codecs/test/videoprocessor_unittest.cc", @@ -1247,7 +1192,6 @@ if (rtc_include_tests) { ":packet_buffer", ":simulcast_test_fixture_impl", ":video_codec_interface", - ":video_codec_tester", ":video_codecs_test_framework", ":video_coding", ":video_coding_legacy", @@ -1270,7 +1214,6 @@ if (rtc_include_tests) { "../../api:rtp_packet_info", "../../api:scoped_refptr", "../../api:simulcast_test_fixture_api", - "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", "../../api/task_queue", "../../api/task_queue:default_task_queue_factory", @@ -1296,6 +1239,7 @@ if (rtc_include_tests) { "../../common_video/generic_frame_descriptor", "../../common_video/test:utilities", "../../media:media_constants", + "../../media:rtc_internal_video_codecs", "../../media:rtc_media_base", "../../rtc_base:checks", "../../rtc_base:gunit_helpers", diff --git a/modules/video_coding/codecs/test/video_codec_analyzer.cc b/modules/video_coding/codecs/test/video_codec_analyzer.cc deleted file mode 100644 index 772c15734a..0000000000 --- a/modules/video_coding/codecs/test/video_codec_analyzer.cc +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/video_codec_analyzer.h" - -#include - -#include "api/task_queue/default_task_queue_factory.h" -#include "api/video/i420_buffer.h" -#include "api/video/video_codec_constants.h" -#include "api/video/video_frame.h" -#include "rtc_base/checks.h" -#include "rtc_base/event.h" -#include "rtc_base/time_utils.h" -#include "third_party/libyuv/include/libyuv/compare.h" - -namespace webrtc { -namespace test { - -namespace { -using Psnr = VideoCodecStats::Frame::Psnr; - -Psnr CalcPsnr(const I420BufferInterface& ref_buffer, - const I420BufferInterface& dec_buffer) { - RTC_CHECK_EQ(ref_buffer.width(), dec_buffer.width()); - RTC_CHECK_EQ(ref_buffer.height(), dec_buffer.height()); - - uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane( - dec_buffer.DataY(), dec_buffer.StrideY(), ref_buffer.DataY(), - ref_buffer.StrideY(), dec_buffer.width(), dec_buffer.height()); - - uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane( - dec_buffer.DataU(), dec_buffer.StrideU(), ref_buffer.DataU(), - ref_buffer.StrideU(), dec_buffer.width() / 2, dec_buffer.height() / 2); - - uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane( - dec_buffer.DataV(), dec_buffer.StrideV(), ref_buffer.DataV(), - ref_buffer.StrideV(), dec_buffer.width() / 2, dec_buffer.height() / 2); - - int num_y_samples = dec_buffer.width() * dec_buffer.height(); - Psnr psnr; - psnr.y = libyuv::SumSquareErrorToPsnr(sse_y, num_y_samples); - psnr.u = libyuv::SumSquareErrorToPsnr(sse_u, num_y_samples / 4); - psnr.v = libyuv::SumSquareErrorToPsnr(sse_v, num_y_samples / 4); - - return psnr; -} - -} // namespace - -VideoCodecAnalyzer::VideoCodecAnalyzer( - ReferenceVideoSource* reference_video_source) - : reference_video_source_(reference_video_source), num_frames_(0) { - sequence_checker_.Detach(); -} - -void VideoCodecAnalyzer::StartEncode(const VideoFrame& input_frame) { - int64_t encode_start_us = rtc::TimeMicros(); - task_queue_.PostTask( - [this, timestamp_rtp = input_frame.timestamp(), encode_start_us]() { - RTC_DCHECK_RUN_ON(&sequence_checker_); - - RTC_CHECK(frame_num_.find(timestamp_rtp) == frame_num_.end()); - frame_num_[timestamp_rtp] = num_frames_++; - - stats_.AddFrame({.frame_num = frame_num_[timestamp_rtp], - .timestamp_rtp = timestamp_rtp, - .encode_start = Timestamp::Micros(encode_start_us)}); - }); -} - -void VideoCodecAnalyzer::FinishEncode(const EncodedImage& frame) { - int64_t encode_finished_us = rtc::TimeMicros(); - - task_queue_.PostTask([this, timestamp_rtp = frame.RtpTimestamp(), - spatial_idx = frame.SpatialIndex().value_or(0), - temporal_idx = frame.TemporalIndex().value_or(0), - width = frame._encodedWidth, - height = frame._encodedHeight, - frame_type = frame._frameType, - frame_size_bytes = frame.size(), qp = frame.qp_, - encode_finished_us]() { - RTC_DCHECK_RUN_ON(&sequence_checker_); - - if (spatial_idx > 0) { - VideoCodecStats::Frame* base_frame = - stats_.GetFrame(timestamp_rtp, /*spatial_idx=*/0); - - stats_.AddFrame({.frame_num = base_frame->frame_num, - .timestamp_rtp = timestamp_rtp, - .spatial_idx = spatial_idx, - .encode_start = base_frame->encode_start}); - } - - VideoCodecStats::Frame* fs = stats_.GetFrame(timestamp_rtp, spatial_idx); - fs->spatial_idx = spatial_idx; - fs->temporal_idx = temporal_idx; - fs->width = width; - fs->height = height; - fs->frame_size = DataSize::Bytes(frame_size_bytes); - fs->qp = qp; - fs->keyframe = frame_type == VideoFrameType::kVideoFrameKey; - fs->encode_time = Timestamp::Micros(encode_finished_us) - fs->encode_start; - fs->encoded = true; - }); -} - -void VideoCodecAnalyzer::StartDecode(const EncodedImage& frame) { - int64_t decode_start_us = rtc::TimeMicros(); - task_queue_.PostTask([this, timestamp_rtp = frame.RtpTimestamp(), - spatial_idx = frame.SpatialIndex().value_or(0), - frame_size_bytes = frame.size(), decode_start_us]() { - RTC_DCHECK_RUN_ON(&sequence_checker_); - - VideoCodecStats::Frame* fs = stats_.GetFrame(timestamp_rtp, spatial_idx); - if (fs == nullptr) { - if (frame_num_.find(timestamp_rtp) == frame_num_.end()) { - frame_num_[timestamp_rtp] = num_frames_++; - } - stats_.AddFrame({.frame_num = frame_num_[timestamp_rtp], - .timestamp_rtp = timestamp_rtp, - .spatial_idx = spatial_idx, - .frame_size = DataSize::Bytes(frame_size_bytes)}); - fs = stats_.GetFrame(timestamp_rtp, spatial_idx); - } - - fs->decode_start = Timestamp::Micros(decode_start_us); - }); -} - -void VideoCodecAnalyzer::FinishDecode(const VideoFrame& frame, - int spatial_idx) { - int64_t decode_finished_us = rtc::TimeMicros(); - task_queue_.PostTask([this, timestamp_rtp = frame.timestamp(), spatial_idx, - width = frame.width(), height = frame.height(), - decode_finished_us]() { - RTC_DCHECK_RUN_ON(&sequence_checker_); - VideoCodecStats::Frame* fs = stats_.GetFrame(timestamp_rtp, spatial_idx); - fs->decode_time = Timestamp::Micros(decode_finished_us) - fs->decode_start; - - if (!fs->encoded) { - fs->width = width; - fs->height = height; - } - - fs->decoded = true; - }); - - if (reference_video_source_ != nullptr) { - // Copy hardware-backed frame into main memory to release output buffers - // which number may be limited in hardware decoders. - rtc::scoped_refptr decoded_buffer = - frame.video_frame_buffer()->ToI420(); - - task_queue_.PostTask([this, decoded_buffer, - timestamp_rtp = frame.timestamp(), spatial_idx]() { - RTC_DCHECK_RUN_ON(&sequence_checker_); - VideoFrame ref_frame = reference_video_source_->GetFrame( - timestamp_rtp, {.width = decoded_buffer->width(), - .height = decoded_buffer->height()}); - rtc::scoped_refptr ref_buffer = - ref_frame.video_frame_buffer()->ToI420(); - - Psnr psnr = CalcPsnr(*decoded_buffer, *ref_buffer); - - VideoCodecStats::Frame* fs = - this->stats_.GetFrame(timestamp_rtp, spatial_idx); - fs->psnr = psnr; - }); - } -} - -std::unique_ptr VideoCodecAnalyzer::GetStats() { - std::unique_ptr stats; - rtc::Event ready; - task_queue_.PostTask([this, &stats, &ready]() mutable { - RTC_DCHECK_RUN_ON(&sequence_checker_); - stats.reset(new VideoCodecStatsImpl(stats_)); - ready.Set(); - }); - ready.Wait(rtc::Event::kForever); - return stats; -} - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_analyzer.h b/modules/video_coding/codecs/test/video_codec_analyzer.h deleted file mode 100644 index 29ca8ee2ff..0000000000 --- a/modules/video_coding/codecs/test/video_codec_analyzer.h +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ -#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ - -#include -#include - -#include "absl/types/optional.h" -#include "api/sequence_checker.h" -#include "api/test/video_codec_tester.h" -#include "api/video/encoded_image.h" -#include "api/video/resolution.h" -#include "api/video/video_frame.h" -#include "modules/video_coding/codecs/test/video_codec_stats_impl.h" -#include "rtc_base/system/no_unique_address.h" -#include "rtc_base/task_queue_for_test.h" - -namespace webrtc { -namespace test { - -// Analyzer measures and collects metrics necessary for evaluation of video -// codec quality and performance. This class is thread-safe. -class VideoCodecAnalyzer { - public: - // An interface that provides reference frames for spatial quality analysis. - class ReferenceVideoSource { - public: - virtual ~ReferenceVideoSource() = default; - - virtual VideoFrame GetFrame(uint32_t timestamp_rtp, - Resolution resolution) = 0; - }; - - explicit VideoCodecAnalyzer( - ReferenceVideoSource* reference_video_source = nullptr); - - void StartEncode(const VideoFrame& frame); - - void FinishEncode(const EncodedImage& frame); - - void StartDecode(const EncodedImage& frame); - - void FinishDecode(const VideoFrame& frame, int spatial_idx); - - std::unique_ptr GetStats(); - - protected: - TaskQueueForTest task_queue_; - - ReferenceVideoSource* const reference_video_source_; - - VideoCodecStatsImpl stats_ RTC_GUARDED_BY(sequence_checker_); - - // Map from RTP timestamp to frame number. - std::map frame_num_ RTC_GUARDED_BY(sequence_checker_); - - // Processed frames counter. - int num_frames_ RTC_GUARDED_BY(sequence_checker_); - - RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; -}; - -} // namespace test -} // namespace webrtc - -#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ diff --git a/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc b/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc deleted file mode 100644 index 03146417da..0000000000 --- a/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/video_codec_analyzer.h" - -#include "absl/types/optional.h" -#include "api/video/i420_buffer.h" -#include "test/gmock.h" -#include "test/gtest.h" -#include "third_party/libyuv/include/libyuv/planar_functions.h" - -namespace webrtc { -namespace test { - -namespace { -using ::testing::Return; -using ::testing::Values; -using Psnr = VideoCodecStats::Frame::Psnr; - -const uint32_t kTimestamp = 3000; -const int kSpatialIdx = 2; - -class MockReferenceVideoSource - : public VideoCodecAnalyzer::ReferenceVideoSource { - public: - MOCK_METHOD(VideoFrame, GetFrame, (uint32_t, Resolution), (override)); -}; - -VideoFrame CreateVideoFrame(uint32_t timestamp_rtp, - uint8_t y = 0, - uint8_t u = 0, - uint8_t v = 0) { - rtc::scoped_refptr buffer(I420Buffer::Create(2, 2)); - - libyuv::I420Rect(buffer->MutableDataY(), buffer->StrideY(), - buffer->MutableDataU(), buffer->StrideU(), - buffer->MutableDataV(), buffer->StrideV(), 0, 0, - buffer->width(), buffer->height(), y, u, v); - - return VideoFrame::Builder() - .set_video_frame_buffer(buffer) - .set_timestamp_rtp(timestamp_rtp) - .build(); -} - -EncodedImage CreateEncodedImage(uint32_t timestamp_rtp, int spatial_idx = 0) { - EncodedImage encoded_image; - encoded_image.SetRtpTimestamp(timestamp_rtp); - encoded_image.SetSpatialIndex(spatial_idx); - return encoded_image; -} -} // namespace - -TEST(VideoCodecAnalyzerTest, StartEncode) { - VideoCodecAnalyzer analyzer; - analyzer.StartEncode(CreateVideoFrame(kTimestamp)); - - auto fs = analyzer.GetStats()->Slice(); - EXPECT_EQ(1u, fs.size()); - EXPECT_EQ(fs[0].timestamp_rtp, kTimestamp); -} - -TEST(VideoCodecAnalyzerTest, FinishEncode) { - VideoCodecAnalyzer analyzer; - analyzer.StartEncode(CreateVideoFrame(kTimestamp)); - - EncodedImage encoded_frame = CreateEncodedImage(kTimestamp, kSpatialIdx); - analyzer.FinishEncode(encoded_frame); - - auto fs = analyzer.GetStats()->Slice(); - EXPECT_EQ(2u, fs.size()); - EXPECT_EQ(kSpatialIdx, fs[1].spatial_idx); -} - -TEST(VideoCodecAnalyzerTest, StartDecode) { - VideoCodecAnalyzer analyzer; - analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); - - auto fs = analyzer.GetStats()->Slice(); - EXPECT_EQ(1u, fs.size()); - EXPECT_EQ(kTimestamp, fs[0].timestamp_rtp); -} - -TEST(VideoCodecAnalyzerTest, FinishDecode) { - VideoCodecAnalyzer analyzer; - analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); - VideoFrame decoded_frame = CreateVideoFrame(kTimestamp); - analyzer.FinishDecode(decoded_frame, kSpatialIdx); - - auto fs = analyzer.GetStats()->Slice(); - EXPECT_EQ(1u, fs.size()); - EXPECT_EQ(decoded_frame.width(), fs[0].width); - EXPECT_EQ(decoded_frame.height(), fs[0].height); -} - -TEST(VideoCodecAnalyzerTest, ReferenceVideoSource) { - MockReferenceVideoSource reference_video_source; - VideoCodecAnalyzer analyzer(&reference_video_source); - analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); - - EXPECT_CALL(reference_video_source, GetFrame) - .WillOnce(Return(CreateVideoFrame(kTimestamp, /*y=*/0, - /*u=*/0, /*v=*/0))); - - analyzer.FinishDecode( - CreateVideoFrame(kTimestamp, /*value_y=*/1, /*value_u=*/2, /*value_v=*/3), - kSpatialIdx); - - auto fs = analyzer.GetStats()->Slice(); - EXPECT_EQ(1u, fs.size()); - EXPECT_TRUE(fs[0].psnr.has_value()); - - const Psnr& psnr = *fs[0].psnr; - EXPECT_NEAR(psnr.y, 48, 1); - EXPECT_NEAR(psnr.u, 42, 1); - EXPECT_NEAR(psnr.v, 38, 1); -} - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_stats_impl.cc b/modules/video_coding/codecs/test/video_codec_stats_impl.cc deleted file mode 100644 index 9808e2a601..0000000000 --- a/modules/video_coding/codecs/test/video_codec_stats_impl.cc +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/video_codec_stats_impl.h" - -#include - -#include "api/numerics/samples_stats_counter.h" -#include "api/test/metrics/metrics_logger.h" -#include "rtc_base/checks.h" -#include "rtc_base/time_utils.h" - -namespace webrtc { -namespace test { -namespace { -using Frame = VideoCodecStats::Frame; -using Stream = VideoCodecStats::Stream; - -constexpr Frequency k90kHz = Frequency::Hertz(90000); - -class LeakyBucket { - public: - LeakyBucket() : level_bits_(0) {} - - // Updates bucket level and returns its current level in bits. Data is remove - // from bucket with rate equal to target bitrate of previous frame. Bucket - // level is tracked with floating point precision. Returned value of bucket - // level is rounded up. - int Update(const Frame& frame) { - RTC_CHECK(frame.target_bitrate) << "Bitrate must be specified."; - - if (prev_frame_) { - RTC_CHECK_GT(frame.timestamp_rtp, prev_frame_->timestamp_rtp) - << "Timestamp must increase."; - TimeDelta passed = - (frame.timestamp_rtp - prev_frame_->timestamp_rtp) / k90kHz; - level_bits_ -= - prev_frame_->target_bitrate->bps() * passed.us() / 1000000.0; - level_bits_ = std::max(level_bits_, 0.0); - } - - prev_frame_ = frame; - - level_bits_ += frame.frame_size.bytes() * 8; - return static_cast(std::ceil(level_bits_)); - } - - private: - absl::optional prev_frame_; - double level_bits_; -}; - -// Merges spatial layer frames into superframes. -std::vector Merge(const std::vector& frames) { - std::vector superframes; - // Map from frame timestamp to index in `superframes` vector. - std::map index; - - for (const auto& f : frames) { - if (index.find(f.timestamp_rtp) == index.end()) { - index[f.timestamp_rtp] = static_cast(superframes.size()); - superframes.push_back(f); - continue; - } - - Frame& sf = superframes[index[f.timestamp_rtp]]; - - sf.width = std::max(sf.width, f.width); - sf.height = std::max(sf.height, f.height); - sf.frame_size += f.frame_size; - sf.keyframe |= f.keyframe; - - sf.encode_time = std::max(sf.encode_time, f.encode_time); - sf.decode_time = std::max(sf.decode_time, f.decode_time); - - if (f.spatial_idx > sf.spatial_idx) { - if (f.qp) { - sf.qp = f.qp; - } - if (f.psnr) { - sf.psnr = f.psnr; - } - } - - sf.spatial_idx = std::max(sf.spatial_idx, f.spatial_idx); - sf.temporal_idx = std::max(sf.temporal_idx, f.temporal_idx); - - sf.encoded |= f.encoded; - sf.decoded |= f.decoded; - } - - return superframes; -} - -Timestamp RtpToTime(uint32_t timestamp_rtp) { - return Timestamp::Micros((timestamp_rtp / k90kHz).us()); -} - -SamplesStatsCounter::StatsSample StatsSample(double value, Timestamp time) { - return SamplesStatsCounter::StatsSample{value, time}; -} - -TimeDelta CalcTotalDuration(const std::vector& frames) { - RTC_CHECK(!frames.empty()); - TimeDelta duration = TimeDelta::Zero(); - if (frames.size() > 1) { - duration += - (frames.rbegin()->timestamp_rtp - frames.begin()->timestamp_rtp) / - k90kHz; - } - - // Add last frame duration. If target frame rate is provided, calculate frame - // duration from it. Otherwise, assume duration of last frame is the same as - // duration of preceding frame. - if (frames.rbegin()->target_framerate) { - duration += 1 / *frames.rbegin()->target_framerate; - } else { - RTC_CHECK_GT(frames.size(), 1u); - duration += (frames.rbegin()->timestamp_rtp - - std::next(frames.rbegin())->timestamp_rtp) / - k90kHz; - } - - return duration; -} -} // namespace - -std::vector VideoCodecStatsImpl::Slice( - absl::optional filter) const { - std::vector frames; - for (const auto& [frame_id, f] : frames_) { - if (filter.has_value()) { - if (filter->first_frame.has_value() && - f.frame_num < *filter->first_frame) { - continue; - } - if (filter->last_frame.has_value() && f.frame_num > *filter->last_frame) { - continue; - } - if (filter->spatial_idx.has_value() && - f.spatial_idx != *filter->spatial_idx) { - continue; - } - if (filter->temporal_idx.has_value() && - f.temporal_idx > *filter->temporal_idx) { - continue; - } - } - frames.push_back(f); - } - return frames; -} - -Stream VideoCodecStatsImpl::Aggregate(const std::vector& frames) const { - std::vector superframes = Merge(frames); - RTC_CHECK(!superframes.empty()); - - LeakyBucket leacky_bucket; - Stream stream; - for (size_t i = 0; i < superframes.size(); ++i) { - Frame& f = superframes[i]; - Timestamp time = RtpToTime(f.timestamp_rtp); - - if (!f.frame_size.IsZero()) { - stream.width.AddSample(StatsSample(f.width, time)); - stream.height.AddSample(StatsSample(f.height, time)); - stream.frame_size_bytes.AddSample( - StatsSample(f.frame_size.bytes(), time)); - stream.keyframe.AddSample(StatsSample(f.keyframe, time)); - if (f.qp) { - stream.qp.AddSample(StatsSample(*f.qp, time)); - } - } - - if (f.encoded) { - stream.encode_time_ms.AddSample(StatsSample(f.encode_time.ms(), time)); - } - - if (f.decoded) { - stream.decode_time_ms.AddSample(StatsSample(f.decode_time.ms(), time)); - } - - if (f.psnr) { - stream.psnr.y.AddSample(StatsSample(f.psnr->y, time)); - stream.psnr.u.AddSample(StatsSample(f.psnr->u, time)); - stream.psnr.v.AddSample(StatsSample(f.psnr->v, time)); - } - - if (f.target_framerate) { - stream.target_framerate_fps.AddSample( - StatsSample(f.target_framerate->millihertz() / 1000.0, time)); - } - - if (f.target_bitrate) { - stream.target_bitrate_kbps.AddSample( - StatsSample(f.target_bitrate->bps() / 1000.0, time)); - - int buffer_level_bits = leacky_bucket.Update(f); - stream.transmission_time_ms.AddSample( - StatsSample(buffer_level_bits * rtc::kNumMillisecsPerSec / - f.target_bitrate->bps(), - RtpToTime(f.timestamp_rtp))); - } - } - - TimeDelta duration = CalcTotalDuration(superframes); - DataRate encoded_bitrate = - DataSize::Bytes(stream.frame_size_bytes.GetSum()) / duration; - - int num_encoded_frames = stream.frame_size_bytes.NumSamples(); - Frequency encoded_framerate = num_encoded_frames / duration; - - absl::optional bitrate_mismatch_pct; - if (auto target_bitrate = superframes.begin()->target_bitrate; - target_bitrate) { - bitrate_mismatch_pct = 100.0 * - (encoded_bitrate.bps() - target_bitrate->bps()) / - target_bitrate->bps(); - } - - absl::optional framerate_mismatch_pct; - if (auto target_framerate = superframes.begin()->target_framerate; - target_framerate) { - framerate_mismatch_pct = - 100.0 * - (encoded_framerate.millihertz() - target_framerate->millihertz()) / - target_framerate->millihertz(); - } - - for (auto& f : superframes) { - Timestamp time = RtpToTime(f.timestamp_rtp); - stream.encoded_bitrate_kbps.AddSample( - StatsSample(encoded_bitrate.bps() / 1000.0, time)); - - stream.encoded_framerate_fps.AddSample( - StatsSample(encoded_framerate.millihertz() / 1000.0, time)); - - if (bitrate_mismatch_pct) { - stream.bitrate_mismatch_pct.AddSample( - StatsSample(*bitrate_mismatch_pct, time)); - } - - if (framerate_mismatch_pct) { - stream.framerate_mismatch_pct.AddSample( - StatsSample(*framerate_mismatch_pct, time)); - } - } - - return stream; -} - -void VideoCodecStatsImpl::AddFrame(const Frame& frame) { - FrameId frame_id{.timestamp_rtp = frame.timestamp_rtp, - .spatial_idx = frame.spatial_idx}; - RTC_CHECK(frames_.find(frame_id) == frames_.end()) - << "Frame with timestamp_rtp=" << frame.timestamp_rtp - << " and spatial_idx=" << frame.spatial_idx << " already exists"; - - frames_[frame_id] = frame; -} - -Frame* VideoCodecStatsImpl::GetFrame(uint32_t timestamp_rtp, int spatial_idx) { - FrameId frame_id{.timestamp_rtp = timestamp_rtp, .spatial_idx = spatial_idx}; - if (frames_.find(frame_id) == frames_.end()) { - return nullptr; - } - return &frames_.find(frame_id)->second; -} - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_stats_impl.h b/modules/video_coding/codecs/test/video_codec_stats_impl.h deleted file mode 100644 index 77471d2ecd..0000000000 --- a/modules/video_coding/codecs/test/video_codec_stats_impl.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_STATS_IMPL_H_ -#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_STATS_IMPL_H_ - -#include -#include -#include - -#include "absl/types/optional.h" -#include "api/test/video_codec_stats.h" - -namespace webrtc { -namespace test { - -// Implementation of `VideoCodecStats`. This class is not thread-safe. -class VideoCodecStatsImpl : public VideoCodecStats { - public: - std::vector Slice( - absl::optional filter = absl::nullopt) const override; - - Stream Aggregate(const std::vector& frames) const override; - - void AddFrame(const Frame& frame); - - // Returns raw pointers to previously added frame. If frame does not exist, - // returns `nullptr`. - Frame* GetFrame(uint32_t timestamp_rtp, int spatial_idx); - - private: - struct FrameId { - uint32_t timestamp_rtp; - int spatial_idx; - - bool operator==(const FrameId& o) const { - return timestamp_rtp == o.timestamp_rtp && spatial_idx == o.spatial_idx; - } - - bool operator<(const FrameId& o) const { - if (timestamp_rtp < o.timestamp_rtp) - return true; - if (timestamp_rtp == o.timestamp_rtp && spatial_idx < o.spatial_idx) - return true; - return false; - } - }; - - std::map frames_; -}; - -} // namespace test -} // namespace webrtc - -#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_STATS_IMPL_H_ diff --git a/modules/video_coding/codecs/test/video_codec_stats_impl_unittest.cc b/modules/video_coding/codecs/test/video_codec_stats_impl_unittest.cc deleted file mode 100644 index ce11d5abe6..0000000000 --- a/modules/video_coding/codecs/test/video_codec_stats_impl_unittest.cc +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/video_codec_stats_impl.h" - -#include - -#include "absl/types/optional.h" -#include "test/gmock.h" -#include "test/gtest.h" - -namespace webrtc { -namespace test { - -namespace { -using ::testing::Return; -using ::testing::Values; -using Filter = VideoCodecStats::Filter; -using Frame = VideoCodecStatsImpl::Frame; -using Stream = VideoCodecStats::Stream; -} // namespace - -TEST(VideoCodecStatsImpl, AddAndGetFrame) { - VideoCodecStatsImpl stats; - stats.AddFrame({.timestamp_rtp = 0, .spatial_idx = 0}); - stats.AddFrame({.timestamp_rtp = 0, .spatial_idx = 1}); - stats.AddFrame({.timestamp_rtp = 1, .spatial_idx = 0}); - - Frame* fs = stats.GetFrame(/*timestamp_rtp=*/0, /*spatial_idx=*/0); - ASSERT_NE(fs, nullptr); - EXPECT_EQ(fs->timestamp_rtp, 0u); - EXPECT_EQ(fs->spatial_idx, 0); - - fs = stats.GetFrame(/*timestamp_rtp=*/0, /*spatial_idx=*/1); - ASSERT_NE(fs, nullptr); - EXPECT_EQ(fs->timestamp_rtp, 0u); - EXPECT_EQ(fs->spatial_idx, 1); - - fs = stats.GetFrame(/*timestamp_rtp=*/1, /*spatial_idx=*/0); - ASSERT_NE(fs, nullptr); - EXPECT_EQ(fs->timestamp_rtp, 1u); - EXPECT_EQ(fs->spatial_idx, 0); - - fs = stats.GetFrame(/*timestamp_rtp=*/1, /*spatial_idx=*/1); - EXPECT_EQ(fs, nullptr); -} - -class VideoCodecStatsImplSlicingTest - : public ::testing::TestWithParam>> {}; - -TEST_P(VideoCodecStatsImplSlicingTest, Slice) { - Filter filter = std::get<0>(GetParam()); - std::vector expected_frames = std::get<1>(GetParam()); - std::vector frames = { - {.frame_num = 0, .timestamp_rtp = 0, .spatial_idx = 0, .temporal_idx = 0}, - {.frame_num = 0, .timestamp_rtp = 0, .spatial_idx = 1, .temporal_idx = 0}, - {.frame_num = 1, .timestamp_rtp = 1, .spatial_idx = 0, .temporal_idx = 1}, - {.frame_num = 1, - .timestamp_rtp = 1, - .spatial_idx = 1, - .temporal_idx = 1}}; - - VideoCodecStatsImpl stats; - stats.AddFrame(frames[0]); - stats.AddFrame(frames[1]); - stats.AddFrame(frames[2]); - stats.AddFrame(frames[3]); - - std::vector slice = stats.Slice(filter); - ASSERT_EQ(slice.size(), expected_frames.size()); - for (size_t i = 0; i < expected_frames.size(); ++i) { - Frame& expected = frames[expected_frames[i]]; - EXPECT_EQ(slice[i].frame_num, expected.frame_num); - EXPECT_EQ(slice[i].timestamp_rtp, expected.timestamp_rtp); - EXPECT_EQ(slice[i].spatial_idx, expected.spatial_idx); - EXPECT_EQ(slice[i].temporal_idx, expected.temporal_idx); - } -} - -INSTANTIATE_TEST_SUITE_P( - All, - VideoCodecStatsImplSlicingTest, - ::testing::Values( - std::make_tuple(Filter{}, std::vector{0, 1, 2, 3}), - std::make_tuple(Filter{.first_frame = 1}, std::vector{2, 3}), - std::make_tuple(Filter{.last_frame = 0}, std::vector{0, 1}), - std::make_tuple(Filter{.spatial_idx = 0}, std::vector{0, 2}), - std::make_tuple(Filter{.temporal_idx = 1}, - std::vector{0, 1, 2, 3}))); - -TEST(VideoCodecStatsImpl, AggregateBitrate) { - std::vector frames = { - {.frame_num = 0, - .timestamp_rtp = 0, - .frame_size = DataSize::Bytes(1000), - .target_bitrate = DataRate::BytesPerSec(1000)}, - {.frame_num = 1, - .timestamp_rtp = 90000, - .frame_size = DataSize::Bytes(2000), - .target_bitrate = DataRate::BytesPerSec(1000)}}; - - Stream stream = VideoCodecStatsImpl().Aggregate(frames); - EXPECT_EQ(stream.encoded_bitrate_kbps.GetAverage(), 12.0); - EXPECT_EQ(stream.bitrate_mismatch_pct.GetAverage(), 50.0); -} - -TEST(VideoCodecStatsImpl, AggregateFramerate) { - std::vector frames = { - {.frame_num = 0, - .timestamp_rtp = 0, - .frame_size = DataSize::Bytes(1), - .target_framerate = Frequency::Hertz(1)}, - {.frame_num = 1, - .timestamp_rtp = 90000, - .frame_size = DataSize::Zero(), - .target_framerate = Frequency::Hertz(1)}}; - - Stream stream = VideoCodecStatsImpl().Aggregate(frames); - EXPECT_EQ(stream.encoded_framerate_fps.GetAverage(), 0.5); - EXPECT_EQ(stream.framerate_mismatch_pct.GetAverage(), -50.0); -} - -TEST(VideoCodecStatsImpl, AggregateTransmissionTime) { - std::vector frames = { - {.frame_num = 0, - .timestamp_rtp = 0, - .frame_size = DataSize::Bytes(2), - .target_bitrate = DataRate::BytesPerSec(1)}, - {.frame_num = 1, - .timestamp_rtp = 90000, - .frame_size = DataSize::Bytes(3), - .target_bitrate = DataRate::BytesPerSec(1)}}; - - Stream stream = VideoCodecStatsImpl().Aggregate(frames); - ASSERT_EQ(stream.transmission_time_ms.NumSamples(), 2); - ASSERT_EQ(stream.transmission_time_ms.GetSamples()[0], 2000); - ASSERT_EQ(stream.transmission_time_ms.GetSamples()[1], 4000); -} - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_test.cc b/modules/video_coding/codecs/test/video_codec_test.cc index 1c8fe97e84..08961a312b 100644 --- a/modules/video_coding/codecs/test/video_codec_test.cc +++ b/modules/video_coding/codecs/test/video_codec_test.cc @@ -8,33 +8,18 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "api/video_codecs/video_codec.h" - -#include #include #include #include #include "absl/flags/flag.h" #include "absl/functional/any_invocable.h" -#include "api/test/create_video_codec_tester.h" #include "api/test/metrics/global_metrics_logger_and_exporter.h" -#include "api/test/video_codec_tester.h" -#include "api/test/videocodec_test_stats.h" #include "api/units/data_rate.h" #include "api/units/frequency.h" -#include "api/video/encoded_image.h" -#include "api/video/i420_buffer.h" #include "api/video/resolution.h" -#include "api/video/video_frame.h" -#include "api/video_codecs/scalability_mode.h" -#include "api/video_codecs/video_decoder.h" -#include "api/video_codecs/video_encoder.h" -#include "media/engine/internal_decoder_factory.h" -#include "media/engine/internal_encoder_factory.h" -#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" -#include "modules/video_coding/include/video_error_codes.h" -#include "modules/video_coding/svc/scalability_mode_util.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" #if defined(WEBRTC_ANDROID) #include "modules/video_coding/codecs/test/android_codec_factory_helper.h" #endif @@ -42,7 +27,15 @@ #include "test/gtest.h" #include "test/test_flags.h" #include "test/testsupport/file_utils.h" -#include "test/testsupport/frame_reader.h" +#include "test/video_codec_tester.h" + +ABSL_FLAG(bool, dump_decoder_input, false, "Dump decoder input."); + +ABSL_FLAG(bool, dump_decoder_output, false, "Dump decoder output."); + +ABSL_FLAG(bool, dump_encoder_input, false, "Dump encoder input."); + +ABSL_FLAG(bool, dump_encoder_output, false, "Dump encoder output."); namespace webrtc { namespace test { @@ -50,6 +43,10 @@ namespace test { namespace { using ::testing::Combine; using ::testing::Values; +using VideoSourceSettings = VideoCodecTester::VideoSourceSettings; +using EncodingSettings = VideoCodecTester::EncodingSettings; +using VideoCodecStats = VideoCodecTester::VideoCodecStats; +using Filter = VideoCodecStats::Filter; using PacingMode = VideoCodecTester::PacingSettings::PacingMode; struct VideoInfo { @@ -58,399 +55,35 @@ struct VideoInfo { Frequency framerate; }; -struct LayerId { - int spatial_idx; - int temporal_idx; - - bool operator==(const LayerId& o) const { - return spatial_idx == o.spatial_idx && temporal_idx == o.temporal_idx; - } - - bool operator<(const LayerId& o) const { - if (spatial_idx < o.spatial_idx) - return true; - if (spatial_idx == o.spatial_idx && temporal_idx < o.temporal_idx) - return true; - return false; - } -}; - -struct EncodingSettings { - ScalabilityMode scalability_mode; - struct LayerSettings { - Resolution resolution; - Frequency framerate; - DataRate bitrate; - }; - std::map layer_settings; - - bool IsSameSettings(const EncodingSettings& other) const { - if (scalability_mode != other.scalability_mode) { - return false; - } - - for (auto [layer_id, layer] : layer_settings) { - const auto& other_layer = other.layer_settings.at(layer_id); - if (layer.resolution != other_layer.resolution) { - return false; - } - } - - return true; - } - - bool IsSameRate(const EncodingSettings& other) const { - for (auto [layer_id, layer] : layer_settings) { - const auto& other_layer = other.layer_settings.at(layer_id); - if (layer.bitrate != other_layer.bitrate || - layer.framerate != other_layer.framerate) { - return false; - } - } - - return true; - } -}; - const VideoInfo kFourPeople_1280x720_30 = { .name = "FourPeople_1280x720_30", .resolution = {.width = 1280, .height = 720}, .framerate = Frequency::Hertz(30)}; -class TestRawVideoSource : public VideoCodecTester::RawVideoSource { - public: - static constexpr Frequency k90kHz = Frequency::Hertz(90000); +static constexpr Frequency k90kHz = Frequency::Hertz(90000); - TestRawVideoSource(VideoInfo video_info, - const std::map& frame_settings, - int num_frames) - : video_info_(video_info), - frame_settings_(frame_settings), - num_frames_(num_frames), - frame_num_(0), - // Start with non-zero timestamp to force using frame RTP timestamps in - // IvfFrameWriter. - timestamp_rtp_(90000) { - // Ensure settings for the first frame are provided. - RTC_CHECK_GT(frame_settings_.size(), 0u); - RTC_CHECK_EQ(frame_settings_.begin()->first, 0); - - frame_reader_ = CreateYuvFrameReader( - ResourcePath(video_info_.name, "yuv"), video_info_.resolution, - YuvFrameReaderImpl::RepeatMode::kPingPong); - RTC_CHECK(frame_reader_); - } - - // Pulls next frame. Frame RTP timestamp is set accordingly to - // `EncodingSettings::framerate`. - absl::optional PullFrame() override { - if (frame_num_ >= num_frames_) { - return absl::nullopt; // End of stream. - } - - const EncodingSettings& encoding_settings = - std::prev(frame_settings_.upper_bound(frame_num_))->second; - - Resolution resolution = - encoding_settings.layer_settings.begin()->second.resolution; - Frequency framerate = - encoding_settings.layer_settings.begin()->second.framerate; - - int pulled_frame; - auto buffer = frame_reader_->PullFrame( - &pulled_frame, resolution, - {.num = static_cast(framerate.millihertz()), - .den = static_cast(video_info_.framerate.millihertz())}); - RTC_CHECK(buffer) << "Cannot pull frame " << frame_num_; - - auto frame = VideoFrame::Builder() - .set_video_frame_buffer(buffer) - .set_timestamp_rtp(timestamp_rtp_) - .set_timestamp_us((timestamp_rtp_ / k90kHz).us()) - .build(); - - pulled_frames_[timestamp_rtp_] = pulled_frame; - timestamp_rtp_ += k90kHz / framerate; - ++frame_num_; - - return frame; - } - - // Reads frame specified by `timestamp_rtp`, scales it to `resolution` and - // returns. Frame with the given `timestamp_rtp` is expected to be pulled - // before. - VideoFrame GetFrame(uint32_t timestamp_rtp, Resolution resolution) override { - RTC_CHECK(pulled_frames_.find(timestamp_rtp) != pulled_frames_.end()) - << "Frame with RTP timestamp " << timestamp_rtp - << " was not pulled before"; - auto buffer = - frame_reader_->ReadFrame(pulled_frames_[timestamp_rtp], resolution); - return VideoFrame::Builder() - .set_video_frame_buffer(buffer) - .set_timestamp_rtp(timestamp_rtp) - .build(); - } - - protected: - VideoInfo video_info_; - std::unique_ptr frame_reader_; - const std::map& frame_settings_; - int num_frames_; - int frame_num_; - uint32_t timestamp_rtp_; - std::map pulled_frames_; -}; - -class TestEncoder : public VideoCodecTester::Encoder, - public EncodedImageCallback { - public: - TestEncoder(std::unique_ptr encoder, - const std::string codec_type, - const std::map& frame_settings) - : encoder_(std::move(encoder)), - codec_type_(codec_type), - frame_settings_(frame_settings), - frame_num_(0) { - // Ensure settings for the first frame is provided. - RTC_CHECK_GT(frame_settings_.size(), 0u); - RTC_CHECK_EQ(frame_settings_.begin()->first, 0); - - encoder_->RegisterEncodeCompleteCallback(this); - } - - void Initialize() override { - const EncodingSettings& first_frame_settings = frame_settings_.at(0); - Configure(first_frame_settings); - SetRates(first_frame_settings); - } - - void Encode(const VideoFrame& frame, EncodeCallback callback) override { - { - MutexLock lock(&mutex_); - callbacks_[frame.timestamp()] = std::move(callback); - } - - if (auto fs = frame_settings_.find(frame_num_); - fs != frame_settings_.begin() && fs != frame_settings_.end()) { - if (!fs->second.IsSameSettings(std::prev(fs)->second)) { - Configure(fs->second); - } else if (!fs->second.IsSameRate(std::prev(fs)->second)) { - SetRates(fs->second); - } - } - - encoder_->Encode(frame, nullptr); - ++frame_num_; - } - - void Flush() override { - // TODO(webrtc:14852): For codecs which buffer frames we need a to - // flush them to get last frames. Add such functionality to VideoEncoder - // API. On Android it will map directly to `MediaCodec.flush()`. - encoder_->Release(); - } - - VideoEncoder* encoder() { return encoder_.get(); } - - protected: - Result OnEncodedImage(const EncodedImage& encoded_image, - const CodecSpecificInfo* codec_specific_info) override { - MutexLock lock(&mutex_); - auto cb = callbacks_.find(encoded_image.RtpTimestamp()); - RTC_CHECK(cb != callbacks_.end()); - cb->second(encoded_image); - - callbacks_.erase(callbacks_.begin(), cb); - return Result(Result::Error::OK); - } - - void Configure(const EncodingSettings& es) { - VideoCodec vc; - const EncodingSettings::LayerSettings& layer_settings = - es.layer_settings.begin()->second; - vc.width = layer_settings.resolution.width; - vc.height = layer_settings.resolution.height; - const DataRate& bitrate = layer_settings.bitrate; - vc.startBitrate = bitrate.kbps(); - vc.maxBitrate = bitrate.kbps(); - vc.minBitrate = 0; - vc.maxFramerate = static_cast(layer_settings.framerate.hertz()); - vc.active = true; - vc.qpMax = 63; - vc.numberOfSimulcastStreams = 0; - vc.mode = webrtc::VideoCodecMode::kRealtimeVideo; - vc.SetFrameDropEnabled(true); - vc.SetScalabilityMode(es.scalability_mode); - - vc.codecType = PayloadStringToCodecType(codec_type_); - if (vc.codecType == kVideoCodecVP8) { - *(vc.VP8()) = VideoEncoder::GetDefaultVp8Settings(); - } else if (vc.codecType == kVideoCodecVP9) { - *(vc.VP9()) = VideoEncoder::GetDefaultVp9Settings(); - } else if (vc.codecType == kVideoCodecH264) { - *(vc.H264()) = VideoEncoder::GetDefaultH264Settings(); - } - - VideoEncoder::Settings ves( - VideoEncoder::Capabilities(/*loss_notification=*/false), - /*number_of_cores=*/1, - /*max_payload_size=*/1440); - - int result = encoder_->InitEncode(&vc, ves); - ASSERT_EQ(result, WEBRTC_VIDEO_CODEC_OK); - - SetRates(es); - } - - void SetRates(const EncodingSettings& es) { - VideoEncoder::RateControlParameters rc; - int num_spatial_layers = - ScalabilityModeToNumSpatialLayers(es.scalability_mode); - int num_temporal_layers = - ScalabilityModeToNumSpatialLayers(es.scalability_mode); - for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { - for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { - auto layer_settings = - es.layer_settings.find({.spatial_idx = sidx, .temporal_idx = tidx}); - RTC_CHECK(layer_settings != es.layer_settings.end()) - << "Bitrate for layer S=" << sidx << " T=" << tidx << " is not set"; - rc.bitrate.SetBitrate(sidx, tidx, layer_settings->second.bitrate.bps()); - } - } - - rc.framerate_fps = - es.layer_settings.begin()->second.framerate.millihertz() / 1000.0; - encoder_->SetRates(rc); - } - - std::unique_ptr encoder_; - const std::string codec_type_; - const std::map& frame_settings_; - int frame_num_; - std::map callbacks_ RTC_GUARDED_BY(mutex_); - Mutex mutex_; -}; - -class TestDecoder : public VideoCodecTester::Decoder, - public DecodedImageCallback { - public: - TestDecoder(std::unique_ptr decoder, - const std::string codec_type) - : decoder_(std::move(decoder)), codec_type_(codec_type) { - decoder_->RegisterDecodeCompleteCallback(this); - } - - void Initialize() override { - VideoDecoder::Settings ds; - ds.set_codec_type(PayloadStringToCodecType(codec_type_)); - ds.set_number_of_cores(1); - ds.set_max_render_resolution({1280, 720}); - - bool result = decoder_->Configure(ds); - ASSERT_TRUE(result); - } - - void Decode(const EncodedImage& frame, DecodeCallback callback) override { - { - MutexLock lock(&mutex_); - callbacks_[frame.RtpTimestamp()] = std::move(callback); - } - - decoder_->Decode(frame, /*render_time_ms=*/0); - } - - void Flush() override { - // TODO(webrtc:14852): For codecs which buffer frames we need a to - // flush them to get last frames. Add such functionality to VideoDecoder - // API. On Android it will map directly to `MediaCodec.flush()`. - decoder_->Release(); - } - - VideoDecoder* decoder() { return decoder_.get(); } - - protected: - int Decoded(VideoFrame& decoded_frame) override { - MutexLock lock(&mutex_); - auto cb = callbacks_.find(decoded_frame.timestamp()); - RTC_CHECK(cb != callbacks_.end()); - cb->second(decoded_frame); - - callbacks_.erase(callbacks_.begin(), cb); - return WEBRTC_VIDEO_CODEC_OK; - } - - std::unique_ptr decoder_; - const std::string codec_type_; - std::map callbacks_ RTC_GUARDED_BY(mutex_); - Mutex mutex_; -}; - -std::unique_ptr CreateVideoSource( - const VideoInfo& video, - const std::map& frame_settings, - int num_frames) { - return std::make_unique(video, frame_settings, - num_frames); -} - -std::unique_ptr CreateEncoder( - std::string type, - std::string impl, - const std::map& frame_settings) { - std::unique_ptr factory; +std::unique_ptr CreateEncoderFactory(std::string impl) { if (impl == "builtin") { - factory = std::make_unique(); - } else if (impl == "mediacodec") { + return CreateBuiltinVideoEncoderFactory(); + } #if defined(WEBRTC_ANDROID) - InitializeAndroidObjects(); - factory = CreateAndroidEncoderFactory(); + InitializeAndroidObjects(); + return CreateAndroidEncoderFactory(); +#else + return nullptr; #endif - } - std::unique_ptr encoder = - factory->CreateVideoEncoder(SdpVideoFormat(type)); - if (encoder == nullptr) { - return nullptr; - } - return std::make_unique(std::move(encoder), type, - frame_settings); } -std::unique_ptr CreateDecoder(std::string type, std::string impl) { - std::unique_ptr factory; +std::unique_ptr CreateDecoderFactory(std::string impl) { if (impl == "builtin") { - factory = std::make_unique(); - } else if (impl == "mediacodec") { + return CreateBuiltinVideoDecoderFactory(); + } #if defined(WEBRTC_ANDROID) - InitializeAndroidObjects(); - factory = CreateAndroidDecoderFactory(); + InitializeAndroidObjects(); + return CreateAndroidDecoderFactory(); +#else + return nullptr; #endif - } - std::unique_ptr decoder = - factory->CreateVideoDecoder(SdpVideoFormat(type)); - if (decoder == nullptr) { - return nullptr; - } - return std::make_unique(std::move(decoder), type); -} - -void SetTargetRates(const std::map& frame_settings, - std::vector& frames) { - for (VideoCodecStats::Frame& f : frames) { - const EncodingSettings& encoding_settings = - std::prev(frame_settings.upper_bound(f.frame_num))->second; - LayerId layer_id = {.spatial_idx = f.spatial_idx, - .temporal_idx = f.temporal_idx}; - RTC_CHECK(encoding_settings.layer_settings.find(layer_id) != - encoding_settings.layer_settings.end()) - << "Frame frame_num=" << f.frame_num - << " belongs to spatial_idx=" << f.spatial_idx - << " temporal_idx=" << f.temporal_idx - << " but settings for this layer are not provided."; - const EncodingSettings::LayerSettings& layer_settings = - encoding_settings.layer_settings.at(layer_id); - f.target_bitrate = layer_settings.bitrate; - f.target_framerate = layer_settings.framerate; - } } std::string TestOutputPath() { @@ -468,113 +101,118 @@ std::unique_ptr RunEncodeDecodeTest( std::string codec_type, std::string codec_impl, const VideoInfo& video_info, - const std::map& frame_settings, - int num_frames, - bool save_codec_input, - bool save_codec_output) { - std::unique_ptr video_source = - CreateVideoSource(video_info, frame_settings, num_frames); + const std::map& encoding_settings) { + VideoSourceSettings source_settings{ + .file_path = ResourcePath(video_info.name, "yuv"), + .resolution = video_info.resolution, + .framerate = video_info.framerate}; - std::unique_ptr encoder = - CreateEncoder(codec_type, codec_impl, frame_settings); - if (encoder == nullptr) { + const SdpVideoFormat& sdp_video_format = + encoding_settings.begin()->second.sdp_video_format; + + std::unique_ptr encoder_factory = + CreateEncoderFactory(codec_impl); + if (!encoder_factory + ->QueryCodecSupport(sdp_video_format, + /*scalability_mode=*/absl::nullopt) + .is_supported) { + RTC_LOG(LS_WARNING) << "No encoder for video format " + << sdp_video_format.ToString(); return nullptr; } - std::unique_ptr decoder = CreateDecoder(codec_type, codec_impl); - if (decoder == nullptr) { - // If platform decoder is not available try built-in one. - if (codec_impl == "builtin") { - return nullptr; - } - - decoder = CreateDecoder(codec_type, "builtin"); - if (decoder == nullptr) { + std::unique_ptr decoder_factory = + CreateDecoderFactory(codec_impl); + if (!decoder_factory + ->QueryCodecSupport(sdp_video_format, + /*reference_scaling=*/false) + .is_supported) { + decoder_factory = CreateDecoderFactory("builtin"); + if (!decoder_factory + ->QueryCodecSupport(sdp_video_format, + /*reference_scaling=*/false) + .is_supported) { + RTC_LOG(LS_WARNING) << "No decoder for video format " + << sdp_video_format.ToString(); return nullptr; } } - RTC_LOG(LS_INFO) << "Encoder implementation: " - << encoder->encoder()->GetEncoderInfo().implementation_name; - RTC_LOG(LS_INFO) << "Decoder implementation: " - << decoder->decoder()->GetDecoderInfo().implementation_name; - - VideoCodecTester::EncoderSettings encoder_settings; - encoder_settings.pacing.mode = - encoder->encoder()->GetEncoderInfo().is_hardware_accelerated - ? PacingMode::kRealTime - : PacingMode::kNoPacing; - - VideoCodecTester::DecoderSettings decoder_settings; - decoder_settings.pacing.mode = - decoder->decoder()->GetDecoderInfo().is_hardware_accelerated - ? PacingMode::kRealTime - : PacingMode::kNoPacing; - std::string output_path = TestOutputPath(); - if (save_codec_input) { + + VideoCodecTester::EncoderSettings encoder_settings; + encoder_settings.pacing_settings.mode = + codec_impl == "builtin" ? PacingMode::kNoPacing : PacingMode::kRealTime; + if (absl::GetFlag(FLAGS_dump_encoder_input)) { encoder_settings.encoder_input_base_path = output_path + "_enc_input"; + } + if (absl::GetFlag(FLAGS_dump_encoder_output)) { + encoder_settings.encoder_output_base_path = output_path + "_enc_output"; + } + + VideoCodecTester::DecoderSettings decoder_settings; + decoder_settings.pacing_settings.mode = + codec_impl == "builtin" ? PacingMode::kNoPacing : PacingMode::kRealTime; + if (absl::GetFlag(FLAGS_dump_decoder_input)) { decoder_settings.decoder_input_base_path = output_path + "_dec_input"; } - if (save_codec_output) { - encoder_settings.encoder_output_base_path = output_path + "_enc_output"; + if (absl::GetFlag(FLAGS_dump_decoder_output)) { decoder_settings.decoder_output_base_path = output_path + "_dec_output"; } - std::unique_ptr tester = CreateVideoCodecTester(); - return tester->RunEncodeDecodeTest(video_source.get(), encoder.get(), - decoder.get(), encoder_settings, - decoder_settings); + return VideoCodecTester::RunEncodeDecodeTest( + source_settings, encoder_factory.get(), decoder_factory.get(), + encoder_settings, decoder_settings, encoding_settings); } std::unique_ptr RunEncodeTest( std::string codec_type, std::string codec_impl, const VideoInfo& video_info, - const std::map& frame_settings, - int num_frames, - bool save_codec_input, - bool save_codec_output) { - std::unique_ptr video_source = - CreateVideoSource(video_info, frame_settings, num_frames); + const std::map& encoding_settings) { + VideoSourceSettings source_settings{ + .file_path = ResourcePath(video_info.name, "yuv"), + .resolution = video_info.resolution, + .framerate = video_info.framerate}; - std::unique_ptr encoder = - CreateEncoder(codec_type, codec_impl, frame_settings); - if (encoder == nullptr) { + const SdpVideoFormat& sdp_video_format = + encoding_settings.begin()->second.sdp_video_format; + + std::unique_ptr encoder_factory = + CreateEncoderFactory(codec_impl); + if (!encoder_factory + ->QueryCodecSupport(sdp_video_format, + /*scalability_mode=*/absl::nullopt) + .is_supported) { + RTC_LOG(LS_WARNING) << "No encoder for video format " + << sdp_video_format.ToString(); return nullptr; } - RTC_LOG(LS_INFO) << "Encoder implementation: " - << encoder->encoder()->GetEncoderInfo().implementation_name; - - VideoCodecTester::EncoderSettings encoder_settings; - encoder_settings.pacing.mode = - encoder->encoder()->GetEncoderInfo().is_hardware_accelerated - ? PacingMode::kRealTime - : PacingMode::kNoPacing; - std::string output_path = TestOutputPath(); - if (save_codec_input) { + VideoCodecTester::EncoderSettings encoder_settings; + encoder_settings.pacing_settings.mode = + codec_impl == "builtin" ? PacingMode::kNoPacing : PacingMode::kRealTime; + if (absl::GetFlag(FLAGS_dump_encoder_input)) { encoder_settings.encoder_input_base_path = output_path + "_enc_input"; } - if (save_codec_output) { + if (absl::GetFlag(FLAGS_dump_encoder_output)) { encoder_settings.encoder_output_base_path = output_path + "_enc_output"; } - std::unique_ptr tester = CreateVideoCodecTester(); - return tester->RunEncodeTest(video_source.get(), encoder.get(), - encoder_settings); + return VideoCodecTester::RunEncodeTest(source_settings, encoder_factory.get(), + encoder_settings, encoding_settings); } -class SpatialQualityTest : public ::testing::TestWithParam< - std::tuple>> { +class SpatialQualityTest : public ::testing::TestWithParam>> { public: static std::string TestParamsToString( const ::testing::TestParamInfo& info) { @@ -590,31 +228,24 @@ class SpatialQualityTest : public ::testing::TestWithParam< TEST_P(SpatialQualityTest, SpatialQuality) { auto [codec_type, codec_impl, video_info, coding_settings] = GetParam(); - auto [width, height, framerate_fps, bitrate_kbps, psnr] = coding_settings; - - std::map frame_settings = { - {0, - {.scalability_mode = ScalabilityMode::kL1T1, - .layer_settings = { - {LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = width, .height = height}, - .framerate = Frequency::MilliHertz(1000 * framerate_fps), - .bitrate = DataRate::KilobitsPerSec(bitrate_kbps)}}}}}}; - + auto [width, height, framerate_fps, bitrate_kbps, expected_min_psnr] = + coding_settings; int duration_s = 10; int num_frames = duration_s * framerate_fps; - std::unique_ptr stats = RunEncodeDecodeTest( - codec_type, codec_impl, video_info, frame_settings, num_frames, - /*save_codec_input=*/false, /*save_codec_output=*/false); + std::map frames_settings = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", width, height, + {bitrate_kbps}, framerate_fps, num_frames); + + std::unique_ptr stats = + RunEncodeDecodeTest(codec_type, codec_impl, video_info, frames_settings); VideoCodecStats::Stream stream; if (stats != nullptr) { - std::vector frames = stats->Slice(); - SetTargetRates(frame_settings, frames); - stream = stats->Aggregate(frames); + stream = stats->Aggregate(Filter{}); if (absl::GetFlag(FLAGS_webrtc_quick_perf_test)) { - EXPECT_GE(stream.psnr.y.GetAverage(), psnr); + EXPECT_GE(stream.psnr.y.GetAverage(), expected_min_psnr); } } @@ -622,9 +253,9 @@ TEST_P(SpatialQualityTest, SpatialQuality) { GetGlobalMetricsLogger(), ::testing::UnitTest::GetInstance()->current_test_info()->name(), /*metadata=*/ - {{"codec_type", codec_type}, - {"codec_impl", codec_impl}, - {"video_name", video_info.name}}); + {{"video_name", video_info.name}, + {"codec_type", codec_type}, + {"codec_impl", codec_impl}}); } INSTANTIATE_TEST_SUITE_P( @@ -671,33 +302,32 @@ TEST_P(BitrateAdaptationTest, BitrateAdaptation) { auto [codec_type, codec_impl, video_info, bitrate_kbps] = GetParam(); int duration_s = 10; // Duration of fixed rate interval. - int first_frame = duration_s * video_info.framerate.millihertz() / 1000; - int num_frames = 2 * duration_s * video_info.framerate.millihertz() / 1000; + int num_frames = + static_cast(duration_s * video_info.framerate.hertz()); - std::map frame_settings = { - {0, - {.layer_settings = {{LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = 640, .height = 360}, - .framerate = video_info.framerate, - .bitrate = DataRate::KilobitsPerSec( - bitrate_kbps.first)}}}}}, - {first_frame, - {.layer_settings = { - {LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = 640, .height = 360}, - .framerate = video_info.framerate, - .bitrate = DataRate::KilobitsPerSec(bitrate_kbps.second)}}}}}}; + std::map encoding_settings = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", + /*width=*/640, /*height=*/360, {bitrate_kbps.first}, + /*framerate_fps=*/30, num_frames); - std::unique_ptr stats = RunEncodeTest( - codec_type, codec_impl, video_info, frame_settings, num_frames, - /*save_codec_input=*/false, /*save_codec_output=*/false); + uint32_t initial_timestamp_rtp = + encoding_settings.rbegin()->first + k90kHz / Frequency::Hertz(30); + + std::map encoding_settings2 = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", + /*width=*/640, /*height=*/360, {bitrate_kbps.second}, + /*framerate_fps=*/30, num_frames, initial_timestamp_rtp); + + encoding_settings.merge(encoding_settings2); + + std::unique_ptr stats = + RunEncodeTest(codec_type, codec_impl, video_info, encoding_settings); VideoCodecStats::Stream stream; if (stats != nullptr) { - std::vector frames = - stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); - SetTargetRates(frame_settings, frames); - stream = stats->Aggregate(frames); + stream = stats->Aggregate({.min_timestamp_rtp = initial_timestamp_rtp}); if (absl::GetFlag(FLAGS_webrtc_quick_perf_test)) { EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); @@ -749,34 +379,34 @@ TEST_P(FramerateAdaptationTest, FramerateAdaptation) { auto [codec_type, codec_impl, video_info, framerate_fps] = GetParam(); int duration_s = 10; // Duration of fixed rate interval. - int first_frame = static_cast(duration_s * framerate_fps.first); - int num_frames = static_cast( - duration_s * (framerate_fps.first + framerate_fps.second)); - std::map frame_settings = { - {0, - {.layer_settings = {{LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = 640, .height = 360}, - .framerate = Frequency::MilliHertz( - 1000 * framerate_fps.first), - .bitrate = DataRate::KilobitsPerSec(512)}}}}}, - {first_frame, - {.layer_settings = { - {LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = 640, .height = 360}, - .framerate = Frequency::MilliHertz(1000 * framerate_fps.second), - .bitrate = DataRate::KilobitsPerSec(512)}}}}}}; + std::map encoding_settings = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", + /*width=*/640, /*height=*/360, + /*layer_bitrates_kbps=*/{512}, framerate_fps.first, + static_cast(duration_s * framerate_fps.first)); - std::unique_ptr stats = RunEncodeTest( - codec_type, codec_impl, video_info, frame_settings, num_frames, - /*save_codec_input=*/false, /*save_codec_output=*/false); + uint32_t initial_timestamp_rtp = + encoding_settings.rbegin()->first + + k90kHz / Frequency::Hertz(framerate_fps.first); + + std::map encoding_settings2 = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", /*width=*/640, + /*height=*/360, + /*layer_bitrates_kbps=*/{512}, framerate_fps.second, + static_cast(duration_s * framerate_fps.second), + initial_timestamp_rtp); + + encoding_settings.merge(encoding_settings2); + + std::unique_ptr stats = + RunEncodeTest(codec_type, codec_impl, video_info, encoding_settings); VideoCodecStats::Stream stream; if (stats != nullptr) { - std::vector frames = - stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); - SetTargetRates(frame_settings, frames); - stream = stats->Aggregate(frames); + stream = stats->Aggregate({.min_timestamp_rtp = initial_timestamp_rtp}); if (absl::GetFlag(FLAGS_webrtc_quick_perf_test)) { EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); @@ -805,7 +435,6 @@ INSTANTIATE_TEST_SUITE_P(All, Values(kFourPeople_1280x720_30), Values(std::pair(30, 15), std::pair(15, 30))), FramerateAdaptationTest::TestParamsToString); - } // namespace test } // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl.cc b/modules/video_coding/codecs/test/video_codec_tester_impl.cc deleted file mode 100644 index f15b1b35f3..0000000000 --- a/modules/video_coding/codecs/test/video_codec_tester_impl.cc +++ /dev/null @@ -1,437 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" - -#include -#include -#include -#include - -#include "api/task_queue/default_task_queue_factory.h" -#include "api/units/frequency.h" -#include "api/units/time_delta.h" -#include "api/units/timestamp.h" -#include "api/video/encoded_image.h" -#include "api/video/i420_buffer.h" -#include "api/video/video_codec_type.h" -#include "api/video/video_frame.h" -#include "modules/video_coding/codecs/test/video_codec_analyzer.h" -#include "modules/video_coding/utility/ivf_file_writer.h" -#include "rtc_base/event.h" -#include "rtc_base/time_utils.h" -#include "system_wrappers/include/sleep.h" -#include "test/testsupport/video_frame_writer.h" - -namespace webrtc { -namespace test { - -namespace { -using RawVideoSource = VideoCodecTester::RawVideoSource; -using CodedVideoSource = VideoCodecTester::CodedVideoSource; -using Decoder = VideoCodecTester::Decoder; -using Encoder = VideoCodecTester::Encoder; -using EncoderSettings = VideoCodecTester::EncoderSettings; -using DecoderSettings = VideoCodecTester::DecoderSettings; -using PacingSettings = VideoCodecTester::PacingSettings; -using PacingMode = PacingSettings::PacingMode; - -constexpr Frequency k90kHz = Frequency::Hertz(90000); - -// A thread-safe wrapper for video source to be shared with the quality analyzer -// that reads reference frames from a separate thread. -class SyncRawVideoSource : public VideoCodecAnalyzer::ReferenceVideoSource { - public: - explicit SyncRawVideoSource(RawVideoSource* video_source) - : video_source_(video_source) {} - - absl::optional PullFrame() { - MutexLock lock(&mutex_); - return video_source_->PullFrame(); - } - - VideoFrame GetFrame(uint32_t timestamp_rtp, Resolution resolution) override { - MutexLock lock(&mutex_); - return video_source_->GetFrame(timestamp_rtp, resolution); - } - - protected: - RawVideoSource* const video_source_ RTC_GUARDED_BY(mutex_); - Mutex mutex_; -}; - -// Pacer calculates delay necessary to keep frame encode or decode call spaced -// from the previous calls by the pacing time. `Delay` is expected to be called -// as close as possible to posting frame encode or decode task. This class is -// not thread safe. -class Pacer { - public: - explicit Pacer(PacingSettings settings) - : settings_(settings), delay_(TimeDelta::Zero()) {} - Timestamp Schedule(Timestamp timestamp) { - Timestamp now = Timestamp::Micros(rtc::TimeMicros()); - if (settings_.mode == PacingMode::kNoPacing) { - return now; - } - - Timestamp scheduled = now; - if (prev_scheduled_) { - scheduled = *prev_scheduled_ + PacingTime(timestamp); - if (scheduled < now) { - scheduled = now; - } - } - - prev_timestamp_ = timestamp; - prev_scheduled_ = scheduled; - return scheduled; - } - - private: - TimeDelta PacingTime(Timestamp timestamp) { - if (settings_.mode == PacingMode::kRealTime) { - return timestamp - *prev_timestamp_; - } - RTC_CHECK_EQ(PacingMode::kConstantRate, settings_.mode); - return 1 / settings_.constant_rate; - } - - PacingSettings settings_; - absl::optional prev_timestamp_; - absl::optional prev_scheduled_; - TimeDelta delay_; -}; - -// Task queue that keeps the number of queued tasks below a certain limit. If -// the limit is reached, posting of a next task is blocked until execution of a -// previously posted task starts. This class is not thread-safe. -class LimitedTaskQueue { - public: - // The codec tester reads frames from video source in the main thread. - // Encoding and decoding are done in separate threads. If encoding or - // decoding is slow, the reading may go far ahead and may buffer too many - // frames in memory. To prevent this we limit the encoding/decoding queue - // size. When the queue is full, the main thread and, hence, reading frames - // from video source is blocked until a previously posted encoding/decoding - // task starts. - static constexpr int kMaxTaskQueueSize = 3; - - LimitedTaskQueue() : queue_size_(0) {} - - void PostScheduledTask(absl::AnyInvocable task, Timestamp start) { - ++queue_size_; - task_queue_.PostTask([this, task = std::move(task), start]() mutable { - int wait_ms = static_cast(start.ms() - rtc::TimeMillis()); - if (wait_ms > 0) { - SleepMs(wait_ms); - } - - std::move(task)(); - --queue_size_; - task_executed_.Set(); - }); - - task_executed_.Reset(); - if (queue_size_ > kMaxTaskQueueSize) { - task_executed_.Wait(rtc::Event::kForever); - } - RTC_CHECK(queue_size_ <= kMaxTaskQueueSize); - } - - void WaitForPreviouslyPostedTasks() { - task_queue_.SendTask([] {}); - } - - TaskQueueForTest task_queue_; - std::atomic_int queue_size_; - rtc::Event task_executed_; -}; - -class TesterY4mWriter { - public: - explicit TesterY4mWriter(absl::string_view base_path) - : base_path_(base_path) {} - - ~TesterY4mWriter() { - task_queue_.SendTask([] {}); - } - - void Write(const VideoFrame& frame, int spatial_idx) { - task_queue_.PostTask([this, frame, spatial_idx] { - if (y4m_writers_.find(spatial_idx) == y4m_writers_.end()) { - std::string file_path = - base_path_ + "_s" + std::to_string(spatial_idx) + ".y4m"; - - Y4mVideoFrameWriterImpl* y4m_writer = new Y4mVideoFrameWriterImpl( - file_path, frame.width(), frame.height(), /*fps=*/30); - RTC_CHECK(y4m_writer); - - y4m_writers_[spatial_idx] = - std::unique_ptr(y4m_writer); - } - - y4m_writers_.at(spatial_idx)->WriteFrame(frame); - }); - } - - protected: - std::string base_path_; - std::map> y4m_writers_; - TaskQueueForTest task_queue_; -}; - -class TesterIvfWriter { - public: - explicit TesterIvfWriter(absl::string_view base_path) - : base_path_(base_path) {} - - ~TesterIvfWriter() { - task_queue_.SendTask([] {}); - } - - void Write(const EncodedImage& encoded_frame) { - task_queue_.PostTask([this, encoded_frame] { - int spatial_idx = encoded_frame.SpatialIndex().value_or(0); - if (ivf_file_writers_.find(spatial_idx) == ivf_file_writers_.end()) { - std::string ivf_path = - base_path_ + "_s" + std::to_string(spatial_idx) + ".ivf"; - - FileWrapper ivf_file = FileWrapper::OpenWriteOnly(ivf_path); - RTC_CHECK(ivf_file.is_open()); - - std::unique_ptr ivf_writer = - IvfFileWriter::Wrap(std::move(ivf_file), /*byte_limit=*/0); - RTC_CHECK(ivf_writer); - - ivf_file_writers_[spatial_idx] = std::move(ivf_writer); - } - - // To play: ffplay -vcodec vp8|vp9|av1|hevc|h264 filename - ivf_file_writers_.at(spatial_idx) - ->WriteFrame(encoded_frame, VideoCodecType::kVideoCodecGeneric); - }); - } - - protected: - std::string base_path_; - std::map> ivf_file_writers_; - TaskQueueForTest task_queue_; -}; - -class TesterDecoder { - public: - TesterDecoder(Decoder* decoder, - VideoCodecAnalyzer* analyzer, - const DecoderSettings& settings) - : decoder_(decoder), - analyzer_(analyzer), - settings_(settings), - pacer_(settings.pacing) { - RTC_CHECK(analyzer_) << "Analyzer must be provided"; - - if (settings.decoder_input_base_path) { - input_writer_ = - std::make_unique(*settings.decoder_input_base_path); - } - - if (settings.decoder_output_base_path) { - output_writer_ = - std::make_unique(*settings.decoder_output_base_path); - } - } - - void Initialize() { - task_queue_.PostScheduledTask([this] { decoder_->Initialize(); }, - Timestamp::Zero()); - task_queue_.WaitForPreviouslyPostedTasks(); - } - - void Decode(const EncodedImage& input_frame) { - Timestamp timestamp = - Timestamp::Micros((input_frame.RtpTimestamp() / k90kHz).us()); - - task_queue_.PostScheduledTask( - [this, input_frame] { - analyzer_->StartDecode(input_frame); - - decoder_->Decode( - input_frame, - [this, spatial_idx = input_frame.SpatialIndex().value_or(0)]( - const VideoFrame& output_frame) { - analyzer_->FinishDecode(output_frame, spatial_idx); - - if (output_writer_) { - output_writer_->Write(output_frame, spatial_idx); - } - }); - - if (input_writer_) { - input_writer_->Write(input_frame); - } - }, - pacer_.Schedule(timestamp)); - } - - void Flush() { - task_queue_.PostScheduledTask([this] { decoder_->Flush(); }, - Timestamp::Zero()); - task_queue_.WaitForPreviouslyPostedTasks(); - } - - protected: - Decoder* const decoder_; - VideoCodecAnalyzer* const analyzer_; - const DecoderSettings& settings_; - Pacer pacer_; - LimitedTaskQueue task_queue_; - std::unique_ptr input_writer_; - std::unique_ptr output_writer_; -}; - -class TesterEncoder { - public: - TesterEncoder(Encoder* encoder, - TesterDecoder* decoder, - VideoCodecAnalyzer* analyzer, - const EncoderSettings& settings) - : encoder_(encoder), - decoder_(decoder), - analyzer_(analyzer), - settings_(settings), - pacer_(settings.pacing) { - RTC_CHECK(analyzer_) << "Analyzer must be provided"; - if (settings.encoder_input_base_path) { - input_writer_ = - std::make_unique(*settings.encoder_input_base_path); - } - - if (settings.encoder_output_base_path) { - output_writer_ = - std::make_unique(*settings.encoder_output_base_path); - } - } - - void Initialize() { - task_queue_.PostScheduledTask([this] { encoder_->Initialize(); }, - Timestamp::Zero()); - task_queue_.WaitForPreviouslyPostedTasks(); - } - - void Encode(const VideoFrame& input_frame) { - Timestamp timestamp = - Timestamp::Micros((input_frame.timestamp() / k90kHz).us()); - - task_queue_.PostScheduledTask( - [this, input_frame] { - analyzer_->StartEncode(input_frame); - encoder_->Encode(input_frame, - [this](const EncodedImage& encoded_frame) { - analyzer_->FinishEncode(encoded_frame); - - if (decoder_ != nullptr) { - decoder_->Decode(encoded_frame); - } - - if (output_writer_ != nullptr) { - output_writer_->Write(encoded_frame); - } - }); - - if (input_writer_) { - input_writer_->Write(input_frame, /*spatial_idx=*/0); - } - }, - pacer_.Schedule(timestamp)); - } - - void Flush() { - task_queue_.PostScheduledTask([this] { encoder_->Flush(); }, - Timestamp::Zero()); - task_queue_.WaitForPreviouslyPostedTasks(); - } - - protected: - Encoder* const encoder_; - TesterDecoder* const decoder_; - VideoCodecAnalyzer* const analyzer_; - const EncoderSettings& settings_; - std::unique_ptr input_writer_; - std::unique_ptr output_writer_; - Pacer pacer_; - LimitedTaskQueue task_queue_; -}; - -} // namespace - -std::unique_ptr VideoCodecTesterImpl::RunDecodeTest( - CodedVideoSource* video_source, - Decoder* decoder, - const DecoderSettings& decoder_settings) { - VideoCodecAnalyzer perf_analyzer; - TesterDecoder tester_decoder(decoder, &perf_analyzer, decoder_settings); - - tester_decoder.Initialize(); - - while (auto frame = video_source->PullFrame()) { - tester_decoder.Decode(*frame); - } - - tester_decoder.Flush(); - - return perf_analyzer.GetStats(); -} - -std::unique_ptr VideoCodecTesterImpl::RunEncodeTest( - RawVideoSource* video_source, - Encoder* encoder, - const EncoderSettings& encoder_settings) { - SyncRawVideoSource sync_source(video_source); - VideoCodecAnalyzer perf_analyzer; - TesterEncoder tester_encoder(encoder, /*decoder=*/nullptr, &perf_analyzer, - encoder_settings); - - tester_encoder.Initialize(); - - while (auto frame = sync_source.PullFrame()) { - tester_encoder.Encode(*frame); - } - - tester_encoder.Flush(); - - return perf_analyzer.GetStats(); -} - -std::unique_ptr VideoCodecTesterImpl::RunEncodeDecodeTest( - RawVideoSource* video_source, - Encoder* encoder, - Decoder* decoder, - const EncoderSettings& encoder_settings, - const DecoderSettings& decoder_settings) { - SyncRawVideoSource sync_source(video_source); - VideoCodecAnalyzer perf_analyzer(&sync_source); - TesterDecoder tester_decoder(decoder, &perf_analyzer, decoder_settings); - TesterEncoder tester_encoder(encoder, &tester_decoder, &perf_analyzer, - encoder_settings); - - tester_encoder.Initialize(); - tester_decoder.Initialize(); - - while (auto frame = sync_source.PullFrame()) { - tester_encoder.Encode(*frame); - } - - tester_encoder.Flush(); - tester_decoder.Flush(); - - return perf_analyzer.GetStats(); -} - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl.h b/modules/video_coding/codecs/test/video_codec_tester_impl.h deleted file mode 100644 index 32191b5a98..0000000000 --- a/modules/video_coding/codecs/test/video_codec_tester_impl.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ -#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ - -#include - -#include "api/test/video_codec_tester.h" - -namespace webrtc { -namespace test { - -// A stateless implementation of `VideoCodecTester`. This class is thread safe. -class VideoCodecTesterImpl : public VideoCodecTester { - public: - std::unique_ptr RunDecodeTest( - CodedVideoSource* video_source, - Decoder* decoder, - const DecoderSettings& decoder_settings) override; - - std::unique_ptr RunEncodeTest( - RawVideoSource* video_source, - Encoder* encoder, - const EncoderSettings& encoder_settings) override; - - std::unique_ptr RunEncodeDecodeTest( - RawVideoSource* video_source, - Encoder* encoder, - Decoder* decoder, - const EncoderSettings& encoder_settings, - const DecoderSettings& decoder_settings) override; -}; - -} // namespace test -} // namespace webrtc - -#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc b/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc deleted file mode 100644 index a8c118ef20..0000000000 --- a/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" - -#include -#include -#include -#include - -#include "api/units/frequency.h" -#include "api/units/time_delta.h" -#include "api/video/encoded_image.h" -#include "api/video/i420_buffer.h" -#include "api/video/video_frame.h" -#include "rtc_base/fake_clock.h" -#include "rtc_base/gunit.h" -#include "rtc_base/task_queue_for_test.h" -#include "rtc_base/time_utils.h" -#include "test/gmock.h" -#include "test/gtest.h" - -namespace webrtc { -namespace test { - -namespace { -using ::testing::_; -using ::testing::Invoke; -using ::testing::InvokeWithoutArgs; -using ::testing::Return; - -using Decoder = VideoCodecTester::Decoder; -using Encoder = VideoCodecTester::Encoder; -using CodedVideoSource = VideoCodecTester::CodedVideoSource; -using RawVideoSource = VideoCodecTester::RawVideoSource; -using DecoderSettings = VideoCodecTester::DecoderSettings; -using EncoderSettings = VideoCodecTester::EncoderSettings; -using PacingSettings = VideoCodecTester::PacingSettings; -using PacingMode = PacingSettings::PacingMode; - -constexpr Frequency k90kHz = Frequency::Hertz(90000); - -struct PacingTestParams { - PacingSettings pacing_settings; - Frequency framerate; - int num_frames; - std::vector expected_delta_ms; -}; - -VideoFrame CreateVideoFrame(uint32_t timestamp_rtp) { - rtc::scoped_refptr buffer(I420Buffer::Create(2, 2)); - return VideoFrame::Builder() - .set_video_frame_buffer(buffer) - .set_timestamp_rtp(timestamp_rtp) - .build(); -} - -EncodedImage CreateEncodedImage(uint32_t timestamp_rtp) { - EncodedImage encoded_image; - encoded_image.SetRtpTimestamp(timestamp_rtp); - return encoded_image; -} - -class MockRawVideoSource : public RawVideoSource { - public: - MockRawVideoSource(int num_frames, Frequency framerate) - : num_frames_(num_frames), frame_num_(0), framerate_(framerate) {} - - absl::optional PullFrame() override { - if (frame_num_ >= num_frames_) { - return absl::nullopt; - } - uint32_t timestamp_rtp = frame_num_ * k90kHz / framerate_; - ++frame_num_; - return CreateVideoFrame(timestamp_rtp); - } - - MOCK_METHOD(VideoFrame, - GetFrame, - (uint32_t timestamp_rtp, Resolution), - (override)); - - private: - int num_frames_; - int frame_num_; - Frequency framerate_; -}; - -class MockCodedVideoSource : public CodedVideoSource { - public: - MockCodedVideoSource(int num_frames, Frequency framerate) - : num_frames_(num_frames), frame_num_(0), framerate_(framerate) {} - - absl::optional PullFrame() override { - if (frame_num_ >= num_frames_) { - return absl::nullopt; - } - uint32_t timestamp_rtp = frame_num_ * k90kHz / framerate_; - ++frame_num_; - return CreateEncodedImage(timestamp_rtp); - } - - private: - int num_frames_; - int frame_num_; - Frequency framerate_; -}; - -class MockDecoder : public Decoder { - public: - MOCK_METHOD(void, Initialize, (), (override)); - MOCK_METHOD(void, - Decode, - (const EncodedImage& frame, DecodeCallback callback), - (override)); - MOCK_METHOD(void, Flush, (), (override)); -}; - -class MockEncoder : public Encoder { - public: - MOCK_METHOD(void, Initialize, (), (override)); - MOCK_METHOD(void, - Encode, - (const VideoFrame& frame, EncodeCallback callback), - (override)); - MOCK_METHOD(void, Flush, (), (override)); -}; - -} // namespace - -class VideoCodecTesterImplPacingTest - : public ::testing::TestWithParam { - public: - VideoCodecTesterImplPacingTest() : test_params_(GetParam()) {} - - protected: - PacingTestParams test_params_; -}; - -TEST_P(VideoCodecTesterImplPacingTest, PaceEncode) { - MockRawVideoSource video_source(test_params_.num_frames, - test_params_.framerate); - MockEncoder encoder; - EncoderSettings encoder_settings; - encoder_settings.pacing = test_params_.pacing_settings; - - VideoCodecTesterImpl tester; - auto fs = - tester.RunEncodeTest(&video_source, &encoder, encoder_settings)->Slice(); - ASSERT_EQ(static_cast(fs.size()), test_params_.num_frames); - - for (size_t i = 1; i < fs.size(); ++i) { - int delta_ms = (fs[i].encode_start - fs[i - 1].encode_start).ms(); - EXPECT_NEAR(delta_ms, test_params_.expected_delta_ms[i - 1], 10); - } -} - -TEST_P(VideoCodecTesterImplPacingTest, PaceDecode) { - MockCodedVideoSource video_source(test_params_.num_frames, - test_params_.framerate); - MockDecoder decoder; - DecoderSettings decoder_settings; - decoder_settings.pacing = test_params_.pacing_settings; - - VideoCodecTesterImpl tester; - auto fs = - tester.RunDecodeTest(&video_source, &decoder, decoder_settings)->Slice(); - ASSERT_EQ(static_cast(fs.size()), test_params_.num_frames); - - for (size_t i = 1; i < fs.size(); ++i) { - int delta_ms = (fs[i].decode_start - fs[i - 1].decode_start).ms(); - EXPECT_NEAR(delta_ms, test_params_.expected_delta_ms[i - 1], 20); - } -} - -INSTANTIATE_TEST_SUITE_P( - DISABLED_All, - VideoCodecTesterImplPacingTest, - ::testing::ValuesIn( - {// No pacing. - PacingTestParams({.pacing_settings = {.mode = PacingMode::kNoPacing}, - .framerate = Frequency::Hertz(10), - .num_frames = 3, - .expected_delta_ms = {0, 0}}), - // Real-time pacing. - PacingTestParams({.pacing_settings = {.mode = PacingMode::kRealTime}, - .framerate = Frequency::Hertz(10), - .num_frames = 3, - .expected_delta_ms = {100, 100}}), - // Pace with specified constant rate. - PacingTestParams( - {.pacing_settings = {.mode = PacingMode::kConstantRate, - .constant_rate = Frequency::Hertz(20)}, - .framerate = Frequency::Hertz(10), - .num_frames = 3, - .expected_delta_ms = {50, 50}})})); -} // namespace test -} // namespace webrtc diff --git a/test/BUILD.gn b/test/BUILD.gn index be8ee1684e..fbc1ab1ad5 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -716,6 +716,7 @@ if (rtc_include_tests) { ":test_main", ":test_support", ":test_support_test_artifacts", + ":video_codec_tester", ":video_test_common", ":video_test_support", ":y4m_frame_generator", @@ -723,11 +724,15 @@ if (rtc_include_tests) { "../api:create_frame_generator", "../api:create_simulcast_test_fixture_api", "../api:frame_generator_api", + "../api:mock_video_codec_factory", + "../api:mock_video_decoder", + "../api:mock_video_encoder", "../api:scoped_refptr", "../api:simulcast_test_fixture_api", "../api/task_queue:task_queue_test", "../api/test/video:function_video_factory", "../api/test/video:video_frame_writer", + "../api/units:data_rate", "../api/units:time_delta", "../api/video:encoded_image", "../api/video:video_frame", @@ -744,6 +749,7 @@ if (rtc_include_tests) { "../modules/video_coding:webrtc_h264", "../modules/video_coding:webrtc_vp8", "../modules/video_coding:webrtc_vp9", + "../modules/video_coding/svc:scalability_mode_util", "../rtc_base:criticalsection", "../rtc_base:rtc_event", "../rtc_base:rtc_task_queue", @@ -757,6 +763,7 @@ if (rtc_include_tests) { "scenario:scenario_unittests", "time_controller:time_controller", "time_controller:time_controller_unittests", + "//third_party/libyuv", ] absl_deps = [ "//third_party/abseil-cpp/absl/flags:flag", @@ -781,6 +788,7 @@ if (rtc_include_tests) { "testsupport/y4m_frame_writer_unittest.cc", "testsupport/yuv_frame_reader_unittest.cc", "testsupport/yuv_frame_writer_unittest.cc", + "video_codec_tester_unittest.cc", ] if (rtc_enable_protobuf) { @@ -1368,3 +1376,46 @@ rtc_library("fake_encoded_frame") { ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } + +rtc_library("video_codec_tester") { + testonly = true + sources = [ + "video_codec_tester.cc", + "video_codec_tester.h", + ] + deps = [ + "../api:array_view", + "../api/numerics:numerics", + "../api/test/metrics:metric", + "../api/test/metrics:metrics_logger", + "../api/units:data_rate", + "../api/units:data_size", + "../api/units:frequency", + "../api/units:time_delta", + "../api/units:timestamp", + "../api/video:builtin_video_bitrate_allocator_factory", + "../api/video:encoded_image", + "../api/video:resolution", + "../api/video:video_bitrate_allocator", + "../api/video:video_frame", + "../api/video_codecs:video_codecs_api", + "../media:media_constants", + "../modules/video_coding:video_codec_interface", + "../modules/video_coding:video_coding_utility", + "../modules/video_coding:webrtc_vp9_helpers", + "../modules/video_coding/codecs/av1:av1_svc_config", + "../modules/video_coding/svc:scalability_mode_util", + "../rtc_base:checks", + "../rtc_base:logging", + "../rtc_base:rtc_event", + "../rtc_base:task_queue_for_test", + "../rtc_base:timeutils", + "../rtc_base/synchronization:mutex", + "../system_wrappers", + "../test:fileutils", + "../test:video_test_support", + "//third_party/libyuv", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} diff --git a/test/video_codec_tester.cc b/test/video_codec_tester.cc new file mode 100644 index 0000000000..26f0a61372 --- /dev/null +++ b/test/video_codec_tester.cc @@ -0,0 +1,1260 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/video_codec_tester.h" + +#include +#include +#include +#include + +#include "api/array_view.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/builtin_video_bitrate_allocator_factory.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_bitrate_allocator.h" +#include "api/video/video_codec_type.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_encoder.h" +#include "media/base/media_constants.h" +#include "modules/video_coding/codecs/av1/av1_svc_config.h" +#include "modules/video_coding/codecs/vp9/svc_config.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "modules/video_coding/utility/ivf_file_writer.h" +#include "rtc_base/event.h" +#include "rtc_base/logging.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/task_queue_for_test.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/sleep.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/frame_reader.h" +#include "test/testsupport/video_frame_writer.h" +#include "third_party/libyuv/include/libyuv/compare.h" + +namespace webrtc { +namespace test { + +namespace { +using CodedVideoSource = VideoCodecTester::CodedVideoSource; +using VideoSourceSettings = VideoCodecTester::VideoSourceSettings; +using EncodingSettings = VideoCodecTester::EncodingSettings; +using LayerSettings = EncodingSettings::LayerSettings; +using LayerId = VideoCodecTester::LayerId; +using EncoderSettings = VideoCodecTester::EncoderSettings; +using DecoderSettings = VideoCodecTester::DecoderSettings; +using PacingSettings = VideoCodecTester::PacingSettings; +using PacingMode = PacingSettings::PacingMode; +using VideoCodecStats = VideoCodecTester::VideoCodecStats; +using DecodeCallback = + absl::AnyInvocable; +using webrtc::test::ImprovementDirection; + +constexpr Frequency k90kHz = Frequency::Hertz(90000); + +const std::set kFullSvcScalabilityModes{ + ScalabilityMode::kL2T1, ScalabilityMode::kL2T1h, ScalabilityMode::kL2T2, + ScalabilityMode::kL2T2h, ScalabilityMode::kL2T3, ScalabilityMode::kL2T3h, + ScalabilityMode::kL3T1, ScalabilityMode::kL3T1h, ScalabilityMode::kL3T2, + ScalabilityMode::kL3T2h, ScalabilityMode::kL3T3, ScalabilityMode::kL3T3h}; + +const std::set kKeySvcScalabilityModes{ + ScalabilityMode::kL2T1_KEY, ScalabilityMode::kL2T2_KEY, + ScalabilityMode::kL2T2_KEY_SHIFT, ScalabilityMode::kL2T3_KEY, + ScalabilityMode::kL3T1_KEY, ScalabilityMode::kL3T2_KEY, + ScalabilityMode::kL3T3_KEY}; + +// A thread-safe raw video frame reader. +class VideoSource { + public: + explicit VideoSource(VideoSourceSettings source_settings) + : source_settings_(source_settings) { + MutexLock lock(&mutex_); + frame_reader_ = CreateYuvFrameReader( + source_settings_.file_path, source_settings_.resolution, + YuvFrameReaderImpl::RepeatMode::kPingPong); + RTC_CHECK(frame_reader_); + } + + // Pulls next frame. + VideoFrame PullFrame(uint32_t timestamp_rtp, + Resolution resolution, + Frequency framerate) { + MutexLock lock(&mutex_); + int frame_num; + auto buffer = frame_reader_->PullFrame( + &frame_num, resolution, + {.num = framerate.millihertz(), + .den = source_settings_.framerate.millihertz()}); + RTC_CHECK(buffer) << "Can not pull frame. RTP timestamp " << timestamp_rtp; + frame_num_[timestamp_rtp] = frame_num; + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .set_timestamp_us((timestamp_rtp / k90kHz).us()) + .build(); + } + + // Reads frame specified by `timestamp_rtp`, scales it to `resolution` and + // returns. Frame with the given `timestamp_rtp` is expected to be pulled + // before. + VideoFrame ReadFrame(uint32_t timestamp_rtp, Resolution resolution) { + MutexLock lock(&mutex_); + RTC_CHECK(frame_num_.find(timestamp_rtp) != frame_num_.end()) + << "Frame with RTP timestamp " << timestamp_rtp + << " was not pulled before"; + auto buffer = + frame_reader_->ReadFrame(frame_num_.at(timestamp_rtp), resolution); + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .build(); + } + + private: + VideoSourceSettings source_settings_; + std::unique_ptr frame_reader_ RTC_GUARDED_BY(mutex_); + std::map frame_num_ RTC_GUARDED_BY(mutex_); + Mutex mutex_; +}; + +// Pacer calculates delay necessary to keep frame encode or decode call spaced +// from the previous calls by the pacing time. `Schedule` is expected to be +// called as close as possible to posting frame encode or decode task. This +// class is not thread safe. +class Pacer { + public: + explicit Pacer(PacingSettings settings) + : settings_(settings), delay_(TimeDelta::Zero()) {} + + Timestamp Schedule(Timestamp timestamp) { + Timestamp now = Timestamp::Micros(rtc::TimeMicros()); + if (settings_.mode == PacingMode::kNoPacing) { + return now; + } + + Timestamp scheduled = now; + if (prev_scheduled_) { + scheduled = *prev_scheduled_ + PacingTime(timestamp); + if (scheduled < now) { + scheduled = now; + } + } + + prev_timestamp_ = timestamp; + prev_scheduled_ = scheduled; + return scheduled; + } + + private: + TimeDelta PacingTime(Timestamp timestamp) { + if (settings_.mode == PacingMode::kRealTime) { + return timestamp - *prev_timestamp_; + } + RTC_CHECK_EQ(PacingMode::kConstantRate, settings_.mode); + return 1 / settings_.constant_rate; + } + + PacingSettings settings_; + absl::optional prev_timestamp_; + absl::optional prev_scheduled_; + TimeDelta delay_; +}; + +class LimitedTaskQueue { + public: + // The codec tester reads frames from video source in the main thread. + // Encoding and decoding are done in separate threads. If encoding or + // decoding is slow, the reading may go far ahead and may buffer too many + // frames in memory. To prevent this we limit the encoding/decoding queue + // size. When the queue is full, the main thread and, hence, reading frames + // from video source is blocked until a previously posted encoding/decoding + // task starts. + static constexpr int kMaxTaskQueueSize = 3; + + LimitedTaskQueue() : queue_size_(0) {} + + void PostScheduledTask(absl::AnyInvocable task, Timestamp start) { + ++queue_size_; + task_queue_.PostTask([this, task = std::move(task), start]() mutable { + // `TaskQueue` doesn't guarantee FIFO order of execution for delayed + // tasks. + int wait_ms = static_cast(start.ms() - rtc::TimeMillis()); + if (wait_ms > 0) { + SleepMs(wait_ms); + } + std::move(task)(); + --queue_size_; + task_executed_.Set(); + }); + + task_executed_.Reset(); + if (queue_size_ > kMaxTaskQueueSize) { + task_executed_.Wait(rtc::Event::kForever); + RTC_CHECK(queue_size_ <= kMaxTaskQueueSize); + } + } + + void PostTaskAndWait(absl::AnyInvocable task) { + PostScheduledTask(std::move(task), Timestamp::Zero()); + task_queue_.WaitForPreviouslyPostedTasks(); + } + + private: + TaskQueueForTest task_queue_; + std::atomic_int queue_size_; + rtc::Event task_executed_; +}; + +class TesterY4mWriter { + public: + explicit TesterY4mWriter(absl::string_view base_path) + : base_path_(base_path) {} + + ~TesterY4mWriter() { + task_queue_.SendTask([] {}); + } + + void Write(const VideoFrame& frame, int spatial_idx) { + task_queue_.PostTask([this, frame, spatial_idx] { + if (y4m_writers_.find(spatial_idx) == y4m_writers_.end()) { + std::string file_path = + base_path_ + "-s" + std::to_string(spatial_idx) + ".y4m"; + Y4mVideoFrameWriterImpl* y4m_writer = new Y4mVideoFrameWriterImpl( + file_path, frame.width(), frame.height(), /*fps=*/30); + RTC_CHECK(y4m_writer); + + y4m_writers_[spatial_idx] = + std::unique_ptr(y4m_writer); + } + + y4m_writers_.at(spatial_idx)->WriteFrame(frame); + }); + } + + private: + std::string base_path_; + std::map> y4m_writers_; + TaskQueueForTest task_queue_; +}; + +class TesterIvfWriter { + public: + explicit TesterIvfWriter(absl::string_view base_path) + : base_path_(base_path) {} + + ~TesterIvfWriter() { + task_queue_.SendTask([] {}); + } + + void Write(const EncodedImage& encoded_frame) { + task_queue_.PostTask([this, encoded_frame] { + int spatial_idx = encoded_frame.SimulcastIndex().value_or(0); + if (ivf_file_writers_.find(spatial_idx) == ivf_file_writers_.end()) { + std::string ivf_path = + base_path_ + "-s" + std::to_string(spatial_idx) + ".ivf"; + FileWrapper ivf_file = FileWrapper::OpenWriteOnly(ivf_path); + RTC_CHECK(ivf_file.is_open()); + + std::unique_ptr ivf_writer = + IvfFileWriter::Wrap(std::move(ivf_file), /*byte_limit=*/0); + RTC_CHECK(ivf_writer); + + ivf_file_writers_[spatial_idx] = std::move(ivf_writer); + } + + // To play: ffplay -vcodec vp8|vp9|av1|hevc|h264 filename + ivf_file_writers_.at(spatial_idx) + ->WriteFrame(encoded_frame, VideoCodecType::kVideoCodecGeneric); + }); + } + + private: + std::string base_path_; + std::map> ivf_file_writers_; + TaskQueueForTest task_queue_; +}; + +class LeakyBucket { + public: + LeakyBucket() : level_bits_(0) {} + + // Updates bucket level and returns its current level in bits. Data is remove + // from bucket with rate equal to target bitrate of previous frame. Bucket + // level is tracked with floating point precision. Returned value of bucket + // level is rounded up. + int Update(const VideoCodecStats::Frame& frame) { + RTC_CHECK(frame.target_bitrate) << "Bitrate must be specified."; + if (prev_frame_) { + RTC_CHECK_GT(frame.timestamp_rtp, prev_frame_->timestamp_rtp) + << "Timestamp must increase."; + TimeDelta passed = + (frame.timestamp_rtp - prev_frame_->timestamp_rtp) / k90kHz; + level_bits_ -= + prev_frame_->target_bitrate->bps() * passed.seconds(); + level_bits_ = std::max(level_bits_, 0.0); + } + prev_frame_ = frame; + level_bits_ += frame.frame_size.bytes() * 8; + return static_cast(std::ceil(level_bits_)); + } + + private: + absl::optional prev_frame_; + double level_bits_; +}; + +class VideoCodecAnalyzer : public VideoCodecTester::VideoCodecStats { + public: + explicit VideoCodecAnalyzer(VideoSource* video_source) + : video_source_(video_source) {} + + void StartEncode(const VideoFrame& video_frame, + const EncodingSettings& encoding_settings) { + int64_t encode_start_us = rtc::TimeMicros(); + task_queue_.PostTask([this, timestamp_rtp = video_frame.timestamp(), + encoding_settings, encode_start_us]() { + RTC_CHECK(frames_.find(timestamp_rtp) == frames_.end()) + << "Duplicate frame. Frame with timestamp " << timestamp_rtp + << " was seen before"; + + Frame frame; + frame.timestamp_rtp = timestamp_rtp; + frame.encode_start = Timestamp::Micros(encode_start_us), + frames_.emplace(timestamp_rtp, + std::map{{/*spatial_idx=*/0, frame}}); + encoding_settings_.emplace(timestamp_rtp, encoding_settings); + }); + } + + void FinishEncode(const EncodedImage& encoded_frame) { + int64_t encode_finished_us = rtc::TimeMicros(); + task_queue_.PostTask( + [this, timestamp_rtp = encoded_frame.RtpTimestamp(), + spatial_idx = encoded_frame.SpatialIndex().value_or(0), + temporal_idx = encoded_frame.TemporalIndex().value_or(0), + width = encoded_frame._encodedWidth, + height = encoded_frame._encodedHeight, + frame_type = encoded_frame._frameType, + frame_size_bytes = encoded_frame.size(), qp = encoded_frame.qp_, + encode_finished_us]() { + if (spatial_idx > 0) { + RTC_CHECK(frames_.find(timestamp_rtp) != frames_.end()) + << "Spatial layer 0 frame with timestamp " << timestamp_rtp + << " was not seen before"; + const Frame& base_frame = + frames_.at(timestamp_rtp).at(/*spatial_idx=*/0); + frames_.at(timestamp_rtp).emplace(spatial_idx, base_frame); + } + + Frame& frame = frames_.at(timestamp_rtp).at(spatial_idx); + frame.layer_id = {.spatial_idx = spatial_idx, + .temporal_idx = temporal_idx}; + frame.width = width; + frame.height = height; + frame.frame_size = DataSize::Bytes(frame_size_bytes); + frame.qp = qp; + frame.keyframe = frame_type == VideoFrameType::kVideoFrameKey; + frame.encode_time = + Timestamp::Micros(encode_finished_us) - frame.encode_start; + frame.encoded = true; + }); + } + + void StartDecode(const EncodedImage& encoded_frame) { + int64_t decode_start_us = rtc::TimeMicros(); + task_queue_.PostTask( + [this, timestamp_rtp = encoded_frame.RtpTimestamp(), + spatial_idx = encoded_frame.SpatialIndex().value_or(0), + frame_size_bytes = encoded_frame.size(), decode_start_us]() { + if (frames_.find(timestamp_rtp) == frames_.end() || + frames_.at(timestamp_rtp).find(spatial_idx) == + frames_.at(timestamp_rtp).end()) { + Frame frame; + frame.timestamp_rtp = timestamp_rtp; + frame.layer_id = {.spatial_idx = spatial_idx}; + frame.frame_size = DataSize::Bytes(frame_size_bytes); + frames_.emplace(timestamp_rtp, + std::map{{spatial_idx, frame}}); + } + + Frame& frame = frames_.at(timestamp_rtp).at(spatial_idx); + frame.decode_start = Timestamp::Micros(decode_start_us); + }); + } + + void FinishDecode(const VideoFrame& decoded_frame, int spatial_idx) { + int64_t decode_finished_us = rtc::TimeMicros(); + task_queue_.PostTask([this, timestamp_rtp = decoded_frame.timestamp(), + spatial_idx, width = decoded_frame.width(), + height = decoded_frame.height(), + decode_finished_us]() { + Frame& frame = frames_.at(timestamp_rtp).at(spatial_idx); + frame.decode_time = + Timestamp::Micros(decode_finished_us) - frame.decode_start; + if (!frame.encoded) { + frame.width = width; + frame.height = height; + } + frame.decoded = true; + }); + + if (video_source_ != nullptr) { + // Copy hardware-backed frame into main memory to release output buffers + // which number may be limited in hardware decoders. + rtc::scoped_refptr decoded_buffer = + decoded_frame.video_frame_buffer()->ToI420(); + + task_queue_.PostTask([this, decoded_buffer, + timestamp_rtp = decoded_frame.timestamp(), + spatial_idx]() { + VideoFrame ref_frame = video_source_->ReadFrame( + timestamp_rtp, {.width = decoded_buffer->width(), + .height = decoded_buffer->height()}); + rtc::scoped_refptr ref_buffer = + ref_frame.video_frame_buffer()->ToI420(); + Frame& frame = frames_.at(timestamp_rtp).at(spatial_idx); + frame.psnr = CalcPsnr(*decoded_buffer, *ref_buffer); + }); + } + } + + std::vector Slice(Filter filter, bool merge) const { + std::vector slice; + for (const auto& [timestamp_rtp, temporal_unit_frames] : frames_) { + if (temporal_unit_frames.empty()) { + continue; + } + + bool is_svc = false; + if (!encoding_settings_.empty()) { + ScalabilityMode scalability_mode = + encoding_settings_.at(timestamp_rtp).scalability_mode; + if (kFullSvcScalabilityModes.count(scalability_mode) > 0 || + (kKeySvcScalabilityModes.count(scalability_mode) > 0 && + temporal_unit_frames.at(0).keyframe)) { + is_svc = true; + } + } + + std::vector subframes; + for (const auto& [spatial_idx, frame] : temporal_unit_frames) { + if (frame.timestamp_rtp < filter.min_timestamp_rtp || + frame.timestamp_rtp > filter.max_timestamp_rtp) { + continue; + } + if (filter.layer_id) { + if ((is_svc && + frame.layer_id.spatial_idx > filter.layer_id->spatial_idx) || + (!is_svc && + frame.layer_id.spatial_idx != filter.layer_id->spatial_idx)) { + continue; + } + if (frame.layer_id.temporal_idx > filter.layer_id->temporal_idx) { + continue; + } + } + subframes.push_back(frame); + } + + if (subframes.empty()) { + continue; + } + + if (!merge) { + std::copy(subframes.begin(), subframes.end(), + std::back_inserter(slice)); + continue; + } + + Frame superframe = subframes.back(); + for (const Frame& frame : + rtc::ArrayView(subframes).subview(0, subframes.size() - 1)) { + superframe.frame_size += frame.frame_size; + superframe.keyframe |= frame.keyframe; + superframe.encode_time = + std::max(superframe.encode_time, frame.encode_time); + superframe.decode_time = + std::max(superframe.decode_time, frame.decode_time); + } + + if (!encoding_settings_.empty()) { + RTC_CHECK(encoding_settings_.find(superframe.timestamp_rtp) != + encoding_settings_.end()) + << "No encoding settings for frame " << superframe.timestamp_rtp; + const EncodingSettings& es = + encoding_settings_.at(superframe.timestamp_rtp); + superframe.target_bitrate = GetTargetBitrate(es, filter.layer_id); + superframe.target_framerate = GetTargetFramerate(es, filter.layer_id); + } + + slice.push_back(superframe); + } + return slice; + } + + Stream Aggregate(Filter filter) const { + std::vector frames = Slice(filter, /*merge=*/true); + Stream stream; + LeakyBucket leaky_bucket; + for (const Frame& frame : frames) { + Timestamp time = Timestamp::Micros((frame.timestamp_rtp / k90kHz).us()); + if (!frame.frame_size.IsZero()) { + stream.width.AddSample(StatsSample(frame.width, time)); + stream.height.AddSample(StatsSample(frame.height, time)); + stream.frame_size_bytes.AddSample( + StatsSample(frame.frame_size.bytes(), time)); + stream.keyframe.AddSample(StatsSample(frame.keyframe, time)); + if (frame.qp) { + stream.qp.AddSample(StatsSample(*frame.qp, time)); + } + } + if (frame.encoded) { + stream.encode_time_ms.AddSample( + StatsSample(frame.encode_time.ms(), time)); + } + if (frame.decoded) { + stream.decode_time_ms.AddSample( + StatsSample(frame.decode_time.ms(), time)); + } + if (frame.psnr) { + stream.psnr.y.AddSample(StatsSample(frame.psnr->y, time)); + stream.psnr.u.AddSample(StatsSample(frame.psnr->u, time)); + stream.psnr.v.AddSample(StatsSample(frame.psnr->v, time)); + } + if (frame.target_framerate) { + stream.target_framerate_fps.AddSample( + StatsSample(frame.target_framerate->hertz(), time)); + } + if (frame.target_bitrate) { + stream.target_bitrate_kbps.AddSample( + StatsSample(frame.target_bitrate->kbps(), time)); + int buffer_level_bits = leaky_bucket.Update(frame); + stream.transmission_time_ms.AddSample(StatsSample( + 1000 * buffer_level_bits / frame.target_bitrate->bps(), + time)); + } + } + + int num_encoded_frames = stream.frame_size_bytes.NumSamples(); + const Frame& first_frame = frames.front(); + + Filter filter_all_layers{.min_timestamp_rtp = filter.min_timestamp_rtp, + .max_timestamp_rtp = filter.max_timestamp_rtp}; + std::vector frames_all_layers = + Slice(filter_all_layers, /*merge=*/true); + const Frame& last_frame = frames_all_layers.back(); + TimeDelta duration = + (last_frame.timestamp_rtp - first_frame.timestamp_rtp) / k90kHz; + if (last_frame.target_framerate) { + duration += 1 / *last_frame.target_framerate; + } + + DataRate encoded_bitrate = + DataSize::Bytes(stream.frame_size_bytes.GetSum()) / duration; + Frequency encoded_framerate = num_encoded_frames / duration; + + double bitrate_mismatch_pct = 0.0; + if (const auto& target_bitrate = first_frame.target_bitrate; + target_bitrate) { + bitrate_mismatch_pct = 100 * (encoded_bitrate / *target_bitrate - 1); + } + double framerate_mismatch_pct = 0.0; + if (const auto& target_framerate = first_frame.target_framerate; + target_framerate) { + framerate_mismatch_pct = + 100 * (encoded_framerate / *target_framerate - 1); + } + + for (Frame& frame : frames) { + Timestamp time = Timestamp::Micros((frame.timestamp_rtp / k90kHz).us()); + stream.encoded_bitrate_kbps.AddSample( + StatsSample(encoded_bitrate.kbps(), time)); + stream.encoded_framerate_fps.AddSample( + StatsSample(encoded_framerate.hertz(), time)); + stream.bitrate_mismatch_pct.AddSample( + StatsSample(bitrate_mismatch_pct, time)); + stream.framerate_mismatch_pct.AddSample( + StatsSample(framerate_mismatch_pct, time)); + } + + return stream; + } + + void Flush() { task_queue_.WaitForPreviouslyPostedTasks(); } + + private: + struct FrameId { + uint32_t timestamp_rtp; + int spatial_idx; + + bool operator==(const FrameId& o) const { + return timestamp_rtp == o.timestamp_rtp && spatial_idx == o.spatial_idx; + } + bool operator<(const FrameId& o) const { + return timestamp_rtp < o.timestamp_rtp || + (timestamp_rtp == o.timestamp_rtp && spatial_idx < o.spatial_idx); + } + }; + + Frame::Psnr CalcPsnr(const I420BufferInterface& ref_buffer, + const I420BufferInterface& dec_buffer) { + RTC_CHECK_EQ(ref_buffer.width(), dec_buffer.width()); + RTC_CHECK_EQ(ref_buffer.height(), dec_buffer.height()); + + uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataY(), dec_buffer.StrideY(), ref_buffer.DataY(), + ref_buffer.StrideY(), dec_buffer.width(), dec_buffer.height()); + + uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataU(), dec_buffer.StrideU(), ref_buffer.DataU(), + ref_buffer.StrideU(), dec_buffer.width() / 2, dec_buffer.height() / 2); + + uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataV(), dec_buffer.StrideV(), ref_buffer.DataV(), + ref_buffer.StrideV(), dec_buffer.width() / 2, dec_buffer.height() / 2); + + int num_y_samples = dec_buffer.width() * dec_buffer.height(); + Frame::Psnr psnr; + psnr.y = libyuv::SumSquareErrorToPsnr(sse_y, num_y_samples); + psnr.u = libyuv::SumSquareErrorToPsnr(sse_u, num_y_samples / 4); + psnr.v = libyuv::SumSquareErrorToPsnr(sse_v, num_y_samples / 4); + return psnr; + } + + DataRate GetTargetBitrate(const EncodingSettings& encoding_settings, + absl::optional layer_id) const { + int base_spatial_idx; + if (layer_id.has_value()) { + bool is_svc = + kFullSvcScalabilityModes.count(encoding_settings.scalability_mode); + base_spatial_idx = is_svc ? 0 : layer_id->spatial_idx; + } else { + int num_spatial_layers = + ScalabilityModeToNumSpatialLayers(encoding_settings.scalability_mode); + int num_temporal_layers = ScalabilityModeToNumTemporalLayers( + encoding_settings.scalability_mode); + layer_id = LayerId({.spatial_idx = num_spatial_layers - 1, + .temporal_idx = num_temporal_layers - 1}); + base_spatial_idx = 0; + } + + DataRate bitrate = DataRate::Zero(); + for (int sidx = base_spatial_idx; sidx <= layer_id->spatial_idx; ++sidx) { + for (int tidx = 0; tidx <= layer_id->temporal_idx; ++tidx) { + auto layer_settings = encoding_settings.layers_settings.find( + {.spatial_idx = sidx, .temporal_idx = tidx}); + RTC_CHECK(layer_settings != encoding_settings.layers_settings.end()) + << "bitrate is not specified for layer sidx=" << sidx + << " tidx=" << tidx; + bitrate += layer_settings->second.bitrate; + } + } + return bitrate; + } + + Frequency GetTargetFramerate(const EncodingSettings& encoding_settings, + absl::optional layer_id) const { + if (layer_id.has_value()) { + auto layer_settings = encoding_settings.layers_settings.find( + {.spatial_idx = layer_id->spatial_idx, + .temporal_idx = layer_id->temporal_idx}); + RTC_CHECK(layer_settings != encoding_settings.layers_settings.end()) + << "framerate is not specified for layer sidx=" + << layer_id->spatial_idx << " tidx=" << layer_id->temporal_idx; + return layer_settings->second.framerate; + } + return encoding_settings.layers_settings.rbegin()->second.framerate; + } + + SamplesStatsCounter::StatsSample StatsSample(double value, + Timestamp time) const { + return SamplesStatsCounter::StatsSample{value, time}; + } + + VideoSource* const video_source_; + TaskQueueForTest task_queue_; + // RTP timestamp -> spatial layer -> Frame + std::map> frames_; + std::map encoding_settings_; +}; + +class Decoder : public DecodedImageCallback { + public: + Decoder(VideoDecoderFactory* decoder_factory, + const DecoderSettings& decoder_settings, + VideoCodecAnalyzer* analyzer) + : decoder_factory_(decoder_factory), + analyzer_(analyzer), + pacer_(decoder_settings.pacing_settings) { + RTC_CHECK(analyzer_) << "Analyzer must be provided"; + + if (decoder_settings.decoder_input_base_path) { + ivf_writer_ = std::make_unique( + *decoder_settings.decoder_input_base_path); + } + + if (decoder_settings.decoder_output_base_path) { + y4m_writer_ = std::make_unique( + *decoder_settings.decoder_output_base_path); + } + } + + void Initialize(const SdpVideoFormat& sdp_video_format) { + decoder_ = decoder_factory_->CreateVideoDecoder(sdp_video_format); + RTC_CHECK(decoder_) << "Could not create decoder for video format " + << sdp_video_format.ToString(); + + task_queue_.PostTaskAndWait([this, &sdp_video_format] { + decoder_->RegisterDecodeCompleteCallback(this); + + VideoDecoder::Settings ds; + ds.set_codec_type(PayloadStringToCodecType(sdp_video_format.name)); + ds.set_number_of_cores(1); + ds.set_max_render_resolution({1280, 720}); + bool result = decoder_->Configure(ds); + RTC_CHECK(result) << "Failed to configure decoder"; + }); + } + + void Decode(const EncodedImage& encoded_frame) { + Timestamp pts = + Timestamp::Micros((encoded_frame.RtpTimestamp() / k90kHz).us()); + + task_queue_.PostScheduledTask( + [this, encoded_frame] { + analyzer_->StartDecode(encoded_frame); + int error = decoder_->Decode(encoded_frame, /*render_time_ms*/ 0); + if (error != 0) { + RTC_LOG(LS_WARNING) + << "Decode failed with error code " << error + << " RTP timestamp " << encoded_frame.RtpTimestamp(); + } + }, + pacer_.Schedule(pts)); + + if (ivf_writer_) { + ivf_writer_->Write(encoded_frame); + } + } + + void Flush() { + // TODO(webrtc:14852): Add Flush() to VideoDecoder API. + task_queue_.PostTaskAndWait([this] { decoder_->Release(); }); + } + + private: + int Decoded(VideoFrame& decoded_frame) override { + analyzer_->FinishDecode(decoded_frame, /*spatial_idx=*/0); + + if (y4m_writer_) { + y4m_writer_->Write(decoded_frame, /*spatial_idx=*/0); + } + + return WEBRTC_VIDEO_CODEC_OK; + } + + VideoDecoderFactory* decoder_factory_; + std::unique_ptr decoder_; + VideoCodecAnalyzer* const analyzer_; + Pacer pacer_; + LimitedTaskQueue task_queue_; + std::unique_ptr ivf_writer_; + std::unique_ptr y4m_writer_; +}; + +class Encoder : public EncodedImageCallback { + public: + using EncodeCallback = + absl::AnyInvocable; + + Encoder(VideoEncoderFactory* encoder_factory, + const EncoderSettings& encoder_settings, + VideoCodecAnalyzer* analyzer) + : encoder_factory_(encoder_factory), + analyzer_(analyzer), + pacer_(encoder_settings.pacing_settings) { + RTC_CHECK(analyzer_) << "Analyzer must be provided"; + + if (encoder_settings.encoder_input_base_path) { + y4m_writer_ = std::make_unique( + *encoder_settings.encoder_input_base_path); + } + + if (encoder_settings.encoder_output_base_path) { + ivf_writer_ = std::make_unique( + *encoder_settings.encoder_output_base_path); + } + } + + void Initialize(const EncodingSettings& encoding_settings) { + encoder_ = encoder_factory_->CreateVideoEncoder( + encoding_settings.sdp_video_format); + RTC_CHECK(encoder_) << "Could not create encoder for video format " + << encoding_settings.sdp_video_format.ToString(); + + task_queue_.PostTaskAndWait([this, encoding_settings] { + encoder_->RegisterEncodeCompleteCallback(this); + Configure(encoding_settings); + SetRates(encoding_settings); + }); + } + + void Encode(const VideoFrame& input_frame, + const EncodingSettings& encoding_settings, + EncodeCallback callback) { + { + MutexLock lock(&mutex_); + callbacks_[input_frame.timestamp()] = std::move(callback); + } + + Timestamp pts = Timestamp::Micros((input_frame.timestamp() / k90kHz).us()); + + task_queue_.PostScheduledTask( + [this, input_frame, encoding_settings] { + analyzer_->StartEncode(input_frame, encoding_settings); + + if (!last_encoding_settings_ || + !IsSameRate(encoding_settings, *last_encoding_settings_)) { + SetRates(encoding_settings); + } + + int error = encoder_->Encode(input_frame, /*frame_types=*/nullptr); + if (error != 0) { + RTC_LOG(LS_WARNING) << "Encode failed with error code " << error + << " RTP timestamp " << input_frame.timestamp(); + } + + last_encoding_settings_ = encoding_settings; + }, + pacer_.Schedule(pts)); + + if (y4m_writer_) { + y4m_writer_->Write(input_frame, /*spatial_idx=*/0); + } + } + + void Flush() { + task_queue_.PostTaskAndWait([this] { encoder_->Release(); }); + } + + private: + Result OnEncodedImage(const EncodedImage& encoded_frame, + const CodecSpecificInfo* codec_specific_info) override { + analyzer_->FinishEncode(encoded_frame); + + { + MutexLock lock(&mutex_); + auto it = callbacks_.find(encoded_frame.RtpTimestamp()); + RTC_CHECK(it != callbacks_.end()); + it->second(encoded_frame); + callbacks_.erase(callbacks_.begin(), it); + } + + if (ivf_writer_ != nullptr) { + ivf_writer_->Write(encoded_frame); + } + + return Result(Result::Error::OK); + } + + void Configure(const EncodingSettings& es) { + const LayerSettings& layer_settings = es.layers_settings.rbegin()->second; + const DataRate& bitrate = layer_settings.bitrate; + + VideoCodec vc; + vc.width = layer_settings.resolution.width; + vc.height = layer_settings.resolution.height; + vc.startBitrate = bitrate.kbps(); + vc.maxBitrate = bitrate.kbps(); + vc.minBitrate = 0; + vc.maxFramerate = layer_settings.framerate.hertz(); + vc.active = true; + vc.numberOfSimulcastStreams = 0; + vc.mode = webrtc::VideoCodecMode::kRealtimeVideo; + vc.SetFrameDropEnabled(true); + vc.SetScalabilityMode(es.scalability_mode); + vc.SetVideoEncoderComplexity(VideoCodecComplexity::kComplexityNormal); + + vc.codecType = PayloadStringToCodecType(es.sdp_video_format.name); + switch (vc.codecType) { + case kVideoCodecVP8: + *(vc.VP8()) = VideoEncoder::GetDefaultVp8Settings(); + vc.VP8()->SetNumberOfTemporalLayers( + ScalabilityModeToNumTemporalLayers(es.scalability_mode)); + vc.qpMax = cricket::kDefaultVideoMaxQpVpx; + // TODO(webrtc:14852): Configure simulcast. + break; + case kVideoCodecVP9: + *(vc.VP9()) = VideoEncoder::GetDefaultVp9Settings(); + // See LibvpxVp9Encoder::ExplicitlyConfiguredSpatialLayers. + vc.spatialLayers[0].targetBitrate = vc.maxBitrate; + vc.qpMax = cricket::kDefaultVideoMaxQpVpx; + break; + case kVideoCodecAV1: + vc.qpMax = cricket::kDefaultVideoMaxQpVpx; + break; + case kVideoCodecH264: + *(vc.H264()) = VideoEncoder::GetDefaultH264Settings(); + vc.qpMax = cricket::kDefaultVideoMaxQpH26x; + break; + case kVideoCodecH265: + vc.qpMax = cricket::kDefaultVideoMaxQpH26x; + break; + case kVideoCodecGeneric: + case kVideoCodecMultiplex: + RTC_CHECK_NOTREACHED(); + break; + } + + VideoEncoder::Settings ves( + VideoEncoder::Capabilities(/*loss_notification=*/false), + /*number_of_cores=*/1, + /*max_payload_size=*/1440); + + int result = encoder_->InitEncode(&vc, ves); + RTC_CHECK(result == WEBRTC_VIDEO_CODEC_OK); + + SetRates(es); + } + + void SetRates(const EncodingSettings& es) { + VideoEncoder::RateControlParameters rc; + int num_spatial_layers = + ScalabilityModeToNumSpatialLayers(es.scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumTemporalLayers(es.scalability_mode); + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + auto layers_settings = es.layers_settings.find( + {.spatial_idx = sidx, .temporal_idx = tidx}); + RTC_CHECK(layers_settings != es.layers_settings.end()) + << "Bitrate for layer S=" << sidx << " T=" << tidx << " is not set"; + rc.bitrate.SetBitrate(sidx, tidx, + layers_settings->second.bitrate.bps()); + } + } + rc.framerate_fps = + es.layers_settings.rbegin()->second.framerate.hertz(); + encoder_->SetRates(rc); + } + + bool IsSameRate(const EncodingSettings& a, const EncodingSettings& b) const { + for (auto [layer_id, layer] : a.layers_settings) { + const auto& other_layer = b.layers_settings.at(layer_id); + if (layer.bitrate != other_layer.bitrate || + layer.framerate != other_layer.framerate) { + return false; + } + } + + return true; + } + + VideoEncoderFactory* const encoder_factory_; + std::unique_ptr encoder_; + VideoCodecAnalyzer* const analyzer_; + Pacer pacer_; + absl::optional last_encoding_settings_; + std::unique_ptr bitrate_allocator_; + LimitedTaskQueue task_queue_; + std::unique_ptr y4m_writer_; + std::unique_ptr ivf_writer_; + std::map sidx_ RTC_GUARDED_BY(mutex_); + std::map callbacks_ RTC_GUARDED_BY(mutex_); + Mutex mutex_; +}; + +std::tuple, ScalabilityMode> +SplitBitrateAndUpdateScalabilityMode(std::string codec_type, + ScalabilityMode scalability_mode, + int width, + int height, + std::vector bitrates_kbps, + double framerate_fps) { + int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumTemporalLayers(scalability_mode); + + if (bitrates_kbps.size() > 1 || + (num_spatial_layers == 1 && num_temporal_layers == 1)) { + RTC_CHECK(bitrates_kbps.size() == + static_cast(num_spatial_layers * num_temporal_layers)) + << "bitrates must be provided for all layers"; + std::vector bitrates; + for (const auto& bitrate_kbps : bitrates_kbps) { + bitrates.push_back(DataRate::KilobitsPerSec(bitrate_kbps)); + } + return std::make_tuple(bitrates, scalability_mode); + } + + VideoCodec vc; + vc.codecType = PayloadStringToCodecType(codec_type); + vc.width = width; + vc.height = height; + vc.startBitrate = bitrates_kbps.front(); + vc.maxBitrate = bitrates_kbps.front(); + vc.minBitrate = 0; + vc.maxFramerate = static_cast(framerate_fps); + vc.numberOfSimulcastStreams = 0; + vc.mode = webrtc::VideoCodecMode::kRealtimeVideo; + vc.SetScalabilityMode(scalability_mode); + + switch (vc.codecType) { + case kVideoCodecVP8: + // TODO(webrtc:14852): Configure simulcast. + *(vc.VP8()) = VideoEncoder::GetDefaultVp8Settings(); + vc.VP8()->SetNumberOfTemporalLayers(num_temporal_layers); + vc.simulcastStream[0].width = vc.width; + vc.simulcastStream[0].height = vc.height; + break; + case kVideoCodecVP9: { + *(vc.VP9()) = VideoEncoder::GetDefaultVp9Settings(); + vc.VP9()->SetNumberOfTemporalLayers(num_temporal_layers); + const std::vector spatialLayers = GetVp9SvcConfig(vc); + for (size_t i = 0; i < spatialLayers.size(); ++i) { + vc.spatialLayers[i] = spatialLayers[i]; + vc.spatialLayers[i].active = true; + } + } break; + case kVideoCodecAV1: { + bool result = + SetAv1SvcConfig(vc, num_spatial_layers, num_temporal_layers); + RTC_CHECK(result) << "SetAv1SvcConfig failed"; + } break; + case kVideoCodecH264: { + *(vc.H264()) = VideoEncoder::GetDefaultH264Settings(); + vc.H264()->SetNumberOfTemporalLayers(num_temporal_layers); + } break; + case kVideoCodecH265: + break; + case kVideoCodecGeneric: + case kVideoCodecMultiplex: + RTC_CHECK_NOTREACHED(); + } + + if (*vc.GetScalabilityMode() != scalability_mode) { + RTC_LOG(LS_WARNING) << "Scalability mode changed from " + << ScalabilityModeToString(scalability_mode) << " to " + << ScalabilityModeToString(*vc.GetScalabilityMode()); + num_spatial_layers = + ScalabilityModeToNumSpatialLayers(*vc.GetScalabilityMode()); + num_temporal_layers = + ScalabilityModeToNumTemporalLayers(*vc.GetScalabilityMode()); + } + + std::unique_ptr bitrate_allocator = + CreateBuiltinVideoBitrateAllocatorFactory()->CreateVideoBitrateAllocator( + vc); + VideoBitrateAllocation bitrate_allocation = + bitrate_allocator->Allocate(VideoBitrateAllocationParameters( + 1000 * bitrates_kbps.front(), framerate_fps)); + + std::vector bitrates; + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + int bitrate_bps = bitrate_allocation.GetBitrate(sidx, tidx); + bitrates.push_back(DataRate::BitsPerSec(bitrate_bps)); + } + } + + return std::make_tuple(bitrates, *vc.GetScalabilityMode()); +} + +} // namespace + +void VideoCodecStats::Stream::LogMetrics( + MetricsLogger* logger, + std::string test_case_name, + std::map metadata) const { + logger->LogMetric("width", test_case_name, width, Unit::kCount, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("height", test_case_name, height, Unit::kCount, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("frame_size_bytes", test_case_name, frame_size_bytes, + Unit::kBytes, ImprovementDirection::kNeitherIsBetter, + metadata); + logger->LogMetric("keyframe", test_case_name, keyframe, Unit::kCount, + ImprovementDirection::kSmallerIsBetter, metadata); + logger->LogMetric("qp", test_case_name, qp, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, metadata); + logger->LogMetric("encode_time_ms", test_case_name, encode_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter, + metadata); + logger->LogMetric("decode_time_ms", test_case_name, decode_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter, + metadata); + // TODO(webrtc:14852): Change to kUnitLess. kKilobitsPerSecond are converted + // to bytes per second in Chromeperf dash. + logger->LogMetric("target_bitrate_kbps", test_case_name, target_bitrate_kbps, + Unit::kKilobitsPerSecond, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("target_framerate_fps", test_case_name, + target_framerate_fps, Unit::kHertz, + ImprovementDirection::kBiggerIsBetter, metadata); + // TODO(webrtc:14852): Change to kUnitLess. kKilobitsPerSecond are converted + // to bytes per second in Chromeperf dash. + logger->LogMetric("encoded_bitrate_kbps", test_case_name, + encoded_bitrate_kbps, Unit::kKilobitsPerSecond, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("encoded_framerate_fps", test_case_name, + encoded_framerate_fps, Unit::kHertz, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("bitrate_mismatch_pct", test_case_name, + bitrate_mismatch_pct, Unit::kPercent, + ImprovementDirection::kNeitherIsBetter, metadata); + logger->LogMetric("framerate_mismatch_pct", test_case_name, + framerate_mismatch_pct, Unit::kPercent, + ImprovementDirection::kNeitherIsBetter, metadata); + logger->LogMetric("transmission_time_ms", test_case_name, + transmission_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metadata); + logger->LogMetric("psnr_y_db", test_case_name, psnr.y, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("psnr_u_db", test_case_name, psnr.u, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("psnr_v_db", test_case_name, psnr.v, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metadata); +} + +// TODO(ssilkin): use Frequency and DataRate for framerate and bitrate. +std::map VideoCodecTester::CreateEncodingSettings( + std::string codec_type, + std::string scalability_name, + int width, + int height, + std::vector layer_bitrates_kbps, + double framerate_fps, + int num_frames, + uint32_t first_timestamp_rtp) { + auto [layer_bitrates, scalability_mode] = + SplitBitrateAndUpdateScalabilityMode( + codec_type, *ScalabilityModeFromString(scalability_name), width, + height, layer_bitrates_kbps, framerate_fps); + + int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumTemporalLayers(scalability_mode); + + std::map layers_settings; + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + int layer_width = width >> (num_spatial_layers - sidx - 1); + int layer_height = height >> (num_spatial_layers - sidx - 1); + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + double layer_framerate_fps = + framerate_fps / (1 << (num_temporal_layers - tidx - 1)); + layers_settings.emplace( + LayerId{.spatial_idx = sidx, .temporal_idx = tidx}, + LayerSettings{ + .resolution = {.width = layer_width, .height = layer_height}, + .framerate = Frequency::MilliHertz(1000 * layer_framerate_fps), + .bitrate = layer_bitrates[sidx * num_temporal_layers + tidx]}); + } + } + + std::map frames_settings; + uint32_t timestamp_rtp = first_timestamp_rtp; + for (int frame_num = 0; frame_num < num_frames; ++frame_num) { + frames_settings.emplace( + timestamp_rtp, + EncodingSettings{.sdp_video_format = SdpVideoFormat(codec_type), + .scalability_mode = scalability_mode, + .layers_settings = layers_settings}); + + timestamp_rtp += k90kHz / Frequency::MilliHertz(1000 * framerate_fps); + } + + return frames_settings; +} + +std::unique_ptr +VideoCodecTester::RunDecodeTest(CodedVideoSource* video_source, + VideoDecoderFactory* decoder_factory, + const DecoderSettings& decoder_settings, + const SdpVideoFormat& sdp_video_format) { + std::unique_ptr analyzer = + std::make_unique(/*video_source=*/nullptr); + Decoder decoder(decoder_factory, decoder_settings, analyzer.get()); + decoder.Initialize(sdp_video_format); + + while (auto frame = video_source->PullFrame()) { + decoder.Decode(*frame); + } + + decoder.Flush(); + analyzer->Flush(); + return std::move(analyzer); +} + +std::unique_ptr +VideoCodecTester::RunEncodeTest( + const VideoSourceSettings& source_settings, + VideoEncoderFactory* encoder_factory, + const EncoderSettings& encoder_settings, + const std::map& encoding_settings) { + VideoSource video_source(source_settings); + std::unique_ptr analyzer = + std::make_unique(/*video_source=*/nullptr); + Encoder encoder(encoder_factory, encoder_settings, analyzer.get()); + encoder.Initialize(encoding_settings.begin()->second); + + for (const auto& [timestamp_rtp, frame_settings] : encoding_settings) { + const EncodingSettings::LayerSettings& top_layer = + frame_settings.layers_settings.rbegin()->second; + VideoFrame source_frame = video_source.PullFrame( + timestamp_rtp, top_layer.resolution, top_layer.framerate); + encoder.Encode(source_frame, frame_settings, + [](const EncodedImage& encoded_frame) {}); + } + + encoder.Flush(); + analyzer->Flush(); + return std::move(analyzer); +} + +std::unique_ptr +VideoCodecTester::RunEncodeDecodeTest( + const VideoSourceSettings& source_settings, + VideoEncoderFactory* encoder_factory, + VideoDecoderFactory* decoder_factory, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings, + const std::map& encoding_settings) { + VideoSource video_source(source_settings); + std::unique_ptr analyzer = + std::make_unique(&video_source); + Decoder decoder(decoder_factory, decoder_settings, analyzer.get()); + Encoder encoder(encoder_factory, encoder_settings, analyzer.get()); + encoder.Initialize(encoding_settings.begin()->second); + decoder.Initialize(encoding_settings.begin()->second.sdp_video_format); + + for (const auto& [timestamp_rtp, frame_settings] : encoding_settings) { + const EncodingSettings::LayerSettings& top_layer = + frame_settings.layers_settings.rbegin()->second; + VideoFrame source_frame = video_source.PullFrame( + timestamp_rtp, top_layer.resolution, top_layer.framerate); + encoder.Encode(source_frame, frame_settings, + [&decoder](const EncodedImage& encoded_frame) { + decoder.Decode(encoded_frame); + }); + } + + encoder.Flush(); + decoder.Flush(); + analyzer->Flush(); + return std::move(analyzer); +} + +} // namespace test +} // namespace webrtc diff --git a/test/video_codec_tester.h b/test/video_codec_tester.h new file mode 100644 index 0000000000..dc72645c18 --- /dev/null +++ b/test/video_codec_tester.h @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_VIDEO_CODEC_TESTER_H_ +#define TEST_VIDEO_CODEC_TESTER_H_ + +#include +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/units/data_rate.h" +#include "api/units/data_size.h" +#include "api/units/frequency.h" +#include "api/video/encoded_image.h" +#include "api/video/resolution.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" + +namespace webrtc { +namespace test { + +class VideoCodecTester { + public: + struct LayerId { + int spatial_idx = 0; + int temporal_idx = 0; + + bool operator==(const LayerId& o) const { + return spatial_idx == o.spatial_idx && temporal_idx == o.temporal_idx; + } + bool operator<(const LayerId& o) const { + return spatial_idx < o.spatial_idx || + (spatial_idx == o.spatial_idx && temporal_idx < o.temporal_idx); + } + }; + + struct EncodingSettings { + SdpVideoFormat sdp_video_format = SdpVideoFormat("VP8"); + ScalabilityMode scalability_mode = ScalabilityMode::kL1T1; + + struct LayerSettings { + Resolution resolution; + Frequency framerate; + DataRate bitrate; + }; + std::map layers_settings; + }; + + class VideoCodecStats { + public: + struct Filter { + uint32_t min_timestamp_rtp = std::numeric_limits::min(); + uint32_t max_timestamp_rtp = std::numeric_limits::max(); + absl::optional layer_id; + }; + + struct Frame { + int frame_num = 0; + uint32_t timestamp_rtp = 0; + LayerId layer_id; + bool encoded = false; + bool decoded = false; + int width = 0; + int height = 0; + DataSize frame_size = DataSize::Zero(); + bool keyframe = false; + absl::optional qp; + Timestamp encode_start = Timestamp::Zero(); + TimeDelta encode_time = TimeDelta::Zero(); + Timestamp decode_start = Timestamp::Zero(); + TimeDelta decode_time = TimeDelta::Zero(); + absl::optional target_bitrate; + absl::optional target_framerate; + + struct Psnr { + double y = 0.0; + double u = 0.0; + double v = 0.0; + }; + absl::optional psnr; + }; + + struct Stream { + SamplesStatsCounter width; + SamplesStatsCounter height; + SamplesStatsCounter frame_size_bytes; + SamplesStatsCounter keyframe; + SamplesStatsCounter qp; + SamplesStatsCounter encode_time_ms; + SamplesStatsCounter decode_time_ms; + SamplesStatsCounter target_bitrate_kbps; + SamplesStatsCounter target_framerate_fps; + SamplesStatsCounter encoded_bitrate_kbps; + SamplesStatsCounter encoded_framerate_fps; + SamplesStatsCounter bitrate_mismatch_pct; + SamplesStatsCounter framerate_mismatch_pct; + SamplesStatsCounter transmission_time_ms; + + struct Psnr { + SamplesStatsCounter y; + SamplesStatsCounter u; + SamplesStatsCounter v; + } psnr; + + // Logs `Stream` metrics to provided `MetricsLogger`. + void LogMetrics(MetricsLogger* logger, + std::string test_case_name, + std::map metadata = {}) const; + }; + + virtual ~VideoCodecStats() = default; + + // Returns frames for the slice specified by `filter`. If `merge` is true, + // also merges frames belonging to the same temporal unit into one + // superframe. + virtual std::vector Slice(Filter filter, bool merge) const = 0; + + // Returns video statistics aggregated for the slice specified by `filter`. + virtual Stream Aggregate(Filter filter) const = 0; + }; + + // Pacing settings for codec input. + struct PacingSettings { + enum PacingMode { + // Pacing is not used. Frames are sent to codec back-to-back. + kNoPacing, + // Pace with the rate equal to the target video frame rate. Pacing time is + // derived from RTP timestamp. + kRealTime, + // Pace with the explicitly provided rate. + kConstantRate, + }; + PacingMode mode = PacingMode::kNoPacing; + // Pacing rate for `kConstantRate` mode. + Frequency constant_rate = Frequency::Zero(); + }; + + struct VideoSourceSettings { + std::string file_path; + Resolution resolution; + Frequency framerate; + }; + + struct DecoderSettings { + PacingSettings pacing_settings; + absl::optional decoder_input_base_path; + absl::optional decoder_output_base_path; + }; + + struct EncoderSettings { + PacingSettings pacing_settings; + absl::optional encoder_input_base_path; + absl::optional encoder_output_base_path; + }; + + virtual ~VideoCodecTester() = default; + + // Interface for a coded video frames source. + class CodedVideoSource { + public: + virtual ~CodedVideoSource() = default; + + // Returns next frame. Returns `absl::nullopt` if the end-of-stream is + // reached. Frames should have RTP timestamps representing desired frame + // rate. + virtual absl::optional PullFrame() = 0; + }; + + // A helper function that creates `EncodingSettings` for `num_frames` frames, + // wraps the settings into RTP timestamp -> settings map and returns the map. + static std::map CreateEncodingSettings( + std::string codec_type, + std::string scalability_name, + int width, + int height, + std::vector bitrates_kbps, + double framerate_fps, + int num_frames, + uint32_t first_timestamp_rtp = 90000); + + // Decodes video, collects and returns decode metrics. + static std::unique_ptr RunDecodeTest( + CodedVideoSource* video_source, + VideoDecoderFactory* decoder_factory, + const DecoderSettings& decoder_settings, + const SdpVideoFormat& sdp_video_format); + + // Encodes video, collects and returns encode metrics. + static std::unique_ptr RunEncodeTest( + const VideoSourceSettings& source_settings, + VideoEncoderFactory* encoder_factory, + const EncoderSettings& encoder_settings, + const std::map& encoding_settings); + + // Encodes and decodes video, collects and returns encode and decode metrics. + static std::unique_ptr RunEncodeDecodeTest( + const VideoSourceSettings& source_settings, + VideoEncoderFactory* encoder_factory, + VideoDecoderFactory* decoder_factory, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings, + const std::map& encoding_settings); +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_VIDEO_CODEC_TESTER_H_ diff --git a/test/video_codec_tester_unittest.cc b/test/video_codec_tester_unittest.cc new file mode 100644 index 0000000000..af31fe2c13 --- /dev/null +++ b/test/video_codec_tester_unittest.cc @@ -0,0 +1,513 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/video_codec_tester.h" + +#include +#include +#include +#include +#include +#include + +#include "api/test/mock_video_decoder.h" +#include "api/test/mock_video_decoder_factory.h" +#include "api/test/mock_video_encoder.h" +#include "api/test/mock_video_encoder_factory.h" +#include "api/units/data_rate.h" +#include "api/units/time_delta.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" +#include "third_party/libyuv/include/libyuv/planar_functions.h" + +namespace webrtc { +namespace test { + +namespace { +using ::testing::_; +using ::testing::ElementsAre; +using ::testing::Field; +using ::testing::Invoke; +using ::testing::InvokeWithoutArgs; +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::SizeIs; + +using VideoCodecStats = VideoCodecTester::VideoCodecStats; +using VideoSourceSettings = VideoCodecTester::VideoSourceSettings; +using CodedVideoSource = VideoCodecTester::CodedVideoSource; +using EncodingSettings = VideoCodecTester::EncodingSettings; +using LayerSettings = EncodingSettings::LayerSettings; +using LayerId = VideoCodecTester::LayerId; +using DecoderSettings = VideoCodecTester::DecoderSettings; +using EncoderSettings = VideoCodecTester::EncoderSettings; +using PacingSettings = VideoCodecTester::PacingSettings; +using PacingMode = PacingSettings::PacingMode; +using Filter = VideoCodecStats::Filter; +using Frame = VideoCodecTester::VideoCodecStats::Frame; +using Stream = VideoCodecTester::VideoCodecStats::Stream; + +constexpr int kWidth = 2; +constexpr int kHeight = 2; +const DataRate kTargetLayerBitrate = DataRate::BytesPerSec(100); +const Frequency kTargetFramerate = Frequency::Hertz(30); +constexpr Frequency k90kHz = Frequency::Hertz(90000); + +rtc::scoped_refptr CreateYuvBuffer(uint8_t y = 0, + uint8_t u = 0, + uint8_t v = 0) { + rtc::scoped_refptr buffer(I420Buffer::Create(2, 2)); + + libyuv::I420Rect(buffer->MutableDataY(), buffer->StrideY(), + buffer->MutableDataU(), buffer->StrideU(), + buffer->MutableDataV(), buffer->StrideV(), 0, 0, + buffer->width(), buffer->height(), y, u, v); + return buffer; +} + +std::string CreateYuvFile(int width, int height, int num_frames) { + std::string path = webrtc::test::TempFilename(webrtc::test::OutputPath(), + "video_codec_tester_unittest"); + FILE* file = fopen(path.c_str(), "wb"); + for (int frame_num = 0; frame_num < num_frames; ++frame_num) { + uint8_t y = (frame_num + 0) & 255; + uint8_t u = (frame_num + 1) & 255; + uint8_t v = (frame_num + 2) & 255; + rtc::scoped_refptr buffer = CreateYuvBuffer(y, u, v); + fwrite(buffer->DataY(), 1, width * height, file); + int chroma_size_bytes = (width + 1) / 2 * (height + 1) / 2; + fwrite(buffer->DataU(), 1, chroma_size_bytes, file); + fwrite(buffer->DataV(), 1, chroma_size_bytes, file); + } + fclose(file); + return path; +} + +std::unique_ptr RunTest(std::vector> frames, + ScalabilityMode scalability_mode) { + int num_frames = static_cast(frames.size()); + std::string source_yuv_path = CreateYuvFile(kWidth, kHeight, num_frames); + VideoSourceSettings source_settings{ + .file_path = source_yuv_path, + .resolution = {.width = kWidth, .height = kHeight}, + .framerate = kTargetFramerate}; + + int num_encoded_frames = 0; + EncodedImageCallback* encoded_frame_callback; + NiceMock encoder_factory; + ON_CALL(encoder_factory, CreateVideoEncoder) + .WillByDefault([&](const SdpVideoFormat&) { + auto encoder = std::make_unique>(); + ON_CALL(*encoder, RegisterEncodeCompleteCallback) + .WillByDefault([&](EncodedImageCallback* callback) { + encoded_frame_callback = callback; + return WEBRTC_VIDEO_CODEC_OK; + }); + ON_CALL(*encoder, Encode) + .WillByDefault([&](const VideoFrame& input_frame, + const std::vector*) { + for (const Frame& frame : frames[num_encoded_frames]) { + EncodedImage encoded_frame; + encoded_frame._encodedWidth = frame.width; + encoded_frame._encodedHeight = frame.height; + encoded_frame.SetFrameType( + frame.keyframe ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta); + encoded_frame.SetRtpTimestamp(input_frame.timestamp()); + encoded_frame.SetSpatialIndex(frame.layer_id.spatial_idx); + encoded_frame.SetTemporalIndex(frame.layer_id.temporal_idx); + encoded_frame.SetEncodedData( + EncodedImageBuffer::Create(frame.frame_size.bytes())); + encoded_frame_callback->OnEncodedImage( + encoded_frame, + /*codec_specific_info=*/nullptr); + } + ++num_encoded_frames; + return WEBRTC_VIDEO_CODEC_OK; + }); + return encoder; + }); + + int num_decoded_frames = 0; + DecodedImageCallback* decode_callback; + NiceMock decoder_factory; + ON_CALL(decoder_factory, CreateVideoDecoder) + .WillByDefault([&](const SdpVideoFormat&) { + auto decoder = std::make_unique>(); + ON_CALL(*decoder, RegisterDecodeCompleteCallback) + .WillByDefault([&](DecodedImageCallback* callback) { + decode_callback = callback; + return WEBRTC_VIDEO_CODEC_OK; + }); + ON_CALL(*decoder, Decode(_, _)) + .WillByDefault([&](const EncodedImage& encoded_frame, int64_t) { + // Make values to be different from source YUV generated in + // `CreateYuvFile`. + uint8_t y = ((num_decoded_frames + 1) * 2) & 255; + uint8_t u = ((num_decoded_frames + 2) * 2) & 255; + uint8_t v = ((num_decoded_frames + 3) * 2) & 255; + rtc::scoped_refptr frame_buffer = + CreateYuvBuffer(y, u, v); + VideoFrame decoded_frame = + VideoFrame::Builder() + .set_video_frame_buffer(frame_buffer) + .set_timestamp_rtp(encoded_frame.RtpTimestamp()) + .build(); + decode_callback->Decoded(decoded_frame); + ++num_decoded_frames; + return WEBRTC_VIDEO_CODEC_OK; + }); + return decoder; + }); + + int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumTemporalLayers(scalability_mode); + + std::map encoding_settings; + for (int frame_num = 0; frame_num < num_frames; ++frame_num) { + std::map layers_settings; + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + layers_settings.emplace( + LayerId{.spatial_idx = sidx, .temporal_idx = tidx}, + LayerSettings{.resolution = {.width = kWidth, .height = kHeight}, + .framerate = kTargetFramerate / + (1 << (num_temporal_layers - 1 - tidx)), + .bitrate = kTargetLayerBitrate}); + } + } + encoding_settings.emplace( + frames[frame_num][0].timestamp_rtp, + EncodingSettings{.scalability_mode = scalability_mode, + .layers_settings = layers_settings}); + } + + EncoderSettings encoder_settings; + DecoderSettings decoder_settings; + std::unique_ptr stats = + VideoCodecTester::RunEncodeDecodeTest( + source_settings, &encoder_factory, &decoder_factory, encoder_settings, + decoder_settings, encoding_settings); + remove(source_yuv_path.c_str()); + return stats; +} + +EncodedImage CreateEncodedImage(uint32_t timestamp_rtp) { + EncodedImage encoded_image; + encoded_image.SetRtpTimestamp(timestamp_rtp); + return encoded_image; +} + +class MockCodedVideoSource : public CodedVideoSource { + public: + MockCodedVideoSource(int num_frames, Frequency framerate) + : num_frames_(num_frames), frame_num_(0), framerate_(framerate) {} + + absl::optional PullFrame() override { + if (frame_num_ >= num_frames_) { + return absl::nullopt; + } + uint32_t timestamp_rtp = frame_num_ * k90kHz / framerate_; + ++frame_num_; + return CreateEncodedImage(timestamp_rtp); + } + + private: + int num_frames_; + int frame_num_; + Frequency framerate_; +}; + +} // namespace + +TEST(VideoCodecTester, Slice) { + std::unique_ptr stats = RunTest( + {{{.timestamp_rtp = 0, .layer_id = {.spatial_idx = 0, .temporal_idx = 0}}, + {.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 1, .temporal_idx = 0}}}, + {{.timestamp_rtp = 1, + .layer_id = {.spatial_idx = 0, .temporal_idx = 1}}}}, + ScalabilityMode::kL2T2); + std::vector slice = stats->Slice(Filter{}, /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0), + Field(&Frame::timestamp_rtp, 0), + Field(&Frame::timestamp_rtp, 1))); + + slice = stats->Slice({.min_timestamp_rtp = 1}, /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 1))); + + slice = stats->Slice({.max_timestamp_rtp = 0}, /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0), + Field(&Frame::timestamp_rtp, 0))); + + slice = stats->Slice({.layer_id = {{.spatial_idx = 0, .temporal_idx = 0}}}, + /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0))); + + slice = stats->Slice({.layer_id = {{.spatial_idx = 0, .temporal_idx = 1}}}, + /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0), + Field(&Frame::timestamp_rtp, 1))); +} + +TEST(VideoCodecTester, Merge) { + std::unique_ptr stats = + RunTest({{{.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 0, .temporal_idx = 0}, + .frame_size = DataSize::Bytes(1), + .keyframe = true}, + {.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 1, .temporal_idx = 0}, + .frame_size = DataSize::Bytes(2)}}, + {{.timestamp_rtp = 1, + .layer_id = {.spatial_idx = 0, .temporal_idx = 1}, + .frame_size = DataSize::Bytes(4)}, + {.timestamp_rtp = 1, + .layer_id = {.spatial_idx = 1, .temporal_idx = 1}, + .frame_size = DataSize::Bytes(8)}}}, + ScalabilityMode::kL2T2_KEY); + + std::vector slice = stats->Slice(Filter{}, /*merge=*/true); + EXPECT_THAT( + slice, + ElementsAre( + AllOf(Field(&Frame::timestamp_rtp, 0), Field(&Frame::keyframe, true), + Field(&Frame::frame_size, DataSize::Bytes(3))), + AllOf(Field(&Frame::timestamp_rtp, 1), Field(&Frame::keyframe, false), + Field(&Frame::frame_size, DataSize::Bytes(12))))); +} + +struct AggregationTestParameters { + Filter filter; + double expected_keyframe_sum; + double expected_encoded_bitrate_kbps; + double expected_encoded_framerate_fps; + double expected_bitrate_mismatch_pct; + double expected_framerate_mismatch_pct; +}; + +class VideoCodecTesterTestAggregation + : public ::testing::TestWithParam {}; + +TEST_P(VideoCodecTesterTestAggregation, Aggregate) { + AggregationTestParameters test_params = GetParam(); + std::unique_ptr stats = + RunTest({{// L0T0 + {.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 0, .temporal_idx = 0}, + .frame_size = DataSize::Bytes(1), + .keyframe = true}, + // L1T0 + {.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 1, .temporal_idx = 0}, + .frame_size = DataSize::Bytes(2)}}, + // Emulate frame drop (frame_size = 0). + {{.timestamp_rtp = 3000, + .layer_id = {.spatial_idx = 0, .temporal_idx = 0}, + .frame_size = DataSize::Zero()}}, + {// L0T1 + {.timestamp_rtp = 87000, + .layer_id = {.spatial_idx = 0, .temporal_idx = 1}, + .frame_size = DataSize::Bytes(4)}, + // L1T1 + {.timestamp_rtp = 87000, + .layer_id = {.spatial_idx = 1, .temporal_idx = 1}, + .frame_size = DataSize::Bytes(8)}}}, + ScalabilityMode::kL2T2_KEY); + + Stream stream = stats->Aggregate(test_params.filter); + EXPECT_EQ(stream.keyframe.GetSum(), test_params.expected_keyframe_sum); + EXPECT_EQ(stream.encoded_bitrate_kbps.GetAverage(), + test_params.expected_encoded_bitrate_kbps); + EXPECT_EQ(stream.encoded_framerate_fps.GetAverage(), + test_params.expected_encoded_framerate_fps); + EXPECT_EQ(stream.bitrate_mismatch_pct.GetAverage(), + test_params.expected_bitrate_mismatch_pct); + EXPECT_EQ(stream.framerate_mismatch_pct.GetAverage(), + test_params.expected_framerate_mismatch_pct); +} + +INSTANTIATE_TEST_SUITE_P( + All, + VideoCodecTesterTestAggregation, + ::testing::Values( + // No filtering. + AggregationTestParameters{ + .filter = {}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(15).kbps(), + .expected_encoded_framerate_fps = 2, + .expected_bitrate_mismatch_pct = + 100 * (15.0 / (kTargetLayerBitrate.bytes_per_sec() * 4) - 1), + .expected_framerate_mismatch_pct = + 100 * (2.0 / kTargetFramerate.hertz() - 1)}, + // L0T0 + AggregationTestParameters{ + .filter = {.layer_id = {{.spatial_idx = 0, .temporal_idx = 0}}}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(1).kbps(), + .expected_encoded_framerate_fps = 1, + .expected_bitrate_mismatch_pct = + 100 * (1.0 / kTargetLayerBitrate.bytes_per_sec() - 1), + .expected_framerate_mismatch_pct = + 100 * (1.0 / (kTargetFramerate.hertz() / 2) - 1)}, + // L0T1 + AggregationTestParameters{ + .filter = {.layer_id = {{.spatial_idx = 0, .temporal_idx = 1}}}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(5).kbps(), + .expected_encoded_framerate_fps = 2, + .expected_bitrate_mismatch_pct = + 100 * (5.0 / (kTargetLayerBitrate.bytes_per_sec() * 2) - 1), + .expected_framerate_mismatch_pct = + 100 * (2.0 / kTargetFramerate.hertz() - 1)}, + // L1T0 + AggregationTestParameters{ + .filter = {.layer_id = {{.spatial_idx = 1, .temporal_idx = 0}}}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(3).kbps(), + .expected_encoded_framerate_fps = 1, + .expected_bitrate_mismatch_pct = + 100 * (3.0 / kTargetLayerBitrate.bytes_per_sec() - 1), + .expected_framerate_mismatch_pct = + 100 * (1.0 / (kTargetFramerate.hertz() / 2) - 1)}, + // L1T1 + AggregationTestParameters{ + .filter = {.layer_id = {{.spatial_idx = 1, .temporal_idx = 1}}}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(11).kbps(), + .expected_encoded_framerate_fps = 2, + .expected_bitrate_mismatch_pct = + 100 * (11.0 / (kTargetLayerBitrate.bytes_per_sec() * 2) - 1), + .expected_framerate_mismatch_pct = + 100 * (2.0 / kTargetFramerate.hertz() - 1)})); + +TEST(VideoCodecTester, Psnr) { + std::unique_ptr stats = + RunTest({{{.timestamp_rtp = 0, .frame_size = DataSize::Bytes(1)}}, + {{.timestamp_rtp = 3000, .frame_size = DataSize::Bytes(1)}}}, + ScalabilityMode::kL1T1); + + std::vector slice = stats->Slice(Filter{}, /*merge=*/false); + ASSERT_THAT(slice, SizeIs(2)); + ASSERT_TRUE(slice[0].psnr.has_value()); + ASSERT_TRUE(slice[1].psnr.has_value()); + EXPECT_NEAR(slice[0].psnr->y, 42, 1); + EXPECT_NEAR(slice[0].psnr->u, 38, 1); + EXPECT_NEAR(slice[0].psnr->v, 36, 1); + EXPECT_NEAR(slice[1].psnr->y, 38, 1); + EXPECT_NEAR(slice[1].psnr->u, 36, 1); + EXPECT_NEAR(slice[1].psnr->v, 34, 1); +} + +class VideoCodecTesterTestPacing + : public ::testing::TestWithParam> { + public: + const int kSourceWidth = 2; + const int kSourceHeight = 2; + const int kNumFrames = 3; + const int kTargetLayerBitrateKbps = 128; + const Frequency kTargetFramerate = Frequency::Hertz(10); + + void SetUp() override { + source_yuv_file_path_ = webrtc::test::TempFilename( + webrtc::test::OutputPath(), "video_codec_tester_impl_unittest"); + FILE* file = fopen(source_yuv_file_path_.c_str(), "wb"); + for (int i = 0; i < 3 * kSourceWidth * kSourceHeight / 2; ++i) { + fwrite("x", 1, 1, file); + } + fclose(file); + } + + protected: + std::string source_yuv_file_path_; +}; + +TEST_P(VideoCodecTesterTestPacing, PaceEncode) { + auto [pacing_settings, expected_delta_ms] = GetParam(); + VideoSourceSettings video_source{ + .file_path = source_yuv_file_path_, + .resolution = {.width = kSourceWidth, .height = kSourceHeight}, + .framerate = kTargetFramerate}; + + NiceMock encoder_factory; + ON_CALL(encoder_factory, CreateVideoEncoder(_)) + .WillByDefault([](const SdpVideoFormat&) { + return std::make_unique>(); + }); + + std::map encoding_settings = + VideoCodecTester::CreateEncodingSettings( + "VP8", "L1T1", kSourceWidth, kSourceHeight, {kTargetLayerBitrateKbps}, + kTargetFramerate.hertz(), kNumFrames); + + EncoderSettings encoder_settings; + encoder_settings.pacing_settings = pacing_settings; + std::vector frames = + VideoCodecTester::RunEncodeTest(video_source, &encoder_factory, + encoder_settings, encoding_settings) + ->Slice(/*filter=*/{}, /*merge=*/false); + ASSERT_THAT(frames, SizeIs(kNumFrames)); + EXPECT_NEAR((frames[1].encode_start - frames[0].encode_start).ms(), + expected_delta_ms, 10); + EXPECT_NEAR((frames[2].encode_start - frames[1].encode_start).ms(), + expected_delta_ms, 10); +} + +TEST_P(VideoCodecTesterTestPacing, PaceDecode) { + auto [pacing_settings, expected_delta_ms] = GetParam(); + MockCodedVideoSource video_source(kNumFrames, kTargetFramerate); + + NiceMock decoder_factory; + ON_CALL(decoder_factory, CreateVideoDecoder(_)) + .WillByDefault([](const SdpVideoFormat&) { + return std::make_unique>(); + }); + + DecoderSettings decoder_settings; + decoder_settings.pacing_settings = pacing_settings; + std::vector frames = + VideoCodecTester::RunDecodeTest(&video_source, &decoder_factory, + decoder_settings, SdpVideoFormat("VP8")) + ->Slice(/*filter=*/{}, /*merge=*/false); + ASSERT_THAT(frames, SizeIs(kNumFrames)); + EXPECT_NEAR((frames[1].decode_start - frames[0].decode_start).ms(), + expected_delta_ms, 10); + EXPECT_NEAR((frames[2].decode_start - frames[1].decode_start).ms(), + expected_delta_ms, 10); +} + +INSTANTIATE_TEST_SUITE_P( + DISABLED_All, + VideoCodecTesterTestPacing, + ::testing::Values( + // No pacing. + std::make_tuple(PacingSettings{.mode = PacingMode::kNoPacing}, + /*expected_delta_ms=*/0), + // Real-time pacing. + std::make_tuple(PacingSettings{.mode = PacingMode::kRealTime}, + /*expected_delta_ms=*/100), + // Pace with specified constant rate. + std::make_tuple(PacingSettings{.mode = PacingMode::kConstantRate, + .constant_rate = Frequency::Hertz(20)}, + /*expected_delta_ms=*/50))); +} // namespace test +} // namespace webrtc