diff --git a/api/BUILD.gn b/api/BUILD.gn index a163315440..ff461b8e73 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -1071,24 +1071,6 @@ if (rtc_include_tests) { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } - rtc_library("video_codec_stats_api") { - visibility = [ "*" ] - testonly = true - sources = [ - "test/video_codec_stats.cc", - "test/video_codec_stats.h", - ] - deps = [ - "../api/numerics:numerics", - "../api/units:data_rate", - "../api/units:data_size", - "../api/units:frequency", - "test/metrics:metric", - "test/metrics:metrics_logger", - ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] - } - rtc_library("videocodec_test_fixture_api") { visibility = [ "*" ] testonly = true @@ -1100,23 +1082,6 @@ if (rtc_include_tests) { ] } - rtc_library("video_codec_tester_api") { - visibility = [ "*" ] - testonly = true - sources = [ "test/video_codec_tester.h" ] - deps = [ - ":video_codec_stats_api", - "../modules/video_coding/svc:scalability_mode_util", - "video:encoded_image", - "video:resolution", - "video:video_frame", - ] - absl_deps = [ - "//third_party/abseil-cpp/absl/functional:any_invocable", - "//third_party/abseil-cpp/absl/types:optional", - ] - } - rtc_library("create_videocodec_test_fixture_api") { visibility = [ "*" ] testonly = true @@ -1132,19 +1097,6 @@ if (rtc_include_tests) { ] } - rtc_library("create_video_codec_tester_api") { - visibility = [ "*" ] - testonly = true - sources = [ - "test/create_video_codec_tester.cc", - "test/create_video_codec_tester.h", - ] - deps = [ - ":video_codec_tester_api", - "../modules/video_coding:video_codec_tester", - ] - } - rtc_source_set("mock_audio_mixer") { visibility = [ "*" ] testonly = true diff --git a/api/test/create_video_codec_tester.cc b/api/test/create_video_codec_tester.cc deleted file mode 100644 index a1efefdb48..0000000000 --- a/api/test/create_video_codec_tester.cc +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "api/test/create_video_codec_tester.h" - -#include -#include - -#include "api/test/video_codec_tester.h" -#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" - -namespace webrtc { -namespace test { - -std::unique_ptr CreateVideoCodecTester() { - return std::make_unique(); -} - -} // namespace test -} // namespace webrtc diff --git a/api/test/create_video_codec_tester.h b/api/test/create_video_codec_tester.h deleted file mode 100644 index c68864ce85..0000000000 --- a/api/test/create_video_codec_tester.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ -#define API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ - -#include - -#include "api/test/video_codec_tester.h" - -namespace webrtc { -namespace test { - -std::unique_ptr CreateVideoCodecTester(); - -} // namespace test -} // namespace webrtc - -#endif // API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ diff --git a/api/test/video_codec_stats.cc b/api/test/video_codec_stats.cc deleted file mode 100644 index fb7226701e..0000000000 --- a/api/test/video_codec_stats.cc +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "api/test/video_codec_stats.h" - -namespace webrtc { -namespace test { - -void VideoCodecStats::Stream::LogMetrics( - MetricsLogger* logger, - std::string test_case_name, - std::map metadata) const { - logger->LogMetric("width", test_case_name, width, Unit::kCount, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("height", test_case_name, height, Unit::kCount, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric( - "frame_size_bytes", test_case_name, frame_size_bytes, Unit::kBytes, - webrtc::test::ImprovementDirection::kNeitherIsBetter, metadata); - - logger->LogMetric("keyframe", test_case_name, keyframe, Unit::kCount, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("qp", test_case_name, qp, Unit::kUnitless, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric( - "encode_time_ms", test_case_name, encode_time_ms, Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter, metadata); - - logger->LogMetric( - "decode_time_ms", test_case_name, decode_time_ms, Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter, metadata); - - logger->LogMetric("target_bitrate_kbps", test_case_name, target_bitrate_kbps, - Unit::kKilobitsPerSecond, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("target_framerate_fps", test_case_name, - target_framerate_fps, Unit::kHertz, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("encoded_bitrate_kbps", test_case_name, - encoded_bitrate_kbps, Unit::kKilobitsPerSecond, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("encoded_framerate_fps", test_case_name, - encoded_framerate_fps, Unit::kHertz, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("bitrate_mismatch_pct", test_case_name, - bitrate_mismatch_pct, Unit::kPercent, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("framerate_mismatch_pct", test_case_name, - framerate_mismatch_pct, Unit::kPercent, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("transmission_time_ms", test_case_name, - transmission_time_ms, Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("psnr_y_db", test_case_name, psnr.y, Unit::kUnitless, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("psnr_u_db", test_case_name, psnr.u, Unit::kUnitless, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("psnr_v_db", test_case_name, psnr.v, Unit::kUnitless, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); -} - -} // namespace test -} // namespace webrtc diff --git a/api/test/video_codec_stats.h b/api/test/video_codec_stats.h deleted file mode 100644 index 80f8287848..0000000000 --- a/api/test/video_codec_stats.h +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef API_TEST_VIDEO_CODEC_STATS_H_ -#define API_TEST_VIDEO_CODEC_STATS_H_ - -#include -#include -#include - -#include "absl/types/optional.h" -#include "api/numerics/samples_stats_counter.h" -#include "api/test/metrics/metric.h" -#include "api/test/metrics/metrics_logger.h" -#include "api/units/data_rate.h" -#include "api/units/data_size.h" -#include "api/units/frequency.h" - -namespace webrtc { -namespace test { - -// Interface for encoded and/or decoded video frame and stream statistics. -class VideoCodecStats { - public: - // Filter for slicing frames. - struct Filter { - absl::optional first_frame; - absl::optional last_frame; - absl::optional spatial_idx; - absl::optional temporal_idx; - }; - - struct Frame { - int frame_num = 0; - uint32_t timestamp_rtp = 0; - - int spatial_idx = 0; - int temporal_idx = 0; - - int width = 0; - int height = 0; - DataSize frame_size = DataSize::Zero(); - bool keyframe = false; - absl::optional qp; - absl::optional base_spatial_idx; - - Timestamp encode_start = Timestamp::Zero(); - TimeDelta encode_time = TimeDelta::Zero(); - Timestamp decode_start = Timestamp::Zero(); - TimeDelta decode_time = TimeDelta::Zero(); - - struct Psnr { - double y = 0.0; - double u = 0.0; - double v = 0.0; - }; - absl::optional psnr; - - absl::optional target_bitrate; - absl::optional target_framerate; - - bool encoded = false; - bool decoded = false; - }; - - struct Stream { - SamplesStatsCounter width; - SamplesStatsCounter height; - SamplesStatsCounter frame_size_bytes; - SamplesStatsCounter keyframe; - SamplesStatsCounter qp; - - SamplesStatsCounter encode_time_ms; - SamplesStatsCounter decode_time_ms; - - SamplesStatsCounter target_bitrate_kbps; - SamplesStatsCounter target_framerate_fps; - - SamplesStatsCounter encoded_bitrate_kbps; - SamplesStatsCounter encoded_framerate_fps; - - SamplesStatsCounter bitrate_mismatch_pct; - SamplesStatsCounter framerate_mismatch_pct; - - SamplesStatsCounter transmission_time_ms; - - struct Psnr { - SamplesStatsCounter y; - SamplesStatsCounter u; - SamplesStatsCounter v; - } psnr; - - // Logs `Stream` metrics to provided `MetricsLogger`. - void LogMetrics(MetricsLogger* logger, - std::string test_case_name, - std::map metadata = {}) const; - }; - - virtual ~VideoCodecStats() = default; - - // Returns frames from interval, spatial and temporal layer specified by given - // `filter`. - virtual std::vector Slice( - absl::optional filter = absl::nullopt) const = 0; - - // Returns video statistics aggregated for given `frames`. - virtual Stream Aggregate(const std::vector& frames) const = 0; -}; - -} // namespace test -} // namespace webrtc - -#endif // API_TEST_VIDEO_CODEC_STATS_H_ diff --git a/api/test/video_codec_tester.h b/api/test/video_codec_tester.h deleted file mode 100644 index c2fb89e2cb..0000000000 --- a/api/test/video_codec_tester.h +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef API_TEST_VIDEO_CODEC_TESTER_H_ -#define API_TEST_VIDEO_CODEC_TESTER_H_ - -#include -#include - -#include "absl/functional/any_invocable.h" -#include "absl/types/optional.h" -#include "api/test/video_codec_stats.h" -#include "api/video/encoded_image.h" -#include "api/video/resolution.h" -#include "api/video/video_frame.h" - -namespace webrtc { -namespace test { - -// Interface for a video codec tester. The interface provides minimalistic set -// of data structures that enables implementation of decode-only, encode-only -// and encode-decode tests. -class VideoCodecTester { - public: - // Pacing settings for codec input. - struct PacingSettings { - enum PacingMode { - // Pacing is not used. Frames are sent to codec back-to-back. - kNoPacing, - // Pace with the rate equal to the target video frame rate. Pacing time is - // derived from RTP timestamp. - kRealTime, - // Pace with the explicitly provided rate. - kConstantRate, - }; - PacingMode mode = PacingMode::kNoPacing; - // Pacing rate for `kConstantRate` mode. - Frequency constant_rate = Frequency::Zero(); - }; - - struct DecoderSettings { - PacingSettings pacing; - absl::optional decoder_input_base_path; - absl::optional decoder_output_base_path; - }; - - struct EncoderSettings { - PacingSettings pacing; - absl::optional encoder_input_base_path; - absl::optional encoder_output_base_path; - }; - - virtual ~VideoCodecTester() = default; - - // Interface for a raw video frames source. - class RawVideoSource { - public: - virtual ~RawVideoSource() = default; - - // Returns next frame. If no more frames to pull, returns `absl::nullopt`. - // For analysis and pacing purposes, frame must have RTP timestamp set. The - // timestamp must represent the target video frame rate and be unique. - virtual absl::optional PullFrame() = 0; - - // Returns early pulled frame with RTP timestamp equal to `timestamp_rtp`. - virtual VideoFrame GetFrame(uint32_t timestamp_rtp, - Resolution resolution) = 0; - }; - - // Interface for a coded video frames source. - class CodedVideoSource { - public: - virtual ~CodedVideoSource() = default; - - // Returns next frame. If no more frames to pull, returns `absl::nullopt`. - // For analysis and pacing purposes, frame must have RTP timestamp set. The - // timestamp must represent the target video frame rate and be unique. - virtual absl::optional PullFrame() = 0; - }; - - // Interface for a video encoder. - class Encoder { - public: - using EncodeCallback = - absl::AnyInvocable; - - virtual ~Encoder() = default; - - virtual void Initialize() = 0; - - virtual void Encode(const VideoFrame& frame, EncodeCallback callback) = 0; - - virtual void Flush() = 0; - }; - - // Interface for a video decoder. - class Decoder { - public: - using DecodeCallback = - absl::AnyInvocable; - - virtual ~Decoder() = default; - - virtual void Initialize() = 0; - - virtual void Decode(const EncodedImage& frame, DecodeCallback callback) = 0; - - virtual void Flush() = 0; - }; - - // Pulls coded video frames from `video_source` and passes them to `decoder`. - // Returns `VideoCodecTestStats` object that contains collected per-frame - // metrics. - virtual std::unique_ptr RunDecodeTest( - CodedVideoSource* video_source, - Decoder* decoder, - const DecoderSettings& decoder_settings) = 0; - - // Pulls raw video frames from `video_source` and passes them to `encoder`. - // Returns `VideoCodecTestStats` object that contains collected per-frame - // metrics. - virtual std::unique_ptr RunEncodeTest( - RawVideoSource* video_source, - Encoder* encoder, - const EncoderSettings& encoder_settings) = 0; - - // Pulls raw video frames from `video_source`, passes them to `encoder` and - // then passes encoded frames to `decoder`. Returns `VideoCodecTestStats` - // object that contains collected per-frame metrics. - virtual std::unique_ptr RunEncodeDecodeTest( - RawVideoSource* video_source, - Encoder* encoder, - Decoder* decoder, - const EncoderSettings& encoder_settings, - const DecoderSettings& decoder_settings) = 0; -}; - -} // namespace test -} // namespace webrtc - -#endif // API_TEST_VIDEO_CODEC_TESTER_H_ diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index a3f2befae9..c75b433cd4 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -851,8 +851,6 @@ if (rtc_include_tests) { "../../api:frame_generator_api", "../../api:scoped_refptr", "../../api:sequence_checker", - "../../api:video_codec_stats_api", - "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", "../../api/numerics:numerics", "../../api/task_queue", @@ -994,46 +992,6 @@ if (rtc_include_tests) { ] } - rtc_library("video_codec_tester") { - testonly = true - sources = [ - "codecs/test/video_codec_analyzer.cc", - "codecs/test/video_codec_analyzer.h", - "codecs/test/video_codec_stats_impl.cc", - "codecs/test/video_codec_stats_impl.h", - "codecs/test/video_codec_tester_impl.cc", - "codecs/test/video_codec_tester_impl.h", - ] - - deps = [ - ":video_coding_utility", - "../../api:sequence_checker", - "../../api:video_codec_stats_api", - "../../api:video_codec_tester_api", - "../../api/numerics:numerics", - "../../api/task_queue:default_task_queue_factory", - "../../api/test/metrics:metrics_logger", - "../../api/units:data_rate", - "../../api/units:frequency", - "../../api/units:time_delta", - "../../api/units:timestamp", - "../../api/video:encoded_image", - "../../api/video:resolution", - "../../api/video:video_codec_constants", - "../../api/video:video_frame", - "../../rtc_base:checks", - "../../rtc_base:rtc_event", - "../../rtc_base:task_queue_for_test", - "../../rtc_base:timeutils", - "../../rtc_base/system:no_unique_address", - "../../system_wrappers", - "../../test:video_test_support", - "//third_party/libyuv", - ] - - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] - } - rtc_test("video_codec_perf_tests") { testonly = true @@ -1041,28 +999,18 @@ if (rtc_include_tests) { deps = [ ":video_codec_interface", - ":video_codec_tester", - "../../api:create_video_codec_tester_api", - "../../api:video_codec_tester_api", - "../../api:videocodec_test_stats_api", "../../api/test/metrics:global_metrics_logger_and_exporter", "../../api/units:data_rate", "../../api/units:frequency", - "../../api/video:encoded_image", "../../api/video:resolution", - "../../api/video:video_frame", - "../../api/video_codecs:scalability_mode", - "../../api/video_codecs:video_codecs_api", - "../../media:rtc_internal_video_codecs", + "../../api/video_codecs:builtin_video_decoder_factory", + "../../api/video_codecs:builtin_video_encoder_factory", "../../rtc_base:logging", "../../test:fileutils", "../../test:test_flags", "../../test:test_main", "../../test:test_support", - "../../test:video_test_support", - "../rtp_rtcp:rtp_rtcp_format", - "svc:scalability_mode_util", - "//third_party/libyuv", + "../../test:video_codec_tester", ] if (is_android) { @@ -1190,9 +1138,6 @@ if (rtc_include_tests) { sources = [ "chain_diff_calculator_unittest.cc", - "codecs/test/video_codec_analyzer_unittest.cc", - "codecs/test/video_codec_stats_impl_unittest.cc", - "codecs/test/video_codec_tester_impl_unittest.cc", "codecs/test/videocodec_test_fixture_config_unittest.cc", "codecs/test/videocodec_test_stats_impl_unittest.cc", "codecs/test/videoprocessor_unittest.cc", @@ -1247,7 +1192,6 @@ if (rtc_include_tests) { ":packet_buffer", ":simulcast_test_fixture_impl", ":video_codec_interface", - ":video_codec_tester", ":video_codecs_test_framework", ":video_coding", ":video_coding_legacy", @@ -1270,7 +1214,6 @@ if (rtc_include_tests) { "../../api:rtp_packet_info", "../../api:scoped_refptr", "../../api:simulcast_test_fixture_api", - "../../api:video_codec_tester_api", "../../api:videocodec_test_fixture_api", "../../api/task_queue", "../../api/task_queue:default_task_queue_factory", @@ -1296,6 +1239,7 @@ if (rtc_include_tests) { "../../common_video/generic_frame_descriptor", "../../common_video/test:utilities", "../../media:media_constants", + "../../media:rtc_internal_video_codecs", "../../media:rtc_media_base", "../../rtc_base:checks", "../../rtc_base:gunit_helpers", diff --git a/modules/video_coding/codecs/test/video_codec_analyzer.cc b/modules/video_coding/codecs/test/video_codec_analyzer.cc deleted file mode 100644 index 772c15734a..0000000000 --- a/modules/video_coding/codecs/test/video_codec_analyzer.cc +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/video_codec_analyzer.h" - -#include - -#include "api/task_queue/default_task_queue_factory.h" -#include "api/video/i420_buffer.h" -#include "api/video/video_codec_constants.h" -#include "api/video/video_frame.h" -#include "rtc_base/checks.h" -#include "rtc_base/event.h" -#include "rtc_base/time_utils.h" -#include "third_party/libyuv/include/libyuv/compare.h" - -namespace webrtc { -namespace test { - -namespace { -using Psnr = VideoCodecStats::Frame::Psnr; - -Psnr CalcPsnr(const I420BufferInterface& ref_buffer, - const I420BufferInterface& dec_buffer) { - RTC_CHECK_EQ(ref_buffer.width(), dec_buffer.width()); - RTC_CHECK_EQ(ref_buffer.height(), dec_buffer.height()); - - uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane( - dec_buffer.DataY(), dec_buffer.StrideY(), ref_buffer.DataY(), - ref_buffer.StrideY(), dec_buffer.width(), dec_buffer.height()); - - uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane( - dec_buffer.DataU(), dec_buffer.StrideU(), ref_buffer.DataU(), - ref_buffer.StrideU(), dec_buffer.width() / 2, dec_buffer.height() / 2); - - uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane( - dec_buffer.DataV(), dec_buffer.StrideV(), ref_buffer.DataV(), - ref_buffer.StrideV(), dec_buffer.width() / 2, dec_buffer.height() / 2); - - int num_y_samples = dec_buffer.width() * dec_buffer.height(); - Psnr psnr; - psnr.y = libyuv::SumSquareErrorToPsnr(sse_y, num_y_samples); - psnr.u = libyuv::SumSquareErrorToPsnr(sse_u, num_y_samples / 4); - psnr.v = libyuv::SumSquareErrorToPsnr(sse_v, num_y_samples / 4); - - return psnr; -} - -} // namespace - -VideoCodecAnalyzer::VideoCodecAnalyzer( - ReferenceVideoSource* reference_video_source) - : reference_video_source_(reference_video_source), num_frames_(0) { - sequence_checker_.Detach(); -} - -void VideoCodecAnalyzer::StartEncode(const VideoFrame& input_frame) { - int64_t encode_start_us = rtc::TimeMicros(); - task_queue_.PostTask( - [this, timestamp_rtp = input_frame.timestamp(), encode_start_us]() { - RTC_DCHECK_RUN_ON(&sequence_checker_); - - RTC_CHECK(frame_num_.find(timestamp_rtp) == frame_num_.end()); - frame_num_[timestamp_rtp] = num_frames_++; - - stats_.AddFrame({.frame_num = frame_num_[timestamp_rtp], - .timestamp_rtp = timestamp_rtp, - .encode_start = Timestamp::Micros(encode_start_us)}); - }); -} - -void VideoCodecAnalyzer::FinishEncode(const EncodedImage& frame) { - int64_t encode_finished_us = rtc::TimeMicros(); - - task_queue_.PostTask([this, timestamp_rtp = frame.RtpTimestamp(), - spatial_idx = frame.SpatialIndex().value_or(0), - temporal_idx = frame.TemporalIndex().value_or(0), - width = frame._encodedWidth, - height = frame._encodedHeight, - frame_type = frame._frameType, - frame_size_bytes = frame.size(), qp = frame.qp_, - encode_finished_us]() { - RTC_DCHECK_RUN_ON(&sequence_checker_); - - if (spatial_idx > 0) { - VideoCodecStats::Frame* base_frame = - stats_.GetFrame(timestamp_rtp, /*spatial_idx=*/0); - - stats_.AddFrame({.frame_num = base_frame->frame_num, - .timestamp_rtp = timestamp_rtp, - .spatial_idx = spatial_idx, - .encode_start = base_frame->encode_start}); - } - - VideoCodecStats::Frame* fs = stats_.GetFrame(timestamp_rtp, spatial_idx); - fs->spatial_idx = spatial_idx; - fs->temporal_idx = temporal_idx; - fs->width = width; - fs->height = height; - fs->frame_size = DataSize::Bytes(frame_size_bytes); - fs->qp = qp; - fs->keyframe = frame_type == VideoFrameType::kVideoFrameKey; - fs->encode_time = Timestamp::Micros(encode_finished_us) - fs->encode_start; - fs->encoded = true; - }); -} - -void VideoCodecAnalyzer::StartDecode(const EncodedImage& frame) { - int64_t decode_start_us = rtc::TimeMicros(); - task_queue_.PostTask([this, timestamp_rtp = frame.RtpTimestamp(), - spatial_idx = frame.SpatialIndex().value_or(0), - frame_size_bytes = frame.size(), decode_start_us]() { - RTC_DCHECK_RUN_ON(&sequence_checker_); - - VideoCodecStats::Frame* fs = stats_.GetFrame(timestamp_rtp, spatial_idx); - if (fs == nullptr) { - if (frame_num_.find(timestamp_rtp) == frame_num_.end()) { - frame_num_[timestamp_rtp] = num_frames_++; - } - stats_.AddFrame({.frame_num = frame_num_[timestamp_rtp], - .timestamp_rtp = timestamp_rtp, - .spatial_idx = spatial_idx, - .frame_size = DataSize::Bytes(frame_size_bytes)}); - fs = stats_.GetFrame(timestamp_rtp, spatial_idx); - } - - fs->decode_start = Timestamp::Micros(decode_start_us); - }); -} - -void VideoCodecAnalyzer::FinishDecode(const VideoFrame& frame, - int spatial_idx) { - int64_t decode_finished_us = rtc::TimeMicros(); - task_queue_.PostTask([this, timestamp_rtp = frame.timestamp(), spatial_idx, - width = frame.width(), height = frame.height(), - decode_finished_us]() { - RTC_DCHECK_RUN_ON(&sequence_checker_); - VideoCodecStats::Frame* fs = stats_.GetFrame(timestamp_rtp, spatial_idx); - fs->decode_time = Timestamp::Micros(decode_finished_us) - fs->decode_start; - - if (!fs->encoded) { - fs->width = width; - fs->height = height; - } - - fs->decoded = true; - }); - - if (reference_video_source_ != nullptr) { - // Copy hardware-backed frame into main memory to release output buffers - // which number may be limited in hardware decoders. - rtc::scoped_refptr decoded_buffer = - frame.video_frame_buffer()->ToI420(); - - task_queue_.PostTask([this, decoded_buffer, - timestamp_rtp = frame.timestamp(), spatial_idx]() { - RTC_DCHECK_RUN_ON(&sequence_checker_); - VideoFrame ref_frame = reference_video_source_->GetFrame( - timestamp_rtp, {.width = decoded_buffer->width(), - .height = decoded_buffer->height()}); - rtc::scoped_refptr ref_buffer = - ref_frame.video_frame_buffer()->ToI420(); - - Psnr psnr = CalcPsnr(*decoded_buffer, *ref_buffer); - - VideoCodecStats::Frame* fs = - this->stats_.GetFrame(timestamp_rtp, spatial_idx); - fs->psnr = psnr; - }); - } -} - -std::unique_ptr VideoCodecAnalyzer::GetStats() { - std::unique_ptr stats; - rtc::Event ready; - task_queue_.PostTask([this, &stats, &ready]() mutable { - RTC_DCHECK_RUN_ON(&sequence_checker_); - stats.reset(new VideoCodecStatsImpl(stats_)); - ready.Set(); - }); - ready.Wait(rtc::Event::kForever); - return stats; -} - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_analyzer.h b/modules/video_coding/codecs/test/video_codec_analyzer.h deleted file mode 100644 index 29ca8ee2ff..0000000000 --- a/modules/video_coding/codecs/test/video_codec_analyzer.h +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ -#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ - -#include -#include - -#include "absl/types/optional.h" -#include "api/sequence_checker.h" -#include "api/test/video_codec_tester.h" -#include "api/video/encoded_image.h" -#include "api/video/resolution.h" -#include "api/video/video_frame.h" -#include "modules/video_coding/codecs/test/video_codec_stats_impl.h" -#include "rtc_base/system/no_unique_address.h" -#include "rtc_base/task_queue_for_test.h" - -namespace webrtc { -namespace test { - -// Analyzer measures and collects metrics necessary for evaluation of video -// codec quality and performance. This class is thread-safe. -class VideoCodecAnalyzer { - public: - // An interface that provides reference frames for spatial quality analysis. - class ReferenceVideoSource { - public: - virtual ~ReferenceVideoSource() = default; - - virtual VideoFrame GetFrame(uint32_t timestamp_rtp, - Resolution resolution) = 0; - }; - - explicit VideoCodecAnalyzer( - ReferenceVideoSource* reference_video_source = nullptr); - - void StartEncode(const VideoFrame& frame); - - void FinishEncode(const EncodedImage& frame); - - void StartDecode(const EncodedImage& frame); - - void FinishDecode(const VideoFrame& frame, int spatial_idx); - - std::unique_ptr GetStats(); - - protected: - TaskQueueForTest task_queue_; - - ReferenceVideoSource* const reference_video_source_; - - VideoCodecStatsImpl stats_ RTC_GUARDED_BY(sequence_checker_); - - // Map from RTP timestamp to frame number. - std::map frame_num_ RTC_GUARDED_BY(sequence_checker_); - - // Processed frames counter. - int num_frames_ RTC_GUARDED_BY(sequence_checker_); - - RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; -}; - -} // namespace test -} // namespace webrtc - -#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_ANALYZER_H_ diff --git a/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc b/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc deleted file mode 100644 index 03146417da..0000000000 --- a/modules/video_coding/codecs/test/video_codec_analyzer_unittest.cc +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/video_codec_analyzer.h" - -#include "absl/types/optional.h" -#include "api/video/i420_buffer.h" -#include "test/gmock.h" -#include "test/gtest.h" -#include "third_party/libyuv/include/libyuv/planar_functions.h" - -namespace webrtc { -namespace test { - -namespace { -using ::testing::Return; -using ::testing::Values; -using Psnr = VideoCodecStats::Frame::Psnr; - -const uint32_t kTimestamp = 3000; -const int kSpatialIdx = 2; - -class MockReferenceVideoSource - : public VideoCodecAnalyzer::ReferenceVideoSource { - public: - MOCK_METHOD(VideoFrame, GetFrame, (uint32_t, Resolution), (override)); -}; - -VideoFrame CreateVideoFrame(uint32_t timestamp_rtp, - uint8_t y = 0, - uint8_t u = 0, - uint8_t v = 0) { - rtc::scoped_refptr buffer(I420Buffer::Create(2, 2)); - - libyuv::I420Rect(buffer->MutableDataY(), buffer->StrideY(), - buffer->MutableDataU(), buffer->StrideU(), - buffer->MutableDataV(), buffer->StrideV(), 0, 0, - buffer->width(), buffer->height(), y, u, v); - - return VideoFrame::Builder() - .set_video_frame_buffer(buffer) - .set_timestamp_rtp(timestamp_rtp) - .build(); -} - -EncodedImage CreateEncodedImage(uint32_t timestamp_rtp, int spatial_idx = 0) { - EncodedImage encoded_image; - encoded_image.SetRtpTimestamp(timestamp_rtp); - encoded_image.SetSpatialIndex(spatial_idx); - return encoded_image; -} -} // namespace - -TEST(VideoCodecAnalyzerTest, StartEncode) { - VideoCodecAnalyzer analyzer; - analyzer.StartEncode(CreateVideoFrame(kTimestamp)); - - auto fs = analyzer.GetStats()->Slice(); - EXPECT_EQ(1u, fs.size()); - EXPECT_EQ(fs[0].timestamp_rtp, kTimestamp); -} - -TEST(VideoCodecAnalyzerTest, FinishEncode) { - VideoCodecAnalyzer analyzer; - analyzer.StartEncode(CreateVideoFrame(kTimestamp)); - - EncodedImage encoded_frame = CreateEncodedImage(kTimestamp, kSpatialIdx); - analyzer.FinishEncode(encoded_frame); - - auto fs = analyzer.GetStats()->Slice(); - EXPECT_EQ(2u, fs.size()); - EXPECT_EQ(kSpatialIdx, fs[1].spatial_idx); -} - -TEST(VideoCodecAnalyzerTest, StartDecode) { - VideoCodecAnalyzer analyzer; - analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); - - auto fs = analyzer.GetStats()->Slice(); - EXPECT_EQ(1u, fs.size()); - EXPECT_EQ(kTimestamp, fs[0].timestamp_rtp); -} - -TEST(VideoCodecAnalyzerTest, FinishDecode) { - VideoCodecAnalyzer analyzer; - analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); - VideoFrame decoded_frame = CreateVideoFrame(kTimestamp); - analyzer.FinishDecode(decoded_frame, kSpatialIdx); - - auto fs = analyzer.GetStats()->Slice(); - EXPECT_EQ(1u, fs.size()); - EXPECT_EQ(decoded_frame.width(), fs[0].width); - EXPECT_EQ(decoded_frame.height(), fs[0].height); -} - -TEST(VideoCodecAnalyzerTest, ReferenceVideoSource) { - MockReferenceVideoSource reference_video_source; - VideoCodecAnalyzer analyzer(&reference_video_source); - analyzer.StartDecode(CreateEncodedImage(kTimestamp, kSpatialIdx)); - - EXPECT_CALL(reference_video_source, GetFrame) - .WillOnce(Return(CreateVideoFrame(kTimestamp, /*y=*/0, - /*u=*/0, /*v=*/0))); - - analyzer.FinishDecode( - CreateVideoFrame(kTimestamp, /*value_y=*/1, /*value_u=*/2, /*value_v=*/3), - kSpatialIdx); - - auto fs = analyzer.GetStats()->Slice(); - EXPECT_EQ(1u, fs.size()); - EXPECT_TRUE(fs[0].psnr.has_value()); - - const Psnr& psnr = *fs[0].psnr; - EXPECT_NEAR(psnr.y, 48, 1); - EXPECT_NEAR(psnr.u, 42, 1); - EXPECT_NEAR(psnr.v, 38, 1); -} - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_stats_impl.cc b/modules/video_coding/codecs/test/video_codec_stats_impl.cc deleted file mode 100644 index 9808e2a601..0000000000 --- a/modules/video_coding/codecs/test/video_codec_stats_impl.cc +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/video_codec_stats_impl.h" - -#include - -#include "api/numerics/samples_stats_counter.h" -#include "api/test/metrics/metrics_logger.h" -#include "rtc_base/checks.h" -#include "rtc_base/time_utils.h" - -namespace webrtc { -namespace test { -namespace { -using Frame = VideoCodecStats::Frame; -using Stream = VideoCodecStats::Stream; - -constexpr Frequency k90kHz = Frequency::Hertz(90000); - -class LeakyBucket { - public: - LeakyBucket() : level_bits_(0) {} - - // Updates bucket level and returns its current level in bits. Data is remove - // from bucket with rate equal to target bitrate of previous frame. Bucket - // level is tracked with floating point precision. Returned value of bucket - // level is rounded up. - int Update(const Frame& frame) { - RTC_CHECK(frame.target_bitrate) << "Bitrate must be specified."; - - if (prev_frame_) { - RTC_CHECK_GT(frame.timestamp_rtp, prev_frame_->timestamp_rtp) - << "Timestamp must increase."; - TimeDelta passed = - (frame.timestamp_rtp - prev_frame_->timestamp_rtp) / k90kHz; - level_bits_ -= - prev_frame_->target_bitrate->bps() * passed.us() / 1000000.0; - level_bits_ = std::max(level_bits_, 0.0); - } - - prev_frame_ = frame; - - level_bits_ += frame.frame_size.bytes() * 8; - return static_cast(std::ceil(level_bits_)); - } - - private: - absl::optional prev_frame_; - double level_bits_; -}; - -// Merges spatial layer frames into superframes. -std::vector Merge(const std::vector& frames) { - std::vector superframes; - // Map from frame timestamp to index in `superframes` vector. - std::map index; - - for (const auto& f : frames) { - if (index.find(f.timestamp_rtp) == index.end()) { - index[f.timestamp_rtp] = static_cast(superframes.size()); - superframes.push_back(f); - continue; - } - - Frame& sf = superframes[index[f.timestamp_rtp]]; - - sf.width = std::max(sf.width, f.width); - sf.height = std::max(sf.height, f.height); - sf.frame_size += f.frame_size; - sf.keyframe |= f.keyframe; - - sf.encode_time = std::max(sf.encode_time, f.encode_time); - sf.decode_time = std::max(sf.decode_time, f.decode_time); - - if (f.spatial_idx > sf.spatial_idx) { - if (f.qp) { - sf.qp = f.qp; - } - if (f.psnr) { - sf.psnr = f.psnr; - } - } - - sf.spatial_idx = std::max(sf.spatial_idx, f.spatial_idx); - sf.temporal_idx = std::max(sf.temporal_idx, f.temporal_idx); - - sf.encoded |= f.encoded; - sf.decoded |= f.decoded; - } - - return superframes; -} - -Timestamp RtpToTime(uint32_t timestamp_rtp) { - return Timestamp::Micros((timestamp_rtp / k90kHz).us()); -} - -SamplesStatsCounter::StatsSample StatsSample(double value, Timestamp time) { - return SamplesStatsCounter::StatsSample{value, time}; -} - -TimeDelta CalcTotalDuration(const std::vector& frames) { - RTC_CHECK(!frames.empty()); - TimeDelta duration = TimeDelta::Zero(); - if (frames.size() > 1) { - duration += - (frames.rbegin()->timestamp_rtp - frames.begin()->timestamp_rtp) / - k90kHz; - } - - // Add last frame duration. If target frame rate is provided, calculate frame - // duration from it. Otherwise, assume duration of last frame is the same as - // duration of preceding frame. - if (frames.rbegin()->target_framerate) { - duration += 1 / *frames.rbegin()->target_framerate; - } else { - RTC_CHECK_GT(frames.size(), 1u); - duration += (frames.rbegin()->timestamp_rtp - - std::next(frames.rbegin())->timestamp_rtp) / - k90kHz; - } - - return duration; -} -} // namespace - -std::vector VideoCodecStatsImpl::Slice( - absl::optional filter) const { - std::vector frames; - for (const auto& [frame_id, f] : frames_) { - if (filter.has_value()) { - if (filter->first_frame.has_value() && - f.frame_num < *filter->first_frame) { - continue; - } - if (filter->last_frame.has_value() && f.frame_num > *filter->last_frame) { - continue; - } - if (filter->spatial_idx.has_value() && - f.spatial_idx != *filter->spatial_idx) { - continue; - } - if (filter->temporal_idx.has_value() && - f.temporal_idx > *filter->temporal_idx) { - continue; - } - } - frames.push_back(f); - } - return frames; -} - -Stream VideoCodecStatsImpl::Aggregate(const std::vector& frames) const { - std::vector superframes = Merge(frames); - RTC_CHECK(!superframes.empty()); - - LeakyBucket leacky_bucket; - Stream stream; - for (size_t i = 0; i < superframes.size(); ++i) { - Frame& f = superframes[i]; - Timestamp time = RtpToTime(f.timestamp_rtp); - - if (!f.frame_size.IsZero()) { - stream.width.AddSample(StatsSample(f.width, time)); - stream.height.AddSample(StatsSample(f.height, time)); - stream.frame_size_bytes.AddSample( - StatsSample(f.frame_size.bytes(), time)); - stream.keyframe.AddSample(StatsSample(f.keyframe, time)); - if (f.qp) { - stream.qp.AddSample(StatsSample(*f.qp, time)); - } - } - - if (f.encoded) { - stream.encode_time_ms.AddSample(StatsSample(f.encode_time.ms(), time)); - } - - if (f.decoded) { - stream.decode_time_ms.AddSample(StatsSample(f.decode_time.ms(), time)); - } - - if (f.psnr) { - stream.psnr.y.AddSample(StatsSample(f.psnr->y, time)); - stream.psnr.u.AddSample(StatsSample(f.psnr->u, time)); - stream.psnr.v.AddSample(StatsSample(f.psnr->v, time)); - } - - if (f.target_framerate) { - stream.target_framerate_fps.AddSample( - StatsSample(f.target_framerate->millihertz() / 1000.0, time)); - } - - if (f.target_bitrate) { - stream.target_bitrate_kbps.AddSample( - StatsSample(f.target_bitrate->bps() / 1000.0, time)); - - int buffer_level_bits = leacky_bucket.Update(f); - stream.transmission_time_ms.AddSample( - StatsSample(buffer_level_bits * rtc::kNumMillisecsPerSec / - f.target_bitrate->bps(), - RtpToTime(f.timestamp_rtp))); - } - } - - TimeDelta duration = CalcTotalDuration(superframes); - DataRate encoded_bitrate = - DataSize::Bytes(stream.frame_size_bytes.GetSum()) / duration; - - int num_encoded_frames = stream.frame_size_bytes.NumSamples(); - Frequency encoded_framerate = num_encoded_frames / duration; - - absl::optional bitrate_mismatch_pct; - if (auto target_bitrate = superframes.begin()->target_bitrate; - target_bitrate) { - bitrate_mismatch_pct = 100.0 * - (encoded_bitrate.bps() - target_bitrate->bps()) / - target_bitrate->bps(); - } - - absl::optional framerate_mismatch_pct; - if (auto target_framerate = superframes.begin()->target_framerate; - target_framerate) { - framerate_mismatch_pct = - 100.0 * - (encoded_framerate.millihertz() - target_framerate->millihertz()) / - target_framerate->millihertz(); - } - - for (auto& f : superframes) { - Timestamp time = RtpToTime(f.timestamp_rtp); - stream.encoded_bitrate_kbps.AddSample( - StatsSample(encoded_bitrate.bps() / 1000.0, time)); - - stream.encoded_framerate_fps.AddSample( - StatsSample(encoded_framerate.millihertz() / 1000.0, time)); - - if (bitrate_mismatch_pct) { - stream.bitrate_mismatch_pct.AddSample( - StatsSample(*bitrate_mismatch_pct, time)); - } - - if (framerate_mismatch_pct) { - stream.framerate_mismatch_pct.AddSample( - StatsSample(*framerate_mismatch_pct, time)); - } - } - - return stream; -} - -void VideoCodecStatsImpl::AddFrame(const Frame& frame) { - FrameId frame_id{.timestamp_rtp = frame.timestamp_rtp, - .spatial_idx = frame.spatial_idx}; - RTC_CHECK(frames_.find(frame_id) == frames_.end()) - << "Frame with timestamp_rtp=" << frame.timestamp_rtp - << " and spatial_idx=" << frame.spatial_idx << " already exists"; - - frames_[frame_id] = frame; -} - -Frame* VideoCodecStatsImpl::GetFrame(uint32_t timestamp_rtp, int spatial_idx) { - FrameId frame_id{.timestamp_rtp = timestamp_rtp, .spatial_idx = spatial_idx}; - if (frames_.find(frame_id) == frames_.end()) { - return nullptr; - } - return &frames_.find(frame_id)->second; -} - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_stats_impl.h b/modules/video_coding/codecs/test/video_codec_stats_impl.h deleted file mode 100644 index 77471d2ecd..0000000000 --- a/modules/video_coding/codecs/test/video_codec_stats_impl.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_STATS_IMPL_H_ -#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_STATS_IMPL_H_ - -#include -#include -#include - -#include "absl/types/optional.h" -#include "api/test/video_codec_stats.h" - -namespace webrtc { -namespace test { - -// Implementation of `VideoCodecStats`. This class is not thread-safe. -class VideoCodecStatsImpl : public VideoCodecStats { - public: - std::vector Slice( - absl::optional filter = absl::nullopt) const override; - - Stream Aggregate(const std::vector& frames) const override; - - void AddFrame(const Frame& frame); - - // Returns raw pointers to previously added frame. If frame does not exist, - // returns `nullptr`. - Frame* GetFrame(uint32_t timestamp_rtp, int spatial_idx); - - private: - struct FrameId { - uint32_t timestamp_rtp; - int spatial_idx; - - bool operator==(const FrameId& o) const { - return timestamp_rtp == o.timestamp_rtp && spatial_idx == o.spatial_idx; - } - - bool operator<(const FrameId& o) const { - if (timestamp_rtp < o.timestamp_rtp) - return true; - if (timestamp_rtp == o.timestamp_rtp && spatial_idx < o.spatial_idx) - return true; - return false; - } - }; - - std::map frames_; -}; - -} // namespace test -} // namespace webrtc - -#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_STATS_IMPL_H_ diff --git a/modules/video_coding/codecs/test/video_codec_stats_impl_unittest.cc b/modules/video_coding/codecs/test/video_codec_stats_impl_unittest.cc deleted file mode 100644 index ce11d5abe6..0000000000 --- a/modules/video_coding/codecs/test/video_codec_stats_impl_unittest.cc +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/video_codec_stats_impl.h" - -#include - -#include "absl/types/optional.h" -#include "test/gmock.h" -#include "test/gtest.h" - -namespace webrtc { -namespace test { - -namespace { -using ::testing::Return; -using ::testing::Values; -using Filter = VideoCodecStats::Filter; -using Frame = VideoCodecStatsImpl::Frame; -using Stream = VideoCodecStats::Stream; -} // namespace - -TEST(VideoCodecStatsImpl, AddAndGetFrame) { - VideoCodecStatsImpl stats; - stats.AddFrame({.timestamp_rtp = 0, .spatial_idx = 0}); - stats.AddFrame({.timestamp_rtp = 0, .spatial_idx = 1}); - stats.AddFrame({.timestamp_rtp = 1, .spatial_idx = 0}); - - Frame* fs = stats.GetFrame(/*timestamp_rtp=*/0, /*spatial_idx=*/0); - ASSERT_NE(fs, nullptr); - EXPECT_EQ(fs->timestamp_rtp, 0u); - EXPECT_EQ(fs->spatial_idx, 0); - - fs = stats.GetFrame(/*timestamp_rtp=*/0, /*spatial_idx=*/1); - ASSERT_NE(fs, nullptr); - EXPECT_EQ(fs->timestamp_rtp, 0u); - EXPECT_EQ(fs->spatial_idx, 1); - - fs = stats.GetFrame(/*timestamp_rtp=*/1, /*spatial_idx=*/0); - ASSERT_NE(fs, nullptr); - EXPECT_EQ(fs->timestamp_rtp, 1u); - EXPECT_EQ(fs->spatial_idx, 0); - - fs = stats.GetFrame(/*timestamp_rtp=*/1, /*spatial_idx=*/1); - EXPECT_EQ(fs, nullptr); -} - -class VideoCodecStatsImplSlicingTest - : public ::testing::TestWithParam>> {}; - -TEST_P(VideoCodecStatsImplSlicingTest, Slice) { - Filter filter = std::get<0>(GetParam()); - std::vector expected_frames = std::get<1>(GetParam()); - std::vector frames = { - {.frame_num = 0, .timestamp_rtp = 0, .spatial_idx = 0, .temporal_idx = 0}, - {.frame_num = 0, .timestamp_rtp = 0, .spatial_idx = 1, .temporal_idx = 0}, - {.frame_num = 1, .timestamp_rtp = 1, .spatial_idx = 0, .temporal_idx = 1}, - {.frame_num = 1, - .timestamp_rtp = 1, - .spatial_idx = 1, - .temporal_idx = 1}}; - - VideoCodecStatsImpl stats; - stats.AddFrame(frames[0]); - stats.AddFrame(frames[1]); - stats.AddFrame(frames[2]); - stats.AddFrame(frames[3]); - - std::vector slice = stats.Slice(filter); - ASSERT_EQ(slice.size(), expected_frames.size()); - for (size_t i = 0; i < expected_frames.size(); ++i) { - Frame& expected = frames[expected_frames[i]]; - EXPECT_EQ(slice[i].frame_num, expected.frame_num); - EXPECT_EQ(slice[i].timestamp_rtp, expected.timestamp_rtp); - EXPECT_EQ(slice[i].spatial_idx, expected.spatial_idx); - EXPECT_EQ(slice[i].temporal_idx, expected.temporal_idx); - } -} - -INSTANTIATE_TEST_SUITE_P( - All, - VideoCodecStatsImplSlicingTest, - ::testing::Values( - std::make_tuple(Filter{}, std::vector{0, 1, 2, 3}), - std::make_tuple(Filter{.first_frame = 1}, std::vector{2, 3}), - std::make_tuple(Filter{.last_frame = 0}, std::vector{0, 1}), - std::make_tuple(Filter{.spatial_idx = 0}, std::vector{0, 2}), - std::make_tuple(Filter{.temporal_idx = 1}, - std::vector{0, 1, 2, 3}))); - -TEST(VideoCodecStatsImpl, AggregateBitrate) { - std::vector frames = { - {.frame_num = 0, - .timestamp_rtp = 0, - .frame_size = DataSize::Bytes(1000), - .target_bitrate = DataRate::BytesPerSec(1000)}, - {.frame_num = 1, - .timestamp_rtp = 90000, - .frame_size = DataSize::Bytes(2000), - .target_bitrate = DataRate::BytesPerSec(1000)}}; - - Stream stream = VideoCodecStatsImpl().Aggregate(frames); - EXPECT_EQ(stream.encoded_bitrate_kbps.GetAverage(), 12.0); - EXPECT_EQ(stream.bitrate_mismatch_pct.GetAverage(), 50.0); -} - -TEST(VideoCodecStatsImpl, AggregateFramerate) { - std::vector frames = { - {.frame_num = 0, - .timestamp_rtp = 0, - .frame_size = DataSize::Bytes(1), - .target_framerate = Frequency::Hertz(1)}, - {.frame_num = 1, - .timestamp_rtp = 90000, - .frame_size = DataSize::Zero(), - .target_framerate = Frequency::Hertz(1)}}; - - Stream stream = VideoCodecStatsImpl().Aggregate(frames); - EXPECT_EQ(stream.encoded_framerate_fps.GetAverage(), 0.5); - EXPECT_EQ(stream.framerate_mismatch_pct.GetAverage(), -50.0); -} - -TEST(VideoCodecStatsImpl, AggregateTransmissionTime) { - std::vector frames = { - {.frame_num = 0, - .timestamp_rtp = 0, - .frame_size = DataSize::Bytes(2), - .target_bitrate = DataRate::BytesPerSec(1)}, - {.frame_num = 1, - .timestamp_rtp = 90000, - .frame_size = DataSize::Bytes(3), - .target_bitrate = DataRate::BytesPerSec(1)}}; - - Stream stream = VideoCodecStatsImpl().Aggregate(frames); - ASSERT_EQ(stream.transmission_time_ms.NumSamples(), 2); - ASSERT_EQ(stream.transmission_time_ms.GetSamples()[0], 2000); - ASSERT_EQ(stream.transmission_time_ms.GetSamples()[1], 4000); -} - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_test.cc b/modules/video_coding/codecs/test/video_codec_test.cc index 1c8fe97e84..08961a312b 100644 --- a/modules/video_coding/codecs/test/video_codec_test.cc +++ b/modules/video_coding/codecs/test/video_codec_test.cc @@ -8,33 +8,18 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "api/video_codecs/video_codec.h" - -#include #include #include #include #include "absl/flags/flag.h" #include "absl/functional/any_invocable.h" -#include "api/test/create_video_codec_tester.h" #include "api/test/metrics/global_metrics_logger_and_exporter.h" -#include "api/test/video_codec_tester.h" -#include "api/test/videocodec_test_stats.h" #include "api/units/data_rate.h" #include "api/units/frequency.h" -#include "api/video/encoded_image.h" -#include "api/video/i420_buffer.h" #include "api/video/resolution.h" -#include "api/video/video_frame.h" -#include "api/video_codecs/scalability_mode.h" -#include "api/video_codecs/video_decoder.h" -#include "api/video_codecs/video_encoder.h" -#include "media/engine/internal_decoder_factory.h" -#include "media/engine/internal_encoder_factory.h" -#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" -#include "modules/video_coding/include/video_error_codes.h" -#include "modules/video_coding/svc/scalability_mode_util.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" #if defined(WEBRTC_ANDROID) #include "modules/video_coding/codecs/test/android_codec_factory_helper.h" #endif @@ -42,7 +27,15 @@ #include "test/gtest.h" #include "test/test_flags.h" #include "test/testsupport/file_utils.h" -#include "test/testsupport/frame_reader.h" +#include "test/video_codec_tester.h" + +ABSL_FLAG(bool, dump_decoder_input, false, "Dump decoder input."); + +ABSL_FLAG(bool, dump_decoder_output, false, "Dump decoder output."); + +ABSL_FLAG(bool, dump_encoder_input, false, "Dump encoder input."); + +ABSL_FLAG(bool, dump_encoder_output, false, "Dump encoder output."); namespace webrtc { namespace test { @@ -50,6 +43,10 @@ namespace test { namespace { using ::testing::Combine; using ::testing::Values; +using VideoSourceSettings = VideoCodecTester::VideoSourceSettings; +using EncodingSettings = VideoCodecTester::EncodingSettings; +using VideoCodecStats = VideoCodecTester::VideoCodecStats; +using Filter = VideoCodecStats::Filter; using PacingMode = VideoCodecTester::PacingSettings::PacingMode; struct VideoInfo { @@ -58,399 +55,35 @@ struct VideoInfo { Frequency framerate; }; -struct LayerId { - int spatial_idx; - int temporal_idx; - - bool operator==(const LayerId& o) const { - return spatial_idx == o.spatial_idx && temporal_idx == o.temporal_idx; - } - - bool operator<(const LayerId& o) const { - if (spatial_idx < o.spatial_idx) - return true; - if (spatial_idx == o.spatial_idx && temporal_idx < o.temporal_idx) - return true; - return false; - } -}; - -struct EncodingSettings { - ScalabilityMode scalability_mode; - struct LayerSettings { - Resolution resolution; - Frequency framerate; - DataRate bitrate; - }; - std::map layer_settings; - - bool IsSameSettings(const EncodingSettings& other) const { - if (scalability_mode != other.scalability_mode) { - return false; - } - - for (auto [layer_id, layer] : layer_settings) { - const auto& other_layer = other.layer_settings.at(layer_id); - if (layer.resolution != other_layer.resolution) { - return false; - } - } - - return true; - } - - bool IsSameRate(const EncodingSettings& other) const { - for (auto [layer_id, layer] : layer_settings) { - const auto& other_layer = other.layer_settings.at(layer_id); - if (layer.bitrate != other_layer.bitrate || - layer.framerate != other_layer.framerate) { - return false; - } - } - - return true; - } -}; - const VideoInfo kFourPeople_1280x720_30 = { .name = "FourPeople_1280x720_30", .resolution = {.width = 1280, .height = 720}, .framerate = Frequency::Hertz(30)}; -class TestRawVideoSource : public VideoCodecTester::RawVideoSource { - public: - static constexpr Frequency k90kHz = Frequency::Hertz(90000); +static constexpr Frequency k90kHz = Frequency::Hertz(90000); - TestRawVideoSource(VideoInfo video_info, - const std::map& frame_settings, - int num_frames) - : video_info_(video_info), - frame_settings_(frame_settings), - num_frames_(num_frames), - frame_num_(0), - // Start with non-zero timestamp to force using frame RTP timestamps in - // IvfFrameWriter. - timestamp_rtp_(90000) { - // Ensure settings for the first frame are provided. - RTC_CHECK_GT(frame_settings_.size(), 0u); - RTC_CHECK_EQ(frame_settings_.begin()->first, 0); - - frame_reader_ = CreateYuvFrameReader( - ResourcePath(video_info_.name, "yuv"), video_info_.resolution, - YuvFrameReaderImpl::RepeatMode::kPingPong); - RTC_CHECK(frame_reader_); - } - - // Pulls next frame. Frame RTP timestamp is set accordingly to - // `EncodingSettings::framerate`. - absl::optional PullFrame() override { - if (frame_num_ >= num_frames_) { - return absl::nullopt; // End of stream. - } - - const EncodingSettings& encoding_settings = - std::prev(frame_settings_.upper_bound(frame_num_))->second; - - Resolution resolution = - encoding_settings.layer_settings.begin()->second.resolution; - Frequency framerate = - encoding_settings.layer_settings.begin()->second.framerate; - - int pulled_frame; - auto buffer = frame_reader_->PullFrame( - &pulled_frame, resolution, - {.num = static_cast(framerate.millihertz()), - .den = static_cast(video_info_.framerate.millihertz())}); - RTC_CHECK(buffer) << "Cannot pull frame " << frame_num_; - - auto frame = VideoFrame::Builder() - .set_video_frame_buffer(buffer) - .set_timestamp_rtp(timestamp_rtp_) - .set_timestamp_us((timestamp_rtp_ / k90kHz).us()) - .build(); - - pulled_frames_[timestamp_rtp_] = pulled_frame; - timestamp_rtp_ += k90kHz / framerate; - ++frame_num_; - - return frame; - } - - // Reads frame specified by `timestamp_rtp`, scales it to `resolution` and - // returns. Frame with the given `timestamp_rtp` is expected to be pulled - // before. - VideoFrame GetFrame(uint32_t timestamp_rtp, Resolution resolution) override { - RTC_CHECK(pulled_frames_.find(timestamp_rtp) != pulled_frames_.end()) - << "Frame with RTP timestamp " << timestamp_rtp - << " was not pulled before"; - auto buffer = - frame_reader_->ReadFrame(pulled_frames_[timestamp_rtp], resolution); - return VideoFrame::Builder() - .set_video_frame_buffer(buffer) - .set_timestamp_rtp(timestamp_rtp) - .build(); - } - - protected: - VideoInfo video_info_; - std::unique_ptr frame_reader_; - const std::map& frame_settings_; - int num_frames_; - int frame_num_; - uint32_t timestamp_rtp_; - std::map pulled_frames_; -}; - -class TestEncoder : public VideoCodecTester::Encoder, - public EncodedImageCallback { - public: - TestEncoder(std::unique_ptr encoder, - const std::string codec_type, - const std::map& frame_settings) - : encoder_(std::move(encoder)), - codec_type_(codec_type), - frame_settings_(frame_settings), - frame_num_(0) { - // Ensure settings for the first frame is provided. - RTC_CHECK_GT(frame_settings_.size(), 0u); - RTC_CHECK_EQ(frame_settings_.begin()->first, 0); - - encoder_->RegisterEncodeCompleteCallback(this); - } - - void Initialize() override { - const EncodingSettings& first_frame_settings = frame_settings_.at(0); - Configure(first_frame_settings); - SetRates(first_frame_settings); - } - - void Encode(const VideoFrame& frame, EncodeCallback callback) override { - { - MutexLock lock(&mutex_); - callbacks_[frame.timestamp()] = std::move(callback); - } - - if (auto fs = frame_settings_.find(frame_num_); - fs != frame_settings_.begin() && fs != frame_settings_.end()) { - if (!fs->second.IsSameSettings(std::prev(fs)->second)) { - Configure(fs->second); - } else if (!fs->second.IsSameRate(std::prev(fs)->second)) { - SetRates(fs->second); - } - } - - encoder_->Encode(frame, nullptr); - ++frame_num_; - } - - void Flush() override { - // TODO(webrtc:14852): For codecs which buffer frames we need a to - // flush them to get last frames. Add such functionality to VideoEncoder - // API. On Android it will map directly to `MediaCodec.flush()`. - encoder_->Release(); - } - - VideoEncoder* encoder() { return encoder_.get(); } - - protected: - Result OnEncodedImage(const EncodedImage& encoded_image, - const CodecSpecificInfo* codec_specific_info) override { - MutexLock lock(&mutex_); - auto cb = callbacks_.find(encoded_image.RtpTimestamp()); - RTC_CHECK(cb != callbacks_.end()); - cb->second(encoded_image); - - callbacks_.erase(callbacks_.begin(), cb); - return Result(Result::Error::OK); - } - - void Configure(const EncodingSettings& es) { - VideoCodec vc; - const EncodingSettings::LayerSettings& layer_settings = - es.layer_settings.begin()->second; - vc.width = layer_settings.resolution.width; - vc.height = layer_settings.resolution.height; - const DataRate& bitrate = layer_settings.bitrate; - vc.startBitrate = bitrate.kbps(); - vc.maxBitrate = bitrate.kbps(); - vc.minBitrate = 0; - vc.maxFramerate = static_cast(layer_settings.framerate.hertz()); - vc.active = true; - vc.qpMax = 63; - vc.numberOfSimulcastStreams = 0; - vc.mode = webrtc::VideoCodecMode::kRealtimeVideo; - vc.SetFrameDropEnabled(true); - vc.SetScalabilityMode(es.scalability_mode); - - vc.codecType = PayloadStringToCodecType(codec_type_); - if (vc.codecType == kVideoCodecVP8) { - *(vc.VP8()) = VideoEncoder::GetDefaultVp8Settings(); - } else if (vc.codecType == kVideoCodecVP9) { - *(vc.VP9()) = VideoEncoder::GetDefaultVp9Settings(); - } else if (vc.codecType == kVideoCodecH264) { - *(vc.H264()) = VideoEncoder::GetDefaultH264Settings(); - } - - VideoEncoder::Settings ves( - VideoEncoder::Capabilities(/*loss_notification=*/false), - /*number_of_cores=*/1, - /*max_payload_size=*/1440); - - int result = encoder_->InitEncode(&vc, ves); - ASSERT_EQ(result, WEBRTC_VIDEO_CODEC_OK); - - SetRates(es); - } - - void SetRates(const EncodingSettings& es) { - VideoEncoder::RateControlParameters rc; - int num_spatial_layers = - ScalabilityModeToNumSpatialLayers(es.scalability_mode); - int num_temporal_layers = - ScalabilityModeToNumSpatialLayers(es.scalability_mode); - for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { - for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { - auto layer_settings = - es.layer_settings.find({.spatial_idx = sidx, .temporal_idx = tidx}); - RTC_CHECK(layer_settings != es.layer_settings.end()) - << "Bitrate for layer S=" << sidx << " T=" << tidx << " is not set"; - rc.bitrate.SetBitrate(sidx, tidx, layer_settings->second.bitrate.bps()); - } - } - - rc.framerate_fps = - es.layer_settings.begin()->second.framerate.millihertz() / 1000.0; - encoder_->SetRates(rc); - } - - std::unique_ptr encoder_; - const std::string codec_type_; - const std::map& frame_settings_; - int frame_num_; - std::map callbacks_ RTC_GUARDED_BY(mutex_); - Mutex mutex_; -}; - -class TestDecoder : public VideoCodecTester::Decoder, - public DecodedImageCallback { - public: - TestDecoder(std::unique_ptr decoder, - const std::string codec_type) - : decoder_(std::move(decoder)), codec_type_(codec_type) { - decoder_->RegisterDecodeCompleteCallback(this); - } - - void Initialize() override { - VideoDecoder::Settings ds; - ds.set_codec_type(PayloadStringToCodecType(codec_type_)); - ds.set_number_of_cores(1); - ds.set_max_render_resolution({1280, 720}); - - bool result = decoder_->Configure(ds); - ASSERT_TRUE(result); - } - - void Decode(const EncodedImage& frame, DecodeCallback callback) override { - { - MutexLock lock(&mutex_); - callbacks_[frame.RtpTimestamp()] = std::move(callback); - } - - decoder_->Decode(frame, /*render_time_ms=*/0); - } - - void Flush() override { - // TODO(webrtc:14852): For codecs which buffer frames we need a to - // flush them to get last frames. Add such functionality to VideoDecoder - // API. On Android it will map directly to `MediaCodec.flush()`. - decoder_->Release(); - } - - VideoDecoder* decoder() { return decoder_.get(); } - - protected: - int Decoded(VideoFrame& decoded_frame) override { - MutexLock lock(&mutex_); - auto cb = callbacks_.find(decoded_frame.timestamp()); - RTC_CHECK(cb != callbacks_.end()); - cb->second(decoded_frame); - - callbacks_.erase(callbacks_.begin(), cb); - return WEBRTC_VIDEO_CODEC_OK; - } - - std::unique_ptr decoder_; - const std::string codec_type_; - std::map callbacks_ RTC_GUARDED_BY(mutex_); - Mutex mutex_; -}; - -std::unique_ptr CreateVideoSource( - const VideoInfo& video, - const std::map& frame_settings, - int num_frames) { - return std::make_unique(video, frame_settings, - num_frames); -} - -std::unique_ptr CreateEncoder( - std::string type, - std::string impl, - const std::map& frame_settings) { - std::unique_ptr factory; +std::unique_ptr CreateEncoderFactory(std::string impl) { if (impl == "builtin") { - factory = std::make_unique(); - } else if (impl == "mediacodec") { + return CreateBuiltinVideoEncoderFactory(); + } #if defined(WEBRTC_ANDROID) - InitializeAndroidObjects(); - factory = CreateAndroidEncoderFactory(); + InitializeAndroidObjects(); + return CreateAndroidEncoderFactory(); +#else + return nullptr; #endif - } - std::unique_ptr encoder = - factory->CreateVideoEncoder(SdpVideoFormat(type)); - if (encoder == nullptr) { - return nullptr; - } - return std::make_unique(std::move(encoder), type, - frame_settings); } -std::unique_ptr CreateDecoder(std::string type, std::string impl) { - std::unique_ptr factory; +std::unique_ptr CreateDecoderFactory(std::string impl) { if (impl == "builtin") { - factory = std::make_unique(); - } else if (impl == "mediacodec") { + return CreateBuiltinVideoDecoderFactory(); + } #if defined(WEBRTC_ANDROID) - InitializeAndroidObjects(); - factory = CreateAndroidDecoderFactory(); + InitializeAndroidObjects(); + return CreateAndroidDecoderFactory(); +#else + return nullptr; #endif - } - std::unique_ptr decoder = - factory->CreateVideoDecoder(SdpVideoFormat(type)); - if (decoder == nullptr) { - return nullptr; - } - return std::make_unique(std::move(decoder), type); -} - -void SetTargetRates(const std::map& frame_settings, - std::vector& frames) { - for (VideoCodecStats::Frame& f : frames) { - const EncodingSettings& encoding_settings = - std::prev(frame_settings.upper_bound(f.frame_num))->second; - LayerId layer_id = {.spatial_idx = f.spatial_idx, - .temporal_idx = f.temporal_idx}; - RTC_CHECK(encoding_settings.layer_settings.find(layer_id) != - encoding_settings.layer_settings.end()) - << "Frame frame_num=" << f.frame_num - << " belongs to spatial_idx=" << f.spatial_idx - << " temporal_idx=" << f.temporal_idx - << " but settings for this layer are not provided."; - const EncodingSettings::LayerSettings& layer_settings = - encoding_settings.layer_settings.at(layer_id); - f.target_bitrate = layer_settings.bitrate; - f.target_framerate = layer_settings.framerate; - } } std::string TestOutputPath() { @@ -468,113 +101,118 @@ std::unique_ptr RunEncodeDecodeTest( std::string codec_type, std::string codec_impl, const VideoInfo& video_info, - const std::map& frame_settings, - int num_frames, - bool save_codec_input, - bool save_codec_output) { - std::unique_ptr video_source = - CreateVideoSource(video_info, frame_settings, num_frames); + const std::map& encoding_settings) { + VideoSourceSettings source_settings{ + .file_path = ResourcePath(video_info.name, "yuv"), + .resolution = video_info.resolution, + .framerate = video_info.framerate}; - std::unique_ptr encoder = - CreateEncoder(codec_type, codec_impl, frame_settings); - if (encoder == nullptr) { + const SdpVideoFormat& sdp_video_format = + encoding_settings.begin()->second.sdp_video_format; + + std::unique_ptr encoder_factory = + CreateEncoderFactory(codec_impl); + if (!encoder_factory + ->QueryCodecSupport(sdp_video_format, + /*scalability_mode=*/absl::nullopt) + .is_supported) { + RTC_LOG(LS_WARNING) << "No encoder for video format " + << sdp_video_format.ToString(); return nullptr; } - std::unique_ptr decoder = CreateDecoder(codec_type, codec_impl); - if (decoder == nullptr) { - // If platform decoder is not available try built-in one. - if (codec_impl == "builtin") { - return nullptr; - } - - decoder = CreateDecoder(codec_type, "builtin"); - if (decoder == nullptr) { + std::unique_ptr decoder_factory = + CreateDecoderFactory(codec_impl); + if (!decoder_factory + ->QueryCodecSupport(sdp_video_format, + /*reference_scaling=*/false) + .is_supported) { + decoder_factory = CreateDecoderFactory("builtin"); + if (!decoder_factory + ->QueryCodecSupport(sdp_video_format, + /*reference_scaling=*/false) + .is_supported) { + RTC_LOG(LS_WARNING) << "No decoder for video format " + << sdp_video_format.ToString(); return nullptr; } } - RTC_LOG(LS_INFO) << "Encoder implementation: " - << encoder->encoder()->GetEncoderInfo().implementation_name; - RTC_LOG(LS_INFO) << "Decoder implementation: " - << decoder->decoder()->GetDecoderInfo().implementation_name; - - VideoCodecTester::EncoderSettings encoder_settings; - encoder_settings.pacing.mode = - encoder->encoder()->GetEncoderInfo().is_hardware_accelerated - ? PacingMode::kRealTime - : PacingMode::kNoPacing; - - VideoCodecTester::DecoderSettings decoder_settings; - decoder_settings.pacing.mode = - decoder->decoder()->GetDecoderInfo().is_hardware_accelerated - ? PacingMode::kRealTime - : PacingMode::kNoPacing; - std::string output_path = TestOutputPath(); - if (save_codec_input) { + + VideoCodecTester::EncoderSettings encoder_settings; + encoder_settings.pacing_settings.mode = + codec_impl == "builtin" ? PacingMode::kNoPacing : PacingMode::kRealTime; + if (absl::GetFlag(FLAGS_dump_encoder_input)) { encoder_settings.encoder_input_base_path = output_path + "_enc_input"; + } + if (absl::GetFlag(FLAGS_dump_encoder_output)) { + encoder_settings.encoder_output_base_path = output_path + "_enc_output"; + } + + VideoCodecTester::DecoderSettings decoder_settings; + decoder_settings.pacing_settings.mode = + codec_impl == "builtin" ? PacingMode::kNoPacing : PacingMode::kRealTime; + if (absl::GetFlag(FLAGS_dump_decoder_input)) { decoder_settings.decoder_input_base_path = output_path + "_dec_input"; } - if (save_codec_output) { - encoder_settings.encoder_output_base_path = output_path + "_enc_output"; + if (absl::GetFlag(FLAGS_dump_decoder_output)) { decoder_settings.decoder_output_base_path = output_path + "_dec_output"; } - std::unique_ptr tester = CreateVideoCodecTester(); - return tester->RunEncodeDecodeTest(video_source.get(), encoder.get(), - decoder.get(), encoder_settings, - decoder_settings); + return VideoCodecTester::RunEncodeDecodeTest( + source_settings, encoder_factory.get(), decoder_factory.get(), + encoder_settings, decoder_settings, encoding_settings); } std::unique_ptr RunEncodeTest( std::string codec_type, std::string codec_impl, const VideoInfo& video_info, - const std::map& frame_settings, - int num_frames, - bool save_codec_input, - bool save_codec_output) { - std::unique_ptr video_source = - CreateVideoSource(video_info, frame_settings, num_frames); + const std::map& encoding_settings) { + VideoSourceSettings source_settings{ + .file_path = ResourcePath(video_info.name, "yuv"), + .resolution = video_info.resolution, + .framerate = video_info.framerate}; - std::unique_ptr encoder = - CreateEncoder(codec_type, codec_impl, frame_settings); - if (encoder == nullptr) { + const SdpVideoFormat& sdp_video_format = + encoding_settings.begin()->second.sdp_video_format; + + std::unique_ptr encoder_factory = + CreateEncoderFactory(codec_impl); + if (!encoder_factory + ->QueryCodecSupport(sdp_video_format, + /*scalability_mode=*/absl::nullopt) + .is_supported) { + RTC_LOG(LS_WARNING) << "No encoder for video format " + << sdp_video_format.ToString(); return nullptr; } - RTC_LOG(LS_INFO) << "Encoder implementation: " - << encoder->encoder()->GetEncoderInfo().implementation_name; - - VideoCodecTester::EncoderSettings encoder_settings; - encoder_settings.pacing.mode = - encoder->encoder()->GetEncoderInfo().is_hardware_accelerated - ? PacingMode::kRealTime - : PacingMode::kNoPacing; - std::string output_path = TestOutputPath(); - if (save_codec_input) { + VideoCodecTester::EncoderSettings encoder_settings; + encoder_settings.pacing_settings.mode = + codec_impl == "builtin" ? PacingMode::kNoPacing : PacingMode::kRealTime; + if (absl::GetFlag(FLAGS_dump_encoder_input)) { encoder_settings.encoder_input_base_path = output_path + "_enc_input"; } - if (save_codec_output) { + if (absl::GetFlag(FLAGS_dump_encoder_output)) { encoder_settings.encoder_output_base_path = output_path + "_enc_output"; } - std::unique_ptr tester = CreateVideoCodecTester(); - return tester->RunEncodeTest(video_source.get(), encoder.get(), - encoder_settings); + return VideoCodecTester::RunEncodeTest(source_settings, encoder_factory.get(), + encoder_settings, encoding_settings); } -class SpatialQualityTest : public ::testing::TestWithParam< - std::tuple>> { +class SpatialQualityTest : public ::testing::TestWithParam>> { public: static std::string TestParamsToString( const ::testing::TestParamInfo& info) { @@ -590,31 +228,24 @@ class SpatialQualityTest : public ::testing::TestWithParam< TEST_P(SpatialQualityTest, SpatialQuality) { auto [codec_type, codec_impl, video_info, coding_settings] = GetParam(); - auto [width, height, framerate_fps, bitrate_kbps, psnr] = coding_settings; - - std::map frame_settings = { - {0, - {.scalability_mode = ScalabilityMode::kL1T1, - .layer_settings = { - {LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = width, .height = height}, - .framerate = Frequency::MilliHertz(1000 * framerate_fps), - .bitrate = DataRate::KilobitsPerSec(bitrate_kbps)}}}}}}; - + auto [width, height, framerate_fps, bitrate_kbps, expected_min_psnr] = + coding_settings; int duration_s = 10; int num_frames = duration_s * framerate_fps; - std::unique_ptr stats = RunEncodeDecodeTest( - codec_type, codec_impl, video_info, frame_settings, num_frames, - /*save_codec_input=*/false, /*save_codec_output=*/false); + std::map frames_settings = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", width, height, + {bitrate_kbps}, framerate_fps, num_frames); + + std::unique_ptr stats = + RunEncodeDecodeTest(codec_type, codec_impl, video_info, frames_settings); VideoCodecStats::Stream stream; if (stats != nullptr) { - std::vector frames = stats->Slice(); - SetTargetRates(frame_settings, frames); - stream = stats->Aggregate(frames); + stream = stats->Aggregate(Filter{}); if (absl::GetFlag(FLAGS_webrtc_quick_perf_test)) { - EXPECT_GE(stream.psnr.y.GetAverage(), psnr); + EXPECT_GE(stream.psnr.y.GetAverage(), expected_min_psnr); } } @@ -622,9 +253,9 @@ TEST_P(SpatialQualityTest, SpatialQuality) { GetGlobalMetricsLogger(), ::testing::UnitTest::GetInstance()->current_test_info()->name(), /*metadata=*/ - {{"codec_type", codec_type}, - {"codec_impl", codec_impl}, - {"video_name", video_info.name}}); + {{"video_name", video_info.name}, + {"codec_type", codec_type}, + {"codec_impl", codec_impl}}); } INSTANTIATE_TEST_SUITE_P( @@ -671,33 +302,32 @@ TEST_P(BitrateAdaptationTest, BitrateAdaptation) { auto [codec_type, codec_impl, video_info, bitrate_kbps] = GetParam(); int duration_s = 10; // Duration of fixed rate interval. - int first_frame = duration_s * video_info.framerate.millihertz() / 1000; - int num_frames = 2 * duration_s * video_info.framerate.millihertz() / 1000; + int num_frames = + static_cast(duration_s * video_info.framerate.hertz()); - std::map frame_settings = { - {0, - {.layer_settings = {{LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = 640, .height = 360}, - .framerate = video_info.framerate, - .bitrate = DataRate::KilobitsPerSec( - bitrate_kbps.first)}}}}}, - {first_frame, - {.layer_settings = { - {LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = 640, .height = 360}, - .framerate = video_info.framerate, - .bitrate = DataRate::KilobitsPerSec(bitrate_kbps.second)}}}}}}; + std::map encoding_settings = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", + /*width=*/640, /*height=*/360, {bitrate_kbps.first}, + /*framerate_fps=*/30, num_frames); - std::unique_ptr stats = RunEncodeTest( - codec_type, codec_impl, video_info, frame_settings, num_frames, - /*save_codec_input=*/false, /*save_codec_output=*/false); + uint32_t initial_timestamp_rtp = + encoding_settings.rbegin()->first + k90kHz / Frequency::Hertz(30); + + std::map encoding_settings2 = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", + /*width=*/640, /*height=*/360, {bitrate_kbps.second}, + /*framerate_fps=*/30, num_frames, initial_timestamp_rtp); + + encoding_settings.merge(encoding_settings2); + + std::unique_ptr stats = + RunEncodeTest(codec_type, codec_impl, video_info, encoding_settings); VideoCodecStats::Stream stream; if (stats != nullptr) { - std::vector frames = - stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); - SetTargetRates(frame_settings, frames); - stream = stats->Aggregate(frames); + stream = stats->Aggregate({.min_timestamp_rtp = initial_timestamp_rtp}); if (absl::GetFlag(FLAGS_webrtc_quick_perf_test)) { EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); @@ -749,34 +379,34 @@ TEST_P(FramerateAdaptationTest, FramerateAdaptation) { auto [codec_type, codec_impl, video_info, framerate_fps] = GetParam(); int duration_s = 10; // Duration of fixed rate interval. - int first_frame = static_cast(duration_s * framerate_fps.first); - int num_frames = static_cast( - duration_s * (framerate_fps.first + framerate_fps.second)); - std::map frame_settings = { - {0, - {.layer_settings = {{LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = 640, .height = 360}, - .framerate = Frequency::MilliHertz( - 1000 * framerate_fps.first), - .bitrate = DataRate::KilobitsPerSec(512)}}}}}, - {first_frame, - {.layer_settings = { - {LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = 640, .height = 360}, - .framerate = Frequency::MilliHertz(1000 * framerate_fps.second), - .bitrate = DataRate::KilobitsPerSec(512)}}}}}}; + std::map encoding_settings = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", + /*width=*/640, /*height=*/360, + /*layer_bitrates_kbps=*/{512}, framerate_fps.first, + static_cast(duration_s * framerate_fps.first)); - std::unique_ptr stats = RunEncodeTest( - codec_type, codec_impl, video_info, frame_settings, num_frames, - /*save_codec_input=*/false, /*save_codec_output=*/false); + uint32_t initial_timestamp_rtp = + encoding_settings.rbegin()->first + + k90kHz / Frequency::Hertz(framerate_fps.first); + + std::map encoding_settings2 = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", /*width=*/640, + /*height=*/360, + /*layer_bitrates_kbps=*/{512}, framerate_fps.second, + static_cast(duration_s * framerate_fps.second), + initial_timestamp_rtp); + + encoding_settings.merge(encoding_settings2); + + std::unique_ptr stats = + RunEncodeTest(codec_type, codec_impl, video_info, encoding_settings); VideoCodecStats::Stream stream; if (stats != nullptr) { - std::vector frames = - stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); - SetTargetRates(frame_settings, frames); - stream = stats->Aggregate(frames); + stream = stats->Aggregate({.min_timestamp_rtp = initial_timestamp_rtp}); if (absl::GetFlag(FLAGS_webrtc_quick_perf_test)) { EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); @@ -805,7 +435,6 @@ INSTANTIATE_TEST_SUITE_P(All, Values(kFourPeople_1280x720_30), Values(std::pair(30, 15), std::pair(15, 30))), FramerateAdaptationTest::TestParamsToString); - } // namespace test } // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl.cc b/modules/video_coding/codecs/test/video_codec_tester_impl.cc deleted file mode 100644 index f15b1b35f3..0000000000 --- a/modules/video_coding/codecs/test/video_codec_tester_impl.cc +++ /dev/null @@ -1,437 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" - -#include -#include -#include -#include - -#include "api/task_queue/default_task_queue_factory.h" -#include "api/units/frequency.h" -#include "api/units/time_delta.h" -#include "api/units/timestamp.h" -#include "api/video/encoded_image.h" -#include "api/video/i420_buffer.h" -#include "api/video/video_codec_type.h" -#include "api/video/video_frame.h" -#include "modules/video_coding/codecs/test/video_codec_analyzer.h" -#include "modules/video_coding/utility/ivf_file_writer.h" -#include "rtc_base/event.h" -#include "rtc_base/time_utils.h" -#include "system_wrappers/include/sleep.h" -#include "test/testsupport/video_frame_writer.h" - -namespace webrtc { -namespace test { - -namespace { -using RawVideoSource = VideoCodecTester::RawVideoSource; -using CodedVideoSource = VideoCodecTester::CodedVideoSource; -using Decoder = VideoCodecTester::Decoder; -using Encoder = VideoCodecTester::Encoder; -using EncoderSettings = VideoCodecTester::EncoderSettings; -using DecoderSettings = VideoCodecTester::DecoderSettings; -using PacingSettings = VideoCodecTester::PacingSettings; -using PacingMode = PacingSettings::PacingMode; - -constexpr Frequency k90kHz = Frequency::Hertz(90000); - -// A thread-safe wrapper for video source to be shared with the quality analyzer -// that reads reference frames from a separate thread. -class SyncRawVideoSource : public VideoCodecAnalyzer::ReferenceVideoSource { - public: - explicit SyncRawVideoSource(RawVideoSource* video_source) - : video_source_(video_source) {} - - absl::optional PullFrame() { - MutexLock lock(&mutex_); - return video_source_->PullFrame(); - } - - VideoFrame GetFrame(uint32_t timestamp_rtp, Resolution resolution) override { - MutexLock lock(&mutex_); - return video_source_->GetFrame(timestamp_rtp, resolution); - } - - protected: - RawVideoSource* const video_source_ RTC_GUARDED_BY(mutex_); - Mutex mutex_; -}; - -// Pacer calculates delay necessary to keep frame encode or decode call spaced -// from the previous calls by the pacing time. `Delay` is expected to be called -// as close as possible to posting frame encode or decode task. This class is -// not thread safe. -class Pacer { - public: - explicit Pacer(PacingSettings settings) - : settings_(settings), delay_(TimeDelta::Zero()) {} - Timestamp Schedule(Timestamp timestamp) { - Timestamp now = Timestamp::Micros(rtc::TimeMicros()); - if (settings_.mode == PacingMode::kNoPacing) { - return now; - } - - Timestamp scheduled = now; - if (prev_scheduled_) { - scheduled = *prev_scheduled_ + PacingTime(timestamp); - if (scheduled < now) { - scheduled = now; - } - } - - prev_timestamp_ = timestamp; - prev_scheduled_ = scheduled; - return scheduled; - } - - private: - TimeDelta PacingTime(Timestamp timestamp) { - if (settings_.mode == PacingMode::kRealTime) { - return timestamp - *prev_timestamp_; - } - RTC_CHECK_EQ(PacingMode::kConstantRate, settings_.mode); - return 1 / settings_.constant_rate; - } - - PacingSettings settings_; - absl::optional prev_timestamp_; - absl::optional prev_scheduled_; - TimeDelta delay_; -}; - -// Task queue that keeps the number of queued tasks below a certain limit. If -// the limit is reached, posting of a next task is blocked until execution of a -// previously posted task starts. This class is not thread-safe. -class LimitedTaskQueue { - public: - // The codec tester reads frames from video source in the main thread. - // Encoding and decoding are done in separate threads. If encoding or - // decoding is slow, the reading may go far ahead and may buffer too many - // frames in memory. To prevent this we limit the encoding/decoding queue - // size. When the queue is full, the main thread and, hence, reading frames - // from video source is blocked until a previously posted encoding/decoding - // task starts. - static constexpr int kMaxTaskQueueSize = 3; - - LimitedTaskQueue() : queue_size_(0) {} - - void PostScheduledTask(absl::AnyInvocable task, Timestamp start) { - ++queue_size_; - task_queue_.PostTask([this, task = std::move(task), start]() mutable { - int wait_ms = static_cast(start.ms() - rtc::TimeMillis()); - if (wait_ms > 0) { - SleepMs(wait_ms); - } - - std::move(task)(); - --queue_size_; - task_executed_.Set(); - }); - - task_executed_.Reset(); - if (queue_size_ > kMaxTaskQueueSize) { - task_executed_.Wait(rtc::Event::kForever); - } - RTC_CHECK(queue_size_ <= kMaxTaskQueueSize); - } - - void WaitForPreviouslyPostedTasks() { - task_queue_.SendTask([] {}); - } - - TaskQueueForTest task_queue_; - std::atomic_int queue_size_; - rtc::Event task_executed_; -}; - -class TesterY4mWriter { - public: - explicit TesterY4mWriter(absl::string_view base_path) - : base_path_(base_path) {} - - ~TesterY4mWriter() { - task_queue_.SendTask([] {}); - } - - void Write(const VideoFrame& frame, int spatial_idx) { - task_queue_.PostTask([this, frame, spatial_idx] { - if (y4m_writers_.find(spatial_idx) == y4m_writers_.end()) { - std::string file_path = - base_path_ + "_s" + std::to_string(spatial_idx) + ".y4m"; - - Y4mVideoFrameWriterImpl* y4m_writer = new Y4mVideoFrameWriterImpl( - file_path, frame.width(), frame.height(), /*fps=*/30); - RTC_CHECK(y4m_writer); - - y4m_writers_[spatial_idx] = - std::unique_ptr(y4m_writer); - } - - y4m_writers_.at(spatial_idx)->WriteFrame(frame); - }); - } - - protected: - std::string base_path_; - std::map> y4m_writers_; - TaskQueueForTest task_queue_; -}; - -class TesterIvfWriter { - public: - explicit TesterIvfWriter(absl::string_view base_path) - : base_path_(base_path) {} - - ~TesterIvfWriter() { - task_queue_.SendTask([] {}); - } - - void Write(const EncodedImage& encoded_frame) { - task_queue_.PostTask([this, encoded_frame] { - int spatial_idx = encoded_frame.SpatialIndex().value_or(0); - if (ivf_file_writers_.find(spatial_idx) == ivf_file_writers_.end()) { - std::string ivf_path = - base_path_ + "_s" + std::to_string(spatial_idx) + ".ivf"; - - FileWrapper ivf_file = FileWrapper::OpenWriteOnly(ivf_path); - RTC_CHECK(ivf_file.is_open()); - - std::unique_ptr ivf_writer = - IvfFileWriter::Wrap(std::move(ivf_file), /*byte_limit=*/0); - RTC_CHECK(ivf_writer); - - ivf_file_writers_[spatial_idx] = std::move(ivf_writer); - } - - // To play: ffplay -vcodec vp8|vp9|av1|hevc|h264 filename - ivf_file_writers_.at(spatial_idx) - ->WriteFrame(encoded_frame, VideoCodecType::kVideoCodecGeneric); - }); - } - - protected: - std::string base_path_; - std::map> ivf_file_writers_; - TaskQueueForTest task_queue_; -}; - -class TesterDecoder { - public: - TesterDecoder(Decoder* decoder, - VideoCodecAnalyzer* analyzer, - const DecoderSettings& settings) - : decoder_(decoder), - analyzer_(analyzer), - settings_(settings), - pacer_(settings.pacing) { - RTC_CHECK(analyzer_) << "Analyzer must be provided"; - - if (settings.decoder_input_base_path) { - input_writer_ = - std::make_unique(*settings.decoder_input_base_path); - } - - if (settings.decoder_output_base_path) { - output_writer_ = - std::make_unique(*settings.decoder_output_base_path); - } - } - - void Initialize() { - task_queue_.PostScheduledTask([this] { decoder_->Initialize(); }, - Timestamp::Zero()); - task_queue_.WaitForPreviouslyPostedTasks(); - } - - void Decode(const EncodedImage& input_frame) { - Timestamp timestamp = - Timestamp::Micros((input_frame.RtpTimestamp() / k90kHz).us()); - - task_queue_.PostScheduledTask( - [this, input_frame] { - analyzer_->StartDecode(input_frame); - - decoder_->Decode( - input_frame, - [this, spatial_idx = input_frame.SpatialIndex().value_or(0)]( - const VideoFrame& output_frame) { - analyzer_->FinishDecode(output_frame, spatial_idx); - - if (output_writer_) { - output_writer_->Write(output_frame, spatial_idx); - } - }); - - if (input_writer_) { - input_writer_->Write(input_frame); - } - }, - pacer_.Schedule(timestamp)); - } - - void Flush() { - task_queue_.PostScheduledTask([this] { decoder_->Flush(); }, - Timestamp::Zero()); - task_queue_.WaitForPreviouslyPostedTasks(); - } - - protected: - Decoder* const decoder_; - VideoCodecAnalyzer* const analyzer_; - const DecoderSettings& settings_; - Pacer pacer_; - LimitedTaskQueue task_queue_; - std::unique_ptr input_writer_; - std::unique_ptr output_writer_; -}; - -class TesterEncoder { - public: - TesterEncoder(Encoder* encoder, - TesterDecoder* decoder, - VideoCodecAnalyzer* analyzer, - const EncoderSettings& settings) - : encoder_(encoder), - decoder_(decoder), - analyzer_(analyzer), - settings_(settings), - pacer_(settings.pacing) { - RTC_CHECK(analyzer_) << "Analyzer must be provided"; - if (settings.encoder_input_base_path) { - input_writer_ = - std::make_unique(*settings.encoder_input_base_path); - } - - if (settings.encoder_output_base_path) { - output_writer_ = - std::make_unique(*settings.encoder_output_base_path); - } - } - - void Initialize() { - task_queue_.PostScheduledTask([this] { encoder_->Initialize(); }, - Timestamp::Zero()); - task_queue_.WaitForPreviouslyPostedTasks(); - } - - void Encode(const VideoFrame& input_frame) { - Timestamp timestamp = - Timestamp::Micros((input_frame.timestamp() / k90kHz).us()); - - task_queue_.PostScheduledTask( - [this, input_frame] { - analyzer_->StartEncode(input_frame); - encoder_->Encode(input_frame, - [this](const EncodedImage& encoded_frame) { - analyzer_->FinishEncode(encoded_frame); - - if (decoder_ != nullptr) { - decoder_->Decode(encoded_frame); - } - - if (output_writer_ != nullptr) { - output_writer_->Write(encoded_frame); - } - }); - - if (input_writer_) { - input_writer_->Write(input_frame, /*spatial_idx=*/0); - } - }, - pacer_.Schedule(timestamp)); - } - - void Flush() { - task_queue_.PostScheduledTask([this] { encoder_->Flush(); }, - Timestamp::Zero()); - task_queue_.WaitForPreviouslyPostedTasks(); - } - - protected: - Encoder* const encoder_; - TesterDecoder* const decoder_; - VideoCodecAnalyzer* const analyzer_; - const EncoderSettings& settings_; - std::unique_ptr input_writer_; - std::unique_ptr output_writer_; - Pacer pacer_; - LimitedTaskQueue task_queue_; -}; - -} // namespace - -std::unique_ptr VideoCodecTesterImpl::RunDecodeTest( - CodedVideoSource* video_source, - Decoder* decoder, - const DecoderSettings& decoder_settings) { - VideoCodecAnalyzer perf_analyzer; - TesterDecoder tester_decoder(decoder, &perf_analyzer, decoder_settings); - - tester_decoder.Initialize(); - - while (auto frame = video_source->PullFrame()) { - tester_decoder.Decode(*frame); - } - - tester_decoder.Flush(); - - return perf_analyzer.GetStats(); -} - -std::unique_ptr VideoCodecTesterImpl::RunEncodeTest( - RawVideoSource* video_source, - Encoder* encoder, - const EncoderSettings& encoder_settings) { - SyncRawVideoSource sync_source(video_source); - VideoCodecAnalyzer perf_analyzer; - TesterEncoder tester_encoder(encoder, /*decoder=*/nullptr, &perf_analyzer, - encoder_settings); - - tester_encoder.Initialize(); - - while (auto frame = sync_source.PullFrame()) { - tester_encoder.Encode(*frame); - } - - tester_encoder.Flush(); - - return perf_analyzer.GetStats(); -} - -std::unique_ptr VideoCodecTesterImpl::RunEncodeDecodeTest( - RawVideoSource* video_source, - Encoder* encoder, - Decoder* decoder, - const EncoderSettings& encoder_settings, - const DecoderSettings& decoder_settings) { - SyncRawVideoSource sync_source(video_source); - VideoCodecAnalyzer perf_analyzer(&sync_source); - TesterDecoder tester_decoder(decoder, &perf_analyzer, decoder_settings); - TesterEncoder tester_encoder(encoder, &tester_decoder, &perf_analyzer, - encoder_settings); - - tester_encoder.Initialize(); - tester_decoder.Initialize(); - - while (auto frame = sync_source.PullFrame()) { - tester_encoder.Encode(*frame); - } - - tester_encoder.Flush(); - tester_decoder.Flush(); - - return perf_analyzer.GetStats(); -} - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl.h b/modules/video_coding/codecs/test/video_codec_tester_impl.h deleted file mode 100644 index 32191b5a98..0000000000 --- a/modules/video_coding/codecs/test/video_codec_tester_impl.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ -#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ - -#include - -#include "api/test/video_codec_tester.h" - -namespace webrtc { -namespace test { - -// A stateless implementation of `VideoCodecTester`. This class is thread safe. -class VideoCodecTesterImpl : public VideoCodecTester { - public: - std::unique_ptr RunDecodeTest( - CodedVideoSource* video_source, - Decoder* decoder, - const DecoderSettings& decoder_settings) override; - - std::unique_ptr RunEncodeTest( - RawVideoSource* video_source, - Encoder* encoder, - const EncoderSettings& encoder_settings) override; - - std::unique_ptr RunEncodeDecodeTest( - RawVideoSource* video_source, - Encoder* encoder, - Decoder* decoder, - const EncoderSettings& encoder_settings, - const DecoderSettings& decoder_settings) override; -}; - -} // namespace test -} // namespace webrtc - -#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEO_CODEC_TESTER_IMPL_H_ diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc b/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc deleted file mode 100644 index a8c118ef20..0000000000 --- a/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" - -#include -#include -#include -#include - -#include "api/units/frequency.h" -#include "api/units/time_delta.h" -#include "api/video/encoded_image.h" -#include "api/video/i420_buffer.h" -#include "api/video/video_frame.h" -#include "rtc_base/fake_clock.h" -#include "rtc_base/gunit.h" -#include "rtc_base/task_queue_for_test.h" -#include "rtc_base/time_utils.h" -#include "test/gmock.h" -#include "test/gtest.h" - -namespace webrtc { -namespace test { - -namespace { -using ::testing::_; -using ::testing::Invoke; -using ::testing::InvokeWithoutArgs; -using ::testing::Return; - -using Decoder = VideoCodecTester::Decoder; -using Encoder = VideoCodecTester::Encoder; -using CodedVideoSource = VideoCodecTester::CodedVideoSource; -using RawVideoSource = VideoCodecTester::RawVideoSource; -using DecoderSettings = VideoCodecTester::DecoderSettings; -using EncoderSettings = VideoCodecTester::EncoderSettings; -using PacingSettings = VideoCodecTester::PacingSettings; -using PacingMode = PacingSettings::PacingMode; - -constexpr Frequency k90kHz = Frequency::Hertz(90000); - -struct PacingTestParams { - PacingSettings pacing_settings; - Frequency framerate; - int num_frames; - std::vector expected_delta_ms; -}; - -VideoFrame CreateVideoFrame(uint32_t timestamp_rtp) { - rtc::scoped_refptr buffer(I420Buffer::Create(2, 2)); - return VideoFrame::Builder() - .set_video_frame_buffer(buffer) - .set_timestamp_rtp(timestamp_rtp) - .build(); -} - -EncodedImage CreateEncodedImage(uint32_t timestamp_rtp) { - EncodedImage encoded_image; - encoded_image.SetRtpTimestamp(timestamp_rtp); - return encoded_image; -} - -class MockRawVideoSource : public RawVideoSource { - public: - MockRawVideoSource(int num_frames, Frequency framerate) - : num_frames_(num_frames), frame_num_(0), framerate_(framerate) {} - - absl::optional PullFrame() override { - if (frame_num_ >= num_frames_) { - return absl::nullopt; - } - uint32_t timestamp_rtp = frame_num_ * k90kHz / framerate_; - ++frame_num_; - return CreateVideoFrame(timestamp_rtp); - } - - MOCK_METHOD(VideoFrame, - GetFrame, - (uint32_t timestamp_rtp, Resolution), - (override)); - - private: - int num_frames_; - int frame_num_; - Frequency framerate_; -}; - -class MockCodedVideoSource : public CodedVideoSource { - public: - MockCodedVideoSource(int num_frames, Frequency framerate) - : num_frames_(num_frames), frame_num_(0), framerate_(framerate) {} - - absl::optional PullFrame() override { - if (frame_num_ >= num_frames_) { - return absl::nullopt; - } - uint32_t timestamp_rtp = frame_num_ * k90kHz / framerate_; - ++frame_num_; - return CreateEncodedImage(timestamp_rtp); - } - - private: - int num_frames_; - int frame_num_; - Frequency framerate_; -}; - -class MockDecoder : public Decoder { - public: - MOCK_METHOD(void, Initialize, (), (override)); - MOCK_METHOD(void, - Decode, - (const EncodedImage& frame, DecodeCallback callback), - (override)); - MOCK_METHOD(void, Flush, (), (override)); -}; - -class MockEncoder : public Encoder { - public: - MOCK_METHOD(void, Initialize, (), (override)); - MOCK_METHOD(void, - Encode, - (const VideoFrame& frame, EncodeCallback callback), - (override)); - MOCK_METHOD(void, Flush, (), (override)); -}; - -} // namespace - -class VideoCodecTesterImplPacingTest - : public ::testing::TestWithParam { - public: - VideoCodecTesterImplPacingTest() : test_params_(GetParam()) {} - - protected: - PacingTestParams test_params_; -}; - -TEST_P(VideoCodecTesterImplPacingTest, PaceEncode) { - MockRawVideoSource video_source(test_params_.num_frames, - test_params_.framerate); - MockEncoder encoder; - EncoderSettings encoder_settings; - encoder_settings.pacing = test_params_.pacing_settings; - - VideoCodecTesterImpl tester; - auto fs = - tester.RunEncodeTest(&video_source, &encoder, encoder_settings)->Slice(); - ASSERT_EQ(static_cast(fs.size()), test_params_.num_frames); - - for (size_t i = 1; i < fs.size(); ++i) { - int delta_ms = (fs[i].encode_start - fs[i - 1].encode_start).ms(); - EXPECT_NEAR(delta_ms, test_params_.expected_delta_ms[i - 1], 10); - } -} - -TEST_P(VideoCodecTesterImplPacingTest, PaceDecode) { - MockCodedVideoSource video_source(test_params_.num_frames, - test_params_.framerate); - MockDecoder decoder; - DecoderSettings decoder_settings; - decoder_settings.pacing = test_params_.pacing_settings; - - VideoCodecTesterImpl tester; - auto fs = - tester.RunDecodeTest(&video_source, &decoder, decoder_settings)->Slice(); - ASSERT_EQ(static_cast(fs.size()), test_params_.num_frames); - - for (size_t i = 1; i < fs.size(); ++i) { - int delta_ms = (fs[i].decode_start - fs[i - 1].decode_start).ms(); - EXPECT_NEAR(delta_ms, test_params_.expected_delta_ms[i - 1], 20); - } -} - -INSTANTIATE_TEST_SUITE_P( - DISABLED_All, - VideoCodecTesterImplPacingTest, - ::testing::ValuesIn( - {// No pacing. - PacingTestParams({.pacing_settings = {.mode = PacingMode::kNoPacing}, - .framerate = Frequency::Hertz(10), - .num_frames = 3, - .expected_delta_ms = {0, 0}}), - // Real-time pacing. - PacingTestParams({.pacing_settings = {.mode = PacingMode::kRealTime}, - .framerate = Frequency::Hertz(10), - .num_frames = 3, - .expected_delta_ms = {100, 100}}), - // Pace with specified constant rate. - PacingTestParams( - {.pacing_settings = {.mode = PacingMode::kConstantRate, - .constant_rate = Frequency::Hertz(20)}, - .framerate = Frequency::Hertz(10), - .num_frames = 3, - .expected_delta_ms = {50, 50}})})); -} // namespace test -} // namespace webrtc diff --git a/test/BUILD.gn b/test/BUILD.gn index be8ee1684e..fbc1ab1ad5 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -716,6 +716,7 @@ if (rtc_include_tests) { ":test_main", ":test_support", ":test_support_test_artifacts", + ":video_codec_tester", ":video_test_common", ":video_test_support", ":y4m_frame_generator", @@ -723,11 +724,15 @@ if (rtc_include_tests) { "../api:create_frame_generator", "../api:create_simulcast_test_fixture_api", "../api:frame_generator_api", + "../api:mock_video_codec_factory", + "../api:mock_video_decoder", + "../api:mock_video_encoder", "../api:scoped_refptr", "../api:simulcast_test_fixture_api", "../api/task_queue:task_queue_test", "../api/test/video:function_video_factory", "../api/test/video:video_frame_writer", + "../api/units:data_rate", "../api/units:time_delta", "../api/video:encoded_image", "../api/video:video_frame", @@ -744,6 +749,7 @@ if (rtc_include_tests) { "../modules/video_coding:webrtc_h264", "../modules/video_coding:webrtc_vp8", "../modules/video_coding:webrtc_vp9", + "../modules/video_coding/svc:scalability_mode_util", "../rtc_base:criticalsection", "../rtc_base:rtc_event", "../rtc_base:rtc_task_queue", @@ -757,6 +763,7 @@ if (rtc_include_tests) { "scenario:scenario_unittests", "time_controller:time_controller", "time_controller:time_controller_unittests", + "//third_party/libyuv", ] absl_deps = [ "//third_party/abseil-cpp/absl/flags:flag", @@ -781,6 +788,7 @@ if (rtc_include_tests) { "testsupport/y4m_frame_writer_unittest.cc", "testsupport/yuv_frame_reader_unittest.cc", "testsupport/yuv_frame_writer_unittest.cc", + "video_codec_tester_unittest.cc", ] if (rtc_enable_protobuf) { @@ -1368,3 +1376,46 @@ rtc_library("fake_encoded_frame") { ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } + +rtc_library("video_codec_tester") { + testonly = true + sources = [ + "video_codec_tester.cc", + "video_codec_tester.h", + ] + deps = [ + "../api:array_view", + "../api/numerics:numerics", + "../api/test/metrics:metric", + "../api/test/metrics:metrics_logger", + "../api/units:data_rate", + "../api/units:data_size", + "../api/units:frequency", + "../api/units:time_delta", + "../api/units:timestamp", + "../api/video:builtin_video_bitrate_allocator_factory", + "../api/video:encoded_image", + "../api/video:resolution", + "../api/video:video_bitrate_allocator", + "../api/video:video_frame", + "../api/video_codecs:video_codecs_api", + "../media:media_constants", + "../modules/video_coding:video_codec_interface", + "../modules/video_coding:video_coding_utility", + "../modules/video_coding:webrtc_vp9_helpers", + "../modules/video_coding/codecs/av1:av1_svc_config", + "../modules/video_coding/svc:scalability_mode_util", + "../rtc_base:checks", + "../rtc_base:logging", + "../rtc_base:rtc_event", + "../rtc_base:task_queue_for_test", + "../rtc_base:timeutils", + "../rtc_base/synchronization:mutex", + "../system_wrappers", + "../test:fileutils", + "../test:video_test_support", + "//third_party/libyuv", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} diff --git a/test/video_codec_tester.cc b/test/video_codec_tester.cc new file mode 100644 index 0000000000..26f0a61372 --- /dev/null +++ b/test/video_codec_tester.cc @@ -0,0 +1,1260 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/video_codec_tester.h" + +#include +#include +#include +#include + +#include "api/array_view.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/builtin_video_bitrate_allocator_factory.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_bitrate_allocator.h" +#include "api/video/video_codec_type.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_encoder.h" +#include "media/base/media_constants.h" +#include "modules/video_coding/codecs/av1/av1_svc_config.h" +#include "modules/video_coding/codecs/vp9/svc_config.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "modules/video_coding/utility/ivf_file_writer.h" +#include "rtc_base/event.h" +#include "rtc_base/logging.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/task_queue_for_test.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/sleep.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/frame_reader.h" +#include "test/testsupport/video_frame_writer.h" +#include "third_party/libyuv/include/libyuv/compare.h" + +namespace webrtc { +namespace test { + +namespace { +using CodedVideoSource = VideoCodecTester::CodedVideoSource; +using VideoSourceSettings = VideoCodecTester::VideoSourceSettings; +using EncodingSettings = VideoCodecTester::EncodingSettings; +using LayerSettings = EncodingSettings::LayerSettings; +using LayerId = VideoCodecTester::LayerId; +using EncoderSettings = VideoCodecTester::EncoderSettings; +using DecoderSettings = VideoCodecTester::DecoderSettings; +using PacingSettings = VideoCodecTester::PacingSettings; +using PacingMode = PacingSettings::PacingMode; +using VideoCodecStats = VideoCodecTester::VideoCodecStats; +using DecodeCallback = + absl::AnyInvocable; +using webrtc::test::ImprovementDirection; + +constexpr Frequency k90kHz = Frequency::Hertz(90000); + +const std::set kFullSvcScalabilityModes{ + ScalabilityMode::kL2T1, ScalabilityMode::kL2T1h, ScalabilityMode::kL2T2, + ScalabilityMode::kL2T2h, ScalabilityMode::kL2T3, ScalabilityMode::kL2T3h, + ScalabilityMode::kL3T1, ScalabilityMode::kL3T1h, ScalabilityMode::kL3T2, + ScalabilityMode::kL3T2h, ScalabilityMode::kL3T3, ScalabilityMode::kL3T3h}; + +const std::set kKeySvcScalabilityModes{ + ScalabilityMode::kL2T1_KEY, ScalabilityMode::kL2T2_KEY, + ScalabilityMode::kL2T2_KEY_SHIFT, ScalabilityMode::kL2T3_KEY, + ScalabilityMode::kL3T1_KEY, ScalabilityMode::kL3T2_KEY, + ScalabilityMode::kL3T3_KEY}; + +// A thread-safe raw video frame reader. +class VideoSource { + public: + explicit VideoSource(VideoSourceSettings source_settings) + : source_settings_(source_settings) { + MutexLock lock(&mutex_); + frame_reader_ = CreateYuvFrameReader( + source_settings_.file_path, source_settings_.resolution, + YuvFrameReaderImpl::RepeatMode::kPingPong); + RTC_CHECK(frame_reader_); + } + + // Pulls next frame. + VideoFrame PullFrame(uint32_t timestamp_rtp, + Resolution resolution, + Frequency framerate) { + MutexLock lock(&mutex_); + int frame_num; + auto buffer = frame_reader_->PullFrame( + &frame_num, resolution, + {.num = framerate.millihertz(), + .den = source_settings_.framerate.millihertz()}); + RTC_CHECK(buffer) << "Can not pull frame. RTP timestamp " << timestamp_rtp; + frame_num_[timestamp_rtp] = frame_num; + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .set_timestamp_us((timestamp_rtp / k90kHz).us()) + .build(); + } + + // Reads frame specified by `timestamp_rtp`, scales it to `resolution` and + // returns. Frame with the given `timestamp_rtp` is expected to be pulled + // before. + VideoFrame ReadFrame(uint32_t timestamp_rtp, Resolution resolution) { + MutexLock lock(&mutex_); + RTC_CHECK(frame_num_.find(timestamp_rtp) != frame_num_.end()) + << "Frame with RTP timestamp " << timestamp_rtp + << " was not pulled before"; + auto buffer = + frame_reader_->ReadFrame(frame_num_.at(timestamp_rtp), resolution); + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .build(); + } + + private: + VideoSourceSettings source_settings_; + std::unique_ptr frame_reader_ RTC_GUARDED_BY(mutex_); + std::map frame_num_ RTC_GUARDED_BY(mutex_); + Mutex mutex_; +}; + +// Pacer calculates delay necessary to keep frame encode or decode call spaced +// from the previous calls by the pacing time. `Schedule` is expected to be +// called as close as possible to posting frame encode or decode task. This +// class is not thread safe. +class Pacer { + public: + explicit Pacer(PacingSettings settings) + : settings_(settings), delay_(TimeDelta::Zero()) {} + + Timestamp Schedule(Timestamp timestamp) { + Timestamp now = Timestamp::Micros(rtc::TimeMicros()); + if (settings_.mode == PacingMode::kNoPacing) { + return now; + } + + Timestamp scheduled = now; + if (prev_scheduled_) { + scheduled = *prev_scheduled_ + PacingTime(timestamp); + if (scheduled < now) { + scheduled = now; + } + } + + prev_timestamp_ = timestamp; + prev_scheduled_ = scheduled; + return scheduled; + } + + private: + TimeDelta PacingTime(Timestamp timestamp) { + if (settings_.mode == PacingMode::kRealTime) { + return timestamp - *prev_timestamp_; + } + RTC_CHECK_EQ(PacingMode::kConstantRate, settings_.mode); + return 1 / settings_.constant_rate; + } + + PacingSettings settings_; + absl::optional prev_timestamp_; + absl::optional prev_scheduled_; + TimeDelta delay_; +}; + +class LimitedTaskQueue { + public: + // The codec tester reads frames from video source in the main thread. + // Encoding and decoding are done in separate threads. If encoding or + // decoding is slow, the reading may go far ahead and may buffer too many + // frames in memory. To prevent this we limit the encoding/decoding queue + // size. When the queue is full, the main thread and, hence, reading frames + // from video source is blocked until a previously posted encoding/decoding + // task starts. + static constexpr int kMaxTaskQueueSize = 3; + + LimitedTaskQueue() : queue_size_(0) {} + + void PostScheduledTask(absl::AnyInvocable task, Timestamp start) { + ++queue_size_; + task_queue_.PostTask([this, task = std::move(task), start]() mutable { + // `TaskQueue` doesn't guarantee FIFO order of execution for delayed + // tasks. + int wait_ms = static_cast(start.ms() - rtc::TimeMillis()); + if (wait_ms > 0) { + SleepMs(wait_ms); + } + std::move(task)(); + --queue_size_; + task_executed_.Set(); + }); + + task_executed_.Reset(); + if (queue_size_ > kMaxTaskQueueSize) { + task_executed_.Wait(rtc::Event::kForever); + RTC_CHECK(queue_size_ <= kMaxTaskQueueSize); + } + } + + void PostTaskAndWait(absl::AnyInvocable task) { + PostScheduledTask(std::move(task), Timestamp::Zero()); + task_queue_.WaitForPreviouslyPostedTasks(); + } + + private: + TaskQueueForTest task_queue_; + std::atomic_int queue_size_; + rtc::Event task_executed_; +}; + +class TesterY4mWriter { + public: + explicit TesterY4mWriter(absl::string_view base_path) + : base_path_(base_path) {} + + ~TesterY4mWriter() { + task_queue_.SendTask([] {}); + } + + void Write(const VideoFrame& frame, int spatial_idx) { + task_queue_.PostTask([this, frame, spatial_idx] { + if (y4m_writers_.find(spatial_idx) == y4m_writers_.end()) { + std::string file_path = + base_path_ + "-s" + std::to_string(spatial_idx) + ".y4m"; + Y4mVideoFrameWriterImpl* y4m_writer = new Y4mVideoFrameWriterImpl( + file_path, frame.width(), frame.height(), /*fps=*/30); + RTC_CHECK(y4m_writer); + + y4m_writers_[spatial_idx] = + std::unique_ptr(y4m_writer); + } + + y4m_writers_.at(spatial_idx)->WriteFrame(frame); + }); + } + + private: + std::string base_path_; + std::map> y4m_writers_; + TaskQueueForTest task_queue_; +}; + +class TesterIvfWriter { + public: + explicit TesterIvfWriter(absl::string_view base_path) + : base_path_(base_path) {} + + ~TesterIvfWriter() { + task_queue_.SendTask([] {}); + } + + void Write(const EncodedImage& encoded_frame) { + task_queue_.PostTask([this, encoded_frame] { + int spatial_idx = encoded_frame.SimulcastIndex().value_or(0); + if (ivf_file_writers_.find(spatial_idx) == ivf_file_writers_.end()) { + std::string ivf_path = + base_path_ + "-s" + std::to_string(spatial_idx) + ".ivf"; + FileWrapper ivf_file = FileWrapper::OpenWriteOnly(ivf_path); + RTC_CHECK(ivf_file.is_open()); + + std::unique_ptr ivf_writer = + IvfFileWriter::Wrap(std::move(ivf_file), /*byte_limit=*/0); + RTC_CHECK(ivf_writer); + + ivf_file_writers_[spatial_idx] = std::move(ivf_writer); + } + + // To play: ffplay -vcodec vp8|vp9|av1|hevc|h264 filename + ivf_file_writers_.at(spatial_idx) + ->WriteFrame(encoded_frame, VideoCodecType::kVideoCodecGeneric); + }); + } + + private: + std::string base_path_; + std::map> ivf_file_writers_; + TaskQueueForTest task_queue_; +}; + +class LeakyBucket { + public: + LeakyBucket() : level_bits_(0) {} + + // Updates bucket level and returns its current level in bits. Data is remove + // from bucket with rate equal to target bitrate of previous frame. Bucket + // level is tracked with floating point precision. Returned value of bucket + // level is rounded up. + int Update(const VideoCodecStats::Frame& frame) { + RTC_CHECK(frame.target_bitrate) << "Bitrate must be specified."; + if (prev_frame_) { + RTC_CHECK_GT(frame.timestamp_rtp, prev_frame_->timestamp_rtp) + << "Timestamp must increase."; + TimeDelta passed = + (frame.timestamp_rtp - prev_frame_->timestamp_rtp) / k90kHz; + level_bits_ -= + prev_frame_->target_bitrate->bps() * passed.seconds(); + level_bits_ = std::max(level_bits_, 0.0); + } + prev_frame_ = frame; + level_bits_ += frame.frame_size.bytes() * 8; + return static_cast(std::ceil(level_bits_)); + } + + private: + absl::optional prev_frame_; + double level_bits_; +}; + +class VideoCodecAnalyzer : public VideoCodecTester::VideoCodecStats { + public: + explicit VideoCodecAnalyzer(VideoSource* video_source) + : video_source_(video_source) {} + + void StartEncode(const VideoFrame& video_frame, + const EncodingSettings& encoding_settings) { + int64_t encode_start_us = rtc::TimeMicros(); + task_queue_.PostTask([this, timestamp_rtp = video_frame.timestamp(), + encoding_settings, encode_start_us]() { + RTC_CHECK(frames_.find(timestamp_rtp) == frames_.end()) + << "Duplicate frame. Frame with timestamp " << timestamp_rtp + << " was seen before"; + + Frame frame; + frame.timestamp_rtp = timestamp_rtp; + frame.encode_start = Timestamp::Micros(encode_start_us), + frames_.emplace(timestamp_rtp, + std::map{{/*spatial_idx=*/0, frame}}); + encoding_settings_.emplace(timestamp_rtp, encoding_settings); + }); + } + + void FinishEncode(const EncodedImage& encoded_frame) { + int64_t encode_finished_us = rtc::TimeMicros(); + task_queue_.PostTask( + [this, timestamp_rtp = encoded_frame.RtpTimestamp(), + spatial_idx = encoded_frame.SpatialIndex().value_or(0), + temporal_idx = encoded_frame.TemporalIndex().value_or(0), + width = encoded_frame._encodedWidth, + height = encoded_frame._encodedHeight, + frame_type = encoded_frame._frameType, + frame_size_bytes = encoded_frame.size(), qp = encoded_frame.qp_, + encode_finished_us]() { + if (spatial_idx > 0) { + RTC_CHECK(frames_.find(timestamp_rtp) != frames_.end()) + << "Spatial layer 0 frame with timestamp " << timestamp_rtp + << " was not seen before"; + const Frame& base_frame = + frames_.at(timestamp_rtp).at(/*spatial_idx=*/0); + frames_.at(timestamp_rtp).emplace(spatial_idx, base_frame); + } + + Frame& frame = frames_.at(timestamp_rtp).at(spatial_idx); + frame.layer_id = {.spatial_idx = spatial_idx, + .temporal_idx = temporal_idx}; + frame.width = width; + frame.height = height; + frame.frame_size = DataSize::Bytes(frame_size_bytes); + frame.qp = qp; + frame.keyframe = frame_type == VideoFrameType::kVideoFrameKey; + frame.encode_time = + Timestamp::Micros(encode_finished_us) - frame.encode_start; + frame.encoded = true; + }); + } + + void StartDecode(const EncodedImage& encoded_frame) { + int64_t decode_start_us = rtc::TimeMicros(); + task_queue_.PostTask( + [this, timestamp_rtp = encoded_frame.RtpTimestamp(), + spatial_idx = encoded_frame.SpatialIndex().value_or(0), + frame_size_bytes = encoded_frame.size(), decode_start_us]() { + if (frames_.find(timestamp_rtp) == frames_.end() || + frames_.at(timestamp_rtp).find(spatial_idx) == + frames_.at(timestamp_rtp).end()) { + Frame frame; + frame.timestamp_rtp = timestamp_rtp; + frame.layer_id = {.spatial_idx = spatial_idx}; + frame.frame_size = DataSize::Bytes(frame_size_bytes); + frames_.emplace(timestamp_rtp, + std::map{{spatial_idx, frame}}); + } + + Frame& frame = frames_.at(timestamp_rtp).at(spatial_idx); + frame.decode_start = Timestamp::Micros(decode_start_us); + }); + } + + void FinishDecode(const VideoFrame& decoded_frame, int spatial_idx) { + int64_t decode_finished_us = rtc::TimeMicros(); + task_queue_.PostTask([this, timestamp_rtp = decoded_frame.timestamp(), + spatial_idx, width = decoded_frame.width(), + height = decoded_frame.height(), + decode_finished_us]() { + Frame& frame = frames_.at(timestamp_rtp).at(spatial_idx); + frame.decode_time = + Timestamp::Micros(decode_finished_us) - frame.decode_start; + if (!frame.encoded) { + frame.width = width; + frame.height = height; + } + frame.decoded = true; + }); + + if (video_source_ != nullptr) { + // Copy hardware-backed frame into main memory to release output buffers + // which number may be limited in hardware decoders. + rtc::scoped_refptr decoded_buffer = + decoded_frame.video_frame_buffer()->ToI420(); + + task_queue_.PostTask([this, decoded_buffer, + timestamp_rtp = decoded_frame.timestamp(), + spatial_idx]() { + VideoFrame ref_frame = video_source_->ReadFrame( + timestamp_rtp, {.width = decoded_buffer->width(), + .height = decoded_buffer->height()}); + rtc::scoped_refptr ref_buffer = + ref_frame.video_frame_buffer()->ToI420(); + Frame& frame = frames_.at(timestamp_rtp).at(spatial_idx); + frame.psnr = CalcPsnr(*decoded_buffer, *ref_buffer); + }); + } + } + + std::vector Slice(Filter filter, bool merge) const { + std::vector slice; + for (const auto& [timestamp_rtp, temporal_unit_frames] : frames_) { + if (temporal_unit_frames.empty()) { + continue; + } + + bool is_svc = false; + if (!encoding_settings_.empty()) { + ScalabilityMode scalability_mode = + encoding_settings_.at(timestamp_rtp).scalability_mode; + if (kFullSvcScalabilityModes.count(scalability_mode) > 0 || + (kKeySvcScalabilityModes.count(scalability_mode) > 0 && + temporal_unit_frames.at(0).keyframe)) { + is_svc = true; + } + } + + std::vector subframes; + for (const auto& [spatial_idx, frame] : temporal_unit_frames) { + if (frame.timestamp_rtp < filter.min_timestamp_rtp || + frame.timestamp_rtp > filter.max_timestamp_rtp) { + continue; + } + if (filter.layer_id) { + if ((is_svc && + frame.layer_id.spatial_idx > filter.layer_id->spatial_idx) || + (!is_svc && + frame.layer_id.spatial_idx != filter.layer_id->spatial_idx)) { + continue; + } + if (frame.layer_id.temporal_idx > filter.layer_id->temporal_idx) { + continue; + } + } + subframes.push_back(frame); + } + + if (subframes.empty()) { + continue; + } + + if (!merge) { + std::copy(subframes.begin(), subframes.end(), + std::back_inserter(slice)); + continue; + } + + Frame superframe = subframes.back(); + for (const Frame& frame : + rtc::ArrayView(subframes).subview(0, subframes.size() - 1)) { + superframe.frame_size += frame.frame_size; + superframe.keyframe |= frame.keyframe; + superframe.encode_time = + std::max(superframe.encode_time, frame.encode_time); + superframe.decode_time = + std::max(superframe.decode_time, frame.decode_time); + } + + if (!encoding_settings_.empty()) { + RTC_CHECK(encoding_settings_.find(superframe.timestamp_rtp) != + encoding_settings_.end()) + << "No encoding settings for frame " << superframe.timestamp_rtp; + const EncodingSettings& es = + encoding_settings_.at(superframe.timestamp_rtp); + superframe.target_bitrate = GetTargetBitrate(es, filter.layer_id); + superframe.target_framerate = GetTargetFramerate(es, filter.layer_id); + } + + slice.push_back(superframe); + } + return slice; + } + + Stream Aggregate(Filter filter) const { + std::vector frames = Slice(filter, /*merge=*/true); + Stream stream; + LeakyBucket leaky_bucket; + for (const Frame& frame : frames) { + Timestamp time = Timestamp::Micros((frame.timestamp_rtp / k90kHz).us()); + if (!frame.frame_size.IsZero()) { + stream.width.AddSample(StatsSample(frame.width, time)); + stream.height.AddSample(StatsSample(frame.height, time)); + stream.frame_size_bytes.AddSample( + StatsSample(frame.frame_size.bytes(), time)); + stream.keyframe.AddSample(StatsSample(frame.keyframe, time)); + if (frame.qp) { + stream.qp.AddSample(StatsSample(*frame.qp, time)); + } + } + if (frame.encoded) { + stream.encode_time_ms.AddSample( + StatsSample(frame.encode_time.ms(), time)); + } + if (frame.decoded) { + stream.decode_time_ms.AddSample( + StatsSample(frame.decode_time.ms(), time)); + } + if (frame.psnr) { + stream.psnr.y.AddSample(StatsSample(frame.psnr->y, time)); + stream.psnr.u.AddSample(StatsSample(frame.psnr->u, time)); + stream.psnr.v.AddSample(StatsSample(frame.psnr->v, time)); + } + if (frame.target_framerate) { + stream.target_framerate_fps.AddSample( + StatsSample(frame.target_framerate->hertz(), time)); + } + if (frame.target_bitrate) { + stream.target_bitrate_kbps.AddSample( + StatsSample(frame.target_bitrate->kbps(), time)); + int buffer_level_bits = leaky_bucket.Update(frame); + stream.transmission_time_ms.AddSample(StatsSample( + 1000 * buffer_level_bits / frame.target_bitrate->bps(), + time)); + } + } + + int num_encoded_frames = stream.frame_size_bytes.NumSamples(); + const Frame& first_frame = frames.front(); + + Filter filter_all_layers{.min_timestamp_rtp = filter.min_timestamp_rtp, + .max_timestamp_rtp = filter.max_timestamp_rtp}; + std::vector frames_all_layers = + Slice(filter_all_layers, /*merge=*/true); + const Frame& last_frame = frames_all_layers.back(); + TimeDelta duration = + (last_frame.timestamp_rtp - first_frame.timestamp_rtp) / k90kHz; + if (last_frame.target_framerate) { + duration += 1 / *last_frame.target_framerate; + } + + DataRate encoded_bitrate = + DataSize::Bytes(stream.frame_size_bytes.GetSum()) / duration; + Frequency encoded_framerate = num_encoded_frames / duration; + + double bitrate_mismatch_pct = 0.0; + if (const auto& target_bitrate = first_frame.target_bitrate; + target_bitrate) { + bitrate_mismatch_pct = 100 * (encoded_bitrate / *target_bitrate - 1); + } + double framerate_mismatch_pct = 0.0; + if (const auto& target_framerate = first_frame.target_framerate; + target_framerate) { + framerate_mismatch_pct = + 100 * (encoded_framerate / *target_framerate - 1); + } + + for (Frame& frame : frames) { + Timestamp time = Timestamp::Micros((frame.timestamp_rtp / k90kHz).us()); + stream.encoded_bitrate_kbps.AddSample( + StatsSample(encoded_bitrate.kbps(), time)); + stream.encoded_framerate_fps.AddSample( + StatsSample(encoded_framerate.hertz(), time)); + stream.bitrate_mismatch_pct.AddSample( + StatsSample(bitrate_mismatch_pct, time)); + stream.framerate_mismatch_pct.AddSample( + StatsSample(framerate_mismatch_pct, time)); + } + + return stream; + } + + void Flush() { task_queue_.WaitForPreviouslyPostedTasks(); } + + private: + struct FrameId { + uint32_t timestamp_rtp; + int spatial_idx; + + bool operator==(const FrameId& o) const { + return timestamp_rtp == o.timestamp_rtp && spatial_idx == o.spatial_idx; + } + bool operator<(const FrameId& o) const { + return timestamp_rtp < o.timestamp_rtp || + (timestamp_rtp == o.timestamp_rtp && spatial_idx < o.spatial_idx); + } + }; + + Frame::Psnr CalcPsnr(const I420BufferInterface& ref_buffer, + const I420BufferInterface& dec_buffer) { + RTC_CHECK_EQ(ref_buffer.width(), dec_buffer.width()); + RTC_CHECK_EQ(ref_buffer.height(), dec_buffer.height()); + + uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataY(), dec_buffer.StrideY(), ref_buffer.DataY(), + ref_buffer.StrideY(), dec_buffer.width(), dec_buffer.height()); + + uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataU(), dec_buffer.StrideU(), ref_buffer.DataU(), + ref_buffer.StrideU(), dec_buffer.width() / 2, dec_buffer.height() / 2); + + uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataV(), dec_buffer.StrideV(), ref_buffer.DataV(), + ref_buffer.StrideV(), dec_buffer.width() / 2, dec_buffer.height() / 2); + + int num_y_samples = dec_buffer.width() * dec_buffer.height(); + Frame::Psnr psnr; + psnr.y = libyuv::SumSquareErrorToPsnr(sse_y, num_y_samples); + psnr.u = libyuv::SumSquareErrorToPsnr(sse_u, num_y_samples / 4); + psnr.v = libyuv::SumSquareErrorToPsnr(sse_v, num_y_samples / 4); + return psnr; + } + + DataRate GetTargetBitrate(const EncodingSettings& encoding_settings, + absl::optional layer_id) const { + int base_spatial_idx; + if (layer_id.has_value()) { + bool is_svc = + kFullSvcScalabilityModes.count(encoding_settings.scalability_mode); + base_spatial_idx = is_svc ? 0 : layer_id->spatial_idx; + } else { + int num_spatial_layers = + ScalabilityModeToNumSpatialLayers(encoding_settings.scalability_mode); + int num_temporal_layers = ScalabilityModeToNumTemporalLayers( + encoding_settings.scalability_mode); + layer_id = LayerId({.spatial_idx = num_spatial_layers - 1, + .temporal_idx = num_temporal_layers - 1}); + base_spatial_idx = 0; + } + + DataRate bitrate = DataRate::Zero(); + for (int sidx = base_spatial_idx; sidx <= layer_id->spatial_idx; ++sidx) { + for (int tidx = 0; tidx <= layer_id->temporal_idx; ++tidx) { + auto layer_settings = encoding_settings.layers_settings.find( + {.spatial_idx = sidx, .temporal_idx = tidx}); + RTC_CHECK(layer_settings != encoding_settings.layers_settings.end()) + << "bitrate is not specified for layer sidx=" << sidx + << " tidx=" << tidx; + bitrate += layer_settings->second.bitrate; + } + } + return bitrate; + } + + Frequency GetTargetFramerate(const EncodingSettings& encoding_settings, + absl::optional layer_id) const { + if (layer_id.has_value()) { + auto layer_settings = encoding_settings.layers_settings.find( + {.spatial_idx = layer_id->spatial_idx, + .temporal_idx = layer_id->temporal_idx}); + RTC_CHECK(layer_settings != encoding_settings.layers_settings.end()) + << "framerate is not specified for layer sidx=" + << layer_id->spatial_idx << " tidx=" << layer_id->temporal_idx; + return layer_settings->second.framerate; + } + return encoding_settings.layers_settings.rbegin()->second.framerate; + } + + SamplesStatsCounter::StatsSample StatsSample(double value, + Timestamp time) const { + return SamplesStatsCounter::StatsSample{value, time}; + } + + VideoSource* const video_source_; + TaskQueueForTest task_queue_; + // RTP timestamp -> spatial layer -> Frame + std::map> frames_; + std::map encoding_settings_; +}; + +class Decoder : public DecodedImageCallback { + public: + Decoder(VideoDecoderFactory* decoder_factory, + const DecoderSettings& decoder_settings, + VideoCodecAnalyzer* analyzer) + : decoder_factory_(decoder_factory), + analyzer_(analyzer), + pacer_(decoder_settings.pacing_settings) { + RTC_CHECK(analyzer_) << "Analyzer must be provided"; + + if (decoder_settings.decoder_input_base_path) { + ivf_writer_ = std::make_unique( + *decoder_settings.decoder_input_base_path); + } + + if (decoder_settings.decoder_output_base_path) { + y4m_writer_ = std::make_unique( + *decoder_settings.decoder_output_base_path); + } + } + + void Initialize(const SdpVideoFormat& sdp_video_format) { + decoder_ = decoder_factory_->CreateVideoDecoder(sdp_video_format); + RTC_CHECK(decoder_) << "Could not create decoder for video format " + << sdp_video_format.ToString(); + + task_queue_.PostTaskAndWait([this, &sdp_video_format] { + decoder_->RegisterDecodeCompleteCallback(this); + + VideoDecoder::Settings ds; + ds.set_codec_type(PayloadStringToCodecType(sdp_video_format.name)); + ds.set_number_of_cores(1); + ds.set_max_render_resolution({1280, 720}); + bool result = decoder_->Configure(ds); + RTC_CHECK(result) << "Failed to configure decoder"; + }); + } + + void Decode(const EncodedImage& encoded_frame) { + Timestamp pts = + Timestamp::Micros((encoded_frame.RtpTimestamp() / k90kHz).us()); + + task_queue_.PostScheduledTask( + [this, encoded_frame] { + analyzer_->StartDecode(encoded_frame); + int error = decoder_->Decode(encoded_frame, /*render_time_ms*/ 0); + if (error != 0) { + RTC_LOG(LS_WARNING) + << "Decode failed with error code " << error + << " RTP timestamp " << encoded_frame.RtpTimestamp(); + } + }, + pacer_.Schedule(pts)); + + if (ivf_writer_) { + ivf_writer_->Write(encoded_frame); + } + } + + void Flush() { + // TODO(webrtc:14852): Add Flush() to VideoDecoder API. + task_queue_.PostTaskAndWait([this] { decoder_->Release(); }); + } + + private: + int Decoded(VideoFrame& decoded_frame) override { + analyzer_->FinishDecode(decoded_frame, /*spatial_idx=*/0); + + if (y4m_writer_) { + y4m_writer_->Write(decoded_frame, /*spatial_idx=*/0); + } + + return WEBRTC_VIDEO_CODEC_OK; + } + + VideoDecoderFactory* decoder_factory_; + std::unique_ptr decoder_; + VideoCodecAnalyzer* const analyzer_; + Pacer pacer_; + LimitedTaskQueue task_queue_; + std::unique_ptr ivf_writer_; + std::unique_ptr y4m_writer_; +}; + +class Encoder : public EncodedImageCallback { + public: + using EncodeCallback = + absl::AnyInvocable; + + Encoder(VideoEncoderFactory* encoder_factory, + const EncoderSettings& encoder_settings, + VideoCodecAnalyzer* analyzer) + : encoder_factory_(encoder_factory), + analyzer_(analyzer), + pacer_(encoder_settings.pacing_settings) { + RTC_CHECK(analyzer_) << "Analyzer must be provided"; + + if (encoder_settings.encoder_input_base_path) { + y4m_writer_ = std::make_unique( + *encoder_settings.encoder_input_base_path); + } + + if (encoder_settings.encoder_output_base_path) { + ivf_writer_ = std::make_unique( + *encoder_settings.encoder_output_base_path); + } + } + + void Initialize(const EncodingSettings& encoding_settings) { + encoder_ = encoder_factory_->CreateVideoEncoder( + encoding_settings.sdp_video_format); + RTC_CHECK(encoder_) << "Could not create encoder for video format " + << encoding_settings.sdp_video_format.ToString(); + + task_queue_.PostTaskAndWait([this, encoding_settings] { + encoder_->RegisterEncodeCompleteCallback(this); + Configure(encoding_settings); + SetRates(encoding_settings); + }); + } + + void Encode(const VideoFrame& input_frame, + const EncodingSettings& encoding_settings, + EncodeCallback callback) { + { + MutexLock lock(&mutex_); + callbacks_[input_frame.timestamp()] = std::move(callback); + } + + Timestamp pts = Timestamp::Micros((input_frame.timestamp() / k90kHz).us()); + + task_queue_.PostScheduledTask( + [this, input_frame, encoding_settings] { + analyzer_->StartEncode(input_frame, encoding_settings); + + if (!last_encoding_settings_ || + !IsSameRate(encoding_settings, *last_encoding_settings_)) { + SetRates(encoding_settings); + } + + int error = encoder_->Encode(input_frame, /*frame_types=*/nullptr); + if (error != 0) { + RTC_LOG(LS_WARNING) << "Encode failed with error code " << error + << " RTP timestamp " << input_frame.timestamp(); + } + + last_encoding_settings_ = encoding_settings; + }, + pacer_.Schedule(pts)); + + if (y4m_writer_) { + y4m_writer_->Write(input_frame, /*spatial_idx=*/0); + } + } + + void Flush() { + task_queue_.PostTaskAndWait([this] { encoder_->Release(); }); + } + + private: + Result OnEncodedImage(const EncodedImage& encoded_frame, + const CodecSpecificInfo* codec_specific_info) override { + analyzer_->FinishEncode(encoded_frame); + + { + MutexLock lock(&mutex_); + auto it = callbacks_.find(encoded_frame.RtpTimestamp()); + RTC_CHECK(it != callbacks_.end()); + it->second(encoded_frame); + callbacks_.erase(callbacks_.begin(), it); + } + + if (ivf_writer_ != nullptr) { + ivf_writer_->Write(encoded_frame); + } + + return Result(Result::Error::OK); + } + + void Configure(const EncodingSettings& es) { + const LayerSettings& layer_settings = es.layers_settings.rbegin()->second; + const DataRate& bitrate = layer_settings.bitrate; + + VideoCodec vc; + vc.width = layer_settings.resolution.width; + vc.height = layer_settings.resolution.height; + vc.startBitrate = bitrate.kbps(); + vc.maxBitrate = bitrate.kbps(); + vc.minBitrate = 0; + vc.maxFramerate = layer_settings.framerate.hertz(); + vc.active = true; + vc.numberOfSimulcastStreams = 0; + vc.mode = webrtc::VideoCodecMode::kRealtimeVideo; + vc.SetFrameDropEnabled(true); + vc.SetScalabilityMode(es.scalability_mode); + vc.SetVideoEncoderComplexity(VideoCodecComplexity::kComplexityNormal); + + vc.codecType = PayloadStringToCodecType(es.sdp_video_format.name); + switch (vc.codecType) { + case kVideoCodecVP8: + *(vc.VP8()) = VideoEncoder::GetDefaultVp8Settings(); + vc.VP8()->SetNumberOfTemporalLayers( + ScalabilityModeToNumTemporalLayers(es.scalability_mode)); + vc.qpMax = cricket::kDefaultVideoMaxQpVpx; + // TODO(webrtc:14852): Configure simulcast. + break; + case kVideoCodecVP9: + *(vc.VP9()) = VideoEncoder::GetDefaultVp9Settings(); + // See LibvpxVp9Encoder::ExplicitlyConfiguredSpatialLayers. + vc.spatialLayers[0].targetBitrate = vc.maxBitrate; + vc.qpMax = cricket::kDefaultVideoMaxQpVpx; + break; + case kVideoCodecAV1: + vc.qpMax = cricket::kDefaultVideoMaxQpVpx; + break; + case kVideoCodecH264: + *(vc.H264()) = VideoEncoder::GetDefaultH264Settings(); + vc.qpMax = cricket::kDefaultVideoMaxQpH26x; + break; + case kVideoCodecH265: + vc.qpMax = cricket::kDefaultVideoMaxQpH26x; + break; + case kVideoCodecGeneric: + case kVideoCodecMultiplex: + RTC_CHECK_NOTREACHED(); + break; + } + + VideoEncoder::Settings ves( + VideoEncoder::Capabilities(/*loss_notification=*/false), + /*number_of_cores=*/1, + /*max_payload_size=*/1440); + + int result = encoder_->InitEncode(&vc, ves); + RTC_CHECK(result == WEBRTC_VIDEO_CODEC_OK); + + SetRates(es); + } + + void SetRates(const EncodingSettings& es) { + VideoEncoder::RateControlParameters rc; + int num_spatial_layers = + ScalabilityModeToNumSpatialLayers(es.scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumTemporalLayers(es.scalability_mode); + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + auto layers_settings = es.layers_settings.find( + {.spatial_idx = sidx, .temporal_idx = tidx}); + RTC_CHECK(layers_settings != es.layers_settings.end()) + << "Bitrate for layer S=" << sidx << " T=" << tidx << " is not set"; + rc.bitrate.SetBitrate(sidx, tidx, + layers_settings->second.bitrate.bps()); + } + } + rc.framerate_fps = + es.layers_settings.rbegin()->second.framerate.hertz(); + encoder_->SetRates(rc); + } + + bool IsSameRate(const EncodingSettings& a, const EncodingSettings& b) const { + for (auto [layer_id, layer] : a.layers_settings) { + const auto& other_layer = b.layers_settings.at(layer_id); + if (layer.bitrate != other_layer.bitrate || + layer.framerate != other_layer.framerate) { + return false; + } + } + + return true; + } + + VideoEncoderFactory* const encoder_factory_; + std::unique_ptr encoder_; + VideoCodecAnalyzer* const analyzer_; + Pacer pacer_; + absl::optional last_encoding_settings_; + std::unique_ptr bitrate_allocator_; + LimitedTaskQueue task_queue_; + std::unique_ptr y4m_writer_; + std::unique_ptr ivf_writer_; + std::map sidx_ RTC_GUARDED_BY(mutex_); + std::map callbacks_ RTC_GUARDED_BY(mutex_); + Mutex mutex_; +}; + +std::tuple, ScalabilityMode> +SplitBitrateAndUpdateScalabilityMode(std::string codec_type, + ScalabilityMode scalability_mode, + int width, + int height, + std::vector bitrates_kbps, + double framerate_fps) { + int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumTemporalLayers(scalability_mode); + + if (bitrates_kbps.size() > 1 || + (num_spatial_layers == 1 && num_temporal_layers == 1)) { + RTC_CHECK(bitrates_kbps.size() == + static_cast(num_spatial_layers * num_temporal_layers)) + << "bitrates must be provided for all layers"; + std::vector bitrates; + for (const auto& bitrate_kbps : bitrates_kbps) { + bitrates.push_back(DataRate::KilobitsPerSec(bitrate_kbps)); + } + return std::make_tuple(bitrates, scalability_mode); + } + + VideoCodec vc; + vc.codecType = PayloadStringToCodecType(codec_type); + vc.width = width; + vc.height = height; + vc.startBitrate = bitrates_kbps.front(); + vc.maxBitrate = bitrates_kbps.front(); + vc.minBitrate = 0; + vc.maxFramerate = static_cast(framerate_fps); + vc.numberOfSimulcastStreams = 0; + vc.mode = webrtc::VideoCodecMode::kRealtimeVideo; + vc.SetScalabilityMode(scalability_mode); + + switch (vc.codecType) { + case kVideoCodecVP8: + // TODO(webrtc:14852): Configure simulcast. + *(vc.VP8()) = VideoEncoder::GetDefaultVp8Settings(); + vc.VP8()->SetNumberOfTemporalLayers(num_temporal_layers); + vc.simulcastStream[0].width = vc.width; + vc.simulcastStream[0].height = vc.height; + break; + case kVideoCodecVP9: { + *(vc.VP9()) = VideoEncoder::GetDefaultVp9Settings(); + vc.VP9()->SetNumberOfTemporalLayers(num_temporal_layers); + const std::vector spatialLayers = GetVp9SvcConfig(vc); + for (size_t i = 0; i < spatialLayers.size(); ++i) { + vc.spatialLayers[i] = spatialLayers[i]; + vc.spatialLayers[i].active = true; + } + } break; + case kVideoCodecAV1: { + bool result = + SetAv1SvcConfig(vc, num_spatial_layers, num_temporal_layers); + RTC_CHECK(result) << "SetAv1SvcConfig failed"; + } break; + case kVideoCodecH264: { + *(vc.H264()) = VideoEncoder::GetDefaultH264Settings(); + vc.H264()->SetNumberOfTemporalLayers(num_temporal_layers); + } break; + case kVideoCodecH265: + break; + case kVideoCodecGeneric: + case kVideoCodecMultiplex: + RTC_CHECK_NOTREACHED(); + } + + if (*vc.GetScalabilityMode() != scalability_mode) { + RTC_LOG(LS_WARNING) << "Scalability mode changed from " + << ScalabilityModeToString(scalability_mode) << " to " + << ScalabilityModeToString(*vc.GetScalabilityMode()); + num_spatial_layers = + ScalabilityModeToNumSpatialLayers(*vc.GetScalabilityMode()); + num_temporal_layers = + ScalabilityModeToNumTemporalLayers(*vc.GetScalabilityMode()); + } + + std::unique_ptr bitrate_allocator = + CreateBuiltinVideoBitrateAllocatorFactory()->CreateVideoBitrateAllocator( + vc); + VideoBitrateAllocation bitrate_allocation = + bitrate_allocator->Allocate(VideoBitrateAllocationParameters( + 1000 * bitrates_kbps.front(), framerate_fps)); + + std::vector bitrates; + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + int bitrate_bps = bitrate_allocation.GetBitrate(sidx, tidx); + bitrates.push_back(DataRate::BitsPerSec(bitrate_bps)); + } + } + + return std::make_tuple(bitrates, *vc.GetScalabilityMode()); +} + +} // namespace + +void VideoCodecStats::Stream::LogMetrics( + MetricsLogger* logger, + std::string test_case_name, + std::map metadata) const { + logger->LogMetric("width", test_case_name, width, Unit::kCount, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("height", test_case_name, height, Unit::kCount, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("frame_size_bytes", test_case_name, frame_size_bytes, + Unit::kBytes, ImprovementDirection::kNeitherIsBetter, + metadata); + logger->LogMetric("keyframe", test_case_name, keyframe, Unit::kCount, + ImprovementDirection::kSmallerIsBetter, metadata); + logger->LogMetric("qp", test_case_name, qp, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, metadata); + logger->LogMetric("encode_time_ms", test_case_name, encode_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter, + metadata); + logger->LogMetric("decode_time_ms", test_case_name, decode_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter, + metadata); + // TODO(webrtc:14852): Change to kUnitLess. kKilobitsPerSecond are converted + // to bytes per second in Chromeperf dash. + logger->LogMetric("target_bitrate_kbps", test_case_name, target_bitrate_kbps, + Unit::kKilobitsPerSecond, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("target_framerate_fps", test_case_name, + target_framerate_fps, Unit::kHertz, + ImprovementDirection::kBiggerIsBetter, metadata); + // TODO(webrtc:14852): Change to kUnitLess. kKilobitsPerSecond are converted + // to bytes per second in Chromeperf dash. + logger->LogMetric("encoded_bitrate_kbps", test_case_name, + encoded_bitrate_kbps, Unit::kKilobitsPerSecond, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("encoded_framerate_fps", test_case_name, + encoded_framerate_fps, Unit::kHertz, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("bitrate_mismatch_pct", test_case_name, + bitrate_mismatch_pct, Unit::kPercent, + ImprovementDirection::kNeitherIsBetter, metadata); + logger->LogMetric("framerate_mismatch_pct", test_case_name, + framerate_mismatch_pct, Unit::kPercent, + ImprovementDirection::kNeitherIsBetter, metadata); + logger->LogMetric("transmission_time_ms", test_case_name, + transmission_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metadata); + logger->LogMetric("psnr_y_db", test_case_name, psnr.y, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("psnr_u_db", test_case_name, psnr.u, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric("psnr_v_db", test_case_name, psnr.v, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metadata); +} + +// TODO(ssilkin): use Frequency and DataRate for framerate and bitrate. +std::map VideoCodecTester::CreateEncodingSettings( + std::string codec_type, + std::string scalability_name, + int width, + int height, + std::vector layer_bitrates_kbps, + double framerate_fps, + int num_frames, + uint32_t first_timestamp_rtp) { + auto [layer_bitrates, scalability_mode] = + SplitBitrateAndUpdateScalabilityMode( + codec_type, *ScalabilityModeFromString(scalability_name), width, + height, layer_bitrates_kbps, framerate_fps); + + int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumTemporalLayers(scalability_mode); + + std::map layers_settings; + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + int layer_width = width >> (num_spatial_layers - sidx - 1); + int layer_height = height >> (num_spatial_layers - sidx - 1); + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + double layer_framerate_fps = + framerate_fps / (1 << (num_temporal_layers - tidx - 1)); + layers_settings.emplace( + LayerId{.spatial_idx = sidx, .temporal_idx = tidx}, + LayerSettings{ + .resolution = {.width = layer_width, .height = layer_height}, + .framerate = Frequency::MilliHertz(1000 * layer_framerate_fps), + .bitrate = layer_bitrates[sidx * num_temporal_layers + tidx]}); + } + } + + std::map frames_settings; + uint32_t timestamp_rtp = first_timestamp_rtp; + for (int frame_num = 0; frame_num < num_frames; ++frame_num) { + frames_settings.emplace( + timestamp_rtp, + EncodingSettings{.sdp_video_format = SdpVideoFormat(codec_type), + .scalability_mode = scalability_mode, + .layers_settings = layers_settings}); + + timestamp_rtp += k90kHz / Frequency::MilliHertz(1000 * framerate_fps); + } + + return frames_settings; +} + +std::unique_ptr +VideoCodecTester::RunDecodeTest(CodedVideoSource* video_source, + VideoDecoderFactory* decoder_factory, + const DecoderSettings& decoder_settings, + const SdpVideoFormat& sdp_video_format) { + std::unique_ptr analyzer = + std::make_unique(/*video_source=*/nullptr); + Decoder decoder(decoder_factory, decoder_settings, analyzer.get()); + decoder.Initialize(sdp_video_format); + + while (auto frame = video_source->PullFrame()) { + decoder.Decode(*frame); + } + + decoder.Flush(); + analyzer->Flush(); + return std::move(analyzer); +} + +std::unique_ptr +VideoCodecTester::RunEncodeTest( + const VideoSourceSettings& source_settings, + VideoEncoderFactory* encoder_factory, + const EncoderSettings& encoder_settings, + const std::map& encoding_settings) { + VideoSource video_source(source_settings); + std::unique_ptr analyzer = + std::make_unique(/*video_source=*/nullptr); + Encoder encoder(encoder_factory, encoder_settings, analyzer.get()); + encoder.Initialize(encoding_settings.begin()->second); + + for (const auto& [timestamp_rtp, frame_settings] : encoding_settings) { + const EncodingSettings::LayerSettings& top_layer = + frame_settings.layers_settings.rbegin()->second; + VideoFrame source_frame = video_source.PullFrame( + timestamp_rtp, top_layer.resolution, top_layer.framerate); + encoder.Encode(source_frame, frame_settings, + [](const EncodedImage& encoded_frame) {}); + } + + encoder.Flush(); + analyzer->Flush(); + return std::move(analyzer); +} + +std::unique_ptr +VideoCodecTester::RunEncodeDecodeTest( + const VideoSourceSettings& source_settings, + VideoEncoderFactory* encoder_factory, + VideoDecoderFactory* decoder_factory, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings, + const std::map& encoding_settings) { + VideoSource video_source(source_settings); + std::unique_ptr analyzer = + std::make_unique(&video_source); + Decoder decoder(decoder_factory, decoder_settings, analyzer.get()); + Encoder encoder(encoder_factory, encoder_settings, analyzer.get()); + encoder.Initialize(encoding_settings.begin()->second); + decoder.Initialize(encoding_settings.begin()->second.sdp_video_format); + + for (const auto& [timestamp_rtp, frame_settings] : encoding_settings) { + const EncodingSettings::LayerSettings& top_layer = + frame_settings.layers_settings.rbegin()->second; + VideoFrame source_frame = video_source.PullFrame( + timestamp_rtp, top_layer.resolution, top_layer.framerate); + encoder.Encode(source_frame, frame_settings, + [&decoder](const EncodedImage& encoded_frame) { + decoder.Decode(encoded_frame); + }); + } + + encoder.Flush(); + decoder.Flush(); + analyzer->Flush(); + return std::move(analyzer); +} + +} // namespace test +} // namespace webrtc diff --git a/test/video_codec_tester.h b/test/video_codec_tester.h new file mode 100644 index 0000000000..dc72645c18 --- /dev/null +++ b/test/video_codec_tester.h @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_VIDEO_CODEC_TESTER_H_ +#define TEST_VIDEO_CODEC_TESTER_H_ + +#include +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/units/data_rate.h" +#include "api/units/data_size.h" +#include "api/units/frequency.h" +#include "api/video/encoded_image.h" +#include "api/video/resolution.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" + +namespace webrtc { +namespace test { + +class VideoCodecTester { + public: + struct LayerId { + int spatial_idx = 0; + int temporal_idx = 0; + + bool operator==(const LayerId& o) const { + return spatial_idx == o.spatial_idx && temporal_idx == o.temporal_idx; + } + bool operator<(const LayerId& o) const { + return spatial_idx < o.spatial_idx || + (spatial_idx == o.spatial_idx && temporal_idx < o.temporal_idx); + } + }; + + struct EncodingSettings { + SdpVideoFormat sdp_video_format = SdpVideoFormat("VP8"); + ScalabilityMode scalability_mode = ScalabilityMode::kL1T1; + + struct LayerSettings { + Resolution resolution; + Frequency framerate; + DataRate bitrate; + }; + std::map layers_settings; + }; + + class VideoCodecStats { + public: + struct Filter { + uint32_t min_timestamp_rtp = std::numeric_limits::min(); + uint32_t max_timestamp_rtp = std::numeric_limits::max(); + absl::optional layer_id; + }; + + struct Frame { + int frame_num = 0; + uint32_t timestamp_rtp = 0; + LayerId layer_id; + bool encoded = false; + bool decoded = false; + int width = 0; + int height = 0; + DataSize frame_size = DataSize::Zero(); + bool keyframe = false; + absl::optional qp; + Timestamp encode_start = Timestamp::Zero(); + TimeDelta encode_time = TimeDelta::Zero(); + Timestamp decode_start = Timestamp::Zero(); + TimeDelta decode_time = TimeDelta::Zero(); + absl::optional target_bitrate; + absl::optional target_framerate; + + struct Psnr { + double y = 0.0; + double u = 0.0; + double v = 0.0; + }; + absl::optional psnr; + }; + + struct Stream { + SamplesStatsCounter width; + SamplesStatsCounter height; + SamplesStatsCounter frame_size_bytes; + SamplesStatsCounter keyframe; + SamplesStatsCounter qp; + SamplesStatsCounter encode_time_ms; + SamplesStatsCounter decode_time_ms; + SamplesStatsCounter target_bitrate_kbps; + SamplesStatsCounter target_framerate_fps; + SamplesStatsCounter encoded_bitrate_kbps; + SamplesStatsCounter encoded_framerate_fps; + SamplesStatsCounter bitrate_mismatch_pct; + SamplesStatsCounter framerate_mismatch_pct; + SamplesStatsCounter transmission_time_ms; + + struct Psnr { + SamplesStatsCounter y; + SamplesStatsCounter u; + SamplesStatsCounter v; + } psnr; + + // Logs `Stream` metrics to provided `MetricsLogger`. + void LogMetrics(MetricsLogger* logger, + std::string test_case_name, + std::map metadata = {}) const; + }; + + virtual ~VideoCodecStats() = default; + + // Returns frames for the slice specified by `filter`. If `merge` is true, + // also merges frames belonging to the same temporal unit into one + // superframe. + virtual std::vector Slice(Filter filter, bool merge) const = 0; + + // Returns video statistics aggregated for the slice specified by `filter`. + virtual Stream Aggregate(Filter filter) const = 0; + }; + + // Pacing settings for codec input. + struct PacingSettings { + enum PacingMode { + // Pacing is not used. Frames are sent to codec back-to-back. + kNoPacing, + // Pace with the rate equal to the target video frame rate. Pacing time is + // derived from RTP timestamp. + kRealTime, + // Pace with the explicitly provided rate. + kConstantRate, + }; + PacingMode mode = PacingMode::kNoPacing; + // Pacing rate for `kConstantRate` mode. + Frequency constant_rate = Frequency::Zero(); + }; + + struct VideoSourceSettings { + std::string file_path; + Resolution resolution; + Frequency framerate; + }; + + struct DecoderSettings { + PacingSettings pacing_settings; + absl::optional decoder_input_base_path; + absl::optional decoder_output_base_path; + }; + + struct EncoderSettings { + PacingSettings pacing_settings; + absl::optional encoder_input_base_path; + absl::optional encoder_output_base_path; + }; + + virtual ~VideoCodecTester() = default; + + // Interface for a coded video frames source. + class CodedVideoSource { + public: + virtual ~CodedVideoSource() = default; + + // Returns next frame. Returns `absl::nullopt` if the end-of-stream is + // reached. Frames should have RTP timestamps representing desired frame + // rate. + virtual absl::optional PullFrame() = 0; + }; + + // A helper function that creates `EncodingSettings` for `num_frames` frames, + // wraps the settings into RTP timestamp -> settings map and returns the map. + static std::map CreateEncodingSettings( + std::string codec_type, + std::string scalability_name, + int width, + int height, + std::vector bitrates_kbps, + double framerate_fps, + int num_frames, + uint32_t first_timestamp_rtp = 90000); + + // Decodes video, collects and returns decode metrics. + static std::unique_ptr RunDecodeTest( + CodedVideoSource* video_source, + VideoDecoderFactory* decoder_factory, + const DecoderSettings& decoder_settings, + const SdpVideoFormat& sdp_video_format); + + // Encodes video, collects and returns encode metrics. + static std::unique_ptr RunEncodeTest( + const VideoSourceSettings& source_settings, + VideoEncoderFactory* encoder_factory, + const EncoderSettings& encoder_settings, + const std::map& encoding_settings); + + // Encodes and decodes video, collects and returns encode and decode metrics. + static std::unique_ptr RunEncodeDecodeTest( + const VideoSourceSettings& source_settings, + VideoEncoderFactory* encoder_factory, + VideoDecoderFactory* decoder_factory, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings, + const std::map& encoding_settings); +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_VIDEO_CODEC_TESTER_H_ diff --git a/test/video_codec_tester_unittest.cc b/test/video_codec_tester_unittest.cc new file mode 100644 index 0000000000..af31fe2c13 --- /dev/null +++ b/test/video_codec_tester_unittest.cc @@ -0,0 +1,513 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/video_codec_tester.h" + +#include +#include +#include +#include +#include +#include + +#include "api/test/mock_video_decoder.h" +#include "api/test/mock_video_decoder_factory.h" +#include "api/test/mock_video_encoder.h" +#include "api/test/mock_video_encoder_factory.h" +#include "api/units/data_rate.h" +#include "api/units/time_delta.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" +#include "third_party/libyuv/include/libyuv/planar_functions.h" + +namespace webrtc { +namespace test { + +namespace { +using ::testing::_; +using ::testing::ElementsAre; +using ::testing::Field; +using ::testing::Invoke; +using ::testing::InvokeWithoutArgs; +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::SizeIs; + +using VideoCodecStats = VideoCodecTester::VideoCodecStats; +using VideoSourceSettings = VideoCodecTester::VideoSourceSettings; +using CodedVideoSource = VideoCodecTester::CodedVideoSource; +using EncodingSettings = VideoCodecTester::EncodingSettings; +using LayerSettings = EncodingSettings::LayerSettings; +using LayerId = VideoCodecTester::LayerId; +using DecoderSettings = VideoCodecTester::DecoderSettings; +using EncoderSettings = VideoCodecTester::EncoderSettings; +using PacingSettings = VideoCodecTester::PacingSettings; +using PacingMode = PacingSettings::PacingMode; +using Filter = VideoCodecStats::Filter; +using Frame = VideoCodecTester::VideoCodecStats::Frame; +using Stream = VideoCodecTester::VideoCodecStats::Stream; + +constexpr int kWidth = 2; +constexpr int kHeight = 2; +const DataRate kTargetLayerBitrate = DataRate::BytesPerSec(100); +const Frequency kTargetFramerate = Frequency::Hertz(30); +constexpr Frequency k90kHz = Frequency::Hertz(90000); + +rtc::scoped_refptr CreateYuvBuffer(uint8_t y = 0, + uint8_t u = 0, + uint8_t v = 0) { + rtc::scoped_refptr buffer(I420Buffer::Create(2, 2)); + + libyuv::I420Rect(buffer->MutableDataY(), buffer->StrideY(), + buffer->MutableDataU(), buffer->StrideU(), + buffer->MutableDataV(), buffer->StrideV(), 0, 0, + buffer->width(), buffer->height(), y, u, v); + return buffer; +} + +std::string CreateYuvFile(int width, int height, int num_frames) { + std::string path = webrtc::test::TempFilename(webrtc::test::OutputPath(), + "video_codec_tester_unittest"); + FILE* file = fopen(path.c_str(), "wb"); + for (int frame_num = 0; frame_num < num_frames; ++frame_num) { + uint8_t y = (frame_num + 0) & 255; + uint8_t u = (frame_num + 1) & 255; + uint8_t v = (frame_num + 2) & 255; + rtc::scoped_refptr buffer = CreateYuvBuffer(y, u, v); + fwrite(buffer->DataY(), 1, width * height, file); + int chroma_size_bytes = (width + 1) / 2 * (height + 1) / 2; + fwrite(buffer->DataU(), 1, chroma_size_bytes, file); + fwrite(buffer->DataV(), 1, chroma_size_bytes, file); + } + fclose(file); + return path; +} + +std::unique_ptr RunTest(std::vector> frames, + ScalabilityMode scalability_mode) { + int num_frames = static_cast(frames.size()); + std::string source_yuv_path = CreateYuvFile(kWidth, kHeight, num_frames); + VideoSourceSettings source_settings{ + .file_path = source_yuv_path, + .resolution = {.width = kWidth, .height = kHeight}, + .framerate = kTargetFramerate}; + + int num_encoded_frames = 0; + EncodedImageCallback* encoded_frame_callback; + NiceMock encoder_factory; + ON_CALL(encoder_factory, CreateVideoEncoder) + .WillByDefault([&](const SdpVideoFormat&) { + auto encoder = std::make_unique>(); + ON_CALL(*encoder, RegisterEncodeCompleteCallback) + .WillByDefault([&](EncodedImageCallback* callback) { + encoded_frame_callback = callback; + return WEBRTC_VIDEO_CODEC_OK; + }); + ON_CALL(*encoder, Encode) + .WillByDefault([&](const VideoFrame& input_frame, + const std::vector*) { + for (const Frame& frame : frames[num_encoded_frames]) { + EncodedImage encoded_frame; + encoded_frame._encodedWidth = frame.width; + encoded_frame._encodedHeight = frame.height; + encoded_frame.SetFrameType( + frame.keyframe ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta); + encoded_frame.SetRtpTimestamp(input_frame.timestamp()); + encoded_frame.SetSpatialIndex(frame.layer_id.spatial_idx); + encoded_frame.SetTemporalIndex(frame.layer_id.temporal_idx); + encoded_frame.SetEncodedData( + EncodedImageBuffer::Create(frame.frame_size.bytes())); + encoded_frame_callback->OnEncodedImage( + encoded_frame, + /*codec_specific_info=*/nullptr); + } + ++num_encoded_frames; + return WEBRTC_VIDEO_CODEC_OK; + }); + return encoder; + }); + + int num_decoded_frames = 0; + DecodedImageCallback* decode_callback; + NiceMock decoder_factory; + ON_CALL(decoder_factory, CreateVideoDecoder) + .WillByDefault([&](const SdpVideoFormat&) { + auto decoder = std::make_unique>(); + ON_CALL(*decoder, RegisterDecodeCompleteCallback) + .WillByDefault([&](DecodedImageCallback* callback) { + decode_callback = callback; + return WEBRTC_VIDEO_CODEC_OK; + }); + ON_CALL(*decoder, Decode(_, _)) + .WillByDefault([&](const EncodedImage& encoded_frame, int64_t) { + // Make values to be different from source YUV generated in + // `CreateYuvFile`. + uint8_t y = ((num_decoded_frames + 1) * 2) & 255; + uint8_t u = ((num_decoded_frames + 2) * 2) & 255; + uint8_t v = ((num_decoded_frames + 3) * 2) & 255; + rtc::scoped_refptr frame_buffer = + CreateYuvBuffer(y, u, v); + VideoFrame decoded_frame = + VideoFrame::Builder() + .set_video_frame_buffer(frame_buffer) + .set_timestamp_rtp(encoded_frame.RtpTimestamp()) + .build(); + decode_callback->Decoded(decoded_frame); + ++num_decoded_frames; + return WEBRTC_VIDEO_CODEC_OK; + }); + return decoder; + }); + + int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumTemporalLayers(scalability_mode); + + std::map encoding_settings; + for (int frame_num = 0; frame_num < num_frames; ++frame_num) { + std::map layers_settings; + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + layers_settings.emplace( + LayerId{.spatial_idx = sidx, .temporal_idx = tidx}, + LayerSettings{.resolution = {.width = kWidth, .height = kHeight}, + .framerate = kTargetFramerate / + (1 << (num_temporal_layers - 1 - tidx)), + .bitrate = kTargetLayerBitrate}); + } + } + encoding_settings.emplace( + frames[frame_num][0].timestamp_rtp, + EncodingSettings{.scalability_mode = scalability_mode, + .layers_settings = layers_settings}); + } + + EncoderSettings encoder_settings; + DecoderSettings decoder_settings; + std::unique_ptr stats = + VideoCodecTester::RunEncodeDecodeTest( + source_settings, &encoder_factory, &decoder_factory, encoder_settings, + decoder_settings, encoding_settings); + remove(source_yuv_path.c_str()); + return stats; +} + +EncodedImage CreateEncodedImage(uint32_t timestamp_rtp) { + EncodedImage encoded_image; + encoded_image.SetRtpTimestamp(timestamp_rtp); + return encoded_image; +} + +class MockCodedVideoSource : public CodedVideoSource { + public: + MockCodedVideoSource(int num_frames, Frequency framerate) + : num_frames_(num_frames), frame_num_(0), framerate_(framerate) {} + + absl::optional PullFrame() override { + if (frame_num_ >= num_frames_) { + return absl::nullopt; + } + uint32_t timestamp_rtp = frame_num_ * k90kHz / framerate_; + ++frame_num_; + return CreateEncodedImage(timestamp_rtp); + } + + private: + int num_frames_; + int frame_num_; + Frequency framerate_; +}; + +} // namespace + +TEST(VideoCodecTester, Slice) { + std::unique_ptr stats = RunTest( + {{{.timestamp_rtp = 0, .layer_id = {.spatial_idx = 0, .temporal_idx = 0}}, + {.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 1, .temporal_idx = 0}}}, + {{.timestamp_rtp = 1, + .layer_id = {.spatial_idx = 0, .temporal_idx = 1}}}}, + ScalabilityMode::kL2T2); + std::vector slice = stats->Slice(Filter{}, /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0), + Field(&Frame::timestamp_rtp, 0), + Field(&Frame::timestamp_rtp, 1))); + + slice = stats->Slice({.min_timestamp_rtp = 1}, /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 1))); + + slice = stats->Slice({.max_timestamp_rtp = 0}, /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0), + Field(&Frame::timestamp_rtp, 0))); + + slice = stats->Slice({.layer_id = {{.spatial_idx = 0, .temporal_idx = 0}}}, + /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0))); + + slice = stats->Slice({.layer_id = {{.spatial_idx = 0, .temporal_idx = 1}}}, + /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0), + Field(&Frame::timestamp_rtp, 1))); +} + +TEST(VideoCodecTester, Merge) { + std::unique_ptr stats = + RunTest({{{.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 0, .temporal_idx = 0}, + .frame_size = DataSize::Bytes(1), + .keyframe = true}, + {.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 1, .temporal_idx = 0}, + .frame_size = DataSize::Bytes(2)}}, + {{.timestamp_rtp = 1, + .layer_id = {.spatial_idx = 0, .temporal_idx = 1}, + .frame_size = DataSize::Bytes(4)}, + {.timestamp_rtp = 1, + .layer_id = {.spatial_idx = 1, .temporal_idx = 1}, + .frame_size = DataSize::Bytes(8)}}}, + ScalabilityMode::kL2T2_KEY); + + std::vector slice = stats->Slice(Filter{}, /*merge=*/true); + EXPECT_THAT( + slice, + ElementsAre( + AllOf(Field(&Frame::timestamp_rtp, 0), Field(&Frame::keyframe, true), + Field(&Frame::frame_size, DataSize::Bytes(3))), + AllOf(Field(&Frame::timestamp_rtp, 1), Field(&Frame::keyframe, false), + Field(&Frame::frame_size, DataSize::Bytes(12))))); +} + +struct AggregationTestParameters { + Filter filter; + double expected_keyframe_sum; + double expected_encoded_bitrate_kbps; + double expected_encoded_framerate_fps; + double expected_bitrate_mismatch_pct; + double expected_framerate_mismatch_pct; +}; + +class VideoCodecTesterTestAggregation + : public ::testing::TestWithParam {}; + +TEST_P(VideoCodecTesterTestAggregation, Aggregate) { + AggregationTestParameters test_params = GetParam(); + std::unique_ptr stats = + RunTest({{// L0T0 + {.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 0, .temporal_idx = 0}, + .frame_size = DataSize::Bytes(1), + .keyframe = true}, + // L1T0 + {.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 1, .temporal_idx = 0}, + .frame_size = DataSize::Bytes(2)}}, + // Emulate frame drop (frame_size = 0). + {{.timestamp_rtp = 3000, + .layer_id = {.spatial_idx = 0, .temporal_idx = 0}, + .frame_size = DataSize::Zero()}}, + {// L0T1 + {.timestamp_rtp = 87000, + .layer_id = {.spatial_idx = 0, .temporal_idx = 1}, + .frame_size = DataSize::Bytes(4)}, + // L1T1 + {.timestamp_rtp = 87000, + .layer_id = {.spatial_idx = 1, .temporal_idx = 1}, + .frame_size = DataSize::Bytes(8)}}}, + ScalabilityMode::kL2T2_KEY); + + Stream stream = stats->Aggregate(test_params.filter); + EXPECT_EQ(stream.keyframe.GetSum(), test_params.expected_keyframe_sum); + EXPECT_EQ(stream.encoded_bitrate_kbps.GetAverage(), + test_params.expected_encoded_bitrate_kbps); + EXPECT_EQ(stream.encoded_framerate_fps.GetAverage(), + test_params.expected_encoded_framerate_fps); + EXPECT_EQ(stream.bitrate_mismatch_pct.GetAverage(), + test_params.expected_bitrate_mismatch_pct); + EXPECT_EQ(stream.framerate_mismatch_pct.GetAverage(), + test_params.expected_framerate_mismatch_pct); +} + +INSTANTIATE_TEST_SUITE_P( + All, + VideoCodecTesterTestAggregation, + ::testing::Values( + // No filtering. + AggregationTestParameters{ + .filter = {}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(15).kbps(), + .expected_encoded_framerate_fps = 2, + .expected_bitrate_mismatch_pct = + 100 * (15.0 / (kTargetLayerBitrate.bytes_per_sec() * 4) - 1), + .expected_framerate_mismatch_pct = + 100 * (2.0 / kTargetFramerate.hertz() - 1)}, + // L0T0 + AggregationTestParameters{ + .filter = {.layer_id = {{.spatial_idx = 0, .temporal_idx = 0}}}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(1).kbps(), + .expected_encoded_framerate_fps = 1, + .expected_bitrate_mismatch_pct = + 100 * (1.0 / kTargetLayerBitrate.bytes_per_sec() - 1), + .expected_framerate_mismatch_pct = + 100 * (1.0 / (kTargetFramerate.hertz() / 2) - 1)}, + // L0T1 + AggregationTestParameters{ + .filter = {.layer_id = {{.spatial_idx = 0, .temporal_idx = 1}}}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(5).kbps(), + .expected_encoded_framerate_fps = 2, + .expected_bitrate_mismatch_pct = + 100 * (5.0 / (kTargetLayerBitrate.bytes_per_sec() * 2) - 1), + .expected_framerate_mismatch_pct = + 100 * (2.0 / kTargetFramerate.hertz() - 1)}, + // L1T0 + AggregationTestParameters{ + .filter = {.layer_id = {{.spatial_idx = 1, .temporal_idx = 0}}}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(3).kbps(), + .expected_encoded_framerate_fps = 1, + .expected_bitrate_mismatch_pct = + 100 * (3.0 / kTargetLayerBitrate.bytes_per_sec() - 1), + .expected_framerate_mismatch_pct = + 100 * (1.0 / (kTargetFramerate.hertz() / 2) - 1)}, + // L1T1 + AggregationTestParameters{ + .filter = {.layer_id = {{.spatial_idx = 1, .temporal_idx = 1}}}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(11).kbps(), + .expected_encoded_framerate_fps = 2, + .expected_bitrate_mismatch_pct = + 100 * (11.0 / (kTargetLayerBitrate.bytes_per_sec() * 2) - 1), + .expected_framerate_mismatch_pct = + 100 * (2.0 / kTargetFramerate.hertz() - 1)})); + +TEST(VideoCodecTester, Psnr) { + std::unique_ptr stats = + RunTest({{{.timestamp_rtp = 0, .frame_size = DataSize::Bytes(1)}}, + {{.timestamp_rtp = 3000, .frame_size = DataSize::Bytes(1)}}}, + ScalabilityMode::kL1T1); + + std::vector slice = stats->Slice(Filter{}, /*merge=*/false); + ASSERT_THAT(slice, SizeIs(2)); + ASSERT_TRUE(slice[0].psnr.has_value()); + ASSERT_TRUE(slice[1].psnr.has_value()); + EXPECT_NEAR(slice[0].psnr->y, 42, 1); + EXPECT_NEAR(slice[0].psnr->u, 38, 1); + EXPECT_NEAR(slice[0].psnr->v, 36, 1); + EXPECT_NEAR(slice[1].psnr->y, 38, 1); + EXPECT_NEAR(slice[1].psnr->u, 36, 1); + EXPECT_NEAR(slice[1].psnr->v, 34, 1); +} + +class VideoCodecTesterTestPacing + : public ::testing::TestWithParam> { + public: + const int kSourceWidth = 2; + const int kSourceHeight = 2; + const int kNumFrames = 3; + const int kTargetLayerBitrateKbps = 128; + const Frequency kTargetFramerate = Frequency::Hertz(10); + + void SetUp() override { + source_yuv_file_path_ = webrtc::test::TempFilename( + webrtc::test::OutputPath(), "video_codec_tester_impl_unittest"); + FILE* file = fopen(source_yuv_file_path_.c_str(), "wb"); + for (int i = 0; i < 3 * kSourceWidth * kSourceHeight / 2; ++i) { + fwrite("x", 1, 1, file); + } + fclose(file); + } + + protected: + std::string source_yuv_file_path_; +}; + +TEST_P(VideoCodecTesterTestPacing, PaceEncode) { + auto [pacing_settings, expected_delta_ms] = GetParam(); + VideoSourceSettings video_source{ + .file_path = source_yuv_file_path_, + .resolution = {.width = kSourceWidth, .height = kSourceHeight}, + .framerate = kTargetFramerate}; + + NiceMock encoder_factory; + ON_CALL(encoder_factory, CreateVideoEncoder(_)) + .WillByDefault([](const SdpVideoFormat&) { + return std::make_unique>(); + }); + + std::map encoding_settings = + VideoCodecTester::CreateEncodingSettings( + "VP8", "L1T1", kSourceWidth, kSourceHeight, {kTargetLayerBitrateKbps}, + kTargetFramerate.hertz(), kNumFrames); + + EncoderSettings encoder_settings; + encoder_settings.pacing_settings = pacing_settings; + std::vector frames = + VideoCodecTester::RunEncodeTest(video_source, &encoder_factory, + encoder_settings, encoding_settings) + ->Slice(/*filter=*/{}, /*merge=*/false); + ASSERT_THAT(frames, SizeIs(kNumFrames)); + EXPECT_NEAR((frames[1].encode_start - frames[0].encode_start).ms(), + expected_delta_ms, 10); + EXPECT_NEAR((frames[2].encode_start - frames[1].encode_start).ms(), + expected_delta_ms, 10); +} + +TEST_P(VideoCodecTesterTestPacing, PaceDecode) { + auto [pacing_settings, expected_delta_ms] = GetParam(); + MockCodedVideoSource video_source(kNumFrames, kTargetFramerate); + + NiceMock decoder_factory; + ON_CALL(decoder_factory, CreateVideoDecoder(_)) + .WillByDefault([](const SdpVideoFormat&) { + return std::make_unique>(); + }); + + DecoderSettings decoder_settings; + decoder_settings.pacing_settings = pacing_settings; + std::vector frames = + VideoCodecTester::RunDecodeTest(&video_source, &decoder_factory, + decoder_settings, SdpVideoFormat("VP8")) + ->Slice(/*filter=*/{}, /*merge=*/false); + ASSERT_THAT(frames, SizeIs(kNumFrames)); + EXPECT_NEAR((frames[1].decode_start - frames[0].decode_start).ms(), + expected_delta_ms, 10); + EXPECT_NEAR((frames[2].decode_start - frames[1].decode_start).ms(), + expected_delta_ms, 10); +} + +INSTANTIATE_TEST_SUITE_P( + DISABLED_All, + VideoCodecTesterTestPacing, + ::testing::Values( + // No pacing. + std::make_tuple(PacingSettings{.mode = PacingMode::kNoPacing}, + /*expected_delta_ms=*/0), + // Real-time pacing. + std::make_tuple(PacingSettings{.mode = PacingMode::kRealTime}, + /*expected_delta_ms=*/100), + // Pace with specified constant rate. + std::make_tuple(PacingSettings{.mode = PacingMode::kConstantRate, + .constant_rate = Frequency::Hertz(20)}, + /*expected_delta_ms=*/50))); +} // namespace test +} // namespace webrtc