diff --git a/api/BUILD.gn b/api/BUILD.gn index 949e5ad5e9..f54766a32b 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -1019,7 +1019,10 @@ if (rtc_include_tests) { rtc_library("video_codec_stats_api") { visibility = [ "*" ] testonly = true - sources = [ "test/video_codec_stats.h" ] + sources = [ + "test/video_codec_stats.cc", + "test/video_codec_stats.h", + ] deps = [ "../api/numerics:numerics", "../api/units:data_rate", diff --git a/api/test/video_codec_stats.cc b/api/test/video_codec_stats.cc new file mode 100644 index 0000000000..fb7226701e --- /dev/null +++ b/api/test/video_codec_stats.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/video_codec_stats.h" + +namespace webrtc { +namespace test { + +void VideoCodecStats::Stream::LogMetrics( + MetricsLogger* logger, + std::string test_case_name, + std::map metadata) const { + logger->LogMetric("width", test_case_name, width, Unit::kCount, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("height", test_case_name, height, Unit::kCount, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric( + "frame_size_bytes", test_case_name, frame_size_bytes, Unit::kBytes, + webrtc::test::ImprovementDirection::kNeitherIsBetter, metadata); + + logger->LogMetric("keyframe", test_case_name, keyframe, Unit::kCount, + webrtc::test::ImprovementDirection::kSmallerIsBetter, + metadata); + + logger->LogMetric("qp", test_case_name, qp, Unit::kUnitless, + webrtc::test::ImprovementDirection::kSmallerIsBetter, + metadata); + + logger->LogMetric( + "encode_time_ms", test_case_name, encode_time_ms, Unit::kMilliseconds, + webrtc::test::ImprovementDirection::kSmallerIsBetter, metadata); + + logger->LogMetric( + "decode_time_ms", test_case_name, decode_time_ms, Unit::kMilliseconds, + webrtc::test::ImprovementDirection::kSmallerIsBetter, metadata); + + logger->LogMetric("target_bitrate_kbps", test_case_name, target_bitrate_kbps, + Unit::kKilobitsPerSecond, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("target_framerate_fps", test_case_name, + target_framerate_fps, Unit::kHertz, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("encoded_bitrate_kbps", test_case_name, + encoded_bitrate_kbps, Unit::kKilobitsPerSecond, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("encoded_framerate_fps", test_case_name, + encoded_framerate_fps, Unit::kHertz, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("bitrate_mismatch_pct", test_case_name, + bitrate_mismatch_pct, Unit::kPercent, + webrtc::test::ImprovementDirection::kSmallerIsBetter, + metadata); + + logger->LogMetric("framerate_mismatch_pct", test_case_name, + framerate_mismatch_pct, Unit::kPercent, + webrtc::test::ImprovementDirection::kSmallerIsBetter, + metadata); + + logger->LogMetric("transmission_time_ms", test_case_name, + transmission_time_ms, Unit::kMilliseconds, + webrtc::test::ImprovementDirection::kSmallerIsBetter, + metadata); + + logger->LogMetric("psnr_y_db", test_case_name, psnr.y, Unit::kUnitless, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("psnr_u_db", test_case_name, psnr.u, Unit::kUnitless, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("psnr_v_db", test_case_name, psnr.v, Unit::kUnitless, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); +} + +} // namespace test +} // namespace webrtc diff --git a/api/test/video_codec_stats.h b/api/test/video_codec_stats.h index 1be1d8e8b8..80f8287848 100644 --- a/api/test/video_codec_stats.h +++ b/api/test/video_codec_stats.h @@ -96,6 +96,11 @@ class VideoCodecStats { SamplesStatsCounter u; SamplesStatsCounter v; } psnr; + + // Logs `Stream` metrics to provided `MetricsLogger`. + void LogMetrics(MetricsLogger* logger, + std::string test_case_name, + std::map metadata = {}) const; }; virtual ~VideoCodecStats() = default; @@ -107,13 +112,6 @@ class VideoCodecStats { // Returns video statistics aggregated for given `frames`. virtual Stream Aggregate(const std::vector& frames) const = 0; - - // Logs `Stream` metrics to provided `MetricsLogger`. - virtual void LogMetrics( - MetricsLogger* logger, - const Stream& stream, - std::string test_case_name, - std::map metadata = {}) const = 0; }; } // namespace test diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 271069c9d0..a5bcc2f802 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -1054,6 +1054,7 @@ if (rtc_include_tests) { "../../api/video_codecs:scalability_mode", "../../api/video_codecs:video_codecs_api", "../../media:rtc_internal_video_codecs", + "../../rtc_base:logging", "../../test:fileutils", "../../test:test_main", "../../test:test_support", diff --git a/modules/video_coding/codecs/test/video_codec_stats_impl.cc b/modules/video_coding/codecs/test/video_codec_stats_impl.cc index 9fa9d35516..9808e2a601 100644 --- a/modules/video_coding/codecs/test/video_codec_stats_impl.cc +++ b/modules/video_coding/codecs/test/video_codec_stats_impl.cc @@ -256,89 +256,6 @@ Stream VideoCodecStatsImpl::Aggregate(const std::vector& frames) const { return stream; } -void VideoCodecStatsImpl::LogMetrics( - MetricsLogger* logger, - const Stream& stream, - std::string test_case_name, - std::map metadata) const { - logger->LogMetric("width", test_case_name, stream.width, Unit::kCount, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("height", test_case_name, stream.height, Unit::kCount, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric( - "frame_size_bytes", test_case_name, stream.frame_size_bytes, Unit::kBytes, - webrtc::test::ImprovementDirection::kNeitherIsBetter, metadata); - - logger->LogMetric("keyframe", test_case_name, stream.keyframe, Unit::kCount, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("qp", test_case_name, stream.qp, Unit::kUnitless, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("encode_time_ms", test_case_name, stream.encode_time_ms, - Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("decode_time_ms", test_case_name, stream.decode_time_ms, - Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("target_bitrate_kbps", test_case_name, - stream.target_bitrate_kbps, Unit::kKilobitsPerSecond, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("target_framerate_fps", test_case_name, - stream.target_framerate_fps, Unit::kHertz, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("encoded_bitrate_kbps", test_case_name, - stream.encoded_bitrate_kbps, Unit::kKilobitsPerSecond, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("encoded_framerate_fps", test_case_name, - stream.encoded_framerate_fps, Unit::kHertz, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("bitrate_mismatch_pct", test_case_name, - stream.bitrate_mismatch_pct, Unit::kPercent, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("framerate_mismatch_pct", test_case_name, - stream.framerate_mismatch_pct, Unit::kPercent, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("transmission_time_ms", test_case_name, - stream.transmission_time_ms, Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("psnr_y_db", test_case_name, stream.psnr.y, Unit::kUnitless, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("psnr_u_db", test_case_name, stream.psnr.u, Unit::kUnitless, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("psnr_v_db", test_case_name, stream.psnr.v, Unit::kUnitless, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); -} - void VideoCodecStatsImpl::AddFrame(const Frame& frame) { FrameId frame_id{.timestamp_rtp = frame.timestamp_rtp, .spatial_idx = frame.spatial_idx}; diff --git a/modules/video_coding/codecs/test/video_codec_stats_impl.h b/modules/video_coding/codecs/test/video_codec_stats_impl.h index 56067610f0..77471d2ecd 100644 --- a/modules/video_coding/codecs/test/video_codec_stats_impl.h +++ b/modules/video_coding/codecs/test/video_codec_stats_impl.h @@ -29,12 +29,6 @@ class VideoCodecStatsImpl : public VideoCodecStats { Stream Aggregate(const std::vector& frames) const override; - void LogMetrics( - MetricsLogger* logger, - const Stream& stream, - std::string test_case_name, - std::map metadata = {}) const override; - void AddFrame(const Frame& frame); // Returns raw pointers to previously added frame. If frame does not exist, diff --git a/modules/video_coding/codecs/test/video_codec_test.cc b/modules/video_coding/codecs/test/video_codec_test.cc index 6414da37d1..bc4efa0a6f 100644 --- a/modules/video_coding/codecs/test/video_codec_test.cc +++ b/modules/video_coding/codecs/test/video_codec_test.cc @@ -37,6 +37,7 @@ #if defined(WEBRTC_ANDROID) #include "modules/video_coding/codecs/test/android_codec_factory_helper.h" #endif +#include "rtc_base/logging.h" #include "test/gtest.h" #include "test/testsupport/file_utils.h" #include "test/testsupport/frame_reader.h" @@ -462,11 +463,29 @@ std::unique_ptr RunEncodeDecodeTest( std::unique_ptr video_source = CreateVideoSource(video_info, frame_settings, num_frames); - // TODO(webrtc:14852): On platforms where only encoder or decoder is - // available, substitute absent codec with software implementation. std::unique_ptr encoder = CreateEncoder(codec_type, codec_impl, frame_settings); + if (encoder == nullptr) { + return nullptr; + } + std::unique_ptr decoder = CreateDecoder(codec_type, codec_impl); + if (decoder == nullptr) { + // If platform decoder is not available try built-in one. + if (codec_impl == "builtin") { + return nullptr; + } + + decoder = CreateDecoder(codec_type, "builtin"); + if (decoder == nullptr) { + return nullptr; + } + } + + RTC_LOG(LS_INFO) << "Encoder implementation: " + << encoder->encoder()->GetEncoderInfo().implementation_name; + RTC_LOG(LS_INFO) << "Decoder implementation: " + << decoder->decoder()->GetDecoderInfo().implementation_name; VideoCodecTester::EncoderSettings encoder_settings; encoder_settings.pacing.mode = @@ -509,6 +528,12 @@ std::unique_ptr RunEncodeTest( std::unique_ptr encoder = CreateEncoder(codec_type, codec_impl, frame_settings); + if (encoder == nullptr) { + return nullptr; + } + + RTC_LOG(LS_INFO) << "Encoder implementation: " + << encoder->encoder()->GetEncoderInfo().implementation_name; VideoCodecTester::EncoderSettings encoder_settings; encoder_settings.pacing.mode = @@ -551,7 +576,7 @@ class SpatialQualityTest : public ::testing::TestWithParam< } }; -TEST_P(SpatialQualityTest, DISABLED_SpatialQuality) { +TEST_P(SpatialQualityTest, SpatialQuality) { auto [codec_type, codec_impl, video_info, coding_settings] = GetParam(); auto [width, height, framerate_fps, bitrate_kbps, psnr] = coding_settings; @@ -571,13 +596,16 @@ TEST_P(SpatialQualityTest, DISABLED_SpatialQuality) { codec_type, codec_impl, video_info, frame_settings, num_frames, /*save_codec_input=*/false, /*save_codec_output=*/false); - std::vector frames = stats->Slice(); - SetTargetRates(frame_settings, frames); - VideoCodecStats::Stream stream = stats->Aggregate(frames); - EXPECT_GE(stream.psnr.y.GetAverage(), psnr); + VideoCodecStats::Stream stream; + if (stats != nullptr) { + std::vector frames = stats->Slice(); + SetTargetRates(frame_settings, frames); + stream = stats->Aggregate(frames); + EXPECT_GE(stream.psnr.y.GetAverage(), psnr); + } - stats->LogMetrics( - GetGlobalMetricsLogger(), stream, + stream.LogMetrics( + GetGlobalMetricsLogger(), ::testing::UnitTest::GetInstance()->current_test_info()->name(), /*metadata=*/ {{"codec_type", codec_type}, @@ -625,7 +653,7 @@ class BitrateAdaptationTest } }; -TEST_P(BitrateAdaptationTest, DISABLED_BitrateAdaptation) { +TEST_P(BitrateAdaptationTest, BitrateAdaptation) { auto [codec_type, codec_impl, video_info, bitrate_kbps] = GetParam(); int duration_s = 10; // Duration of fixed rate interval. @@ -650,15 +678,18 @@ TEST_P(BitrateAdaptationTest, DISABLED_BitrateAdaptation) { codec_type, codec_impl, video_info, frame_settings, num_frames, /*save_codec_input=*/false, /*save_codec_output=*/false); - std::vector frames = - stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); - SetTargetRates(frame_settings, frames); - VideoCodecStats::Stream stream = stats->Aggregate(frames); - EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); - EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); + VideoCodecStats::Stream stream; + if (stats != nullptr) { + std::vector frames = + stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); + SetTargetRates(frame_settings, frames); + stream = stats->Aggregate(frames); + EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); + EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); + } - stats->LogMetrics( - GetGlobalMetricsLogger(), stream, + stream.LogMetrics( + GetGlobalMetricsLogger(), ::testing::UnitTest::GetInstance()->current_test_info()->name(), /*metadata=*/ {{"codec_type", codec_type}, @@ -698,7 +729,7 @@ class FramerateAdaptationTest } }; -TEST_P(FramerateAdaptationTest, DISABLED_FramerateAdaptation) { +TEST_P(FramerateAdaptationTest, FramerateAdaptation) { auto [codec_type, codec_impl, video_info, framerate_fps] = GetParam(); int duration_s = 10; // Duration of fixed rate interval. @@ -724,15 +755,18 @@ TEST_P(FramerateAdaptationTest, DISABLED_FramerateAdaptation) { codec_type, codec_impl, video_info, frame_settings, num_frames, /*save_codec_input=*/false, /*save_codec_output=*/false); - std::vector frames = - stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); - SetTargetRates(frame_settings, frames); - VideoCodecStats::Stream stream = stats->Aggregate(frames); - EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); - EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); + VideoCodecStats::Stream stream; + if (stats != nullptr) { + std::vector frames = + stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); + SetTargetRates(frame_settings, frames); + stream = stats->Aggregate(frames); + EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); + EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); + } - stats->LogMetrics( - GetGlobalMetricsLogger(), stream, + stream.LogMetrics( + GetGlobalMetricsLogger(), ::testing::UnitTest::GetInstance()->current_test_info()->name(), /*metadata=*/ {{"codec_type", codec_type},