From 26d1b262770e768055ce2ece63b2f165bbf25f29 Mon Sep 17 00:00:00 2001 From: Sergey Silkin Date: Wed, 12 Apr 2023 13:59:57 +0200 Subject: [PATCH] Log metrics even if test failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set of codecs for testing is hardcoded to AV1, VP8, VP9, H264, H265. Some codecs may not be available due to lack of support on the platform or due to some issue in our code which would be a regression. Reporting zero metrics for failed tests would allow the perf tool to detect such a regression. This also enables codec tests by default. The tests should not run on bots since video_codec_perf_tests binary is not included in any test suits yet. Bug: webrtc:14852 Change-Id: I967160069055036f93e595d328c4d5f1ca483be9 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/300868 Reviewed-by: Åsa Persson Commit-Queue: Sergey Silkin Reviewed-by: Mirko Bonadei Cr-Commit-Position: refs/heads/main@{#39840} --- api/BUILD.gn | 5 +- api/test/video_codec_stats.cc | 97 +++++++++++++++++++ api/test/video_codec_stats.h | 12 +-- modules/video_coding/BUILD.gn | 1 + .../codecs/test/video_codec_stats_impl.cc | 83 ---------------- .../codecs/test/video_codec_stats_impl.h | 6 -- .../codecs/test/video_codec_test.cc | 88 +++++++++++------ 7 files changed, 168 insertions(+), 124 deletions(-) create mode 100644 api/test/video_codec_stats.cc diff --git a/api/BUILD.gn b/api/BUILD.gn index 949e5ad5e9..f54766a32b 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -1019,7 +1019,10 @@ if (rtc_include_tests) { rtc_library("video_codec_stats_api") { visibility = [ "*" ] testonly = true - sources = [ "test/video_codec_stats.h" ] + sources = [ + "test/video_codec_stats.cc", + "test/video_codec_stats.h", + ] deps = [ "../api/numerics:numerics", "../api/units:data_rate", diff --git a/api/test/video_codec_stats.cc b/api/test/video_codec_stats.cc new file mode 100644 index 0000000000..fb7226701e --- /dev/null +++ b/api/test/video_codec_stats.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/video_codec_stats.h" + +namespace webrtc { +namespace test { + +void VideoCodecStats::Stream::LogMetrics( + MetricsLogger* logger, + std::string test_case_name, + std::map metadata) const { + logger->LogMetric("width", test_case_name, width, Unit::kCount, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("height", test_case_name, height, Unit::kCount, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric( + "frame_size_bytes", test_case_name, frame_size_bytes, Unit::kBytes, + webrtc::test::ImprovementDirection::kNeitherIsBetter, metadata); + + logger->LogMetric("keyframe", test_case_name, keyframe, Unit::kCount, + webrtc::test::ImprovementDirection::kSmallerIsBetter, + metadata); + + logger->LogMetric("qp", test_case_name, qp, Unit::kUnitless, + webrtc::test::ImprovementDirection::kSmallerIsBetter, + metadata); + + logger->LogMetric( + "encode_time_ms", test_case_name, encode_time_ms, Unit::kMilliseconds, + webrtc::test::ImprovementDirection::kSmallerIsBetter, metadata); + + logger->LogMetric( + "decode_time_ms", test_case_name, decode_time_ms, Unit::kMilliseconds, + webrtc::test::ImprovementDirection::kSmallerIsBetter, metadata); + + logger->LogMetric("target_bitrate_kbps", test_case_name, target_bitrate_kbps, + Unit::kKilobitsPerSecond, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("target_framerate_fps", test_case_name, + target_framerate_fps, Unit::kHertz, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("encoded_bitrate_kbps", test_case_name, + encoded_bitrate_kbps, Unit::kKilobitsPerSecond, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("encoded_framerate_fps", test_case_name, + encoded_framerate_fps, Unit::kHertz, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("bitrate_mismatch_pct", test_case_name, + bitrate_mismatch_pct, Unit::kPercent, + webrtc::test::ImprovementDirection::kSmallerIsBetter, + metadata); + + logger->LogMetric("framerate_mismatch_pct", test_case_name, + framerate_mismatch_pct, Unit::kPercent, + webrtc::test::ImprovementDirection::kSmallerIsBetter, + metadata); + + logger->LogMetric("transmission_time_ms", test_case_name, + transmission_time_ms, Unit::kMilliseconds, + webrtc::test::ImprovementDirection::kSmallerIsBetter, + metadata); + + logger->LogMetric("psnr_y_db", test_case_name, psnr.y, Unit::kUnitless, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("psnr_u_db", test_case_name, psnr.u, Unit::kUnitless, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); + + logger->LogMetric("psnr_v_db", test_case_name, psnr.v, Unit::kUnitless, + webrtc::test::ImprovementDirection::kBiggerIsBetter, + metadata); +} + +} // namespace test +} // namespace webrtc diff --git a/api/test/video_codec_stats.h b/api/test/video_codec_stats.h index 1be1d8e8b8..80f8287848 100644 --- a/api/test/video_codec_stats.h +++ b/api/test/video_codec_stats.h @@ -96,6 +96,11 @@ class VideoCodecStats { SamplesStatsCounter u; SamplesStatsCounter v; } psnr; + + // Logs `Stream` metrics to provided `MetricsLogger`. + void LogMetrics(MetricsLogger* logger, + std::string test_case_name, + std::map metadata = {}) const; }; virtual ~VideoCodecStats() = default; @@ -107,13 +112,6 @@ class VideoCodecStats { // Returns video statistics aggregated for given `frames`. virtual Stream Aggregate(const std::vector& frames) const = 0; - - // Logs `Stream` metrics to provided `MetricsLogger`. - virtual void LogMetrics( - MetricsLogger* logger, - const Stream& stream, - std::string test_case_name, - std::map metadata = {}) const = 0; }; } // namespace test diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 271069c9d0..a5bcc2f802 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -1054,6 +1054,7 @@ if (rtc_include_tests) { "../../api/video_codecs:scalability_mode", "../../api/video_codecs:video_codecs_api", "../../media:rtc_internal_video_codecs", + "../../rtc_base:logging", "../../test:fileutils", "../../test:test_main", "../../test:test_support", diff --git a/modules/video_coding/codecs/test/video_codec_stats_impl.cc b/modules/video_coding/codecs/test/video_codec_stats_impl.cc index 9fa9d35516..9808e2a601 100644 --- a/modules/video_coding/codecs/test/video_codec_stats_impl.cc +++ b/modules/video_coding/codecs/test/video_codec_stats_impl.cc @@ -256,89 +256,6 @@ Stream VideoCodecStatsImpl::Aggregate(const std::vector& frames) const { return stream; } -void VideoCodecStatsImpl::LogMetrics( - MetricsLogger* logger, - const Stream& stream, - std::string test_case_name, - std::map metadata) const { - logger->LogMetric("width", test_case_name, stream.width, Unit::kCount, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("height", test_case_name, stream.height, Unit::kCount, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric( - "frame_size_bytes", test_case_name, stream.frame_size_bytes, Unit::kBytes, - webrtc::test::ImprovementDirection::kNeitherIsBetter, metadata); - - logger->LogMetric("keyframe", test_case_name, stream.keyframe, Unit::kCount, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("qp", test_case_name, stream.qp, Unit::kUnitless, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("encode_time_ms", test_case_name, stream.encode_time_ms, - Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("decode_time_ms", test_case_name, stream.decode_time_ms, - Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("target_bitrate_kbps", test_case_name, - stream.target_bitrate_kbps, Unit::kKilobitsPerSecond, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("target_framerate_fps", test_case_name, - stream.target_framerate_fps, Unit::kHertz, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("encoded_bitrate_kbps", test_case_name, - stream.encoded_bitrate_kbps, Unit::kKilobitsPerSecond, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("encoded_framerate_fps", test_case_name, - stream.encoded_framerate_fps, Unit::kHertz, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("bitrate_mismatch_pct", test_case_name, - stream.bitrate_mismatch_pct, Unit::kPercent, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("framerate_mismatch_pct", test_case_name, - stream.framerate_mismatch_pct, Unit::kPercent, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("transmission_time_ms", test_case_name, - stream.transmission_time_ms, Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter, - metadata); - - logger->LogMetric("psnr_y_db", test_case_name, stream.psnr.y, Unit::kUnitless, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("psnr_u_db", test_case_name, stream.psnr.u, Unit::kUnitless, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); - - logger->LogMetric("psnr_v_db", test_case_name, stream.psnr.v, Unit::kUnitless, - webrtc::test::ImprovementDirection::kBiggerIsBetter, - metadata); -} - void VideoCodecStatsImpl::AddFrame(const Frame& frame) { FrameId frame_id{.timestamp_rtp = frame.timestamp_rtp, .spatial_idx = frame.spatial_idx}; diff --git a/modules/video_coding/codecs/test/video_codec_stats_impl.h b/modules/video_coding/codecs/test/video_codec_stats_impl.h index 56067610f0..77471d2ecd 100644 --- a/modules/video_coding/codecs/test/video_codec_stats_impl.h +++ b/modules/video_coding/codecs/test/video_codec_stats_impl.h @@ -29,12 +29,6 @@ class VideoCodecStatsImpl : public VideoCodecStats { Stream Aggregate(const std::vector& frames) const override; - void LogMetrics( - MetricsLogger* logger, - const Stream& stream, - std::string test_case_name, - std::map metadata = {}) const override; - void AddFrame(const Frame& frame); // Returns raw pointers to previously added frame. If frame does not exist, diff --git a/modules/video_coding/codecs/test/video_codec_test.cc b/modules/video_coding/codecs/test/video_codec_test.cc index 6414da37d1..bc4efa0a6f 100644 --- a/modules/video_coding/codecs/test/video_codec_test.cc +++ b/modules/video_coding/codecs/test/video_codec_test.cc @@ -37,6 +37,7 @@ #if defined(WEBRTC_ANDROID) #include "modules/video_coding/codecs/test/android_codec_factory_helper.h" #endif +#include "rtc_base/logging.h" #include "test/gtest.h" #include "test/testsupport/file_utils.h" #include "test/testsupport/frame_reader.h" @@ -462,11 +463,29 @@ std::unique_ptr RunEncodeDecodeTest( std::unique_ptr video_source = CreateVideoSource(video_info, frame_settings, num_frames); - // TODO(webrtc:14852): On platforms where only encoder or decoder is - // available, substitute absent codec with software implementation. std::unique_ptr encoder = CreateEncoder(codec_type, codec_impl, frame_settings); + if (encoder == nullptr) { + return nullptr; + } + std::unique_ptr decoder = CreateDecoder(codec_type, codec_impl); + if (decoder == nullptr) { + // If platform decoder is not available try built-in one. + if (codec_impl == "builtin") { + return nullptr; + } + + decoder = CreateDecoder(codec_type, "builtin"); + if (decoder == nullptr) { + return nullptr; + } + } + + RTC_LOG(LS_INFO) << "Encoder implementation: " + << encoder->encoder()->GetEncoderInfo().implementation_name; + RTC_LOG(LS_INFO) << "Decoder implementation: " + << decoder->decoder()->GetDecoderInfo().implementation_name; VideoCodecTester::EncoderSettings encoder_settings; encoder_settings.pacing.mode = @@ -509,6 +528,12 @@ std::unique_ptr RunEncodeTest( std::unique_ptr encoder = CreateEncoder(codec_type, codec_impl, frame_settings); + if (encoder == nullptr) { + return nullptr; + } + + RTC_LOG(LS_INFO) << "Encoder implementation: " + << encoder->encoder()->GetEncoderInfo().implementation_name; VideoCodecTester::EncoderSettings encoder_settings; encoder_settings.pacing.mode = @@ -551,7 +576,7 @@ class SpatialQualityTest : public ::testing::TestWithParam< } }; -TEST_P(SpatialQualityTest, DISABLED_SpatialQuality) { +TEST_P(SpatialQualityTest, SpatialQuality) { auto [codec_type, codec_impl, video_info, coding_settings] = GetParam(); auto [width, height, framerate_fps, bitrate_kbps, psnr] = coding_settings; @@ -571,13 +596,16 @@ TEST_P(SpatialQualityTest, DISABLED_SpatialQuality) { codec_type, codec_impl, video_info, frame_settings, num_frames, /*save_codec_input=*/false, /*save_codec_output=*/false); - std::vector frames = stats->Slice(); - SetTargetRates(frame_settings, frames); - VideoCodecStats::Stream stream = stats->Aggregate(frames); - EXPECT_GE(stream.psnr.y.GetAverage(), psnr); + VideoCodecStats::Stream stream; + if (stats != nullptr) { + std::vector frames = stats->Slice(); + SetTargetRates(frame_settings, frames); + stream = stats->Aggregate(frames); + EXPECT_GE(stream.psnr.y.GetAverage(), psnr); + } - stats->LogMetrics( - GetGlobalMetricsLogger(), stream, + stream.LogMetrics( + GetGlobalMetricsLogger(), ::testing::UnitTest::GetInstance()->current_test_info()->name(), /*metadata=*/ {{"codec_type", codec_type}, @@ -625,7 +653,7 @@ class BitrateAdaptationTest } }; -TEST_P(BitrateAdaptationTest, DISABLED_BitrateAdaptation) { +TEST_P(BitrateAdaptationTest, BitrateAdaptation) { auto [codec_type, codec_impl, video_info, bitrate_kbps] = GetParam(); int duration_s = 10; // Duration of fixed rate interval. @@ -650,15 +678,18 @@ TEST_P(BitrateAdaptationTest, DISABLED_BitrateAdaptation) { codec_type, codec_impl, video_info, frame_settings, num_frames, /*save_codec_input=*/false, /*save_codec_output=*/false); - std::vector frames = - stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); - SetTargetRates(frame_settings, frames); - VideoCodecStats::Stream stream = stats->Aggregate(frames); - EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); - EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); + VideoCodecStats::Stream stream; + if (stats != nullptr) { + std::vector frames = + stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); + SetTargetRates(frame_settings, frames); + stream = stats->Aggregate(frames); + EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); + EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); + } - stats->LogMetrics( - GetGlobalMetricsLogger(), stream, + stream.LogMetrics( + GetGlobalMetricsLogger(), ::testing::UnitTest::GetInstance()->current_test_info()->name(), /*metadata=*/ {{"codec_type", codec_type}, @@ -698,7 +729,7 @@ class FramerateAdaptationTest } }; -TEST_P(FramerateAdaptationTest, DISABLED_FramerateAdaptation) { +TEST_P(FramerateAdaptationTest, FramerateAdaptation) { auto [codec_type, codec_impl, video_info, framerate_fps] = GetParam(); int duration_s = 10; // Duration of fixed rate interval. @@ -724,15 +755,18 @@ TEST_P(FramerateAdaptationTest, DISABLED_FramerateAdaptation) { codec_type, codec_impl, video_info, frame_settings, num_frames, /*save_codec_input=*/false, /*save_codec_output=*/false); - std::vector frames = - stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); - SetTargetRates(frame_settings, frames); - VideoCodecStats::Stream stream = stats->Aggregate(frames); - EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); - EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); + VideoCodecStats::Stream stream; + if (stats != nullptr) { + std::vector frames = + stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); + SetTargetRates(frame_settings, frames); + stream = stats->Aggregate(frames); + EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); + EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); + } - stats->LogMetrics( - GetGlobalMetricsLogger(), stream, + stream.LogMetrics( + GetGlobalMetricsLogger(), ::testing::UnitTest::GetInstance()->current_test_info()->name(), /*metadata=*/ {{"codec_type", codec_type},