admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Log metrics even if test failed

Browse Source 
Set of codecs for testing is hardcoded to AV1, VP8, VP9, H264, H265. Some codecs may not be available due to lack of support on the platform or due to some issue in our code which would be a regression. Reporting zero metrics for failed tests would allow the perf tool to detect such a regression.

This also enables codec tests by default. The tests should not run on bots since video_codec_perf_tests binary is not included in any test suits yet.

Bug: webrtc:14852
Change-Id: I967160069055036f93e595d328c4d5f1ca483be9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/300868
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39840}
This commit is contained in:
Sergey Silkin 2023-04-12 13:59:57 +02:00 committed by WebRTC LUCI CQ
parent 5248228d79
commit 26d1b26277
7 changed files with 168 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/BUILD.gn
View File

@ -1019,7 +1019,10 @@ if (rtc_include_tests) {
rtc_library("video_codec_stats_api") {
visibility = [ "*" ]
testonly = true
sources = [ "test/video_codec_stats.h" ]
sources = [
"test/video_codec_stats.cc",
"test/video_codec_stats.h",
]
deps = [
"../api/numerics:numerics",
"../api/units:data_rate",

97
api/test/video_codec_stats.cc Normal file
View File

@ -0,0 +1,97 @@
/*
* Copyright (c) 2023 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/test/video_codec_stats.h"
namespace webrtc {
namespace test {
void VideoCodecStats::Stream::LogMetrics(
MetricsLogger* logger,
std::string test_case_name,
std::map<std::string, std::string> metadata) const {
logger->LogMetric("width", test_case_name, width, Unit::kCount,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("height", test_case_name, height, Unit::kCount,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric(
"frame_size_bytes", test_case_name, frame_size_bytes, Unit::kBytes,
webrtc::test::ImprovementDirection::kNeitherIsBetter, metadata);
logger->LogMetric("keyframe", test_case_name, keyframe, Unit::kCount,
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("qp", test_case_name, qp, Unit::kUnitless,
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric(
"encode_time_ms", test_case_name, encode_time_ms, Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter, metadata);
logger->LogMetric(
"decode_time_ms", test_case_name, decode_time_ms, Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter, metadata);
logger->LogMetric("target_bitrate_kbps", test_case_name, target_bitrate_kbps,
Unit::kKilobitsPerSecond,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("target_framerate_fps", test_case_name,
target_framerate_fps, Unit::kHertz,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("encoded_bitrate_kbps", test_case_name,
encoded_bitrate_kbps, Unit::kKilobitsPerSecond,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("encoded_framerate_fps", test_case_name,
encoded_framerate_fps, Unit::kHertz,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("bitrate_mismatch_pct", test_case_name,
bitrate_mismatch_pct, Unit::kPercent,
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("framerate_mismatch_pct", test_case_name,
framerate_mismatch_pct, Unit::kPercent,
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("transmission_time_ms", test_case_name,
transmission_time_ms, Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("psnr_y_db", test_case_name, psnr.y, Unit::kUnitless,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("psnr_u_db", test_case_name, psnr.u, Unit::kUnitless,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("psnr_v_db", test_case_name, psnr.v, Unit::kUnitless,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
}
} // namespace test
} // namespace webrtc

12
api/test/video_codec_stats.h
View File

@ -96,6 +96,11 @@ class VideoCodecStats {
SamplesStatsCounter u;
SamplesStatsCounter v;
} psnr;
// Logs `Stream` metrics to provided `MetricsLogger`.
void LogMetrics(MetricsLogger* logger,
std::string test_case_name,
std::map<std::string, std::string> metadata = {}) const;
};
virtual ~VideoCodecStats() = default;
@ -107,13 +112,6 @@ class VideoCodecStats {
// Returns video statistics aggregated for given `frames`.
virtual Stream Aggregate(const std::vector<Frame>& frames) const = 0;
// Logs `Stream` metrics to provided `MetricsLogger`.
virtual void LogMetrics(
MetricsLogger* logger,
const Stream& stream,
std::string test_case_name,
std::map<std::string, std::string> metadata = {}) const = 0;
};
} // namespace test

1
modules/video_coding/BUILD.gn
View File

@ -1054,6 +1054,7 @@ if (rtc_include_tests) {
"../../api/video_codecs:scalability_mode",
"../../api/video_codecs:video_codecs_api",
"../../media:rtc_internal_video_codecs",
"../../rtc_base:logging",
"../../test:fileutils",
"../../test:test_main",
"../../test:test_support",

83
modules/video_coding/codecs/test/video_codec_stats_impl.cc
View File

@ -256,89 +256,6 @@ Stream VideoCodecStatsImpl::Aggregate(const std::vector<Frame>& frames) const {
return stream;
}
void VideoCodecStatsImpl::LogMetrics(
MetricsLogger* logger,
const Stream& stream,
std::string test_case_name,
std::map<std::string, std::string> metadata) const {
logger->LogMetric("width", test_case_name, stream.width, Unit::kCount,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("height", test_case_name, stream.height, Unit::kCount,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric(
"frame_size_bytes", test_case_name, stream.frame_size_bytes, Unit::kBytes,
webrtc::test::ImprovementDirection::kNeitherIsBetter, metadata);
logger->LogMetric("keyframe", test_case_name, stream.keyframe, Unit::kCount,
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("qp", test_case_name, stream.qp, Unit::kUnitless,
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("encode_time_ms", test_case_name, stream.encode_time_ms,
Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("decode_time_ms", test_case_name, stream.decode_time_ms,
Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("target_bitrate_kbps", test_case_name,
stream.target_bitrate_kbps, Unit::kKilobitsPerSecond,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("target_framerate_fps", test_case_name,
stream.target_framerate_fps, Unit::kHertz,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("encoded_bitrate_kbps", test_case_name,
stream.encoded_bitrate_kbps, Unit::kKilobitsPerSecond,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("encoded_framerate_fps", test_case_name,
stream.encoded_framerate_fps, Unit::kHertz,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("bitrate_mismatch_pct", test_case_name,
stream.bitrate_mismatch_pct, Unit::kPercent,
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("framerate_mismatch_pct", test_case_name,
stream.framerate_mismatch_pct, Unit::kPercent,
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("transmission_time_ms", test_case_name,
stream.transmission_time_ms, Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("psnr_y_db", test_case_name, stream.psnr.y, Unit::kUnitless,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("psnr_u_db", test_case_name, stream.psnr.u, Unit::kUnitless,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("psnr_v_db", test_case_name, stream.psnr.v, Unit::kUnitless,
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
}
void VideoCodecStatsImpl::AddFrame(const Frame& frame) {
FrameId frame_id{.timestamp_rtp = frame.timestamp_rtp,
.spatial_idx = frame.spatial_idx};

6
modules/video_coding/codecs/test/video_codec_stats_impl.h
View File

@ -29,12 +29,6 @@ class VideoCodecStatsImpl : public VideoCodecStats {
Stream Aggregate(const std::vector<Frame>& frames) const override;
void LogMetrics(
MetricsLogger* logger,
const Stream& stream,
std::string test_case_name,
std::map<std::string, std::string> metadata = {}) const override;
void AddFrame(const Frame& frame);
// Returns raw pointers to previously added frame. If frame does not exist,

88
modules/video_coding/codecs/test/video_codec_test.cc
View File

@ -37,6 +37,7 @@
#if defined(WEBRTC_ANDROID)
#include "modules/video_coding/codecs/test/android_codec_factory_helper.h"
#endif
#include "rtc_base/logging.h"
#include "test/gtest.h"
#include "test/testsupport/file_utils.h"
#include "test/testsupport/frame_reader.h"
@ -462,11 +463,29 @@ std::unique_ptr<VideoCodecStats> RunEncodeDecodeTest(
std::unique_ptr<TestRawVideoSource> video_source =
CreateVideoSource(video_info, frame_settings, num_frames);
// TODO(webrtc:14852): On platforms where only encoder or decoder is
// available, substitute absent codec with software implementation.
std::unique_ptr<TestEncoder> encoder =
CreateEncoder(codec_type, codec_impl, frame_settings);
if (encoder == nullptr) {
return nullptr;
}
std::unique_ptr<TestDecoder> decoder = CreateDecoder(codec_type, codec_impl);
if (decoder == nullptr) {
// If platform decoder is not available try built-in one.
if (codec_impl == "builtin") {
return nullptr;
}
decoder = CreateDecoder(codec_type, "builtin");
if (decoder == nullptr) {
return nullptr;
}
}
RTC_LOG(LS_INFO) << "Encoder implementation: "
<< encoder->encoder()->GetEncoderInfo().implementation_name;
RTC_LOG(LS_INFO) << "Decoder implementation: "
<< decoder->decoder()->GetDecoderInfo().implementation_name;
VideoCodecTester::EncoderSettings encoder_settings;
encoder_settings.pacing.mode =
@ -509,6 +528,12 @@ std::unique_ptr<VideoCodecStats> RunEncodeTest(
std::unique_ptr<TestEncoder> encoder =
CreateEncoder(codec_type, codec_impl, frame_settings);
if (encoder == nullptr) {
return nullptr;
}
RTC_LOG(LS_INFO) << "Encoder implementation: "
<< encoder->encoder()->GetEncoderInfo().implementation_name;
VideoCodecTester::EncoderSettings encoder_settings;
encoder_settings.pacing.mode =
@ -551,7 +576,7 @@ class SpatialQualityTest : public ::testing::TestWithParam<
}
};
TEST_P(SpatialQualityTest, DISABLED_SpatialQuality) {
TEST_P(SpatialQualityTest, SpatialQuality) {
auto [codec_type, codec_impl, video_info, coding_settings] = GetParam();
auto [width, height, framerate_fps, bitrate_kbps, psnr] = coding_settings;
@ -571,13 +596,16 @@ TEST_P(SpatialQualityTest, DISABLED_SpatialQuality) {
codec_type, codec_impl, video_info, frame_settings, num_frames,
/*save_codec_input=*/false, /*save_codec_output=*/false);
std::vector<VideoCodecStats::Frame> frames = stats->Slice();
SetTargetRates(frame_settings, frames);
VideoCodecStats::Stream stream = stats->Aggregate(frames);
EXPECT_GE(stream.psnr.y.GetAverage(), psnr);
VideoCodecStats::Stream stream;
if (stats != nullptr) {
std::vector<VideoCodecStats::Frame> frames = stats->Slice();
SetTargetRates(frame_settings, frames);
stream = stats->Aggregate(frames);
EXPECT_GE(stream.psnr.y.GetAverage(), psnr);
}
stats->LogMetrics(
GetGlobalMetricsLogger(), stream,
stream.LogMetrics(
GetGlobalMetricsLogger(),
::testing::UnitTest::GetInstance()->current_test_info()->name(),
/*metadata=*/
{{"codec_type", codec_type},
@ -625,7 +653,7 @@ class BitrateAdaptationTest
}
};
TEST_P(BitrateAdaptationTest, DISABLED_BitrateAdaptation) {
TEST_P(BitrateAdaptationTest, BitrateAdaptation) {
auto [codec_type, codec_impl, video_info, bitrate_kbps] = GetParam();
int duration_s = 10; // Duration of fixed rate interval.
@ -650,15 +678,18 @@ TEST_P(BitrateAdaptationTest, DISABLED_BitrateAdaptation) {
codec_type, codec_impl, video_info, frame_settings, num_frames,
/*save_codec_input=*/false, /*save_codec_output=*/false);
std::vector<VideoCodecStats::Frame> frames =
stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame});
SetTargetRates(frame_settings, frames);
VideoCodecStats::Stream stream = stats->Aggregate(frames);
EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10);
EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10);
VideoCodecStats::Stream stream;
if (stats != nullptr) {
std::vector<VideoCodecStats::Frame> frames =
stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame});
SetTargetRates(frame_settings, frames);
stream = stats->Aggregate(frames);
EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10);
EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10);
}
stats->LogMetrics(
GetGlobalMetricsLogger(), stream,
stream.LogMetrics(
GetGlobalMetricsLogger(),
::testing::UnitTest::GetInstance()->current_test_info()->name(),
/*metadata=*/
{{"codec_type", codec_type},
@ -698,7 +729,7 @@ class FramerateAdaptationTest
}
};
TEST_P(FramerateAdaptationTest, DISABLED_FramerateAdaptation) {
TEST_P(FramerateAdaptationTest, FramerateAdaptation) {
auto [codec_type, codec_impl, video_info, framerate_fps] = GetParam();
int duration_s = 10; // Duration of fixed rate interval.
@ -724,15 +755,18 @@ TEST_P(FramerateAdaptationTest, DISABLED_FramerateAdaptation) {
codec_type, codec_impl, video_info, frame_settings, num_frames,
/*save_codec_input=*/false, /*save_codec_output=*/false);
std::vector<VideoCodecStats::Frame> frames =
stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame});
SetTargetRates(frame_settings, frames);
VideoCodecStats::Stream stream = stats->Aggregate(frames);
EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10);
EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10);
VideoCodecStats::Stream stream;
if (stats != nullptr) {
std::vector<VideoCodecStats::Frame> frames =
stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame});
SetTargetRates(frame_settings, frames);
stream = stats->Aggregate(frames);
EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10);
EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10);
}
stats->LogMetrics(
GetGlobalMetricsLogger(), stream,
stream.LogMetrics(
GetGlobalMetricsLogger(),
::testing::UnitTest::GetInstance()->current_test_info()->name(),
/*metadata=*/
{{"codec_type", codec_type},