admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add rate adaptation tests

Browse Source 
Bug: b/261160916, webrtc:14852
Change-Id: I58b3647218c961dcf0305c3902f79adb448b73e0
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/295866
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39489}
This commit is contained in:
Sergey Silkin 2023-03-06 12:40:57 +01:00 committed by WebRTC LUCI CQ
parent 812186d060
commit 9259b5f72c
4 changed files with 444 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
api/test/video_codec_stats.h
View File

@ -11,6 +11,7 @@
#ifndef API_TEST_VIDEO_CODEC_STATS_H_
#define API_TEST_VIDEO_CODEC_STATS_H_
#include <map>
#include <string>
#include <vector>
@ -108,9 +109,11 @@ class VideoCodecStats {
virtual Stream Aggregate(const std::vector<Frame>& frames) const = 0;
// Logs `Stream` metrics to provided `MetricsLogger`.
virtual void LogMetrics(MetricsLogger* logger,
const Stream& stream,
std::string test_case_name) const = 0;
virtual void LogMetrics(
MetricsLogger* logger,
const Stream& stream,
std::string test_case_name,
std::map<std::string, std::string> metadata = {}) const = 0;
};
} // namespace test

62
modules/video_coding/codecs/test/video_codec_stats_impl.cc
View File

@ -256,69 +256,87 @@ Stream VideoCodecStatsImpl::Aggregate(const std::vector<Frame>& frames) const {
return stream;
}
void VideoCodecStatsImpl::LogMetrics(MetricsLogger* logger,
const Stream& stream,
std::string test_case_name) const {
void VideoCodecStatsImpl::LogMetrics(
MetricsLogger* logger,
const Stream& stream,
std::string test_case_name,
std::map<std::string, std::string> metadata) const {
logger->LogMetric("width", test_case_name, stream.width, Unit::kCount,
webrtc::test::ImprovementDirection::kBiggerIsBetter);
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("height", test_case_name, stream.height, Unit::kCount,
webrtc::test::ImprovementDirection::kBiggerIsBetter);
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("frame_size_bytes", test_case_name, stream.frame_size_bytes,
Unit::kBytes,
webrtc::test::ImprovementDirection::kNeitherIsBetter);
logger->LogMetric(
"frame_size_bytes", test_case_name, stream.frame_size_bytes, Unit::kBytes,
webrtc::test::ImprovementDirection::kNeitherIsBetter, metadata);
logger->LogMetric("keyframe", test_case_name, stream.keyframe, Unit::kCount,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("qp", test_case_name, stream.qp, Unit::kUnitless,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("encode_time_ms", test_case_name, stream.encode_time_ms,
Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("decode_time_ms", test_case_name, stream.decode_time_ms,
Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("target_bitrate_kbps", test_case_name,
stream.target_bitrate_kbps, Unit::kKilobitsPerSecond,
webrtc::test::ImprovementDirection::kBiggerIsBetter);
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("target_framerate_fps", test_case_name,
stream.target_framerate_fps, Unit::kHertz,
webrtc::test::ImprovementDirection::kBiggerIsBetter);
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("encoded_bitrate_kbps", test_case_name,
stream.encoded_bitrate_kbps, Unit::kKilobitsPerSecond,
webrtc::test::ImprovementDirection::kBiggerIsBetter);
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("encoded_framerate_fps", test_case_name,
stream.encoded_framerate_fps, Unit::kHertz,
webrtc::test::ImprovementDirection::kBiggerIsBetter);
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("bitrate_mismatch_pct", test_case_name,
stream.bitrate_mismatch_pct, Unit::kPercent,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("framerate_mismatch_pct", test_case_name,
stream.framerate_mismatch_pct, Unit::kPercent,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("transmission_time_ms", test_case_name,
stream.transmission_time_ms, Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
webrtc::test::ImprovementDirection::kSmallerIsBetter,
metadata);
logger->LogMetric("psnr_y_db", test_case_name, stream.psnr.y, Unit::kUnitless,
webrtc::test::ImprovementDirection::kBiggerIsBetter);
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("psnr_u_db", test_case_name, stream.psnr.u, Unit::kUnitless,
webrtc::test::ImprovementDirection::kBiggerIsBetter);
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
logger->LogMetric("psnr_v_db", test_case_name, stream.psnr.v, Unit::kUnitless,
webrtc::test::ImprovementDirection::kBiggerIsBetter);
webrtc::test::ImprovementDirection::kBiggerIsBetter,
metadata);
}
void VideoCodecStatsImpl::AddFrame(const Frame& frame) {

8
modules/video_coding/codecs/test/video_codec_stats_impl.h
View File

@ -29,9 +29,11 @@ class VideoCodecStatsImpl : public VideoCodecStats {
Stream Aggregate(const std::vector<Frame>& frames) const override;
void LogMetrics(MetricsLogger* logger,
const Stream& stream,
std::string test_case_name) const override;
void LogMetrics(
MetricsLogger* logger,
const Stream& stream,
std::string test_case_name,
std::map<std::string, std::string> metadata = {}) const override;
void AddFrame(const Frame& frame);

588
modules/video_coding/codecs/test/video_codec_test.cc
View File

@ -52,12 +52,7 @@ using PacingMode = VideoCodecTester::PacingSettings::PacingMode;
struct VideoInfo {
std::string name;
Resolution resolution;
};
struct CodecInfo {
std::string type;
std::string encoder;
std::string decoder;
Frequency framerate;
};
struct LayerId {
@ -71,7 +66,7 @@ struct LayerId {
bool operator<(const LayerId& o) const {
if (spatial_idx < o.spatial_idx)
return true;
if (temporal_idx < o.temporal_idx)
if (spatial_idx == o.spatial_idx && temporal_idx < o.temporal_idx)
return true;
return false;
}
@ -79,91 +74,61 @@ struct LayerId {
struct EncodingSettings {
ScalabilityMode scalability_mode;
// Spatial layer resolution.
std::map<int, Resolution> resolution;
// Top temporal layer frame rate.
Frequency framerate;
// Bitrate of spatial and temporal layers.
std::map<LayerId, DataRate> bitrate;
std::string ToString() const {
return std::string(ScalabilityModeToString(scalability_mode)) + "_" +
std::to_string(resolution.begin()->second.width) + "x" +
std::to_string(resolution.begin()->second.height) + "_" +
std::to_string(framerate.hertz()) + "fps" + "_" +
std::to_string(bitrate.begin()->second.kbps()) + "kbps";
}
struct LayerSettings {
Resolution resolution;
Frequency framerate;
DataRate bitrate;
};
std::map<LayerId, LayerSettings> layer_settings;
};
struct EncodingTestSettings {
std::string name;
int num_frames = 1;
std::map<int, EncodingSettings> frame_settings;
};
struct DecodingTestSettings {
std::string name;
};
struct EncodeDecodeTestParams {
CodecInfo codec;
VideoInfo video;
VideoCodecTester::EncoderSettings encoder_settings;
VideoCodecTester::DecoderSettings decoder_settings;
EncodingTestSettings encoding_settings;
DecodingTestSettings decoding_settings;
struct Expectations {
double min_apsnr_y;
} test_expectations;
};
const EncodingSettings kQvga64Kbps30Fps = {
.scalability_mode = ScalabilityMode::kL1T1,
.resolution = {{0, {.width = 320, .height = 180}}},
.framerate = Frequency::Hertz(30),
.bitrate = {
{{.spatial_idx = 0, .temporal_idx = 0}, DataRate::KilobitsPerSec(64)}}};
const EncodingTestSettings kConstantRateQvga64Kbps30Fps = {
.name = "ConstantRate",
.num_frames = 300,
.frame_settings = {{/*frame_num=*/0, kQvga64Kbps30Fps}}};
const VideoInfo kFourPeople_1280x720_30 = {
.name = "FourPeople_1280x720_30",
.resolution = {.width = 1280, .height = 720}};
.resolution = {.width = 1280, .height = 720},
.framerate = Frequency::Hertz(30)};
class TestRawVideoSource : public VideoCodecTester::RawVideoSource {
public:
static constexpr Frequency k90kHz = Frequency::Hertz(90000);
TestRawVideoSource(std::unique_ptr<FrameReader> frame_reader,
const EncodingTestSettings& test_settings)
: frame_reader_(std::move(frame_reader)),
test_settings_(test_settings),
TestRawVideoSource(VideoInfo video_info,
const std::map<int, EncodingSettings>& frame_settings,
int num_frames)
: video_info_(video_info),
frame_settings_(frame_settings),
num_frames_(num_frames),
frame_num_(0),
timestamp_rtp_(0) {
// Ensure settings for the first frame are provided.
RTC_CHECK_GT(test_settings_.frame_settings.size(), 0u);
RTC_CHECK_EQ(test_settings_.frame_settings.begin()->first, 0);
RTC_CHECK_GT(frame_settings_.size(), 0u);
RTC_CHECK_EQ(frame_settings_.begin()->first, 0);
frame_reader_ = CreateYuvFrameReader(
ResourcePath(video_info_.name, "yuv"), video_info_.resolution,
YuvFrameReaderImpl::RepeatMode::kPingPong);
RTC_CHECK(frame_reader_);
}
// Pulls next frame. Frame RTP timestamp is set accordingly to
// `EncodingSettings::framerate`.
absl::optional<VideoFrame> PullFrame() override {
if (frame_num_ >= test_settings_.num_frames) {
// End of stream.
return absl::nullopt;
if (frame_num_ >= num_frames_) {
return absl::nullopt; // End of stream.
}
EncodingSettings frame_settings =
std::prev(test_settings_.frame_settings.upper_bound(frame_num_))
->second;
const EncodingSettings& encoding_settings =
std::prev(frame_settings_.upper_bound(frame_num_))->second;
Resolution resolution =
encoding_settings.layer_settings.begin()->second.resolution;
Frequency framerate =
encoding_settings.layer_settings.begin()->second.framerate;
int pulled_frame;
auto buffer = frame_reader_->PullFrame(
&pulled_frame, frame_settings.resolution.rbegin()->second,
{.num = 30, .den = static_cast<int>(frame_settings.framerate.hertz())});
&pulled_frame, resolution,
{.num = static_cast<int>(video_info_.framerate.millihertz()),
.den = static_cast<int>(framerate.millihertz())});
RTC_CHECK(buffer) << "Cannot pull frame " << frame_num_;
auto frame = VideoFrame::Builder()
@ -172,7 +137,7 @@ class TestRawVideoSource : public VideoCodecTester::RawVideoSource {
.build();
pulled_frames_[timestamp_rtp_] = pulled_frame;
timestamp_rtp_ += k90kHz / frame_settings.framerate;
timestamp_rtp_ += k90kHz / framerate;
++frame_num_;
return frame;
@ -194,8 +159,10 @@ class TestRawVideoSource : public VideoCodecTester::RawVideoSource {
}
protected:
VideoInfo video_info_;
std::unique_ptr<FrameReader> frame_reader_;
const EncodingTestSettings& test_settings_;
const std::map<int, EncodingSettings>& frame_settings_;
int num_frames_;
int frame_num_;
uint32_t timestamp_rtp_;
std::map<uint32_t, int> pulled_frames_;
@ -205,10 +172,10 @@ class TestEncoder : public VideoCodecTester::Encoder,
public EncodedImageCallback {
public:
TestEncoder(std::unique_ptr<VideoEncoder> encoder,
const CodecInfo& codec_info,
const std::string codec_type,
const std::map<int, EncodingSettings>& frame_settings)
: encoder_(std::move(encoder)),
codec_info_(codec_info),
codec_type_(codec_type),
frame_settings_(frame_settings),
frame_num_(0) {
// Ensure settings for the first frame is provided.
@ -244,6 +211,8 @@ class TestEncoder : public VideoCodecTester::Encoder,
encoder_->Release();
}
VideoEncoder* encoder() { return encoder_.get(); }
protected:
Result OnEncodedImage(const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info) override {
@ -257,21 +226,23 @@ class TestEncoder : public VideoCodecTester::Encoder,
void Configure(const EncodingSettings& es) {
VideoCodec vc;
const Resolution& resolution = es.resolution.rbegin()->second;
vc.width = resolution.width;
vc.height = resolution.height;
const DataRate& bitrate = es.bitrate.rbegin()->second;
const EncodingSettings::LayerSettings& layer_settings =
es.layer_settings.begin()->second;
vc.width = layer_settings.resolution.width;
vc.height = layer_settings.resolution.height;
const DataRate& bitrate = layer_settings.bitrate;
vc.startBitrate = bitrate.kbps();
vc.maxBitrate = bitrate.kbps();
vc.minBitrate = 0;
vc.maxFramerate = static_cast<uint32_t>(es.framerate.hertz());
vc.maxFramerate = static_cast<uint32_t>(layer_settings.framerate.hertz());
vc.active = true;
vc.qpMax = 0;
vc.qpMax = 63;
vc.numberOfSimulcastStreams = 0;
vc.mode = webrtc::VideoCodecMode::kRealtimeVideo;
vc.SetFrameDropEnabled(true);
vc.SetScalabilityMode(es.scalability_mode);
vc.codecType = PayloadStringToCodecType(codec_info_.type);
vc.codecType = PayloadStringToCodecType(codec_type_);
if (vc.codecType == kVideoCodecVP8) {
*(vc.VP8()) = VideoEncoder::GetDefaultVp8Settings();
} else if (vc.codecType == kVideoCodecVP9) {
@ -286,7 +257,7 @@ class TestEncoder : public VideoCodecTester::Encoder,
/*max_payload_size=*/1440);
int result = encoder_->InitEncode(&vc, ves);
RTC_CHECK_EQ(result, WEBRTC_VIDEO_CODEC_OK);
ASSERT_EQ(result, WEBRTC_VIDEO_CODEC_OK);
}
void SetRates(const EncodingSettings& es) {
@ -297,30 +268,49 @@ class TestEncoder : public VideoCodecTester::Encoder,
ScalabilityModeToNumSpatialLayers(es.scalability_mode);
for (int sidx = 0; sidx < num_spatial_layers; ++sidx) {
for (int tidx = 0; tidx < num_temporal_layers; ++tidx) {
LayerId layer_id = {.spatial_idx = sidx, .temporal_idx = tidx};
RTC_CHECK(es.bitrate.find(layer_id) != es.bitrate.end())
auto layer_settings =
es.layer_settings.find({.spatial_idx = sidx, .temporal_idx = tidx});
RTC_CHECK(layer_settings != es.layer_settings.end())
<< "Bitrate for layer S=" << sidx << " T=" << tidx << " is not set";
rc.bitrate.SetBitrate(sidx, tidx, es.bitrate.at(layer_id).bps());
rc.bitrate.SetBitrate(sidx, tidx, layer_settings->second.bitrate.bps());
}
}
rc.framerate_fps = es.framerate.millihertz() / 1000.0;
rc.framerate_fps =
es.layer_settings.begin()->second.framerate.millihertz() / 1000.0;
encoder_->SetRates(rc);
}
bool ConfigChanged(const EncodingSettings& es,
const EncodingSettings& prev_es) const {
return es.scalability_mode != prev_es.scalability_mode ||
es.resolution != prev_es.resolution;
if (es.scalability_mode != prev_es.scalability_mode) {
return true;
}
for (auto [layer_id, layer_settings] : es.layer_settings) {
const auto& prev_layer_settings = prev_es.layer_settings.at(layer_id);
if (layer_settings.resolution != prev_layer_settings.resolution) {
return true;
}
}
return false;
}
bool RateChanged(const EncodingSettings& es,
const EncodingSettings& prev_es) const {
return es.bitrate != prev_es.bitrate || es.framerate != prev_es.framerate;
for (auto [layer_id, layer_settings] : es.layer_settings) {
const auto& prev_layer_settings = prev_es.layer_settings.at(layer_id);
if (layer_settings.bitrate != prev_layer_settings.bitrate ||
layer_settings.framerate != prev_layer_settings.framerate) {
return true;
}
}
return false;
}
std::unique_ptr<VideoEncoder> encoder_;
const CodecInfo& codec_info_;
const std::string codec_type_;
const std::map<int, EncodingSettings>& frame_settings_;
int frame_num_;
std::map<uint32_t, EncodeCallback> callbacks_;
@ -330,8 +320,8 @@ class TestDecoder : public VideoCodecTester::Decoder,
public DecodedImageCallback {
public:
TestDecoder(std::unique_ptr<VideoDecoder> decoder,
const CodecInfo& codec_info)
: decoder_(std::move(decoder)), codec_info_(codec_info), frame_num_(0) {
const std::string codec_type)
: decoder_(std::move(decoder)), codec_type_(codec_type), frame_num_(0) {
decoder_->RegisterDecodeCompleteCallback(this);
}
@ -354,15 +344,17 @@ class TestDecoder : public VideoCodecTester::Decoder,
decoder_->Release();
}
VideoDecoder* decoder() { return decoder_.get(); }
protected:
void Configure() {
VideoDecoder::Settings ds;
ds.set_codec_type(PayloadStringToCodecType(codec_info_.type));
ds.set_codec_type(PayloadStringToCodecType(codec_type_));
ds.set_number_of_cores(1);
ds.set_max_render_resolution({1280, 720});
bool result = decoder_->Configure(ds);
RTC_CHECK(result);
ASSERT_TRUE(result);
}
int Decoded(VideoFrame& decoded_frame) override {
@ -375,152 +367,358 @@ class TestDecoder : public VideoCodecTester::Decoder,
}
std::unique_ptr<VideoDecoder> decoder_;
const CodecInfo& codec_info_;
const std::string codec_type_;
int frame_num_;
std::map<uint32_t, DecodeCallback> callbacks_;
};
std::unique_ptr<VideoCodecTester::Encoder> CreateEncoder(
const CodecInfo& codec_info,
std::unique_ptr<TestRawVideoSource> CreateVideoSource(
const VideoInfo& video,
const std::map<int, EncodingSettings>& frame_settings,
int num_frames) {
return std::make_unique<TestRawVideoSource>(video, frame_settings,
num_frames);
}
std::unique_ptr<TestEncoder> CreateEncoder(
std::string type,
std::string impl,
const std::map<int, EncodingSettings>& frame_settings) {
std::unique_ptr<VideoEncoderFactory> factory;
if (codec_info.encoder == "libvpx" || codec_info.encoder == "libaom" ||
codec_info.encoder == "openh264") {
if (impl == "builtin") {
factory = CreateBuiltinVideoEncoderFactory();
} else if (codec_info.encoder == "mediacodec") {
} else if (impl == "mediacodec") {
#if defined(WEBRTC_ANDROID)
InitializeAndroidObjects();
factory = CreateAndroidEncoderFactory();
#endif
}
RTC_CHECK(factory);
auto encoder = factory->CreateVideoEncoder(SdpVideoFormat(codec_info.type));
return std::make_unique<TestEncoder>(std::move(encoder), codec_info,
std::unique_ptr<VideoEncoder> encoder =
factory->CreateVideoEncoder(SdpVideoFormat(type));
if (encoder == nullptr) {
return nullptr;
}
return std::make_unique<TestEncoder>(std::move(encoder), type,
frame_settings);
}
std::unique_ptr<VideoCodecTester::Decoder> CreateDecoder(
const CodecInfo& codec_info) {
std::unique_ptr<TestDecoder> CreateDecoder(std::string type, std::string impl) {
std::unique_ptr<VideoDecoderFactory> factory;
if (codec_info.decoder == "libvpx" || codec_info.decoder == "dav1d" ||
codec_info.decoder == "ffmpeg") {
if (impl == "builtin") {
factory = CreateBuiltinVideoDecoderFactory();
} else if (codec_info.decoder == "mediacodec") {
} else if (impl == "mediacodec") {
#if defined(WEBRTC_ANDROID)
InitializeAndroidObjects();
factory = CreateAndroidDecoderFactory();
#endif
}
RTC_CHECK(factory);
auto decoder = factory->CreateVideoDecoder(SdpVideoFormat(codec_info.type));
return std::make_unique<TestDecoder>(std::move(decoder), codec_info);
std::unique_ptr<VideoDecoder> decoder =
factory->CreateVideoDecoder(SdpVideoFormat(type));
if (decoder == nullptr) {
return nullptr;
}
return std::make_unique<TestDecoder>(std::move(decoder), type);
}
void SetTargetRates(const std::map<int, EncodingSettings>& frame_settings,
std::vector<VideoCodecStats::Frame>& frames) {
for (VideoCodecStats::Frame& f : frames) {
const EncodingSettings& settings =
const EncodingSettings& encoding_settings =
std::prev(frame_settings.upper_bound(f.frame_num))->second;
LayerId layer_id = {.spatial_idx = f.spatial_idx,
.temporal_idx = f.temporal_idx};
f.target_bitrate = settings.bitrate.at(layer_id);
f.target_framerate = settings.framerate / (1 << f.temporal_idx);
RTC_CHECK(encoding_settings.layer_settings.find(layer_id) !=
encoding_settings.layer_settings.end())
<< "Frame frame_num=" << f.frame_num
<< " belongs to spatial_idx=" << f.spatial_idx
<< " temporal_idx=" << f.temporal_idx
<< " but settings for this layer are not provided.";
const EncodingSettings::LayerSettings& layer_settings =
encoding_settings.layer_settings.at(layer_id);
f.target_bitrate = layer_settings.bitrate;
f.target_framerate = layer_settings.framerate;
}
}
} // namespace
class EncodeDecodeTest
: public ::testing::TestWithParam<EncodeDecodeTestParams> {
std::unique_ptr<VideoCodecStats> RunEncodeDecodeTest(
std::string codec_type,
std::string codec_impl,
const VideoInfo& video_info,
const std::map<int, EncodingSettings>& frame_settings,
int num_frames) {
std::unique_ptr<TestRawVideoSource> video_source =
CreateVideoSource(video_info, frame_settings, num_frames);
// TODO(webrtc:14852): On platforms where only encoder or decoder is
// available, substitute absent codec with software implementation.
std::unique_ptr<TestEncoder> encoder =
CreateEncoder(codec_type, codec_impl, frame_settings);
std::unique_ptr<TestDecoder> decoder = CreateDecoder(codec_type, codec_impl);
VideoCodecTester::EncoderSettings encoder_settings;
encoder_settings.pacing.mode =
encoder->encoder()->GetEncoderInfo().is_hardware_accelerated
? PacingMode::kRealTime
: PacingMode::kNoPacing;
VideoCodecTester::DecoderSettings decoder_settings;
decoder_settings.pacing.mode =
decoder->decoder()->GetDecoderInfo().is_hardware_accelerated
? PacingMode::kRealTime
: PacingMode::kNoPacing;
std::unique_ptr<VideoCodecTester> tester = CreateVideoCodecTester();
return tester->RunEncodeDecodeTest(video_source.get(), encoder.get(),
decoder.get(), encoder_settings,
decoder_settings);
}
std::unique_ptr<VideoCodecStats> RunEncodeTest(
std::string codec_type,
std::string codec_impl,
const VideoInfo& video_info,
const std::map<int, EncodingSettings>& frame_settings,
int num_frames) {
std::unique_ptr<TestRawVideoSource> video_source =
CreateVideoSource(video_info, frame_settings, num_frames);
std::unique_ptr<TestEncoder> encoder =
CreateEncoder(codec_type, codec_impl, frame_settings);
VideoCodecTester::EncoderSettings encoder_settings;
encoder_settings.pacing.mode =
encoder->encoder()->GetEncoderInfo().is_hardware_accelerated
? PacingMode::kRealTime
: PacingMode::kNoPacing;
std::unique_ptr<VideoCodecTester> tester = CreateVideoCodecTester();
return tester->RunEncodeTest(video_source.get(), encoder.get(),
encoder_settings);
}
class SpatialQualityTest : public ::testing::TestWithParam<
std::tuple</*codec_type=*/std::string,
/*codec_impl=*/std::string,
VideoInfo,
std::tuple</*width=*/int,
/*height=*/int,
/*framerate_fps=*/double,
/*bitrate_kbps=*/int,
/*min_psnr=*/double>>> {
public:
EncodeDecodeTest() : test_params_(GetParam()) {}
void SetUp() override {
std::unique_ptr<FrameReader> frame_reader =
CreateYuvFrameReader(ResourcePath(test_params_.video.name, "yuv"),
test_params_.video.resolution,
YuvFrameReaderImpl::RepeatMode::kPingPong);
video_source_ = std::make_unique<TestRawVideoSource>(
std::move(frame_reader), test_params_.encoding_settings);
encoder_ = CreateEncoder(test_params_.codec,
test_params_.encoding_settings.frame_settings);
decoder_ = CreateDecoder(test_params_.codec);
tester_ = CreateVideoCodecTester();
static std::string TestParamsToString(
const ::testing::TestParamInfo<SpatialQualityTest::ParamType>& info) {
auto [codec_type, codec_impl, video_info, coding_settings] = info.param;
auto [width, height, framerate_fps, bitrate_kbps, psnr] = coding_settings;
return std::string(codec_type + codec_impl + video_info.name +
std::to_string(width) + "x" + std::to_string(height) +
"p" +
std::to_string(static_cast<int>(1000 * framerate_fps)) +
"mhz" + std::to_string(bitrate_kbps) + "kbps");
}
static std::string TestParametersToStr(
const ::testing::TestParamInfo<EncodeDecodeTest::ParamType>& info) {
return std::string(info.param.encoding_settings.name +
info.param.codec.type + info.param.codec.encoder +
info.param.codec.decoder);
}
protected:
EncodeDecodeTestParams test_params_;
std::unique_ptr<TestRawVideoSource> video_source_;
std::unique_ptr<VideoCodecTester::Encoder> encoder_;
std::unique_ptr<VideoCodecTester::Decoder> decoder_;
std::unique_ptr<VideoCodecTester> tester_;
};
TEST_P(EncodeDecodeTest, DISABLED_TestEncodeDecode) {
std::unique_ptr<VideoCodecStats> stats = tester_->RunEncodeDecodeTest(
video_source_.get(), encoder_.get(), decoder_.get(),
test_params_.encoder_settings, test_params_.decoder_settings);
TEST_P(SpatialQualityTest, DISABLED_SpatialQuality) {
auto [codec_type, codec_impl, video_info, coding_settings] = GetParam();
auto [width, height, framerate_fps, bitrate_kbps, psnr] = coding_settings;
const auto& frame_settings = test_params_.encoding_settings.frame_settings;
for (auto fs = frame_settings.begin(); fs != frame_settings.end(); ++fs) {
int first_frame = fs->first;
int last_frame = std::next(fs) != frame_settings.end()
? std::next(fs)->first - 1
: test_params_.encoding_settings.num_frames - 1;
VideoCodecStats::Filter slicer = {.first_frame = first_frame,
.last_frame = last_frame};
std::vector<VideoCodecStats::Frame> frames = stats->Slice(slicer);
SetTargetRates(frame_settings, frames);
VideoCodecStats::Stream stream = stats->Aggregate(frames);
EXPECT_GE(stream.psnr.y.GetAverage(),
test_params_.test_expectations.min_apsnr_y);
std::map<int, EncodingSettings> frame_settings = {
{0,
{.scalability_mode = ScalabilityMode::kL1T1,
.layer_settings = {
{LayerId{.spatial_idx = 0, .temporal_idx = 0},
{.resolution = {.width = width, .height = height},
.framerate = Frequency::MilliHertz(1000 * framerate_fps),
.bitrate = DataRate::KilobitsPerSec(bitrate_kbps)}}}}}};
stats->LogMetrics(
GetGlobalMetricsLogger(), stream,
std::string(
::testing::UnitTest::GetInstance()->current_test_info()->name()) +
"_" + fs->second.ToString());
}
int duration_s = 10;
int num_frames = duration_s * framerate_fps;
std::unique_ptr<VideoCodecStats> stats = RunEncodeDecodeTest(
codec_type, codec_impl, video_info, frame_settings, num_frames);
std::vector<VideoCodecStats::Frame> frames = stats->Slice();
SetTargetRates(frame_settings, frames);
VideoCodecStats::Stream stream = stats->Aggregate(frames);
EXPECT_GE(stream.psnr.y.GetAverage(), psnr);
stats->LogMetrics(
GetGlobalMetricsLogger(), stream,
::testing::UnitTest::GetInstance()->current_test_info()->name(),
/*metadata=*/
{{"codec_type", codec_type},
{"codec_impl", codec_impl},
{"video_name", video_info.name}});
}
INSTANTIATE_TEST_SUITE_P(
ConstantRate,
EncodeDecodeTest,
::testing::ValuesIn({
EncodeDecodeTestParams({
.codec = {.type = "VP8", .encoder = "libvpx", .decoder = "libvpx"},
.video = kFourPeople_1280x720_30,
.encoder_settings = {.pacing = {.mode = PacingMode::kNoPacing}},
.decoder_settings = {.pacing = {.mode = PacingMode::kNoPacing}},
.encoding_settings = kConstantRateQvga64Kbps30Fps,
.test_expectations = {.min_apsnr_y = 30.0},
})
All,
SpatialQualityTest,
Combine(Values("AV1", "VP9", "VP8", "H264"),
#if defined(WEBRTC_ANDROID)
,
EncodeDecodeTestParams({
.codec = {.type = "VP8",
.encoder = "mediacodec",
.decoder = "mediacodec"},
.video = kFourPeople_1280x720_30,
.encoder_settings = {.pacing = {.mode = PacingMode::kRealTime}},
.decoder_settings = {.pacing = {.mode = PacingMode::kRealTime}},
.encoding_settings = kConstantRateQvga64Kbps30Fps,
.test_expectations = {.min_apsnr_y = 30.0},
})
Values("builtin", "mediacodec"),
#else
Values("builtin"),
#endif
}),
EncodeDecodeTest::TestParametersToStr);
Values(kFourPeople_1280x720_30),
Values(std::make_tuple(320, 180, 30, 32, 29),
std::make_tuple(320, 180, 30, 64, 30),
std::make_tuple(320, 180, 30, 128, 33),
std::make_tuple(320, 180, 30, 256, 36),
std::make_tuple(640, 360, 30, 128, 31),
std::make_tuple(640, 360, 30, 256, 33),
std::make_tuple(640, 360, 30, 384, 35),
std::make_tuple(640, 360, 30, 512, 36),
std::make_tuple(1280, 720, 30, 256, 33),
std::make_tuple(1280, 720, 30, 512, 35),
std::make_tuple(1280, 720, 30, 1024, 37),
std::make_tuple(1280, 720, 30, 2048, 39))),
SpatialQualityTest::TestParamsToString);
class BitrateAdaptationTest
: public ::testing::TestWithParam<
std::tuple</*codec_type=*/std::string,
/*codec_impl=*/std::string,
VideoInfo,
std::pair</*bitrate_kbps=*/int, /*bitrate_kbps=*/int>>> {
public:
static std::string TestParamsToString(
const ::testing::TestParamInfo<BitrateAdaptationTest::ParamType>& info) {
auto [codec_type, codec_impl, video_info, bitrate_kbps] = info.param;
return std::string(codec_type + codec_impl + video_info.name +
std::to_string(bitrate_kbps.first) + "kbps" +
std::to_string(bitrate_kbps.second) + "kbps");
}
};
TEST_P(BitrateAdaptationTest, DISABLED_BitrateAdaptation) {
auto [codec_type, codec_impl, video_info, bitrate_kbps] = GetParam();
int duration_s = 10; // Duration of fixed rate interval.
int first_frame = duration_s * video_info.framerate.millihertz() / 1000;
int num_frames = 2 * duration_s * video_info.framerate.millihertz() / 1000;
std::map<int, EncodingSettings> frame_settings = {
{0,
{.layer_settings = {{LayerId{.spatial_idx = 0, .temporal_idx = 0},
{.resolution = {.width = 640, .height = 360},
.framerate = video_info.framerate,
.bitrate = DataRate::KilobitsPerSec(
bitrate_kbps.first)}}}}},
{first_frame,
{.layer_settings = {
{LayerId{.spatial_idx = 0, .temporal_idx = 0},
{.resolution = {.width = 640, .height = 360},
.framerate = video_info.framerate,
.bitrate = DataRate::KilobitsPerSec(bitrate_kbps.second)}}}}}};
std::unique_ptr<VideoCodecStats> stats = RunEncodeTest(
codec_type, codec_impl, video_info, frame_settings, num_frames);
std::vector<VideoCodecStats::Frame> frames =
stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame});
SetTargetRates(frame_settings, frames);
VideoCodecStats::Stream stream = stats->Aggregate(frames);
EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10);
EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10);
stats->LogMetrics(
GetGlobalMetricsLogger(), stream,
::testing::UnitTest::GetInstance()->current_test_info()->name(),
/*metadata=*/
{{"codec_type", codec_type},
{"codec_impl", codec_impl},
{"video_name", video_info.name},
{"rate_profile", std::to_string(bitrate_kbps.first) + "," +
std::to_string(bitrate_kbps.second)}});
}
INSTANTIATE_TEST_SUITE_P(All,
BitrateAdaptationTest,
Combine(Values("AV1", "VP9", "VP8", "H264"),
#if defined(WEBRTC_ANDROID)
Values("builtin", "mediacodec"),
#else
Values("builtin"),
#endif
Values(kFourPeople_1280x720_30),
Values(std::pair(1024, 512),
std::pair(512, 1024))),
BitrateAdaptationTest::TestParamsToString);
class FramerateAdaptationTest
: public ::testing::TestWithParam<std::tuple</*codec_type=*/std::string,
/*codec_impl=*/std::string,
VideoInfo,
std::pair<double, double>>> {
public:
static std::string TestParamsToString(
const ::testing::TestParamInfo<FramerateAdaptationTest::ParamType>&
info) {
auto [codec_type, codec_impl, video_info, framerate_fps] = info.param;
return std::string(
codec_type + codec_impl + video_info.name +
std::to_string(static_cast<int>(1000 * framerate_fps.first)) + "mhz" +
std::to_string(static_cast<int>(1000 * framerate_fps.second)) + "mhz");
}
};
TEST_P(FramerateAdaptationTest, DISABLED_FramerateAdaptation) {
auto [codec_type, codec_impl, video_info, framerate_fps] = GetParam();
int duration_s = 10; // Duration of fixed rate interval.
int first_frame = static_cast<int>(duration_s * framerate_fps.first);
int num_frames = static_cast<int>(
duration_s * (framerate_fps.first + framerate_fps.second));
std::map<int, EncodingSettings> frame_settings = {
{0,
{.layer_settings = {{LayerId{.spatial_idx = 0, .temporal_idx = 0},
{.resolution = {.width = 640, .height = 360},
.framerate = Frequency::MilliHertz(
1000 * framerate_fps.first),
.bitrate = DataRate::KilobitsPerSec(512)}}}}},
{first_frame,
{.layer_settings = {
{LayerId{.spatial_idx = 0, .temporal_idx = 0},
{.resolution = {.width = 640, .height = 360},
.framerate = Frequency::MilliHertz(1000 * framerate_fps.second),
.bitrate = DataRate::KilobitsPerSec(512)}}}}}};
std::unique_ptr<VideoCodecStats> stats = RunEncodeTest(
codec_type, codec_impl, video_info, frame_settings, num_frames);
std::vector<VideoCodecStats::Frame> frames =
stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame});
SetTargetRates(frame_settings, frames);
VideoCodecStats::Stream stream = stats->Aggregate(frames);
EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10);
EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10);
stats->LogMetrics(
GetGlobalMetricsLogger(), stream,
::testing::UnitTest::GetInstance()->current_test_info()->name(),
/*metadata=*/
{{"codec_type", codec_type},
{"codec_impl", codec_impl},
{"video_name", video_info.name},
{"rate_profile", std::to_string(framerate_fps.first) + "," +
std::to_string(framerate_fps.second)}});
}
INSTANTIATE_TEST_SUITE_P(All,
FramerateAdaptationTest,
Combine(Values("AV1", "VP9", "VP8", "H264"),
#if defined(WEBRTC_ANDROID)
Values("builtin", "mediacodec"),
#else
Values("builtin"),
#endif
Values(kFourPeople_1280x720_30),
Values(std::pair(30, 15), std::pair(15, 30))),
FramerateAdaptationTest::TestParamsToString);
} // namespace test
} // namespace webrtc