admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
webrtc_m130/test/video_codec_tester_unittest.cc
Sergey Silkin d431156c0e Move codecs handling from test to tester 
* Pass codec factories to the video codec tester instead of creating and wrapping codecs into a tester-specific wrappers in video_codec_test.cc. The motivation for this change is to simplify the tests by moving complexity to the tester.

* Merge codec stats and analysis into the tester and move the tester. The merge fixes circular deps issues. Modularization is not strictly needed for testing framework like the video codec tester. It is still possible to unit test underlaying modules with rather small overhead.

* Move the video codec tester from api/ to test/. test/ is accessible from outside of WebRTC which enables reusing the tester in downstream projects.

Test output ~matches before and after this refactoring. There is a small difference that is caused by changes in qpMax: 63 -> 56 (kDefaultVideoMaxQpVpx). 56 is what WebRTC uses by default for VPx/AV1 encoders.

Bug: webrtc:14852
Change-Id: I762707b7144fcff870119ad741ebe7091ea109ba
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/327260
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41144}
2023-11-13 16:48:49 +00:00

514 lines
21 KiB
C++
Raw Blame History

/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/video_codec_tester.h"
#include <map>
#include <memory>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
#include "api/test/mock_video_decoder.h"
#include "api/test/mock_video_decoder_factory.h"
#include "api/test/mock_video_encoder.h"
#include "api/test/mock_video_encoder_factory.h"
#include "api/units/data_rate.h"
#include "api/units/time_delta.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_frame.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/svc/scalability_mode_util.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/testsupport/file_utils.h"
#include "third_party/libyuv/include/libyuv/planar_functions.h"
namespace webrtc {
namespace test {
namespace {
using ::testing::_;
using ::testing::ElementsAre;
using ::testing::Field;
using ::testing::Invoke;
using ::testing::InvokeWithoutArgs;
using ::testing::NiceMock;
using ::testing::Return;
using ::testing::SizeIs;
using VideoCodecStats = VideoCodecTester::VideoCodecStats;
using VideoSourceSettings = VideoCodecTester::VideoSourceSettings;
using CodedVideoSource = VideoCodecTester::CodedVideoSource;
using EncodingSettings = VideoCodecTester::EncodingSettings;
using LayerSettings = EncodingSettings::LayerSettings;
using LayerId = VideoCodecTester::LayerId;
using DecoderSettings = VideoCodecTester::DecoderSettings;
using EncoderSettings = VideoCodecTester::EncoderSettings;
using PacingSettings = VideoCodecTester::PacingSettings;
using PacingMode = PacingSettings::PacingMode;
using Filter = VideoCodecStats::Filter;
using Frame = VideoCodecTester::VideoCodecStats::Frame;
using Stream = VideoCodecTester::VideoCodecStats::Stream;
constexpr int kWidth = 2;
constexpr int kHeight = 2;
const DataRate kTargetLayerBitrate = DataRate::BytesPerSec(100);
const Frequency kTargetFramerate = Frequency::Hertz(30);
constexpr Frequency k90kHz = Frequency::Hertz(90000);
rtc::scoped_refptr<I420Buffer> CreateYuvBuffer(uint8_t y = 0,
uint8_t u = 0,
uint8_t v = 0) {
rtc::scoped_refptr<I420Buffer> buffer(I420Buffer::Create(2, 2));
libyuv::I420Rect(buffer->MutableDataY(), buffer->StrideY(),
buffer->MutableDataU(), buffer->StrideU(),
buffer->MutableDataV(), buffer->StrideV(), 0, 0,
buffer->width(), buffer->height(), y, u, v);
return buffer;
}
std::string CreateYuvFile(int width, int height, int num_frames) {
std::string path = webrtc::test::TempFilename(webrtc::test::OutputPath(),
"video_codec_tester_unittest");
FILE* file = fopen(path.c_str(), "wb");
for (int frame_num = 0; frame_num < num_frames; ++frame_num) {
uint8_t y = (frame_num + 0) & 255;
uint8_t u = (frame_num + 1) & 255;
uint8_t v = (frame_num + 2) & 255;
rtc::scoped_refptr<I420Buffer> buffer = CreateYuvBuffer(y, u, v);
fwrite(buffer->DataY(), 1, width * height, file);
int chroma_size_bytes = (width + 1) / 2 * (height + 1) / 2;
fwrite(buffer->DataU(), 1, chroma_size_bytes, file);
fwrite(buffer->DataV(), 1, chroma_size_bytes, file);
}
fclose(file);
return path;
}
std::unique_ptr<VideoCodecStats> RunTest(std::vector<std::vector<Frame>> frames,
ScalabilityMode scalability_mode) {
int num_frames = static_cast<int>(frames.size());
std::string source_yuv_path = CreateYuvFile(kWidth, kHeight, num_frames);
VideoSourceSettings source_settings{
.file_path = source_yuv_path,
.resolution = {.width = kWidth, .height = kHeight},
.framerate = kTargetFramerate};
int num_encoded_frames = 0;
EncodedImageCallback* encoded_frame_callback;
NiceMock<MockVideoEncoderFactory> encoder_factory;
ON_CALL(encoder_factory, CreateVideoEncoder)
.WillByDefault([&](const SdpVideoFormat&) {
auto encoder = std::make_unique<NiceMock<MockVideoEncoder>>();
ON_CALL(*encoder, RegisterEncodeCompleteCallback)
.WillByDefault([&](EncodedImageCallback* callback) {
encoded_frame_callback = callback;
return WEBRTC_VIDEO_CODEC_OK;
});
ON_CALL(*encoder, Encode)
.WillByDefault([&](const VideoFrame& input_frame,
const std::vector<VideoFrameType>*) {
for (const Frame& frame : frames[num_encoded_frames]) {
EncodedImage encoded_frame;
encoded_frame._encodedWidth = frame.width;
encoded_frame._encodedHeight = frame.height;
encoded_frame.SetFrameType(
frame.keyframe ? VideoFrameType::kVideoFrameKey
: VideoFrameType::kVideoFrameDelta);
encoded_frame.SetRtpTimestamp(input_frame.timestamp());
encoded_frame.SetSpatialIndex(frame.layer_id.spatial_idx);
encoded_frame.SetTemporalIndex(frame.layer_id.temporal_idx);
encoded_frame.SetEncodedData(
EncodedImageBuffer::Create(frame.frame_size.bytes()));
encoded_frame_callback->OnEncodedImage(
encoded_frame,
/*codec_specific_info=*/nullptr);
}
++num_encoded_frames;
return WEBRTC_VIDEO_CODEC_OK;
});
return encoder;
});
int num_decoded_frames = 0;
DecodedImageCallback* decode_callback;
NiceMock<MockVideoDecoderFactory> decoder_factory;
ON_CALL(decoder_factory, CreateVideoDecoder)
.WillByDefault([&](const SdpVideoFormat&) {
auto decoder = std::make_unique<NiceMock<MockVideoDecoder>>();
ON_CALL(*decoder, RegisterDecodeCompleteCallback)
.WillByDefault([&](DecodedImageCallback* callback) {
decode_callback = callback;
return WEBRTC_VIDEO_CODEC_OK;
});
ON_CALL(*decoder, Decode(_, _))
.WillByDefault([&](const EncodedImage& encoded_frame, int64_t) {
// Make values to be different from source YUV generated in
// `CreateYuvFile`.
uint8_t y = ((num_decoded_frames + 1) * 2) & 255;
uint8_t u = ((num_decoded_frames + 2) * 2) & 255;
uint8_t v = ((num_decoded_frames + 3) * 2) & 255;
rtc::scoped_refptr<I420Buffer> frame_buffer =
CreateYuvBuffer(y, u, v);
VideoFrame decoded_frame =
VideoFrame::Builder()
.set_video_frame_buffer(frame_buffer)
.set_timestamp_rtp(encoded_frame.RtpTimestamp())
.build();
decode_callback->Decoded(decoded_frame);
++num_decoded_frames;
return WEBRTC_VIDEO_CODEC_OK;
});
return decoder;
});
int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode);
int num_temporal_layers =
ScalabilityModeToNumTemporalLayers(scalability_mode);
std::map<uint32_t, EncodingSettings> encoding_settings;
for (int frame_num = 0; frame_num < num_frames; ++frame_num) {
std::map<LayerId, LayerSettings> layers_settings;
for (int sidx = 0; sidx < num_spatial_layers; ++sidx) {
for (int tidx = 0; tidx < num_temporal_layers; ++tidx) {
layers_settings.emplace(
LayerId{.spatial_idx = sidx, .temporal_idx = tidx},
LayerSettings{.resolution = {.width = kWidth, .height = kHeight},
.framerate = kTargetFramerate /
(1 << (num_temporal_layers - 1 - tidx)),
.bitrate = kTargetLayerBitrate});
}
}
encoding_settings.emplace(
frames[frame_num][0].timestamp_rtp,
EncodingSettings{.scalability_mode = scalability_mode,
.layers_settings = layers_settings});
}
EncoderSettings encoder_settings;
DecoderSettings decoder_settings;
std::unique_ptr<VideoCodecStats> stats =
VideoCodecTester::RunEncodeDecodeTest(
source_settings, &encoder_factory, &decoder_factory, encoder_settings,
decoder_settings, encoding_settings);
remove(source_yuv_path.c_str());
return stats;
}
EncodedImage CreateEncodedImage(uint32_t timestamp_rtp) {
EncodedImage encoded_image;
encoded_image.SetRtpTimestamp(timestamp_rtp);
return encoded_image;
}
class MockCodedVideoSource : public CodedVideoSource {
public:
MockCodedVideoSource(int num_frames, Frequency framerate)
: num_frames_(num_frames), frame_num_(0), framerate_(framerate) {}
absl::optional<EncodedImage> PullFrame() override {
if (frame_num_ >= num_frames_) {
return absl::nullopt;
}
uint32_t timestamp_rtp = frame_num_ * k90kHz / framerate_;
++frame_num_;
return CreateEncodedImage(timestamp_rtp);
}
private:
int num_frames_;
int frame_num_;
Frequency framerate_;
};
} // namespace
TEST(VideoCodecTester, Slice) {
std::unique_ptr<VideoCodecStats> stats = RunTest(
{{{.timestamp_rtp = 0, .layer_id = {.spatial_idx = 0, .temporal_idx = 0}},
{.timestamp_rtp = 0,
.layer_id = {.spatial_idx = 1, .temporal_idx = 0}}},
{{.timestamp_rtp = 1,
.layer_id = {.spatial_idx = 0, .temporal_idx = 1}}}},
ScalabilityMode::kL2T2);
std::vector<Frame> slice = stats->Slice(Filter{}, /*merge=*/false);
EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0),
Field(&Frame::timestamp_rtp, 0),
Field(&Frame::timestamp_rtp, 1)));
slice = stats->Slice({.min_timestamp_rtp = 1}, /*merge=*/false);
EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 1)));
slice = stats->Slice({.max_timestamp_rtp = 0}, /*merge=*/false);
EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0),
Field(&Frame::timestamp_rtp, 0)));
slice = stats->Slice({.layer_id = {{.spatial_idx = 0, .temporal_idx = 0}}},
/*merge=*/false);
EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0)));
slice = stats->Slice({.layer_id = {{.spatial_idx = 0, .temporal_idx = 1}}},
/*merge=*/false);
EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0),
Field(&Frame::timestamp_rtp, 1)));
}
TEST(VideoCodecTester, Merge) {
std::unique_ptr<VideoCodecStats> stats =
RunTest({{{.timestamp_rtp = 0,
.layer_id = {.spatial_idx = 0, .temporal_idx = 0},
.frame_size = DataSize::Bytes(1),
.keyframe = true},
{.timestamp_rtp = 0,
.layer_id = {.spatial_idx = 1, .temporal_idx = 0},
.frame_size = DataSize::Bytes(2)}},
{{.timestamp_rtp = 1,
.layer_id = {.spatial_idx = 0, .temporal_idx = 1},
.frame_size = DataSize::Bytes(4)},
{.timestamp_rtp = 1,
.layer_id = {.spatial_idx = 1, .temporal_idx = 1},
.frame_size = DataSize::Bytes(8)}}},
ScalabilityMode::kL2T2_KEY);
std::vector<Frame> slice = stats->Slice(Filter{}, /*merge=*/true);
EXPECT_THAT(
slice,
ElementsAre(
AllOf(Field(&Frame::timestamp_rtp, 0), Field(&Frame::keyframe, true),
Field(&Frame::frame_size, DataSize::Bytes(3))),
AllOf(Field(&Frame::timestamp_rtp, 1), Field(&Frame::keyframe, false),
Field(&Frame::frame_size, DataSize::Bytes(12)))));
}
struct AggregationTestParameters {
Filter filter;
double expected_keyframe_sum;
double expected_encoded_bitrate_kbps;
double expected_encoded_framerate_fps;
double expected_bitrate_mismatch_pct;
double expected_framerate_mismatch_pct;
};
class VideoCodecTesterTestAggregation
: public ::testing::TestWithParam<AggregationTestParameters> {};
TEST_P(VideoCodecTesterTestAggregation, Aggregate) {
AggregationTestParameters test_params = GetParam();
std::unique_ptr<VideoCodecStats> stats =
RunTest({{// L0T0
{.timestamp_rtp = 0,
.layer_id = {.spatial_idx = 0, .temporal_idx = 0},
.frame_size = DataSize::Bytes(1),
.keyframe = true},
// L1T0
{.timestamp_rtp = 0,
.layer_id = {.spatial_idx = 1, .temporal_idx = 0},
.frame_size = DataSize::Bytes(2)}},
// Emulate frame drop (frame_size = 0).
{{.timestamp_rtp = 3000,
.layer_id = {.spatial_idx = 0, .temporal_idx = 0},
.frame_size = DataSize::Zero()}},
{// L0T1
{.timestamp_rtp = 87000,
.layer_id = {.spatial_idx = 0, .temporal_idx = 1},
.frame_size = DataSize::Bytes(4)},
// L1T1
{.timestamp_rtp = 87000,
.layer_id = {.spatial_idx = 1, .temporal_idx = 1},
.frame_size = DataSize::Bytes(8)}}},
ScalabilityMode::kL2T2_KEY);
Stream stream = stats->Aggregate(test_params.filter);
EXPECT_EQ(stream.keyframe.GetSum(), test_params.expected_keyframe_sum);
EXPECT_EQ(stream.encoded_bitrate_kbps.GetAverage(),
test_params.expected_encoded_bitrate_kbps);
EXPECT_EQ(stream.encoded_framerate_fps.GetAverage(),
test_params.expected_encoded_framerate_fps);
EXPECT_EQ(stream.bitrate_mismatch_pct.GetAverage(),
test_params.expected_bitrate_mismatch_pct);
EXPECT_EQ(stream.framerate_mismatch_pct.GetAverage(),
test_params.expected_framerate_mismatch_pct);
}
INSTANTIATE_TEST_SUITE_P(
All,
VideoCodecTesterTestAggregation,
::testing::Values(
// No filtering.
AggregationTestParameters{
.filter = {},
.expected_keyframe_sum = 1,
.expected_encoded_bitrate_kbps =
DataRate::BytesPerSec(15).kbps<double>(),
.expected_encoded_framerate_fps = 2,
.expected_bitrate_mismatch_pct =
100 * (15.0 / (kTargetLayerBitrate.bytes_per_sec() * 4) - 1),
.expected_framerate_mismatch_pct =
100 * (2.0 / kTargetFramerate.hertz() - 1)},
// L0T0
AggregationTestParameters{
.filter = {.layer_id = {{.spatial_idx = 0, .temporal_idx = 0}}},
.expected_keyframe_sum = 1,
.expected_encoded_bitrate_kbps =
DataRate::BytesPerSec(1).kbps<double>(),
.expected_encoded_framerate_fps = 1,
.expected_bitrate_mismatch_pct =
100 * (1.0 / kTargetLayerBitrate.bytes_per_sec() - 1),
.expected_framerate_mismatch_pct =
100 * (1.0 / (kTargetFramerate.hertz() / 2) - 1)},
// L0T1
AggregationTestParameters{
.filter = {.layer_id = {{.spatial_idx = 0, .temporal_idx = 1}}},
.expected_keyframe_sum = 1,
.expected_encoded_bitrate_kbps =
DataRate::BytesPerSec(5).kbps<double>(),
.expected_encoded_framerate_fps = 2,
.expected_bitrate_mismatch_pct =
100 * (5.0 / (kTargetLayerBitrate.bytes_per_sec() * 2) - 1),
.expected_framerate_mismatch_pct =
100 * (2.0 / kTargetFramerate.hertz() - 1)},
// L1T0
AggregationTestParameters{
.filter = {.layer_id = {{.spatial_idx = 1, .temporal_idx = 0}}},
.expected_keyframe_sum = 1,
.expected_encoded_bitrate_kbps =
DataRate::BytesPerSec(3).kbps<double>(),
.expected_encoded_framerate_fps = 1,
.expected_bitrate_mismatch_pct =
100 * (3.0 / kTargetLayerBitrate.bytes_per_sec() - 1),
.expected_framerate_mismatch_pct =
100 * (1.0 / (kTargetFramerate.hertz() / 2) - 1)},
// L1T1
AggregationTestParameters{
.filter = {.layer_id = {{.spatial_idx = 1, .temporal_idx = 1}}},
.expected_keyframe_sum = 1,
.expected_encoded_bitrate_kbps =
DataRate::BytesPerSec(11).kbps<double>(),
.expected_encoded_framerate_fps = 2,
.expected_bitrate_mismatch_pct =
100 * (11.0 / (kTargetLayerBitrate.bytes_per_sec() * 2) - 1),
.expected_framerate_mismatch_pct =
100 * (2.0 / kTargetFramerate.hertz() - 1)}));
TEST(VideoCodecTester, Psnr) {
std::unique_ptr<VideoCodecStats> stats =
RunTest({{{.timestamp_rtp = 0, .frame_size = DataSize::Bytes(1)}},
{{.timestamp_rtp = 3000, .frame_size = DataSize::Bytes(1)}}},
ScalabilityMode::kL1T1);
std::vector<Frame> slice = stats->Slice(Filter{}, /*merge=*/false);
ASSERT_THAT(slice, SizeIs(2));
ASSERT_TRUE(slice[0].psnr.has_value());
ASSERT_TRUE(slice[1].psnr.has_value());
EXPECT_NEAR(slice[0].psnr->y, 42, 1);
EXPECT_NEAR(slice[0].psnr->u, 38, 1);
EXPECT_NEAR(slice[0].psnr->v, 36, 1);
EXPECT_NEAR(slice[1].psnr->y, 38, 1);
EXPECT_NEAR(slice[1].psnr->u, 36, 1);
EXPECT_NEAR(slice[1].psnr->v, 34, 1);
}
class VideoCodecTesterTestPacing
: public ::testing::TestWithParam<std::tuple<PacingSettings, int>> {
public:
const int kSourceWidth = 2;
const int kSourceHeight = 2;
const int kNumFrames = 3;
const int kTargetLayerBitrateKbps = 128;
const Frequency kTargetFramerate = Frequency::Hertz(10);
void SetUp() override {
source_yuv_file_path_ = webrtc::test::TempFilename(
webrtc::test::OutputPath(), "video_codec_tester_impl_unittest");
FILE* file = fopen(source_yuv_file_path_.c_str(), "wb");
for (int i = 0; i < 3 * kSourceWidth * kSourceHeight / 2; ++i) {
fwrite("x", 1, 1, file);
}
fclose(file);
}
protected:
std::string source_yuv_file_path_;
};
TEST_P(VideoCodecTesterTestPacing, PaceEncode) {
auto [pacing_settings, expected_delta_ms] = GetParam();
VideoSourceSettings video_source{
.file_path = source_yuv_file_path_,
.resolution = {.width = kSourceWidth, .height = kSourceHeight},
.framerate = kTargetFramerate};
NiceMock<MockVideoEncoderFactory> encoder_factory;
ON_CALL(encoder_factory, CreateVideoEncoder(_))
.WillByDefault([](const SdpVideoFormat&) {
return std::make_unique<NiceMock<MockVideoEncoder>>();
});
std::map<uint32_t, EncodingSettings> encoding_settings =
VideoCodecTester::CreateEncodingSettings(
"VP8", "L1T1", kSourceWidth, kSourceHeight, {kTargetLayerBitrateKbps},
kTargetFramerate.hertz(), kNumFrames);
EncoderSettings encoder_settings;
encoder_settings.pacing_settings = pacing_settings;
std::vector<Frame> frames =
VideoCodecTester::RunEncodeTest(video_source, &encoder_factory,
encoder_settings, encoding_settings)
->Slice(/*filter=*/{}, /*merge=*/false);
ASSERT_THAT(frames, SizeIs(kNumFrames));
EXPECT_NEAR((frames[1].encode_start - frames[0].encode_start).ms(),
expected_delta_ms, 10);
EXPECT_NEAR((frames[2].encode_start - frames[1].encode_start).ms(),
expected_delta_ms, 10);
}
TEST_P(VideoCodecTesterTestPacing, PaceDecode) {
auto [pacing_settings, expected_delta_ms] = GetParam();
MockCodedVideoSource video_source(kNumFrames, kTargetFramerate);
NiceMock<MockVideoDecoderFactory> decoder_factory;
ON_CALL(decoder_factory, CreateVideoDecoder(_))
.WillByDefault([](const SdpVideoFormat&) {
return std::make_unique<NiceMock<MockVideoDecoder>>();
});
DecoderSettings decoder_settings;
decoder_settings.pacing_settings = pacing_settings;
std::vector<Frame> frames =
VideoCodecTester::RunDecodeTest(&video_source, &decoder_factory,
decoder_settings, SdpVideoFormat("VP8"))
->Slice(/*filter=*/{}, /*merge=*/false);
ASSERT_THAT(frames, SizeIs(kNumFrames));
EXPECT_NEAR((frames[1].decode_start - frames[0].decode_start).ms(),
expected_delta_ms, 10);
EXPECT_NEAR((frames[2].decode_start - frames[1].decode_start).ms(),
expected_delta_ms, 10);
}
INSTANTIATE_TEST_SUITE_P(
DISABLED_All,
VideoCodecTesterTestPacing,
::testing::Values(
// No pacing.
std::make_tuple(PacingSettings{.mode = PacingMode::kNoPacing},
/*expected_delta_ms=*/0),
// Real-time pacing.
std::make_tuple(PacingSettings{.mode = PacingMode::kRealTime},
/*expected_delta_ms=*/100),
// Pace with specified constant rate.
std::make_tuple(PacingSettings{.mode = PacingMode::kConstantRate,
.constant_rate = Frequency::Hertz(20)},
/*expected_delta_ms=*/50)));
} // namespace test
} // namespace webrtc
Reference in New Issue View Git Blame Copy Permalink