admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add OutputFunctions to FrameEncodeSettings.

Browse Source 
Bug: b/336978562
Change-Id: I9fe2d78368f4619532e64670fc9167ddfe253ffe
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/350920
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42350}
This commit is contained in:
philipel 2024-05-20 16:49:13 +02:00 committed by WebRTC LUCI CQ
parent b57178b836
commit d194ec398b
7 changed files with 359 additions and 281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/video_codecs/BUILD.gn
View File

@ -289,6 +289,7 @@ rtc_source_set("video_encoder_interface") {
deps = [
":video_encoding_general",
"../../api/units:data_rate",
"../../api/units:data_size",
"../../api/units:time_delta",
"../../api/units:timestamp",
"../../api/video:encoded_image",

30
api/video_codecs/libaom_av1_encoder_factory.cc
View File

@ -270,8 +270,8 @@ bool ValidateEncodeParams(
const VideoEncoderInterface::FrameEncodeSettings& settings =
frame_settings[i];
if (!settings.result_callback) {
RTC_LOG(LS_ERROR) << "No result callback function provided.";
if (!settings.frame_output) {
RTC_LOG(LS_ERROR) << "No frame output provided.";
return false;
}
@ -618,10 +618,10 @@ aom_svc_params_t GetSvcParams(
void DoErrorCallback(std::vector<FrameEncodeSettings>& frame_settings) {
for (FrameEncodeSettings& settings : frame_settings) {
if (settings.result_callback) {
std::move(settings.result_callback)({});
if (settings.frame_output) {
settings.frame_output->EncodeComplete({});
// To avoid invoking any callback more than once.
settings.result_callback = {};
settings.frame_output = nullptr;
}
}
}
@ -769,6 +769,7 @@ void LibaomAv1Encoder::Encode(
EncodedData result;
aom_codec_iter_t iter = nullptr;
bool bitstream_produced = false;
while (const aom_codec_cx_pkt_t* pkt =
aom_codec_get_cx_data(&ctx_, &iter)) {
if (pkt->kind == AOM_CODEC_CX_FRAME_PKT && pkt->data.frame.sz > 0) {
@ -777,20 +778,27 @@ void LibaomAv1Encoder::Encode(
result.frame_type = pkt->data.frame.flags & AOM_EFLAG_FORCE_KF
? FrameType::kKeyframe
: FrameType::kDeltaFrame;
result.bitstream_data = EncodedImageBuffer::Create(
static_cast<uint8_t*>(pkt->data.frame.buf), pkt->data.frame.sz);
rtc::ArrayView<uint8_t> output_buffer =
settings.frame_output->GetBitstreamOutputBuffer(
DataSize::Bytes(pkt->data.frame.sz));
if (output_buffer.size() != pkt->data.frame.sz) {
DoErrorCallback(frame_settings);
return;
}
memcpy(output_buffer.data(), pkt->data.frame.buf, pkt->data.frame.sz);
bitstream_produced = true;
break;
}
}
if (result.bitstream_data == nullptr) {
if (!bitstream_produced) {
DoErrorCallback(frame_settings);
return;
} else {
RTC_CHECK(settings.result_callback);
std::move(settings.result_callback)(result);
RTC_CHECK(settings.frame_output);
settings.frame_output->EncodeComplete(result);
// To avoid invoking any callback more than once.
settings.result_callback = {};
settings.frame_output = nullptr;
}
}
}

511
api/video_codecs/libaom_av1_encoder_factory_test.cc
View File

@ -29,20 +29,14 @@
namespace webrtc {
namespace {
using ::testing::_;
using ::testing::ElementsAre;
using ::testing::Eq;
using ::testing::Field;
using ::testing::Gt;
using ::testing::IsEmpty;
using ::testing::Lt;
using ::testing::MockFunction;
using ::testing::NotNull;
using ::testing::Not;
using Cbr = VideoEncoderInterface::FrameEncodeSettings::Cbr;
using Cqp = VideoEncoderInterface::FrameEncodeSettings::Cqp;
using EncodedData = VideoEncoderInterface::EncodedData;
using EncodeResult = VideoEncoderInterface::EncodeResult;
using EncodeResultCallback = VideoEncoderInterface::EncodeResultCallback;
using FrameType = VideoEncoderInterface::FrameType;
std::unique_ptr<test::FrameReader> CreateFrameReader() {
@ -58,24 +52,6 @@ std::string OutPath() {
return res;
}
class EncodeResults {
public:
EncodeResultCallback Cb() {
return [&](const EncodeResult& result) { results_.push_back(result); };
}
EncodedData* FrameAt(int index) {
if (index < 0 || index > static_cast<int>(results_.size())) {
RTC_CHECK(false);
return nullptr;
}
return absl::get_if<EncodedData>(&results_[index]);
}
private:
std::vector<EncodeResult> results_;
};
class Av1Decoder : public DecodedImageCallback {
public:
Av1Decoder() : Av1Decoder("") {}
@ -106,12 +82,12 @@ class Av1Decoder : public DecodedImageCallback {
return 0;
}
VideoFrame Decode(const EncodedData& encoded_data) {
VideoFrame Decode(rtc::ArrayView<uint8_t> bitstream_data) {
EncodedImage img;
img.SetEncodedData(encoded_data.bitstream_data);
img.SetEncodedData(EncodedImageBuffer::Create(bitstream_data.data(),
bitstream_data.size()));
if (raw_out_file_) {
fwrite(encoded_data.bitstream_data->data(), 1,
encoded_data.bitstream_data->size(), raw_out_file_);
fwrite(bitstream_data.data(), 1, bitstream_data.size(), raw_out_file_);
}
decoder_->Decode(img, /*dont_care=*/0);
VideoFrame res(std::move(*decode_result_));
@ -125,8 +101,29 @@ class Av1Decoder : public DecodedImageCallback {
FILE* raw_out_file_ = nullptr;
};
struct EncOut {
std::vector<uint8_t> bitstream;
EncodeResult res;
};
class FrameEncoderSettingsBuilder {
public:
FrameEncoderSettingsBuilder() {
class IgnoredOutput : public VideoEncoderInterface::FrameOutput {
public:
rtc::ArrayView<uint8_t> GetBitstreamOutputBuffer(DataSize size) override {
unread_.resize(size.bytes());
return unread_;
}
void EncodeComplete(const EncodeResult& encode_result) override {}
private:
std::vector<uint8_t> unread_;
};
frame_encode_settings_.frame_output = std::make_unique<IgnoredOutput>();
}
FrameEncoderSettingsBuilder& Key() {
frame_encode_settings_.frame_type = FrameType::kKeyframe;
return *this;
@ -173,13 +170,13 @@ class FrameEncoderSettingsBuilder {
return *this;
}
FrameEncoderSettingsBuilder& Cb(EncodeResultCallback cb) {
frame_encode_settings_.result_callback = std::move(cb);
FrameEncoderSettingsBuilder& Effort(int effort_level) {
frame_encode_settings_.effort_level = effort_level;
return *this;
}
FrameEncoderSettingsBuilder& Effort(int effort_level) {
frame_encode_settings_.effort_level = effort_level;
FrameEncoderSettingsBuilder& Out(EncOut& out) {
frame_encode_settings_.frame_output = std::make_unique<FrameOut>(out);
return *this;
}
@ -188,6 +185,18 @@ class FrameEncoderSettingsBuilder {
}
private:
struct FrameOut : public VideoEncoderInterface::FrameOutput {
explicit FrameOut(EncOut& e) : eo(e) {}
rtc::ArrayView<uint8_t> GetBitstreamOutputBuffer(DataSize size) override {
eo.bitstream.resize(size.bytes());
return rtc::ArrayView<uint8_t>(eo.bitstream);
}
void EncodeComplete(const EncodeResult& encode_result) override {
eo.res = encode_result;
}
EncOut& eo;
};
VideoEncoderInterface::FrameEncodeSettings frame_encode_settings_;
};
@ -220,6 +229,18 @@ MATCHER_P2(ResolutionIs, width, height, "") {
return arg.width == width && arg.height == height;
}
MATCHER_P(QpIs, qp, "") {
if (auto ed = absl::get_if<EncodedData>(&arg.res)) {
return ed->encoded_qp == qp;
}
return false;
}
MATCHER(HasBitstreamAndMetaData, "") {
return !arg.bitstream.empty() &&
absl::holds_alternative<EncodedData>(arg.res);
}
double Psnr(const rtc::scoped_refptr<I420BufferInterface>& ref_buffer,
const VideoFrame& decoded_frame) {
return I420PSNR(*ref_buffer, *decoded_frame.video_frame_buffer()->ToI420());
@ -267,75 +288,77 @@ TEST(LibaomAv1EncoderFactory, QpRange) {
TEST(LibaomAv1Encoder, KeyframeUpdatesSpecifiedBuffer) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
Av1Decoder dec;
auto raw_key = frame_reader->PullFrame();
auto raw_delta = frame_reader->PullFrame();
EncOut key;
enc->Encode(raw_key, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(5).Key().Cb(res.Cb())}));
ASSERT_THAT(res.FrameAt(0), NotNull());
VideoFrame decoded_key = dec.Decode(*res.FrameAt(0));
ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(5).Key().Out(key)}));
ASSERT_THAT(key.bitstream, Not(IsEmpty()));
VideoFrame decoded_key = dec.Decode(key.bitstream);
EXPECT_THAT(Resolution(decoded_key), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(raw_key, decoded_key), Gt(40));
EncOut delta;
enc->Encode(raw_delta, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Cb(res.Cb())}));
ASSERT_THAT(res.FrameAt(1), Eq(nullptr));
ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Out(delta)}));
EXPECT_THAT(delta, Not(HasBitstreamAndMetaData()));
}
TEST(LibaomAv1Encoder, MidTemporalUnitKeyframeResetsBuffers) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
Av1Decoder dec;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Cb(res.Cb())}));
ASSERT_THAT(res.FrameAt(2), NotNull());
EncOut tu0_s2;
enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key(),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Out(tu0_s2)}));
EXPECT_THAT(tu0_s2, HasBitstreamAndMetaData());
EncOut tu1_s0;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Ref({0}).Cb(res.Cb()),
Fb().Rate(kCbr).Res(320, 180).S(1).Upd(1).Key().Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Cb(res.Cb())}));
ASSERT_THAT(res.FrameAt(3), Eq(nullptr));
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Ref({0}).Out(tu1_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Upd(1).Key(),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0})}));
EXPECT_THAT(tu1_s0, Not(HasBitstreamAndMetaData()));
}
TEST(LibaomAv1Encoder, ResolutionSwitching) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
rtc::scoped_refptr<I420Buffer> in0 = frame_reader->PullFrame();
EncOut tu0;
enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(320, 180).Upd(0).Key().Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(320, 180).Upd(0).Key().Out(tu0)}));
rtc::scoped_refptr<I420Buffer> in1 = frame_reader->PullFrame();
EncOut tu1;
enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Out(tu1)}));
rtc::scoped_refptr<I420Buffer> in2 = frame_reader->PullFrame();
EncOut tu2;
enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Out(tu2)}));
Av1Decoder dec;
VideoFrame f0 = dec.Decode(*res.FrameAt(0));
VideoFrame f0 = dec.Decode(tu0.bitstream);
EXPECT_THAT(Resolution(f0), ResolutionIs(320, 180));
// TD:
// EXPECT_THAT(Psnr(in0, f0), Gt(40));
VideoFrame f1 = dec.Decode(*res.FrameAt(1));
VideoFrame f1 = dec.Decode(tu1.bitstream);
EXPECT_THAT(Resolution(f1), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(in1, f1), Gt(40));
VideoFrame f2 = dec.Decode(*res.FrameAt(2));
VideoFrame f2 = dec.Decode(tu2.bitstream);
EXPECT_THAT(Resolution(f2), ResolutionIs(160, 90));
// TD:
// EXPECT_THAT(Psnr(in2, f2), Gt(40));
@ -344,38 +367,40 @@ TEST(LibaomAv1Encoder, ResolutionSwitching) {
TEST(LibaomAv1Encoder, InputResolutionSwitching) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
rtc::scoped_refptr<I420Buffer> in0 = frame_reader->PullFrame();
EncOut tu0;
enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).Upd(0).Key().Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(160, 90).Upd(0).Key().Out(tu0)}));
rtc::scoped_refptr<I420Buffer> in1 = frame_reader->PullFrame(
/*frame_num=*/nullptr,
/*resolution=*/{320, 180},
/*framerate_scale=*/{1, 1});
EncOut tu1;
enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Out(tu1)}));
rtc::scoped_refptr<I420Buffer> in2 = frame_reader->PullFrame(
/*frame_num=*/nullptr,
/*resolution=*/{160, 90},
/*framerate_scale=*/{1, 1});
EncOut tu2;
enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Out(tu2)}));
Av1Decoder dec;
VideoFrame f0 = dec.Decode(*res.FrameAt(0));
VideoFrame f0 = dec.Decode(tu0.bitstream);
EXPECT_THAT(Resolution(f0), ResolutionIs(160, 90));
// TD:
// EXPECT_THAT(Psnr(in0, f0), Gt(40));
VideoFrame f1 = dec.Decode(*res.FrameAt(1));
VideoFrame f1 = dec.Decode(tu1.bitstream);
EXPECT_THAT(Resolution(f1), ResolutionIs(160, 90));
// TD:
// EXPECT_THAT(Psnr(in1, f1), Gt(40));
VideoFrame f2 = dec.Decode(*res.FrameAt(2));
VideoFrame f2 = dec.Decode(tu2.bitstream);
EXPECT_THAT(Resolution(f2), ResolutionIs(160, 90));
EXPECT_THAT(Psnr(in2, f2), Gt(40));
}
@ -383,116 +408,132 @@ TEST(LibaomAv1Encoder, InputResolutionSwitching) {
TEST(LibaomAv1Encoder, TempoSpatial) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
const Cbr k10Fps{.duration = TimeDelta::Millis(100),
.target_bitrate = DataRate::KilobitsPerSec(500)};
const Cbr k20Fps{.duration = TimeDelta::Millis(50),
.target_bitrate = DataRate::KilobitsPerSec(500)};
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec(
{Fb().Rate(k10Fps).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()),
Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()),
Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())}));
{Fb().Rate(k10Fps).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1),
Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)}));
EncOut tu1_s2;
enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(50)},
ToVec({Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(
res.Cb())}));
ToVec({Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({2}).Upd(2).Out(
tu1_s2)}));
rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame();
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
enc->Encode(
frame, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec(
{Fb().Rate(k10Fps).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Cb(res.Cb()),
Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Cb(
res.Cb())}));
{Fb().Rate(k10Fps).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Out(tu2_s1),
Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Out(
tu2_s2)}));
Av1Decoder dec;
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(5))), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(tu1_s2.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180));
VideoFrame f = dec.Decode(*res.FrameAt(6));
VideoFrame f = dec.Decode(tu2_s2.bitstream);
EXPECT_THAT(Resolution(f), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(frame, f), Gt(40));
}
TEST(DISABLED_LibaomAv1Encoder, InvertedTempoSpatial) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
EncOut tu0_s0;
EncOut tu0_s1;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Upd(0).Key().Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(1).Ref({0}).Upd(1).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(kCbr).Res(640, 360).S(1).Ref({0}).Upd(1).Out(tu0_s1)}));
EncOut tu1_s0;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Out(tu1_s0)}));
EncOut tu2_s0;
EncOut tu2_s1;
rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame();
enc->Encode(
frame, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(1).Ref({1, 0}).Upd(1).Cb(
res.Cb())}));
ToVec(
{Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(640, 360).S(1).Ref({1, 0}).Upd(1).Out(tu2_s1)}));
Av1Decoder dec;
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(tu1_s0.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(640, 360));
}
TEST(LibaomAv1Encoder, SkipMidLayer) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)}));
EncOut tu1_s0;
EncOut tu1_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu1_s2)}));
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame();
enc->Encode(
frame, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Cb(
res.Cb())}));
ToVec(
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Out(tu2_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Out(tu2_s2)}));
Av1Decoder dec;
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(5))), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(6))), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(tu1_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu1_s2.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180));
VideoFrame f = dec.Decode(*res.FrameAt(7));
VideoFrame f = dec.Decode(tu2_s2.bitstream);
EXPECT_THAT(Resolution(f), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(frame, f), Gt(40));
}
@ -500,47 +541,55 @@ TEST(LibaomAv1Encoder, SkipMidLayer) {
TEST(LibaomAv1Encoder, L3T1) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
Av1Decoder dec;
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)}));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360));
auto tu1_frame = frame_reader->PullFrame();
EncOut tu1_s0;
EncOut tu1_s1;
EncOut tu1_s2;
enc->Encode(
tu1_frame, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Cb(
res.Cb())}));
ToVec(
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Out(tu1_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Out(tu1_s2)}));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu1_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu1_s1.bitstream)), ResolutionIs(320, 180));
VideoFrame f_tu1 = dec.Decode(*res.FrameAt(5));
EXPECT_THAT(Resolution(f_tu1), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(tu1_frame, f_tu1), Gt(40));
VideoFrame f_tu1_s2 = dec.Decode(tu1_s2.bitstream);
EXPECT_THAT(Resolution(f_tu1_s2), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(tu1_frame, f_tu1_s2), Gt(40));
auto tu2_frame = frame_reader->PullFrame();
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
enc->Encode(
tu2_frame, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Cb(
res.Cb())}));
ToVec(
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Out(tu2_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Out(tu2_s2)}));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(6))), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(7))), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180));
VideoFrame f_tu2 = dec.Decode(*res.FrameAt(8));
VideoFrame f_tu2 = dec.Decode(tu2_s2.bitstream);
EXPECT_THAT(Resolution(f_tu2), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(tu2_frame, f_tu2), Gt(40));
}
@ -548,108 +597,124 @@ TEST(LibaomAv1Encoder, L3T1) {
TEST(LibaomAv1Encoder, L3T1_KEY) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
Av1Decoder dec_s0;
Av1Decoder dec_s1;
Av1Decoder dec_s2;
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(0))),
EXPECT_THAT(Resolution(dec_s0.Decode(tu0_s0.bitstream)),
ResolutionIs(160, 90));
dec_s1.Decode(*res.FrameAt(0));
EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(1))),
dec_s1.Decode(tu0_s0.bitstream);
EXPECT_THAT(Resolution(dec_s1.Decode(tu0_s1.bitstream)),
ResolutionIs(320, 180));
dec_s2.Decode(*res.FrameAt(0));
dec_s2.Decode(*res.FrameAt(1));
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(2))),
dec_s2.Decode(tu0_s0.bitstream);
dec_s2.Decode(tu0_s1.bitstream);
EXPECT_THAT(Resolution(dec_s2.Decode(tu0_s2.bitstream)),
ResolutionIs(640, 360));
EncOut tu1_s0;
EncOut tu1_s1;
EncOut tu1_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu1_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu1_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(3))),
EXPECT_THAT(Resolution(dec_s0.Decode(tu1_s0.bitstream)),
ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(4))),
EXPECT_THAT(Resolution(dec_s1.Decode(tu1_s1.bitstream)),
ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(5))),
EXPECT_THAT(Resolution(dec_s2.Decode(tu1_s2.bitstream)),
ResolutionIs(640, 360));
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu2_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu2_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(6))),
EXPECT_THAT(Resolution(dec_s0.Decode(tu2_s0.bitstream)),
ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(7))),
EXPECT_THAT(Resolution(dec_s1.Decode(tu2_s1.bitstream)),
ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(8))),
EXPECT_THAT(Resolution(dec_s2.Decode(tu2_s2.bitstream)),
ResolutionIs(640, 360));
}
TEST(LibaomAv1Encoder, S3T1) {
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
Av1Decoder dec_s0;
Av1Decoder dec_s1;
Av1Decoder dec_s2;
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Start().Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(320, 180).S(1).Start().Upd(1).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Start().Upd(2).Cb(res.Cb())}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(0))),
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Start().Upd(0).Out(tu0_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Start().Upd(1).Out(tu0_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Start().Upd(2).Out(tu0_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(tu0_s0.bitstream)),
ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(1))),
EXPECT_THAT(Resolution(dec_s1.Decode(tu0_s1.bitstream)),
ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(2))),
EXPECT_THAT(Resolution(dec_s2.Decode(tu0_s2.bitstream)),
ResolutionIs(640, 360));
EncOut tu1_s0;
EncOut tu1_s1;
EncOut tu1_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu1_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu1_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(3))),
EXPECT_THAT(Resolution(dec_s0.Decode(tu1_s0.bitstream)),
ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(4))),
EXPECT_THAT(Resolution(dec_s1.Decode(tu1_s1.bitstream)),
ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(5))),
EXPECT_THAT(Resolution(dec_s2.Decode(tu1_s2.bitstream)),
ResolutionIs(640, 360));
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())}));
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu2_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu2_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(6))),
EXPECT_THAT(Resolution(dec_s0.Decode(tu2_s0.bitstream)),
ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(7))),
EXPECT_THAT(Resolution(dec_s1.Decode(tu2_s1.bitstream)),
ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(8))),
EXPECT_THAT(Resolution(dec_s2.Decode(tu2_s2.bitstream)),
ResolutionIs(640, 360));
}
@ -665,11 +730,11 @@ TEST(LibaomAv1Encoder, HigherEffortLevelYieldsHigherQualityFrames) {
for (int i = effort_range.first; i <= effort_range.second; ++i) {
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
enc->Encode(frame_in, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(0).Key().Effort(i).Cb(
res.Cb())}));
double psnr = Psnr(frame_in, dec.Decode(*res.FrameAt(0)));
EncOut tu0;
enc->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(0).Key().Effort(i).Out(tu0)}));
double psnr = Psnr(frame_in, dec.Decode(tu0.bitstream));
EXPECT_THAT(psnr, Gt(psnr_last));
psnr_last = psnr;
}
@ -699,24 +764,24 @@ TEST(LibaomAv1Encoder, KeyframeAndStartrameAreApproximatelyEqual) {
DataSize total_size_key = DataSize::Zero();
DataSize total_size_start = DataSize::Zero();
TimeDelta total_duration = TimeDelta::Zero();
EncodeResults res_key;
EncodeResults res_start;
auto frame_in = frame_reader->PullFrame();
EncOut key;
EncOut start;
enc_key->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Cb(
res_key.Cb())}));
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Out(key)}));
enc_start->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Start().Upd(0).Cb(
res_start.Cb())}));
total_size_key +=
DataSize::Bytes(res_key.FrameAt(0)->bitstream_data->size());
total_size_start +=
DataSize::Bytes(res_start.FrameAt(0)->bitstream_data->size());
ToVec(
{Fb().Rate(kRate).Res(640, 360).S(sid).Start().Upd(0).Out(start)}));
total_size_key += DataSize::Bytes(key.bitstream.size());
total_size_start += DataSize::Bytes(start.bitstream.size());
total_duration += kRate.duration;
dec_key.Decode(*res_key.FrameAt(0));
dec_start.Decode(*res_start.FrameAt(0));
dec_key.Decode(key.bitstream);
dec_start.Decode(start.bitstream);
EXPECT_NEAR(total_size_key.bytes(), total_size_start.bytes(),
0.1 * total_size_key.bytes());
@ -725,19 +790,18 @@ TEST(LibaomAv1Encoder, KeyframeAndStartrameAreApproximatelyEqual) {
frame_in = frame_reader->PullFrame();
enc_key->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Cb(
res_key.Cb())}));
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Out(
key)}));
enc_start->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Cb(
res_start.Cb())}));
total_size_key +=
DataSize::Bytes(res_key.FrameAt(f)->bitstream_data->size());
total_size_start +=
DataSize::Bytes(res_start.FrameAt(f)->bitstream_data->size());
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Out(
start)}));
total_size_key += DataSize::Bytes(key.bitstream.size());
total_size_start += DataSize::Bytes(start.bitstream.size());
total_duration += kRate.duration;
dec_key.Decode(*res_key.FrameAt(f));
dec_start.Decode(*res_start.FrameAt(f));
dec_key.Decode(key.bitstream);
dec_start.Decode(start.bitstream);
}
double key_encode_kbps = (total_size_key / total_duration).kbps();
@ -757,30 +821,28 @@ TEST(LibaomAv1Encoder, BitrateConsistentAcrossSpatialLayers) {
for (int sid = 0; sid < max_spatial_layers; ++sid) {
std::string out_name = "cbr_sl_";
out_name += std::to_string(sid);
Av1Decoder dec(out_name);
auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
DataSize total_size = DataSize::Zero();
TimeDelta total_duration = TimeDelta::Zero();
EncodeResults res;
enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Cb(
res.Cb())}));
total_size += DataSize::Bytes(res.FrameAt(0)->bitstream_data->size());
EncOut out;
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Out(out)}));
total_size += DataSize::Bytes(out.bitstream.size());
total_duration += kRate.duration;
dec.Decode(*res.FrameAt(0));
for (int f = 1; f < 30; ++f) {
enc->Encode(
frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(f * 100)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Cb(
res.Cb())}));
total_size += DataSize::Bytes(res.FrameAt(f)->bitstream_data->size());
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Out(
out)}));
total_size += DataSize::Bytes(out.bitstream.size());
total_duration += kRate.duration;
dec.Decode(*res.FrameAt(f));
}
double encode_kbps = (total_size / total_duration).kbps();
@ -799,11 +861,9 @@ TEST(LibaomAv1Encoder, ConstantQp) {
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCqpEncoderSettings, {});
std::string out_name = "cqp_sl_";
out_name += std::to_string(sid);
Av1Decoder dec(out_name);
DataSize total_size = DataSize::Zero();
auto frame_reader = CreateFrameReader();
EncodeResults res;
EncOut out;
enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(Cqp{.target_qp = kQp})
@ -811,10 +871,8 @@ TEST(LibaomAv1Encoder, ConstantQp) {
.S(sid)
.Upd(0)
.Key()
.Cb(res.Cb())}));
EXPECT_THAT(res.FrameAt(0)->encoded_qp, Eq(kQp));
total_size += DataSize::Bytes(res.FrameAt(0)->bitstream_data->size());
dec.Decode(*res.FrameAt(0));
.Out(out)}));
EXPECT_THAT(out, QpIs(kQp));
for (int f = 1; f < 10; ++f) {
enc->Encode(frame_reader->PullFrame(),
@ -824,9 +882,8 @@ TEST(LibaomAv1Encoder, ConstantQp) {
.S(sid)
.Ref({0})
.Upd(0)
.Cb(res.Cb())}));
EXPECT_THAT(res.FrameAt(f)->encoded_qp, Eq(kQp - f));
dec.Decode(*res.FrameAt(f));
.Out(out)}));
EXPECT_THAT(out, QpIs(kQp - f));
}
}
}

61
api/video_codecs/simple_encoder_wrapper.cc
View File

@ -147,7 +147,6 @@ void SimpleEncoderWrapper::Encode(
svc_controller_->NextFrameConfig(force_keyframe);
std::vector<FrameEncodeSettings> encode_settings;
std::vector<GenericFrameInfo> frame_infos;
bool include_dependency_structure = false;
for (size_t s = 0; s < configs.size(); ++s) {
const ScalableVideoController::LayerFrameConfig& config = configs[s];
@ -176,36 +175,46 @@ void SimpleEncoderWrapper::Encode(
if (settings.reference_buffers.empty()) {
settings.frame_type = FrameType::kKeyframe;
include_dependency_structure = true;
}
absl::optional<FrameDependencyStructure> dependency_structure;
if (include_dependency_structure) {
dependency_structure = svc_controller_->DependencyStructure();
}
struct FrameOut : public VideoEncoderInterface::FrameOutput {
rtc::ArrayView<uint8_t> GetBitstreamOutputBuffer(DataSize size) override {
bitstream.resize(size.bytes());
return bitstream;
}
settings.result_callback =
[cb = callback, ds = std::move(dependency_structure),
info = std::move(frame_infos[settings.spatial_id])](
const VideoEncoderInterface::EncodeResult& result) mutable {
auto* data =
absl::get_if<VideoEncoderInterface::EncodedData>(&result);
void EncodeComplete(
const VideoEncoderInterface::EncodeResult& result) override {
auto* data = absl::get_if<VideoEncoderInterface::EncodedData>(&result);
EncodeResult res;
if (!data) {
res.oh_no = true;
cb(res);
return;
}
SimpleEncoderWrapper::EncodeResult res;
if (!data) {
res.oh_no = true;
callback(res);
return;
}
res.frame_type = data->frame_type;
res.bitstream_data = std::move(data->bitstream_data);
res.generic_frame_info = info;
if (res.frame_type == FrameType::kKeyframe) {
res.dependency_structure = ds;
}
cb(res);
};
res.frame_type = data->frame_type;
res.bitstream_data = std::move(bitstream);
res.generic_frame_info = frame_info;
if (res.frame_type == FrameType::kKeyframe) {
res.dependency_structure = svc_controller->DependencyStructure();
}
callback(res);
}
std::vector<uint8_t> bitstream;
EncodeResultCallback callback;
GenericFrameInfo frame_info;
ScalableVideoController* svc_controller;
};
auto out = std::make_unique<FrameOut>();
out->callback = callback;
out->frame_info = std::move(frame_infos[settings.spatial_id]);
out->svc_controller = svc_controller_.get();
settings.frame_output = std::move(out);
}
encoder_->Encode(std::move(frame_buffer),

2
api/video_codecs/simple_encoder_wrapper.h
View File

@ -27,7 +27,7 @@ class SimpleEncoderWrapper {
public:
struct EncodeResult {
bool oh_no = false;
rtc::scoped_refptr<EncodedImageBufferInterface> bitstream_data;
std::vector<uint8_t> bitstream_data;
FrameType frame_type;
GenericFrameInfo generic_frame_info;
absl::optional<FrameDependencyStructure> dependency_structure;

12
api/video_codecs/simple_encoder_wrapper_unittests.cc
View File

@ -152,7 +152,7 @@ TEST(SimpleEncoderWrapper, EncodeL1T1) {
++num_callbacks;
ASSERT_THAT(result.oh_no, Eq(false));
EXPECT_THAT(result.dependency_structure, Ne(absl::nullopt));
EXPECT_THAT(result.bitstream_data, NotNull());
EXPECT_THAT(result.bitstream_data, Not(IsEmpty()));
EXPECT_THAT(result.frame_type, Eq(FrameType::kKeyframe));
EXPECT_THAT(result.generic_frame_info.spatial_id, Eq(0));
EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0));
@ -164,7 +164,7 @@ TEST(SimpleEncoderWrapper, EncodeL1T1) {
++num_callbacks;
ASSERT_THAT(result.oh_no, Eq(false));
EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt));
EXPECT_THAT(result.bitstream_data, NotNull());
EXPECT_THAT(result.bitstream_data, Not(IsEmpty()));
EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame));
EXPECT_THAT(result.generic_frame_info.spatial_id, Eq(0));
EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0));
@ -197,13 +197,13 @@ TEST(SimpleEncoderWrapper, EncodeL2T2_KEY) {
if (result.generic_frame_info.spatial_id == 0) {
++num_callbacks;
EXPECT_THAT(result.dependency_structure, Ne(absl::nullopt));
EXPECT_THAT(result.bitstream_data, NotNull());
EXPECT_THAT(result.bitstream_data, Not(IsEmpty()));
EXPECT_THAT(result.frame_type, Eq(FrameType::kKeyframe));
EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0));
} else if (result.generic_frame_info.spatial_id == 1) {
++num_callbacks;
EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt));
EXPECT_THAT(result.bitstream_data, NotNull());
EXPECT_THAT(result.bitstream_data, Not(IsEmpty()));
EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame));
EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0));
}
@ -216,13 +216,13 @@ TEST(SimpleEncoderWrapper, EncodeL2T2_KEY) {
if (result.generic_frame_info.spatial_id == 0) {
++num_callbacks;
EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt));
EXPECT_THAT(result.bitstream_data, NotNull());
EXPECT_THAT(result.bitstream_data, Not(IsEmpty()));
EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame));
EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(1));
} else if (result.generic_frame_info.spatial_id == 1) {
++num_callbacks;
EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt));
EXPECT_THAT(result.bitstream_data, NotNull());
EXPECT_THAT(result.bitstream_data, Not(IsEmpty()));
EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame));
EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(1));
}

23
api/video_codecs/video_encoder_interface.h
View File

@ -21,6 +21,7 @@
#include "absl/types/optional.h"
#include "absl/types/variant.h"
#include "api/units/data_rate.h"
#include "api/units/data_size.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "api/video/encoded_image.h"
@ -37,21 +38,23 @@ class VideoEncoderInterface {
virtual ~VideoEncoderInterface() = default;
enum class FrameType { kKeyframe, kStartFrame, kDeltaFrame };
struct TemporalUnitSettings {
VideoCodecMode content_hint = VideoCodecMode::kRealtimeVideo;
Timestamp presentation_timestamp;
};
// Results from calling Encode. Called once for each configured frame.
struct EncodingError {};
struct EncodedData {
rtc::scoped_refptr<EncodedImageBufferInterface> bitstream_data;
FrameType frame_type;
int encoded_qp;
};
using EncodeResult = absl::variant<EncodingError, EncodedData>;
using EncodeResultCallback =
absl::AnyInvocable<void(const EncodeResult& result) &&>;
struct FrameOutput {
virtual ~FrameOutput() = default;
virtual rtc::ArrayView<uint8_t> GetBitstreamOutputBuffer(DataSize size) = 0;
virtual void EncodeComplete(const EncodeResult& encode_result) = 0;
};
struct TemporalUnitSettings {
VideoCodecMode content_hint = VideoCodecMode::kRealtimeVideo;
Timestamp presentation_timestamp;
};
struct FrameEncodeSettings {
struct Cbr {
@ -73,7 +76,7 @@ class VideoEncoderInterface {
absl::optional<int> update_buffer;
int effort_level = 0;
EncodeResultCallback result_callback;
std::unique_ptr<FrameOutput> frame_output;
};
virtual void Encode(rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer,