admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add OutputFunctions to FrameEncodeSettings.

Browse Source 
Bug: b/336978562
Change-Id: I9fe2d78368f4619532e64670fc9167ddfe253ffe
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/350920
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42350}
This commit is contained in:
philipel 2024-05-20 16:49:13 +02:00 committed by WebRTC LUCI CQ
parent b57178b836
commit d194ec398b
7 changed files with 359 additions and 281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/video_codecs/BUILD.gn
View File

@ -289,6 +289,7 @@ rtc_source_set("video_encoder_interface") {
deps = [ deps = [
":video_encoding_general", ":video_encoding_general",
"../../api/units:data_rate", "../../api/units:data_rate",
"../../api/units:data_size",
"../../api/units:time_delta", "../../api/units:time_delta",
"../../api/units:timestamp", "../../api/units:timestamp",
"../../api/video:encoded_image", "../../api/video:encoded_image",

30
api/video_codecs/libaom_av1_encoder_factory.cc
View File

@ -270,8 +270,8 @@ bool ValidateEncodeParams(
const VideoEncoderInterface::FrameEncodeSettings& settings = const VideoEncoderInterface::FrameEncodeSettings& settings =
frame_settings[i]; frame_settings[i];
if (!settings.result_callback) { if (!settings.frame_output) {
RTC_LOG(LS_ERROR) << "No result callback function provided."; RTC_LOG(LS_ERROR) << "No frame output provided.";
return false; return false;
} }
@ -618,10 +618,10 @@ aom_svc_params_t GetSvcParams(
void DoErrorCallback(std::vector<FrameEncodeSettings>& frame_settings) { void DoErrorCallback(std::vector<FrameEncodeSettings>& frame_settings) {
for (FrameEncodeSettings& settings : frame_settings) { for (FrameEncodeSettings& settings : frame_settings) {
if (settings.result_callback) { if (settings.frame_output) {
std::move(settings.result_callback)({}); settings.frame_output->EncodeComplete({});
// To avoid invoking any callback more than once. // To avoid invoking any callback more than once.
settings.result_callback = {}; settings.frame_output = nullptr;
} }
} }
} }
@ -769,6 +769,7 @@ void LibaomAv1Encoder::Encode(
EncodedData result; EncodedData result;
aom_codec_iter_t iter = nullptr; aom_codec_iter_t iter = nullptr;
bool bitstream_produced = false;
while (const aom_codec_cx_pkt_t* pkt = while (const aom_codec_cx_pkt_t* pkt =
aom_codec_get_cx_data(&ctx_, &iter)) { aom_codec_get_cx_data(&ctx_, &iter)) {
if (pkt->kind == AOM_CODEC_CX_FRAME_PKT && pkt->data.frame.sz > 0) { if (pkt->kind == AOM_CODEC_CX_FRAME_PKT && pkt->data.frame.sz > 0) {
@ -777,20 +778,27 @@ void LibaomAv1Encoder::Encode(
result.frame_type = pkt->data.frame.flags & AOM_EFLAG_FORCE_KF result.frame_type = pkt->data.frame.flags & AOM_EFLAG_FORCE_KF
? FrameType::kKeyframe ? FrameType::kKeyframe
: FrameType::kDeltaFrame; : FrameType::kDeltaFrame;
result.bitstream_data = EncodedImageBuffer::Create( rtc::ArrayView<uint8_t> output_buffer =
static_cast<uint8_t*>(pkt->data.frame.buf), pkt->data.frame.sz); settings.frame_output->GetBitstreamOutputBuffer(
DataSize::Bytes(pkt->data.frame.sz));
if (output_buffer.size() != pkt->data.frame.sz) {
DoErrorCallback(frame_settings);
return;
}
memcpy(output_buffer.data(), pkt->data.frame.buf, pkt->data.frame.sz);
bitstream_produced = true;
break; break;
} }
} }
if (result.bitstream_data == nullptr) { if (!bitstream_produced) {
DoErrorCallback(frame_settings); DoErrorCallback(frame_settings);
return; return;
} else { } else {
RTC_CHECK(settings.result_callback); RTC_CHECK(settings.frame_output);
std::move(settings.result_callback)(result); settings.frame_output->EncodeComplete(result);
// To avoid invoking any callback more than once. // To avoid invoking any callback more than once.
settings.result_callback = {}; settings.frame_output = nullptr;
} }
} }
} }

511
api/video_codecs/libaom_av1_encoder_factory_test.cc
View File

@ -29,20 +29,14 @@
namespace webrtc { namespace webrtc {
namespace { namespace {
using ::testing::_;
using ::testing::ElementsAre;
using ::testing::Eq; using ::testing::Eq;
using ::testing::Field;
using ::testing::Gt; using ::testing::Gt;
using ::testing::IsEmpty; using ::testing::IsEmpty;
using ::testing::Lt; using ::testing::Not;
using ::testing::MockFunction;
using ::testing::NotNull;
using Cbr = VideoEncoderInterface::FrameEncodeSettings::Cbr; using Cbr = VideoEncoderInterface::FrameEncodeSettings::Cbr;
using Cqp = VideoEncoderInterface::FrameEncodeSettings::Cqp; using Cqp = VideoEncoderInterface::FrameEncodeSettings::Cqp;
using EncodedData = VideoEncoderInterface::EncodedData; using EncodedData = VideoEncoderInterface::EncodedData;
using EncodeResult = VideoEncoderInterface::EncodeResult; using EncodeResult = VideoEncoderInterface::EncodeResult;
using EncodeResultCallback = VideoEncoderInterface::EncodeResultCallback;
using FrameType = VideoEncoderInterface::FrameType; using FrameType = VideoEncoderInterface::FrameType;
std::unique_ptr<test::FrameReader> CreateFrameReader() { std::unique_ptr<test::FrameReader> CreateFrameReader() {
@ -58,24 +52,6 @@ std::string OutPath() {
return res; return res;
} }
class EncodeResults {
public:
EncodeResultCallback Cb() {
return [&](const EncodeResult& result) { results_.push_back(result); };
}
EncodedData* FrameAt(int index) {
if (index < 0 || index > static_cast<int>(results_.size())) {
RTC_CHECK(false);
return nullptr;
}
return absl::get_if<EncodedData>(&results_[index]);
}
private:
std::vector<EncodeResult> results_;
};
class Av1Decoder : public DecodedImageCallback { class Av1Decoder : public DecodedImageCallback {
public: public:
Av1Decoder() : Av1Decoder("") {} Av1Decoder() : Av1Decoder("") {}
@ -106,12 +82,12 @@ class Av1Decoder : public DecodedImageCallback {
return 0; return 0;
} }
VideoFrame Decode(const EncodedData& encoded_data) { VideoFrame Decode(rtc::ArrayView<uint8_t> bitstream_data) {
EncodedImage img; EncodedImage img;
img.SetEncodedData(encoded_data.bitstream_data); img.SetEncodedData(EncodedImageBuffer::Create(bitstream_data.data(),
bitstream_data.size()));
if (raw_out_file_) { if (raw_out_file_) {
fwrite(encoded_data.bitstream_data->data(), 1, fwrite(bitstream_data.data(), 1, bitstream_data.size(), raw_out_file_);
encoded_data.bitstream_data->size(), raw_out_file_);
} }
decoder_->Decode(img, /*dont_care=*/0); decoder_->Decode(img, /*dont_care=*/0);
VideoFrame res(std::move(*decode_result_)); VideoFrame res(std::move(*decode_result_));
@ -125,8 +101,29 @@ class Av1Decoder : public DecodedImageCallback {
FILE* raw_out_file_ = nullptr; FILE* raw_out_file_ = nullptr;
}; };
struct EncOut {
std::vector<uint8_t> bitstream;
EncodeResult res;
};
class FrameEncoderSettingsBuilder { class FrameEncoderSettingsBuilder {
public: public:
FrameEncoderSettingsBuilder() {
class IgnoredOutput : public VideoEncoderInterface::FrameOutput {
public:
rtc::ArrayView<uint8_t> GetBitstreamOutputBuffer(DataSize size) override {
unread_.resize(size.bytes());
return unread_;
}
void EncodeComplete(const EncodeResult& encode_result) override {}
private:
std::vector<uint8_t> unread_;
};
frame_encode_settings_.frame_output = std::make_unique<IgnoredOutput>();
}
FrameEncoderSettingsBuilder& Key() { FrameEncoderSettingsBuilder& Key() {
frame_encode_settings_.frame_type = FrameType::kKeyframe; frame_encode_settings_.frame_type = FrameType::kKeyframe;
return *this; return *this;
@ -173,13 +170,13 @@ class FrameEncoderSettingsBuilder {
return *this; return *this;
} }
FrameEncoderSettingsBuilder& Cb(EncodeResultCallback cb) { FrameEncoderSettingsBuilder& Effort(int effort_level) {
frame_encode_settings_.result_callback = std::move(cb); frame_encode_settings_.effort_level = effort_level;
return *this; return *this;
} }
FrameEncoderSettingsBuilder& Effort(int effort_level) { FrameEncoderSettingsBuilder& Out(EncOut& out) {
frame_encode_settings_.effort_level = effort_level; frame_encode_settings_.frame_output = std::make_unique<FrameOut>(out);
return *this; return *this;
} }
@ -188,6 +185,18 @@ class FrameEncoderSettingsBuilder {
} }
private: private:
struct FrameOut : public VideoEncoderInterface::FrameOutput {
explicit FrameOut(EncOut& e) : eo(e) {}
rtc::ArrayView<uint8_t> GetBitstreamOutputBuffer(DataSize size) override {
eo.bitstream.resize(size.bytes());
return rtc::ArrayView<uint8_t>(eo.bitstream);
}
void EncodeComplete(const EncodeResult& encode_result) override {
eo.res = encode_result;
}
EncOut& eo;
};
VideoEncoderInterface::FrameEncodeSettings frame_encode_settings_; VideoEncoderInterface::FrameEncodeSettings frame_encode_settings_;
}; };
@ -220,6 +229,18 @@ MATCHER_P2(ResolutionIs, width, height, "") {
return arg.width == width && arg.height == height; return arg.width == width && arg.height == height;
} }
MATCHER_P(QpIs, qp, "") {
if (auto ed = absl::get_if<EncodedData>(&arg.res)) {
return ed->encoded_qp == qp;
}
return false;
}
MATCHER(HasBitstreamAndMetaData, "") {
return !arg.bitstream.empty() &&
absl::holds_alternative<EncodedData>(arg.res);
}
double Psnr(const rtc::scoped_refptr<I420BufferInterface>& ref_buffer, double Psnr(const rtc::scoped_refptr<I420BufferInterface>& ref_buffer,
const VideoFrame& decoded_frame) { const VideoFrame& decoded_frame) {
return I420PSNR(*ref_buffer, *decoded_frame.video_frame_buffer()->ToI420()); return I420PSNR(*ref_buffer, *decoded_frame.video_frame_buffer()->ToI420());
@ -267,75 +288,77 @@ TEST(LibaomAv1EncoderFactory, QpRange) {
TEST(LibaomAv1Encoder, KeyframeUpdatesSpecifiedBuffer) { TEST(LibaomAv1Encoder, KeyframeUpdatesSpecifiedBuffer) {
auto frame_reader = CreateFrameReader(); auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
Av1Decoder dec; Av1Decoder dec;
auto raw_key = frame_reader->PullFrame(); auto raw_key = frame_reader->PullFrame();
auto raw_delta = frame_reader->PullFrame(); auto raw_delta = frame_reader->PullFrame();
EncOut key;
enc->Encode(raw_key, {.presentation_timestamp = Timestamp::Millis(0)}, enc->Encode(raw_key, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(5).Key().Cb(res.Cb())})); ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(5).Key().Out(key)}));
ASSERT_THAT(res.FrameAt(0), NotNull()); ASSERT_THAT(key.bitstream, Not(IsEmpty()));
VideoFrame decoded_key = dec.Decode(*res.FrameAt(0)); VideoFrame decoded_key = dec.Decode(key.bitstream);
EXPECT_THAT(Resolution(decoded_key), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(decoded_key), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(raw_key, decoded_key), Gt(40)); EXPECT_THAT(Psnr(raw_key, decoded_key), Gt(40));
EncOut delta;
enc->Encode(raw_delta, {.presentation_timestamp = Timestamp::Millis(100)}, enc->Encode(raw_delta, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Cb(res.Cb())})); ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Out(delta)}));
ASSERT_THAT(res.FrameAt(1), Eq(nullptr)); EXPECT_THAT(delta, Not(HasBitstreamAndMetaData()));
} }
TEST(LibaomAv1Encoder, MidTemporalUnitKeyframeResetsBuffers) { TEST(LibaomAv1Encoder, MidTemporalUnitKeyframeResetsBuffers) {
auto frame_reader = CreateFrameReader(); auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
Av1Decoder dec;
enc->Encode( EncOut tu0_s2;
frame_reader->PullFrame(), enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)}, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key(),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Cb(res.Cb()), Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Cb(res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Out(tu0_s2)}));
ASSERT_THAT(res.FrameAt(2), NotNull()); EXPECT_THAT(tu0_s2, HasBitstreamAndMetaData());
EncOut tu1_s0;
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)}, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Ref({0}).Cb(res.Cb()), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Ref({0}).Out(tu1_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Upd(1).Key().Cb(res.Cb()), Fb().Rate(kCbr).Res(320, 180).S(1).Upd(1).Key(),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Cb(res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0})}));
ASSERT_THAT(res.FrameAt(3), Eq(nullptr)); EXPECT_THAT(tu1_s0, Not(HasBitstreamAndMetaData()));
} }
TEST(LibaomAv1Encoder, ResolutionSwitching) { TEST(LibaomAv1Encoder, ResolutionSwitching) {
auto frame_reader = CreateFrameReader(); auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
rtc::scoped_refptr<I420Buffer> in0 = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> in0 = frame_reader->PullFrame();
EncOut tu0;
enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)}, enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(320, 180).Upd(0).Key().Cb(res.Cb())})); ToVec({Fb().Rate(kCbr).Res(320, 180).Upd(0).Key().Out(tu0)}));
rtc::scoped_refptr<I420Buffer> in1 = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> in1 = frame_reader->PullFrame();
EncOut tu1;
enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)}, enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Cb(res.Cb())})); ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Out(tu1)}));
rtc::scoped_refptr<I420Buffer> in2 = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> in2 = frame_reader->PullFrame();
EncOut tu2;
enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)}, enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Cb(res.Cb())})); ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Out(tu2)}));
Av1Decoder dec; Av1Decoder dec;
VideoFrame f0 = dec.Decode(*res.FrameAt(0)); VideoFrame f0 = dec.Decode(tu0.bitstream);
EXPECT_THAT(Resolution(f0), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(f0), ResolutionIs(320, 180));
// TD: // TD:
// EXPECT_THAT(Psnr(in0, f0), Gt(40)); // EXPECT_THAT(Psnr(in0, f0), Gt(40));
VideoFrame f1 = dec.Decode(*res.FrameAt(1)); VideoFrame f1 = dec.Decode(tu1.bitstream);
EXPECT_THAT(Resolution(f1), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(f1), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(in1, f1), Gt(40)); EXPECT_THAT(Psnr(in1, f1), Gt(40));
VideoFrame f2 = dec.Decode(*res.FrameAt(2)); VideoFrame f2 = dec.Decode(tu2.bitstream);
EXPECT_THAT(Resolution(f2), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(f2), ResolutionIs(160, 90));
// TD: // TD:
// EXPECT_THAT(Psnr(in2, f2), Gt(40)); // EXPECT_THAT(Psnr(in2, f2), Gt(40));
@ -344,38 +367,40 @@ TEST(LibaomAv1Encoder, ResolutionSwitching) {
TEST(LibaomAv1Encoder, InputResolutionSwitching) { TEST(LibaomAv1Encoder, InputResolutionSwitching) {
auto frame_reader = CreateFrameReader(); auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
rtc::scoped_refptr<I420Buffer> in0 = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> in0 = frame_reader->PullFrame();
EncOut tu0;
enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)}, enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).Upd(0).Key().Cb(res.Cb())})); ToVec({Fb().Rate(kCbr).Res(160, 90).Upd(0).Key().Out(tu0)}));
rtc::scoped_refptr<I420Buffer> in1 = frame_reader->PullFrame( rtc::scoped_refptr<I420Buffer> in1 = frame_reader->PullFrame(
/*frame_num=*/nullptr, /*frame_num=*/nullptr,
/*resolution=*/{320, 180}, /*resolution=*/{320, 180},
/*framerate_scale=*/{1, 1}); /*framerate_scale=*/{1, 1});
EncOut tu1;
enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)}, enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Cb(res.Cb())})); ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Out(tu1)}));
rtc::scoped_refptr<I420Buffer> in2 = frame_reader->PullFrame( rtc::scoped_refptr<I420Buffer> in2 = frame_reader->PullFrame(
/*frame_num=*/nullptr, /*frame_num=*/nullptr,
/*resolution=*/{160, 90}, /*resolution=*/{160, 90},
/*framerate_scale=*/{1, 1}); /*framerate_scale=*/{1, 1});
EncOut tu2;
enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)}, enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Cb(res.Cb())})); ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Out(tu2)}));
Av1Decoder dec; Av1Decoder dec;
VideoFrame f0 = dec.Decode(*res.FrameAt(0)); VideoFrame f0 = dec.Decode(tu0.bitstream);
EXPECT_THAT(Resolution(f0), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(f0), ResolutionIs(160, 90));
// TD: // TD:
// EXPECT_THAT(Psnr(in0, f0), Gt(40)); // EXPECT_THAT(Psnr(in0, f0), Gt(40));
VideoFrame f1 = dec.Decode(*res.FrameAt(1)); VideoFrame f1 = dec.Decode(tu1.bitstream);
EXPECT_THAT(Resolution(f1), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(f1), ResolutionIs(160, 90));
// TD: // TD:
// EXPECT_THAT(Psnr(in1, f1), Gt(40)); // EXPECT_THAT(Psnr(in1, f1), Gt(40));
VideoFrame f2 = dec.Decode(*res.FrameAt(2)); VideoFrame f2 = dec.Decode(tu2.bitstream);
EXPECT_THAT(Resolution(f2), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(f2), ResolutionIs(160, 90));
EXPECT_THAT(Psnr(in2, f2), Gt(40)); EXPECT_THAT(Psnr(in2, f2), Gt(40));
} }
@ -383,116 +408,132 @@ TEST(LibaomAv1Encoder, InputResolutionSwitching) {
TEST(LibaomAv1Encoder, TempoSpatial) { TEST(LibaomAv1Encoder, TempoSpatial) {
auto frame_reader = CreateFrameReader(); auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
const Cbr k10Fps{.duration = TimeDelta::Millis(100), const Cbr k10Fps{.duration = TimeDelta::Millis(100),
.target_bitrate = DataRate::KilobitsPerSec(500)}; .target_bitrate = DataRate::KilobitsPerSec(500)};
const Cbr k20Fps{.duration = TimeDelta::Millis(50), const Cbr k20Fps{.duration = TimeDelta::Millis(50),
.target_bitrate = DataRate::KilobitsPerSec(500)}; .target_bitrate = DataRate::KilobitsPerSec(500)};
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)}, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec( ToVec(
{Fb().Rate(k10Fps).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()), {Fb().Rate(k10Fps).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()), Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1),
Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())})); Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)}));
EncOut tu1_s2;
enc->Encode(frame_reader->PullFrame(), enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(50)}, {.presentation_timestamp = Timestamp::Millis(50)},
ToVec({Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({2}).Upd(2).Cb( ToVec({Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({2}).Upd(2).Out(
res.Cb())})); tu1_s2)}));
rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame();
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
enc->Encode( enc->Encode(
frame, {.presentation_timestamp = Timestamp::Millis(100)}, frame, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec( ToVec(
{Fb().Rate(k10Fps).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), {Fb().Rate(k10Fps).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Cb(res.Cb()), Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Out(tu2_s1),
Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Cb( Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Out(
res.Cb())})); tu2_s2)}));
Av1Decoder dec; Av1Decoder dec;
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(dec.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(dec.Decode(tu1_s2.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(5))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180));
VideoFrame f = dec.Decode(*res.FrameAt(6)); VideoFrame f = dec.Decode(tu2_s2.bitstream);
EXPECT_THAT(Resolution(f), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(f), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(frame, f), Gt(40)); EXPECT_THAT(Psnr(frame, f), Gt(40));
} }
TEST(DISABLED_LibaomAv1Encoder, InvertedTempoSpatial) { TEST(DISABLED_LibaomAv1Encoder, InvertedTempoSpatial) {
auto frame_reader = CreateFrameReader(); auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
EncOut tu0_s0;
EncOut tu0_s1;
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)}, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Upd(0).Key().Cb(res.Cb()), ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(kCbr).Res(640, 360).S(1).Ref({0}).Upd(1).Cb(res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(1).Ref({0}).Upd(1).Out(tu0_s1)}));
EncOut tu1_s0;
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)}, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Cb(res.Cb())})); ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Out(tu1_s0)}));
EncOut tu2_s0;
EncOut tu2_s1;
rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame();
enc->Encode( enc->Encode(
frame, {.presentation_timestamp = Timestamp::Millis(200)}, frame, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Cb(res.Cb()), ToVec(
Fb().Rate(kCbr).Res(640, 360).S(1).Ref({1, 0}).Upd(1).Cb( {Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Out(tu2_s0),
res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(1).Ref({1, 0}).Upd(1).Out(tu2_s1)}));
Av1Decoder dec; Av1Decoder dec;
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(tu1_s0.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(640, 360));
} }
TEST(LibaomAv1Encoder, SkipMidLayer) { TEST(LibaomAv1Encoder, SkipMidLayer) {
auto frame_reader = CreateFrameReader(); auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)}, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()), Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)}));
EncOut tu1_s0;
EncOut tu1_s2;
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)}, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu1_s2)}));
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame();
enc->Encode( enc->Encode(
frame, {.presentation_timestamp = Timestamp::Millis(200)}, frame, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), ToVec(
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Cb(res.Cb()), {Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Cb( Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Out(tu2_s1),
res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Out(tu2_s2)}));
Av1Decoder dec; Av1Decoder dec;
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(dec.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(dec.Decode(tu1_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(dec.Decode(tu1_s2.bitstream)), ResolutionIs(640, 360));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(5))), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(6))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180));
VideoFrame f = dec.Decode(*res.FrameAt(7)); VideoFrame f = dec.Decode(tu2_s2.bitstream);
EXPECT_THAT(Resolution(f), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(f), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(frame, f), Gt(40)); EXPECT_THAT(Psnr(frame, f), Gt(40));
} }
@ -500,47 +541,55 @@ TEST(LibaomAv1Encoder, SkipMidLayer) {
TEST(LibaomAv1Encoder, L3T1) { TEST(LibaomAv1Encoder, L3T1) {
auto frame_reader = CreateFrameReader(); auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
Av1Decoder dec; Av1Decoder dec;
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)}, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()), Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)}));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(dec.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360));
auto tu1_frame = frame_reader->PullFrame(); auto tu1_frame = frame_reader->PullFrame();
EncOut tu1_s0;
EncOut tu1_s1;
EncOut tu1_s2;
enc->Encode( enc->Encode(
tu1_frame, {.presentation_timestamp = Timestamp::Millis(100)}, tu1_frame, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), ToVec(
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Cb(res.Cb()), {Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Cb( Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Out(tu1_s1),
res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Out(tu1_s2)}));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(dec.Decode(tu1_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(tu1_s1.bitstream)), ResolutionIs(320, 180));
VideoFrame f_tu1 = dec.Decode(*res.FrameAt(5)); VideoFrame f_tu1_s2 = dec.Decode(tu1_s2.bitstream);
EXPECT_THAT(Resolution(f_tu1), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(f_tu1_s2), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(tu1_frame, f_tu1), Gt(40)); EXPECT_THAT(Psnr(tu1_frame, f_tu1_s2), Gt(40));
auto tu2_frame = frame_reader->PullFrame(); auto tu2_frame = frame_reader->PullFrame();
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
enc->Encode( enc->Encode(
tu2_frame, {.presentation_timestamp = Timestamp::Millis(200)}, tu2_frame, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), ToVec(
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Cb(res.Cb()), {Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Cb( Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Out(tu2_s1),
res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Out(tu2_s2)}));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(6))), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(7))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180));
VideoFrame f_tu2 = dec.Decode(*res.FrameAt(8)); VideoFrame f_tu2 = dec.Decode(tu2_s2.bitstream);
EXPECT_THAT(Resolution(f_tu2), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(f_tu2), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(tu2_frame, f_tu2), Gt(40)); EXPECT_THAT(Psnr(tu2_frame, f_tu2), Gt(40));
} }
@ -548,108 +597,124 @@ TEST(LibaomAv1Encoder, L3T1) {
TEST(LibaomAv1Encoder, L3T1_KEY) { TEST(LibaomAv1Encoder, L3T1_KEY) {
auto frame_reader = CreateFrameReader(); auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
Av1Decoder dec_s0; Av1Decoder dec_s0;
Av1Decoder dec_s1; Av1Decoder dec_s1;
Av1Decoder dec_s2; Av1Decoder dec_s2;
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)}, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()), Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(0))), EXPECT_THAT(Resolution(dec_s0.Decode(tu0_s0.bitstream)),
ResolutionIs(160, 90)); ResolutionIs(160, 90));
dec_s1.Decode(*res.FrameAt(0)); dec_s1.Decode(tu0_s0.bitstream);
EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(1))), EXPECT_THAT(Resolution(dec_s1.Decode(tu0_s1.bitstream)),
ResolutionIs(320, 180)); ResolutionIs(320, 180));
dec_s2.Decode(*res.FrameAt(0)); dec_s2.Decode(tu0_s0.bitstream);
dec_s2.Decode(*res.FrameAt(1)); dec_s2.Decode(tu0_s1.bitstream);
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(2))), EXPECT_THAT(Resolution(dec_s2.Decode(tu0_s2.bitstream)),
ResolutionIs(640, 360)); ResolutionIs(640, 360));
EncOut tu1_s0;
EncOut tu1_s1;
EncOut tu1_s2;
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)}, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()), Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu1_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu1_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(3))), EXPECT_THAT(Resolution(dec_s0.Decode(tu1_s0.bitstream)),
ResolutionIs(160, 90)); ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(4))), EXPECT_THAT(Resolution(dec_s1.Decode(tu1_s1.bitstream)),
ResolutionIs(320, 180)); ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(5))), EXPECT_THAT(Resolution(dec_s2.Decode(tu1_s2.bitstream)),
ResolutionIs(640, 360)); ResolutionIs(640, 360));
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(200)}, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()), Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu2_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu2_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(6))), EXPECT_THAT(Resolution(dec_s0.Decode(tu2_s0.bitstream)),
ResolutionIs(160, 90)); ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(7))), EXPECT_THAT(Resolution(dec_s1.Decode(tu2_s1.bitstream)),
ResolutionIs(320, 180)); ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(8))), EXPECT_THAT(Resolution(dec_s2.Decode(tu2_s2.bitstream)),
ResolutionIs(640, 360)); ResolutionIs(640, 360));
} }
TEST(LibaomAv1Encoder, S3T1) { TEST(LibaomAv1Encoder, S3T1) {
auto frame_reader = CreateFrameReader(); auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res;
Av1Decoder dec_s0; Av1Decoder dec_s0;
Av1Decoder dec_s1; Av1Decoder dec_s1;
Av1Decoder dec_s2; Av1Decoder dec_s2;
EncOut tu0_s0;
EncOut tu0_s1;
EncOut tu0_s2;
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)}, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Start().Upd(0).Cb(res.Cb()), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Start().Upd(0).Out(tu0_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Start().Upd(1).Cb(res.Cb()), Fb().Rate(kCbr).Res(320, 180).S(1).Start().Upd(1).Out(tu0_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Start().Upd(2).Cb(res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Start().Upd(2).Out(tu0_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(0))), EXPECT_THAT(Resolution(dec_s0.Decode(tu0_s0.bitstream)),
ResolutionIs(160, 90)); ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(1))), EXPECT_THAT(Resolution(dec_s1.Decode(tu0_s1.bitstream)),
ResolutionIs(320, 180)); ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(2))), EXPECT_THAT(Resolution(dec_s2.Decode(tu0_s2.bitstream)),
ResolutionIs(640, 360)); ResolutionIs(640, 360));
EncOut tu1_s0;
EncOut tu1_s1;
EncOut tu1_s2;
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(100)}, {.presentation_timestamp = Timestamp::Millis(100)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()), Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu1_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu1_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(3))), EXPECT_THAT(Resolution(dec_s0.Decode(tu1_s0.bitstream)),
ResolutionIs(160, 90)); ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(4))), EXPECT_THAT(Resolution(dec_s1.Decode(tu1_s1.bitstream)),
ResolutionIs(320, 180)); ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(5))), EXPECT_THAT(Resolution(dec_s2.Decode(tu1_s2.bitstream)),
ResolutionIs(640, 360)); ResolutionIs(640, 360));
EncOut tu2_s0;
EncOut tu2_s1;
EncOut tu2_s2;
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(200)}, {.presentation_timestamp = Timestamp::Millis(200)},
ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0),
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()), Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu2_s1),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())})); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu2_s2)}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(6))), EXPECT_THAT(Resolution(dec_s0.Decode(tu2_s0.bitstream)),
ResolutionIs(160, 90)); ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(7))), EXPECT_THAT(Resolution(dec_s1.Decode(tu2_s1.bitstream)),
ResolutionIs(320, 180)); ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(8))), EXPECT_THAT(Resolution(dec_s2.Decode(tu2_s2.bitstream)),
ResolutionIs(640, 360)); ResolutionIs(640, 360));
} }
@ -665,11 +730,11 @@ TEST(LibaomAv1Encoder, HigherEffortLevelYieldsHigherQualityFrames) {
for (int i = effort_range.first; i <= effort_range.second; ++i) { for (int i = effort_range.first; i <= effort_range.second; ++i) {
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res; EncOut tu0;
enc->Encode(frame_in, {.presentation_timestamp = Timestamp::Millis(0)}, enc->Encode(
ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(0).Key().Effort(i).Cb( frame_in, {.presentation_timestamp = Timestamp::Millis(0)},
res.Cb())})); ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(0).Key().Effort(i).Out(tu0)}));
double psnr = Psnr(frame_in, dec.Decode(*res.FrameAt(0))); double psnr = Psnr(frame_in, dec.Decode(tu0.bitstream));
EXPECT_THAT(psnr, Gt(psnr_last)); EXPECT_THAT(psnr, Gt(psnr_last));
psnr_last = psnr; psnr_last = psnr;
} }
@ -699,24 +764,24 @@ TEST(LibaomAv1Encoder, KeyframeAndStartrameAreApproximatelyEqual) {
DataSize total_size_key = DataSize::Zero(); DataSize total_size_key = DataSize::Zero();
DataSize total_size_start = DataSize::Zero(); DataSize total_size_start = DataSize::Zero();
TimeDelta total_duration = TimeDelta::Zero(); TimeDelta total_duration = TimeDelta::Zero();
EncodeResults res_key;
EncodeResults res_start;
auto frame_in = frame_reader->PullFrame(); auto frame_in = frame_reader->PullFrame();
EncOut key;
EncOut start;
enc_key->Encode( enc_key->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(0)}, frame_in, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Cb( ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Out(key)}));
res_key.Cb())}));
enc_start->Encode( enc_start->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(0)}, frame_in, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Start().Upd(0).Cb( ToVec(
res_start.Cb())})); {Fb().Rate(kRate).Res(640, 360).S(sid).Start().Upd(0).Out(start)}));
total_size_key +=
DataSize::Bytes(res_key.FrameAt(0)->bitstream_data->size()); total_size_key += DataSize::Bytes(key.bitstream.size());
total_size_start += total_size_start += DataSize::Bytes(start.bitstream.size());
DataSize::Bytes(res_start.FrameAt(0)->bitstream_data->size());
total_duration += kRate.duration; total_duration += kRate.duration;
dec_key.Decode(*res_key.FrameAt(0)); dec_key.Decode(key.bitstream);
dec_start.Decode(*res_start.FrameAt(0)); dec_start.Decode(start.bitstream);
EXPECT_NEAR(total_size_key.bytes(), total_size_start.bytes(), EXPECT_NEAR(total_size_key.bytes(), total_size_start.bytes(),
0.1 * total_size_key.bytes()); 0.1 * total_size_key.bytes());
@ -725,19 +790,18 @@ TEST(LibaomAv1Encoder, KeyframeAndStartrameAreApproximatelyEqual) {
frame_in = frame_reader->PullFrame(); frame_in = frame_reader->PullFrame();
enc_key->Encode( enc_key->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)}, frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Cb( ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Out(
res_key.Cb())})); key)}));
enc_start->Encode( enc_start->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)}, frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Cb( ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Out(
res_start.Cb())})); start)}));
total_size_key += total_size_key += DataSize::Bytes(key.bitstream.size());
DataSize::Bytes(res_key.FrameAt(f)->bitstream_data->size()); total_size_start += DataSize::Bytes(start.bitstream.size());
total_size_start +=
DataSize::Bytes(res_start.FrameAt(f)->bitstream_data->size());
total_duration += kRate.duration; total_duration += kRate.duration;
dec_key.Decode(*res_key.FrameAt(f)); dec_key.Decode(key.bitstream);
dec_start.Decode(*res_start.FrameAt(f)); dec_start.Decode(start.bitstream);
} }
double key_encode_kbps = (total_size_key / total_duration).kbps(); double key_encode_kbps = (total_size_key / total_duration).kbps();
@ -757,30 +821,28 @@ TEST(LibaomAv1Encoder, BitrateConsistentAcrossSpatialLayers) {
for (int sid = 0; sid < max_spatial_layers; ++sid) { for (int sid = 0; sid < max_spatial_layers; ++sid) {
std::string out_name = "cbr_sl_"; std::string out_name = "cbr_sl_";
out_name += std::to_string(sid); out_name += std::to_string(sid);
Av1Decoder dec(out_name);
auto frame_reader = CreateFrameReader(); auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
DataSize total_size = DataSize::Zero(); DataSize total_size = DataSize::Zero();
TimeDelta total_duration = TimeDelta::Zero(); TimeDelta total_duration = TimeDelta::Zero();
EncodeResults res;
enc->Encode(frame_reader->PullFrame(), EncOut out;
{.presentation_timestamp = Timestamp::Millis(0)}, enc->Encode(
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Cb( frame_reader->PullFrame(),
res.Cb())})); {.presentation_timestamp = Timestamp::Millis(0)},
total_size += DataSize::Bytes(res.FrameAt(0)->bitstream_data->size()); ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Out(out)}));
total_size += DataSize::Bytes(out.bitstream.size());
total_duration += kRate.duration; total_duration += kRate.duration;
dec.Decode(*res.FrameAt(0));
for (int f = 1; f < 30; ++f) { for (int f = 1; f < 30; ++f) {
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(f * 100)}, {.presentation_timestamp = Timestamp::Millis(f * 100)},
ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Cb( ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Out(
res.Cb())})); out)}));
total_size += DataSize::Bytes(res.FrameAt(f)->bitstream_data->size()); total_size += DataSize::Bytes(out.bitstream.size());
total_duration += kRate.duration; total_duration += kRate.duration;
dec.Decode(*res.FrameAt(f));
} }
double encode_kbps = (total_size / total_duration).kbps(); double encode_kbps = (total_size / total_duration).kbps();
@ -799,11 +861,9 @@ TEST(LibaomAv1Encoder, ConstantQp) {
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCqpEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCqpEncoderSettings, {});
std::string out_name = "cqp_sl_"; std::string out_name = "cqp_sl_";
out_name += std::to_string(sid); out_name += std::to_string(sid);
Av1Decoder dec(out_name);
DataSize total_size = DataSize::Zero();
auto frame_reader = CreateFrameReader(); auto frame_reader = CreateFrameReader();
EncodeResults res;
EncOut out;
enc->Encode(frame_reader->PullFrame(), enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)}, {.presentation_timestamp = Timestamp::Millis(0)},
ToVec({Fb().Rate(Cqp{.target_qp = kQp}) ToVec({Fb().Rate(Cqp{.target_qp = kQp})
@ -811,10 +871,8 @@ TEST(LibaomAv1Encoder, ConstantQp) {
.S(sid) .S(sid)
.Upd(0) .Upd(0)
.Key() .Key()
.Cb(res.Cb())})); .Out(out)}));
EXPECT_THAT(res.FrameAt(0)->encoded_qp, Eq(kQp)); EXPECT_THAT(out, QpIs(kQp));
total_size += DataSize::Bytes(res.FrameAt(0)->bitstream_data->size());
dec.Decode(*res.FrameAt(0));
for (int f = 1; f < 10; ++f) { for (int f = 1; f < 10; ++f) {
enc->Encode(frame_reader->PullFrame(), enc->Encode(frame_reader->PullFrame(),
@ -824,9 +882,8 @@ TEST(LibaomAv1Encoder, ConstantQp) {
.S(sid) .S(sid)
.Ref({0}) .Ref({0})
.Upd(0) .Upd(0)
.Cb(res.Cb())})); .Out(out)}));
EXPECT_THAT(res.FrameAt(f)->encoded_qp, Eq(kQp - f)); EXPECT_THAT(out, QpIs(kQp - f));
dec.Decode(*res.FrameAt(f));
} }
} }
} }

61
api/video_codecs/simple_encoder_wrapper.cc
View File

@ -147,7 +147,6 @@ void SimpleEncoderWrapper::Encode(
svc_controller_->NextFrameConfig(force_keyframe); svc_controller_->NextFrameConfig(force_keyframe);
std::vector<FrameEncodeSettings> encode_settings; std::vector<FrameEncodeSettings> encode_settings;
std::vector<GenericFrameInfo> frame_infos; std::vector<GenericFrameInfo> frame_infos;
bool include_dependency_structure = false;
for (size_t s = 0; s < configs.size(); ++s) { for (size_t s = 0; s < configs.size(); ++s) {
const ScalableVideoController::LayerFrameConfig& config = configs[s]; const ScalableVideoController::LayerFrameConfig& config = configs[s];
@ -176,36 +175,46 @@ void SimpleEncoderWrapper::Encode(
if (settings.reference_buffers.empty()) { if (settings.reference_buffers.empty()) {
settings.frame_type = FrameType::kKeyframe; settings.frame_type = FrameType::kKeyframe;
include_dependency_structure = true;
} }
absl::optional<FrameDependencyStructure> dependency_structure; struct FrameOut : public VideoEncoderInterface::FrameOutput {
if (include_dependency_structure) { rtc::ArrayView<uint8_t> GetBitstreamOutputBuffer(DataSize size) override {
dependency_structure = svc_controller_->DependencyStructure(); bitstream.resize(size.bytes());
} return bitstream;
}
settings.result_callback = void EncodeComplete(
[cb = callback, ds = std::move(dependency_structure), const VideoEncoderInterface::EncodeResult& result) override {
info = std::move(frame_infos[settings.spatial_id])]( auto* data = absl::get_if<VideoEncoderInterface::EncodedData>(&result);
const VideoEncoderInterface::EncodeResult& result) mutable {
auto* data =
absl::get_if<VideoEncoderInterface::EncodedData>(&result);
EncodeResult res; SimpleEncoderWrapper::EncodeResult res;
if (!data) { if (!data) {
res.oh_no = true; res.oh_no = true;
cb(res); callback(res);
return; return;
} }
res.frame_type = data->frame_type; res.frame_type = data->frame_type;
res.bitstream_data = std::move(data->bitstream_data); res.bitstream_data = std::move(bitstream);
res.generic_frame_info = info; res.generic_frame_info = frame_info;
if (res.frame_type == FrameType::kKeyframe) { if (res.frame_type == FrameType::kKeyframe) {
res.dependency_structure = ds; res.dependency_structure = svc_controller->DependencyStructure();
} }
cb(res); callback(res);
}; }
std::vector<uint8_t> bitstream;
EncodeResultCallback callback;
GenericFrameInfo frame_info;
ScalableVideoController* svc_controller;
};
auto out = std::make_unique<FrameOut>();
out->callback = callback;
out->frame_info = std::move(frame_infos[settings.spatial_id]);
out->svc_controller = svc_controller_.get();
settings.frame_output = std::move(out);
} }
encoder_->Encode(std::move(frame_buffer), encoder_->Encode(std::move(frame_buffer),

2
api/video_codecs/simple_encoder_wrapper.h
View File

@ -27,7 +27,7 @@ class SimpleEncoderWrapper {
public: public:
struct EncodeResult { struct EncodeResult {
bool oh_no = false; bool oh_no = false;
rtc::scoped_refptr<EncodedImageBufferInterface> bitstream_data; std::vector<uint8_t> bitstream_data;
FrameType frame_type; FrameType frame_type;
GenericFrameInfo generic_frame_info; GenericFrameInfo generic_frame_info;
absl::optional<FrameDependencyStructure> dependency_structure; absl::optional<FrameDependencyStructure> dependency_structure;

12
api/video_codecs/simple_encoder_wrapper_unittests.cc
View File

@ -152,7 +152,7 @@ TEST(SimpleEncoderWrapper, EncodeL1T1) {
++num_callbacks; ++num_callbacks;
ASSERT_THAT(result.oh_no, Eq(false)); ASSERT_THAT(result.oh_no, Eq(false));
EXPECT_THAT(result.dependency_structure, Ne(absl::nullopt)); EXPECT_THAT(result.dependency_structure, Ne(absl::nullopt));
EXPECT_THAT(result.bitstream_data, NotNull()); EXPECT_THAT(result.bitstream_data, Not(IsEmpty()));
EXPECT_THAT(result.frame_type, Eq(FrameType::kKeyframe)); EXPECT_THAT(result.frame_type, Eq(FrameType::kKeyframe));
EXPECT_THAT(result.generic_frame_info.spatial_id, Eq(0)); EXPECT_THAT(result.generic_frame_info.spatial_id, Eq(0));
EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0)); EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0));
@ -164,7 +164,7 @@ TEST(SimpleEncoderWrapper, EncodeL1T1) {
++num_callbacks; ++num_callbacks;
ASSERT_THAT(result.oh_no, Eq(false)); ASSERT_THAT(result.oh_no, Eq(false));
EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt)); EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt));
EXPECT_THAT(result.bitstream_data, NotNull()); EXPECT_THAT(result.bitstream_data, Not(IsEmpty()));
EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame)); EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame));
EXPECT_THAT(result.generic_frame_info.spatial_id, Eq(0)); EXPECT_THAT(result.generic_frame_info.spatial_id, Eq(0));
EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0)); EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0));
@ -197,13 +197,13 @@ TEST(SimpleEncoderWrapper, EncodeL2T2_KEY) {
if (result.generic_frame_info.spatial_id == 0) { if (result.generic_frame_info.spatial_id == 0) {
++num_callbacks; ++num_callbacks;
EXPECT_THAT(result.dependency_structure, Ne(absl::nullopt)); EXPECT_THAT(result.dependency_structure, Ne(absl::nullopt));
EXPECT_THAT(result.bitstream_data, NotNull()); EXPECT_THAT(result.bitstream_data, Not(IsEmpty()));
EXPECT_THAT(result.frame_type, Eq(FrameType::kKeyframe)); EXPECT_THAT(result.frame_type, Eq(FrameType::kKeyframe));
EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0)); EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0));
} else if (result.generic_frame_info.spatial_id == 1) { } else if (result.generic_frame_info.spatial_id == 1) {
++num_callbacks; ++num_callbacks;
EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt)); EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt));
EXPECT_THAT(result.bitstream_data, NotNull()); EXPECT_THAT(result.bitstream_data, Not(IsEmpty()));
EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame)); EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame));
EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0)); EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0));
} }
@ -216,13 +216,13 @@ TEST(SimpleEncoderWrapper, EncodeL2T2_KEY) {
if (result.generic_frame_info.spatial_id == 0) { if (result.generic_frame_info.spatial_id == 0) {
++num_callbacks; ++num_callbacks;
EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt)); EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt));
EXPECT_THAT(result.bitstream_data, NotNull()); EXPECT_THAT(result.bitstream_data, Not(IsEmpty()));
EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame)); EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame));
EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(1)); EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(1));
} else if (result.generic_frame_info.spatial_id == 1) { } else if (result.generic_frame_info.spatial_id == 1) {
++num_callbacks; ++num_callbacks;
EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt)); EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt));
EXPECT_THAT(result.bitstream_data, NotNull()); EXPECT_THAT(result.bitstream_data, Not(IsEmpty()));
EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame)); EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame));
EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(1)); EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(1));
} }

23
api/video_codecs/video_encoder_interface.h
View File

@ -21,6 +21,7 @@
#include "absl/types/optional.h" #include "absl/types/optional.h"
#include "absl/types/variant.h" #include "absl/types/variant.h"
#include "api/units/data_rate.h" #include "api/units/data_rate.h"
#include "api/units/data_size.h"
#include "api/units/time_delta.h" #include "api/units/time_delta.h"
#include "api/units/timestamp.h" #include "api/units/timestamp.h"
#include "api/video/encoded_image.h" #include "api/video/encoded_image.h"
@ -37,21 +38,23 @@ class VideoEncoderInterface {
virtual ~VideoEncoderInterface() = default; virtual ~VideoEncoderInterface() = default;
enum class FrameType { kKeyframe, kStartFrame, kDeltaFrame }; enum class FrameType { kKeyframe, kStartFrame, kDeltaFrame };
struct TemporalUnitSettings {
VideoCodecMode content_hint = VideoCodecMode::kRealtimeVideo;
Timestamp presentation_timestamp;
};
// Results from calling Encode. Called once for each configured frame.
struct EncodingError {}; struct EncodingError {};
struct EncodedData { struct EncodedData {
rtc::scoped_refptr<EncodedImageBufferInterface> bitstream_data;
FrameType frame_type; FrameType frame_type;
int encoded_qp; int encoded_qp;
}; };
using EncodeResult = absl::variant<EncodingError, EncodedData>; using EncodeResult = absl::variant<EncodingError, EncodedData>;
using EncodeResultCallback =
absl::AnyInvocable<void(const EncodeResult& result) &&>; struct FrameOutput {
virtual ~FrameOutput() = default;
virtual rtc::ArrayView<uint8_t> GetBitstreamOutputBuffer(DataSize size) = 0;
virtual void EncodeComplete(const EncodeResult& encode_result) = 0;
};
struct TemporalUnitSettings {
VideoCodecMode content_hint = VideoCodecMode::kRealtimeVideo;
Timestamp presentation_timestamp;
};
struct FrameEncodeSettings { struct FrameEncodeSettings {
struct Cbr { struct Cbr {
@ -73,7 +76,7 @@ class VideoEncoderInterface {
absl::optional<int> update_buffer; absl::optional<int> update_buffer;
int effort_level = 0; int effort_level = 0;
EncodeResultCallback result_callback; std::unique_ptr<FrameOutput> frame_output;
}; };
virtual void Encode(rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer, virtual void Encode(rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer,