diff --git a/api/video_codecs/BUILD.gn b/api/video_codecs/BUILD.gn index a36bf442e6..093b0bd3d2 100644 --- a/api/video_codecs/BUILD.gn +++ b/api/video_codecs/BUILD.gn @@ -289,6 +289,7 @@ rtc_source_set("video_encoder_interface") { deps = [ ":video_encoding_general", "../../api/units:data_rate", + "../../api/units:data_size", "../../api/units:time_delta", "../../api/units:timestamp", "../../api/video:encoded_image", diff --git a/api/video_codecs/libaom_av1_encoder_factory.cc b/api/video_codecs/libaom_av1_encoder_factory.cc index 87dd5d5685..4ee8c7faef 100644 --- a/api/video_codecs/libaom_av1_encoder_factory.cc +++ b/api/video_codecs/libaom_av1_encoder_factory.cc @@ -270,8 +270,8 @@ bool ValidateEncodeParams( const VideoEncoderInterface::FrameEncodeSettings& settings = frame_settings[i]; - if (!settings.result_callback) { - RTC_LOG(LS_ERROR) << "No result callback function provided."; + if (!settings.frame_output) { + RTC_LOG(LS_ERROR) << "No frame output provided."; return false; } @@ -618,10 +618,10 @@ aom_svc_params_t GetSvcParams( void DoErrorCallback(std::vector& frame_settings) { for (FrameEncodeSettings& settings : frame_settings) { - if (settings.result_callback) { - std::move(settings.result_callback)({}); + if (settings.frame_output) { + settings.frame_output->EncodeComplete({}); // To avoid invoking any callback more than once. - settings.result_callback = {}; + settings.frame_output = nullptr; } } } @@ -769,6 +769,7 @@ void LibaomAv1Encoder::Encode( EncodedData result; aom_codec_iter_t iter = nullptr; + bool bitstream_produced = false; while (const aom_codec_cx_pkt_t* pkt = aom_codec_get_cx_data(&ctx_, &iter)) { if (pkt->kind == AOM_CODEC_CX_FRAME_PKT && pkt->data.frame.sz > 0) { @@ -777,20 +778,27 @@ void LibaomAv1Encoder::Encode( result.frame_type = pkt->data.frame.flags & AOM_EFLAG_FORCE_KF ? FrameType::kKeyframe : FrameType::kDeltaFrame; - result.bitstream_data = EncodedImageBuffer::Create( - static_cast(pkt->data.frame.buf), pkt->data.frame.sz); + rtc::ArrayView output_buffer = + settings.frame_output->GetBitstreamOutputBuffer( + DataSize::Bytes(pkt->data.frame.sz)); + if (output_buffer.size() != pkt->data.frame.sz) { + DoErrorCallback(frame_settings); + return; + } + memcpy(output_buffer.data(), pkt->data.frame.buf, pkt->data.frame.sz); + bitstream_produced = true; break; } } - if (result.bitstream_data == nullptr) { + if (!bitstream_produced) { DoErrorCallback(frame_settings); return; } else { - RTC_CHECK(settings.result_callback); - std::move(settings.result_callback)(result); + RTC_CHECK(settings.frame_output); + settings.frame_output->EncodeComplete(result); // To avoid invoking any callback more than once. - settings.result_callback = {}; + settings.frame_output = nullptr; } } } diff --git a/api/video_codecs/libaom_av1_encoder_factory_test.cc b/api/video_codecs/libaom_av1_encoder_factory_test.cc index 02be3c2b66..38630b2f4c 100644 --- a/api/video_codecs/libaom_av1_encoder_factory_test.cc +++ b/api/video_codecs/libaom_av1_encoder_factory_test.cc @@ -29,20 +29,14 @@ namespace webrtc { namespace { -using ::testing::_; -using ::testing::ElementsAre; using ::testing::Eq; -using ::testing::Field; using ::testing::Gt; using ::testing::IsEmpty; -using ::testing::Lt; -using ::testing::MockFunction; -using ::testing::NotNull; +using ::testing::Not; using Cbr = VideoEncoderInterface::FrameEncodeSettings::Cbr; using Cqp = VideoEncoderInterface::FrameEncodeSettings::Cqp; using EncodedData = VideoEncoderInterface::EncodedData; using EncodeResult = VideoEncoderInterface::EncodeResult; -using EncodeResultCallback = VideoEncoderInterface::EncodeResultCallback; using FrameType = VideoEncoderInterface::FrameType; std::unique_ptr CreateFrameReader() { @@ -58,24 +52,6 @@ std::string OutPath() { return res; } -class EncodeResults { - public: - EncodeResultCallback Cb() { - return [&](const EncodeResult& result) { results_.push_back(result); }; - } - - EncodedData* FrameAt(int index) { - if (index < 0 || index > static_cast(results_.size())) { - RTC_CHECK(false); - return nullptr; - } - return absl::get_if(&results_[index]); - } - - private: - std::vector results_; -}; - class Av1Decoder : public DecodedImageCallback { public: Av1Decoder() : Av1Decoder("") {} @@ -106,12 +82,12 @@ class Av1Decoder : public DecodedImageCallback { return 0; } - VideoFrame Decode(const EncodedData& encoded_data) { + VideoFrame Decode(rtc::ArrayView bitstream_data) { EncodedImage img; - img.SetEncodedData(encoded_data.bitstream_data); + img.SetEncodedData(EncodedImageBuffer::Create(bitstream_data.data(), + bitstream_data.size())); if (raw_out_file_) { - fwrite(encoded_data.bitstream_data->data(), 1, - encoded_data.bitstream_data->size(), raw_out_file_); + fwrite(bitstream_data.data(), 1, bitstream_data.size(), raw_out_file_); } decoder_->Decode(img, /*dont_care=*/0); VideoFrame res(std::move(*decode_result_)); @@ -125,8 +101,29 @@ class Av1Decoder : public DecodedImageCallback { FILE* raw_out_file_ = nullptr; }; +struct EncOut { + std::vector bitstream; + EncodeResult res; +}; + class FrameEncoderSettingsBuilder { public: + FrameEncoderSettingsBuilder() { + class IgnoredOutput : public VideoEncoderInterface::FrameOutput { + public: + rtc::ArrayView GetBitstreamOutputBuffer(DataSize size) override { + unread_.resize(size.bytes()); + return unread_; + } + void EncodeComplete(const EncodeResult& encode_result) override {} + + private: + std::vector unread_; + }; + + frame_encode_settings_.frame_output = std::make_unique(); + } + FrameEncoderSettingsBuilder& Key() { frame_encode_settings_.frame_type = FrameType::kKeyframe; return *this; @@ -173,13 +170,13 @@ class FrameEncoderSettingsBuilder { return *this; } - FrameEncoderSettingsBuilder& Cb(EncodeResultCallback cb) { - frame_encode_settings_.result_callback = std::move(cb); + FrameEncoderSettingsBuilder& Effort(int effort_level) { + frame_encode_settings_.effort_level = effort_level; return *this; } - FrameEncoderSettingsBuilder& Effort(int effort_level) { - frame_encode_settings_.effort_level = effort_level; + FrameEncoderSettingsBuilder& Out(EncOut& out) { + frame_encode_settings_.frame_output = std::make_unique(out); return *this; } @@ -188,6 +185,18 @@ class FrameEncoderSettingsBuilder { } private: + struct FrameOut : public VideoEncoderInterface::FrameOutput { + explicit FrameOut(EncOut& e) : eo(e) {} + rtc::ArrayView GetBitstreamOutputBuffer(DataSize size) override { + eo.bitstream.resize(size.bytes()); + return rtc::ArrayView(eo.bitstream); + } + void EncodeComplete(const EncodeResult& encode_result) override { + eo.res = encode_result; + } + EncOut& eo; + }; + VideoEncoderInterface::FrameEncodeSettings frame_encode_settings_; }; @@ -220,6 +229,18 @@ MATCHER_P2(ResolutionIs, width, height, "") { return arg.width == width && arg.height == height; } +MATCHER_P(QpIs, qp, "") { + if (auto ed = absl::get_if(&arg.res)) { + return ed->encoded_qp == qp; + } + return false; +} + +MATCHER(HasBitstreamAndMetaData, "") { + return !arg.bitstream.empty() && + absl::holds_alternative(arg.res); +} + double Psnr(const rtc::scoped_refptr& ref_buffer, const VideoFrame& decoded_frame) { return I420PSNR(*ref_buffer, *decoded_frame.video_frame_buffer()->ToI420()); @@ -267,75 +288,77 @@ TEST(LibaomAv1EncoderFactory, QpRange) { TEST(LibaomAv1Encoder, KeyframeUpdatesSpecifiedBuffer) { auto frame_reader = CreateFrameReader(); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); - EncodeResults res; Av1Decoder dec; auto raw_key = frame_reader->PullFrame(); auto raw_delta = frame_reader->PullFrame(); + EncOut key; enc->Encode(raw_key, {.presentation_timestamp = Timestamp::Millis(0)}, - ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(5).Key().Cb(res.Cb())})); - ASSERT_THAT(res.FrameAt(0), NotNull()); - VideoFrame decoded_key = dec.Decode(*res.FrameAt(0)); + ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(5).Key().Out(key)})); + ASSERT_THAT(key.bitstream, Not(IsEmpty())); + VideoFrame decoded_key = dec.Decode(key.bitstream); EXPECT_THAT(Resolution(decoded_key), ResolutionIs(640, 360)); EXPECT_THAT(Psnr(raw_key, decoded_key), Gt(40)); + EncOut delta; enc->Encode(raw_delta, {.presentation_timestamp = Timestamp::Millis(100)}, - ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Cb(res.Cb())})); - ASSERT_THAT(res.FrameAt(1), Eq(nullptr)); + ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Out(delta)})); + EXPECT_THAT(delta, Not(HasBitstreamAndMetaData())); } TEST(LibaomAv1Encoder, MidTemporalUnitKeyframeResetsBuffers) { auto frame_reader = CreateFrameReader(); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); - EncodeResults res; - Av1Decoder dec; - enc->Encode( - frame_reader->PullFrame(), - {.presentation_timestamp = Timestamp::Millis(0)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()), - Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Cb(res.Cb())})); - ASSERT_THAT(res.FrameAt(2), NotNull()); + EncOut tu0_s2; + enc->Encode(frame_reader->PullFrame(), + {.presentation_timestamp = Timestamp::Millis(0)}, + ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key(), + Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}), + Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Out(tu0_s2)})); + EXPECT_THAT(tu0_s2, HasBitstreamAndMetaData()); + EncOut tu1_s0; enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(100)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Ref({0}).Cb(res.Cb()), - Fb().Rate(kCbr).Res(320, 180).S(1).Upd(1).Key().Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Cb(res.Cb())})); - ASSERT_THAT(res.FrameAt(3), Eq(nullptr)); + ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Ref({0}).Out(tu1_s0), + Fb().Rate(kCbr).Res(320, 180).S(1).Upd(1).Key(), + Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0})})); + EXPECT_THAT(tu1_s0, Not(HasBitstreamAndMetaData())); } TEST(LibaomAv1Encoder, ResolutionSwitching) { auto frame_reader = CreateFrameReader(); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); - EncodeResults res; rtc::scoped_refptr in0 = frame_reader->PullFrame(); + EncOut tu0; enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)}, - ToVec({Fb().Rate(kCbr).Res(320, 180).Upd(0).Key().Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(320, 180).Upd(0).Key().Out(tu0)})); rtc::scoped_refptr in1 = frame_reader->PullFrame(); + EncOut tu1; enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)}, - ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Out(tu1)})); rtc::scoped_refptr in2 = frame_reader->PullFrame(); + EncOut tu2; enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Out(tu2)})); Av1Decoder dec; - VideoFrame f0 = dec.Decode(*res.FrameAt(0)); + VideoFrame f0 = dec.Decode(tu0.bitstream); EXPECT_THAT(Resolution(f0), ResolutionIs(320, 180)); // TD: // EXPECT_THAT(Psnr(in0, f0), Gt(40)); - VideoFrame f1 = dec.Decode(*res.FrameAt(1)); + VideoFrame f1 = dec.Decode(tu1.bitstream); EXPECT_THAT(Resolution(f1), ResolutionIs(640, 360)); EXPECT_THAT(Psnr(in1, f1), Gt(40)); - VideoFrame f2 = dec.Decode(*res.FrameAt(2)); + VideoFrame f2 = dec.Decode(tu2.bitstream); EXPECT_THAT(Resolution(f2), ResolutionIs(160, 90)); // TD: // EXPECT_THAT(Psnr(in2, f2), Gt(40)); @@ -344,38 +367,40 @@ TEST(LibaomAv1Encoder, ResolutionSwitching) { TEST(LibaomAv1Encoder, InputResolutionSwitching) { auto frame_reader = CreateFrameReader(); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); - EncodeResults res; rtc::scoped_refptr in0 = frame_reader->PullFrame(); + EncOut tu0; enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).Upd(0).Key().Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(160, 90).Upd(0).Key().Out(tu0)})); rtc::scoped_refptr in1 = frame_reader->PullFrame( /*frame_num=*/nullptr, /*resolution=*/{320, 180}, /*framerate_scale=*/{1, 1}); + EncOut tu1; enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Out(tu1)})); rtc::scoped_refptr in2 = frame_reader->PullFrame( /*frame_num=*/nullptr, /*resolution=*/{160, 90}, /*framerate_scale=*/{1, 1}); + EncOut tu2; enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Out(tu2)})); Av1Decoder dec; - VideoFrame f0 = dec.Decode(*res.FrameAt(0)); + VideoFrame f0 = dec.Decode(tu0.bitstream); EXPECT_THAT(Resolution(f0), ResolutionIs(160, 90)); // TD: // EXPECT_THAT(Psnr(in0, f0), Gt(40)); - VideoFrame f1 = dec.Decode(*res.FrameAt(1)); + VideoFrame f1 = dec.Decode(tu1.bitstream); EXPECT_THAT(Resolution(f1), ResolutionIs(160, 90)); // TD: // EXPECT_THAT(Psnr(in1, f1), Gt(40)); - VideoFrame f2 = dec.Decode(*res.FrameAt(2)); + VideoFrame f2 = dec.Decode(tu2.bitstream); EXPECT_THAT(Resolution(f2), ResolutionIs(160, 90)); EXPECT_THAT(Psnr(in2, f2), Gt(40)); } @@ -383,116 +408,132 @@ TEST(LibaomAv1Encoder, InputResolutionSwitching) { TEST(LibaomAv1Encoder, TempoSpatial) { auto frame_reader = CreateFrameReader(); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); - EncodeResults res; const Cbr k10Fps{.duration = TimeDelta::Millis(100), .target_bitrate = DataRate::KilobitsPerSec(500)}; const Cbr k20Fps{.duration = TimeDelta::Millis(50), .target_bitrate = DataRate::KilobitsPerSec(500)}; + EncOut tu0_s0; + EncOut tu0_s1; + EncOut tu0_s2; enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(0)}, ToVec( - {Fb().Rate(k10Fps).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()), - Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()), - Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())})); + {Fb().Rate(k10Fps).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0), + Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1), + Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)})); + EncOut tu1_s2; enc->Encode(frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(50)}, - ToVec({Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({2}).Upd(2).Cb( - res.Cb())})); + ToVec({Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({2}).Upd(2).Out( + tu1_s2)})); rtc::scoped_refptr frame = frame_reader->PullFrame(); + EncOut tu2_s0; + EncOut tu2_s1; + EncOut tu2_s2; enc->Encode( frame, {.presentation_timestamp = Timestamp::Millis(100)}, ToVec( - {Fb().Rate(k10Fps).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), - Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Cb(res.Cb()), - Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Cb( - res.Cb())})); + {Fb().Rate(k10Fps).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0), + Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Out(tu2_s1), + Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Out( + tu2_s2)})); Av1Decoder dec; - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(320, 180)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(640, 360)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(640, 360)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(160, 90)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(5))), ResolutionIs(320, 180)); + EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90)); + EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180)); + EXPECT_THAT(Resolution(dec.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360)); + EXPECT_THAT(Resolution(dec.Decode(tu1_s2.bitstream)), ResolutionIs(640, 360)); + EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90)); + EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180)); - VideoFrame f = dec.Decode(*res.FrameAt(6)); + VideoFrame f = dec.Decode(tu2_s2.bitstream); EXPECT_THAT(Resolution(f), ResolutionIs(640, 360)); - EXPECT_THAT(Psnr(frame, f), Gt(40)); } TEST(DISABLED_LibaomAv1Encoder, InvertedTempoSpatial) { auto frame_reader = CreateFrameReader(); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); - EncodeResults res; + EncOut tu0_s0; + EncOut tu0_s1; enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(0)}, - ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Upd(0).Key().Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(1).Ref({0}).Upd(1).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Upd(0).Key().Out(tu0_s0), + Fb().Rate(kCbr).Res(640, 360).S(1).Ref({0}).Upd(1).Out(tu0_s1)})); + EncOut tu1_s0; enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(100)}, - ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Out(tu1_s0)})); + EncOut tu2_s0; + EncOut tu2_s1; rtc::scoped_refptr frame = frame_reader->PullFrame(); enc->Encode( frame, {.presentation_timestamp = Timestamp::Millis(200)}, - ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(1).Ref({1, 0}).Upd(1).Cb( - res.Cb())})); + ToVec( + {Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Out(tu2_s0), + Fb().Rate(kCbr).Res(640, 360).S(1).Ref({1, 0}).Upd(1).Out(tu2_s1)})); Av1Decoder dec; - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(320, 180)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(640, 360)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(320, 180)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(320, 180)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(640, 360)); + EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(320, 180)); + EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(640, 360)); + EXPECT_THAT(Resolution(dec.Decode(tu1_s0.bitstream)), ResolutionIs(320, 180)); + EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(320, 180)); + EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(640, 360)); } TEST(LibaomAv1Encoder, SkipMidLayer) { auto frame_reader = CreateFrameReader(); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); - EncodeResults res; + EncOut tu0_s0; + EncOut tu0_s1; + EncOut tu0_s2; enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(0)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()), - Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0), + Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1), + Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)})); + EncOut tu1_s0; + EncOut tu1_s2; enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(100)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0), + Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu1_s2)})); + EncOut tu2_s0; + EncOut tu2_s1; + EncOut tu2_s2; rtc::scoped_refptr frame = frame_reader->PullFrame(); enc->Encode( frame, {.presentation_timestamp = Timestamp::Millis(200)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), - Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Cb( - res.Cb())})); + ToVec( + {Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0), + Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Out(tu2_s1), + Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Out(tu2_s2)})); Av1Decoder dec; - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(320, 180)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(640, 360)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(160, 90)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(640, 360)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(5))), ResolutionIs(160, 90)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(6))), ResolutionIs(320, 180)); + EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90)); + EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180)); + EXPECT_THAT(Resolution(dec.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360)); + EXPECT_THAT(Resolution(dec.Decode(tu1_s0.bitstream)), ResolutionIs(160, 90)); + EXPECT_THAT(Resolution(dec.Decode(tu1_s2.bitstream)), ResolutionIs(640, 360)); + EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90)); + EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180)); - VideoFrame f = dec.Decode(*res.FrameAt(7)); + VideoFrame f = dec.Decode(tu2_s2.bitstream); EXPECT_THAT(Resolution(f), ResolutionIs(640, 360)); EXPECT_THAT(Psnr(frame, f), Gt(40)); } @@ -500,47 +541,55 @@ TEST(LibaomAv1Encoder, SkipMidLayer) { TEST(LibaomAv1Encoder, L3T1) { auto frame_reader = CreateFrameReader(); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); - EncodeResults res; Av1Decoder dec; + EncOut tu0_s0; + EncOut tu0_s1; + EncOut tu0_s2; enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(0)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()), - Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0), + Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1), + Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)})); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(320, 180)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(640, 360)); + EXPECT_THAT(Resolution(dec.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90)); + EXPECT_THAT(Resolution(dec.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180)); + EXPECT_THAT(Resolution(dec.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360)); auto tu1_frame = frame_reader->PullFrame(); + EncOut tu1_s0; + EncOut tu1_s1; + EncOut tu1_s2; enc->Encode( tu1_frame, {.presentation_timestamp = Timestamp::Millis(100)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), - Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Cb( - res.Cb())})); + ToVec( + {Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0), + Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Out(tu1_s1), + Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Out(tu1_s2)})); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(160, 90)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(320, 180)); + EXPECT_THAT(Resolution(dec.Decode(tu1_s0.bitstream)), ResolutionIs(160, 90)); + EXPECT_THAT(Resolution(dec.Decode(tu1_s1.bitstream)), ResolutionIs(320, 180)); - VideoFrame f_tu1 = dec.Decode(*res.FrameAt(5)); - EXPECT_THAT(Resolution(f_tu1), ResolutionIs(640, 360)); - EXPECT_THAT(Psnr(tu1_frame, f_tu1), Gt(40)); + VideoFrame f_tu1_s2 = dec.Decode(tu1_s2.bitstream); + EXPECT_THAT(Resolution(f_tu1_s2), ResolutionIs(640, 360)); + EXPECT_THAT(Psnr(tu1_frame, f_tu1_s2), Gt(40)); auto tu2_frame = frame_reader->PullFrame(); + EncOut tu2_s0; + EncOut tu2_s1; + EncOut tu2_s2; enc->Encode( tu2_frame, {.presentation_timestamp = Timestamp::Millis(200)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), - Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Cb( - res.Cb())})); + ToVec( + {Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0), + Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Out(tu2_s1), + Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Out(tu2_s2)})); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(6))), ResolutionIs(160, 90)); - EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(7))), ResolutionIs(320, 180)); + EXPECT_THAT(Resolution(dec.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90)); + EXPECT_THAT(Resolution(dec.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180)); - VideoFrame f_tu2 = dec.Decode(*res.FrameAt(8)); + VideoFrame f_tu2 = dec.Decode(tu2_s2.bitstream); EXPECT_THAT(Resolution(f_tu2), ResolutionIs(640, 360)); EXPECT_THAT(Psnr(tu2_frame, f_tu2), Gt(40)); } @@ -548,108 +597,124 @@ TEST(LibaomAv1Encoder, L3T1) { TEST(LibaomAv1Encoder, L3T1_KEY) { auto frame_reader = CreateFrameReader(); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); - EncodeResults res; Av1Decoder dec_s0; Av1Decoder dec_s1; Av1Decoder dec_s2; + EncOut tu0_s0; + EncOut tu0_s1; + EncOut tu0_s2; enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(0)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()), - Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Out(tu0_s0), + Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Out(tu0_s1), + Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Out(tu0_s2)})); - EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(0))), + EXPECT_THAT(Resolution(dec_s0.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90)); - dec_s1.Decode(*res.FrameAt(0)); - EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(1))), + dec_s1.Decode(tu0_s0.bitstream); + EXPECT_THAT(Resolution(dec_s1.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180)); - dec_s2.Decode(*res.FrameAt(0)); - dec_s2.Decode(*res.FrameAt(1)); - EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(2))), + dec_s2.Decode(tu0_s0.bitstream); + dec_s2.Decode(tu0_s1.bitstream); + EXPECT_THAT(Resolution(dec_s2.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360)); + EncOut tu1_s0; + EncOut tu1_s1; + EncOut tu1_s2; enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(100)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), - Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0), + Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu1_s1), + Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu1_s2)})); - EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(3))), + EXPECT_THAT(Resolution(dec_s0.Decode(tu1_s0.bitstream)), ResolutionIs(160, 90)); - EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(4))), + EXPECT_THAT(Resolution(dec_s1.Decode(tu1_s1.bitstream)), ResolutionIs(320, 180)); - EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(5))), + EXPECT_THAT(Resolution(dec_s2.Decode(tu1_s2.bitstream)), ResolutionIs(640, 360)); + EncOut tu2_s0; + EncOut tu2_s1; + EncOut tu2_s2; enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(200)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), - Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0), + Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu2_s1), + Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu2_s2)})); - EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(6))), + EXPECT_THAT(Resolution(dec_s0.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90)); - EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(7))), + EXPECT_THAT(Resolution(dec_s1.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180)); - EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(8))), + EXPECT_THAT(Resolution(dec_s2.Decode(tu2_s2.bitstream)), ResolutionIs(640, 360)); } TEST(LibaomAv1Encoder, S3T1) { auto frame_reader = CreateFrameReader(); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); - EncodeResults res; Av1Decoder dec_s0; Av1Decoder dec_s1; Av1Decoder dec_s2; + EncOut tu0_s0; + EncOut tu0_s1; + EncOut tu0_s2; enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(0)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Start().Upd(0).Cb(res.Cb()), - Fb().Rate(kCbr).Res(320, 180).S(1).Start().Upd(1).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Start().Upd(2).Cb(res.Cb())})); - EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(0))), + ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Start().Upd(0).Out(tu0_s0), + Fb().Rate(kCbr).Res(320, 180).S(1).Start().Upd(1).Out(tu0_s1), + Fb().Rate(kCbr).Res(640, 360).S(2).Start().Upd(2).Out(tu0_s2)})); + EXPECT_THAT(Resolution(dec_s0.Decode(tu0_s0.bitstream)), ResolutionIs(160, 90)); - EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(1))), + EXPECT_THAT(Resolution(dec_s1.Decode(tu0_s1.bitstream)), ResolutionIs(320, 180)); - EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(2))), + EXPECT_THAT(Resolution(dec_s2.Decode(tu0_s2.bitstream)), ResolutionIs(640, 360)); + EncOut tu1_s0; + EncOut tu1_s1; + EncOut tu1_s2; enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(100)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), - Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu1_s0), + Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu1_s1), + Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu1_s2)})); - EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(3))), + EXPECT_THAT(Resolution(dec_s0.Decode(tu1_s0.bitstream)), ResolutionIs(160, 90)); - EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(4))), + EXPECT_THAT(Resolution(dec_s1.Decode(tu1_s1.bitstream)), ResolutionIs(320, 180)); - EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(5))), + EXPECT_THAT(Resolution(dec_s2.Decode(tu1_s2.bitstream)), ResolutionIs(640, 360)); + EncOut tu2_s0; + EncOut tu2_s1; + EncOut tu2_s2; enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(200)}, - ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()), - Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()), - Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())})); + ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Out(tu2_s0), + Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Out(tu2_s1), + Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Out(tu2_s2)})); - EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(6))), + EXPECT_THAT(Resolution(dec_s0.Decode(tu2_s0.bitstream)), ResolutionIs(160, 90)); - EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(7))), + EXPECT_THAT(Resolution(dec_s1.Decode(tu2_s1.bitstream)), ResolutionIs(320, 180)); - EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(8))), + EXPECT_THAT(Resolution(dec_s2.Decode(tu2_s2.bitstream)), ResolutionIs(640, 360)); } @@ -665,11 +730,11 @@ TEST(LibaomAv1Encoder, HigherEffortLevelYieldsHigherQualityFrames) { for (int i = effort_range.first; i <= effort_range.second; ++i) { auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); - EncodeResults res; - enc->Encode(frame_in, {.presentation_timestamp = Timestamp::Millis(0)}, - ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(0).Key().Effort(i).Cb( - res.Cb())})); - double psnr = Psnr(frame_in, dec.Decode(*res.FrameAt(0))); + EncOut tu0; + enc->Encode( + frame_in, {.presentation_timestamp = Timestamp::Millis(0)}, + ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(0).Key().Effort(i).Out(tu0)})); + double psnr = Psnr(frame_in, dec.Decode(tu0.bitstream)); EXPECT_THAT(psnr, Gt(psnr_last)); psnr_last = psnr; } @@ -699,24 +764,24 @@ TEST(LibaomAv1Encoder, KeyframeAndStartrameAreApproximatelyEqual) { DataSize total_size_key = DataSize::Zero(); DataSize total_size_start = DataSize::Zero(); TimeDelta total_duration = TimeDelta::Zero(); - EncodeResults res_key; - EncodeResults res_start; auto frame_in = frame_reader->PullFrame(); + + EncOut key; + EncOut start; enc_key->Encode( frame_in, {.presentation_timestamp = Timestamp::Millis(0)}, - ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Cb( - res_key.Cb())})); + ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Out(key)})); enc_start->Encode( frame_in, {.presentation_timestamp = Timestamp::Millis(0)}, - ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Start().Upd(0).Cb( - res_start.Cb())})); - total_size_key += - DataSize::Bytes(res_key.FrameAt(0)->bitstream_data->size()); - total_size_start += - DataSize::Bytes(res_start.FrameAt(0)->bitstream_data->size()); + ToVec( + {Fb().Rate(kRate).Res(640, 360).S(sid).Start().Upd(0).Out(start)})); + + total_size_key += DataSize::Bytes(key.bitstream.size()); + total_size_start += DataSize::Bytes(start.bitstream.size()); + total_duration += kRate.duration; - dec_key.Decode(*res_key.FrameAt(0)); - dec_start.Decode(*res_start.FrameAt(0)); + dec_key.Decode(key.bitstream); + dec_start.Decode(start.bitstream); EXPECT_NEAR(total_size_key.bytes(), total_size_start.bytes(), 0.1 * total_size_key.bytes()); @@ -725,19 +790,18 @@ TEST(LibaomAv1Encoder, KeyframeAndStartrameAreApproximatelyEqual) { frame_in = frame_reader->PullFrame(); enc_key->Encode( frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)}, - ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Cb( - res_key.Cb())})); + ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Out( + key)})); enc_start->Encode( frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)}, - ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Cb( - res_start.Cb())})); - total_size_key += - DataSize::Bytes(res_key.FrameAt(f)->bitstream_data->size()); - total_size_start += - DataSize::Bytes(res_start.FrameAt(f)->bitstream_data->size()); + ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Out( + start)})); + total_size_key += DataSize::Bytes(key.bitstream.size()); + total_size_start += DataSize::Bytes(start.bitstream.size()); + total_duration += kRate.duration; - dec_key.Decode(*res_key.FrameAt(f)); - dec_start.Decode(*res_start.FrameAt(f)); + dec_key.Decode(key.bitstream); + dec_start.Decode(start.bitstream); } double key_encode_kbps = (total_size_key / total_duration).kbps(); @@ -757,30 +821,28 @@ TEST(LibaomAv1Encoder, BitrateConsistentAcrossSpatialLayers) { for (int sid = 0; sid < max_spatial_layers; ++sid) { std::string out_name = "cbr_sl_"; out_name += std::to_string(sid); - Av1Decoder dec(out_name); auto frame_reader = CreateFrameReader(); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); DataSize total_size = DataSize::Zero(); TimeDelta total_duration = TimeDelta::Zero(); - EncodeResults res; - enc->Encode(frame_reader->PullFrame(), - {.presentation_timestamp = Timestamp::Millis(0)}, - ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Cb( - res.Cb())})); - total_size += DataSize::Bytes(res.FrameAt(0)->bitstream_data->size()); + + EncOut out; + enc->Encode( + frame_reader->PullFrame(), + {.presentation_timestamp = Timestamp::Millis(0)}, + ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Out(out)})); + total_size += DataSize::Bytes(out.bitstream.size()); total_duration += kRate.duration; - dec.Decode(*res.FrameAt(0)); for (int f = 1; f < 30; ++f) { enc->Encode( frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(f * 100)}, - ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Cb( - res.Cb())})); - total_size += DataSize::Bytes(res.FrameAt(f)->bitstream_data->size()); + ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Out( + out)})); + total_size += DataSize::Bytes(out.bitstream.size()); total_duration += kRate.duration; - dec.Decode(*res.FrameAt(f)); } double encode_kbps = (total_size / total_duration).kbps(); @@ -799,11 +861,9 @@ TEST(LibaomAv1Encoder, ConstantQp) { auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCqpEncoderSettings, {}); std::string out_name = "cqp_sl_"; out_name += std::to_string(sid); - Av1Decoder dec(out_name); - DataSize total_size = DataSize::Zero(); auto frame_reader = CreateFrameReader(); - EncodeResults res; + EncOut out; enc->Encode(frame_reader->PullFrame(), {.presentation_timestamp = Timestamp::Millis(0)}, ToVec({Fb().Rate(Cqp{.target_qp = kQp}) @@ -811,10 +871,8 @@ TEST(LibaomAv1Encoder, ConstantQp) { .S(sid) .Upd(0) .Key() - .Cb(res.Cb())})); - EXPECT_THAT(res.FrameAt(0)->encoded_qp, Eq(kQp)); - total_size += DataSize::Bytes(res.FrameAt(0)->bitstream_data->size()); - dec.Decode(*res.FrameAt(0)); + .Out(out)})); + EXPECT_THAT(out, QpIs(kQp)); for (int f = 1; f < 10; ++f) { enc->Encode(frame_reader->PullFrame(), @@ -824,9 +882,8 @@ TEST(LibaomAv1Encoder, ConstantQp) { .S(sid) .Ref({0}) .Upd(0) - .Cb(res.Cb())})); - EXPECT_THAT(res.FrameAt(f)->encoded_qp, Eq(kQp - f)); - dec.Decode(*res.FrameAt(f)); + .Out(out)})); + EXPECT_THAT(out, QpIs(kQp - f)); } } } diff --git a/api/video_codecs/simple_encoder_wrapper.cc b/api/video_codecs/simple_encoder_wrapper.cc index c3627e07d1..96ce98a129 100644 --- a/api/video_codecs/simple_encoder_wrapper.cc +++ b/api/video_codecs/simple_encoder_wrapper.cc @@ -147,7 +147,6 @@ void SimpleEncoderWrapper::Encode( svc_controller_->NextFrameConfig(force_keyframe); std::vector encode_settings; std::vector frame_infos; - bool include_dependency_structure = false; for (size_t s = 0; s < configs.size(); ++s) { const ScalableVideoController::LayerFrameConfig& config = configs[s]; @@ -176,36 +175,46 @@ void SimpleEncoderWrapper::Encode( if (settings.reference_buffers.empty()) { settings.frame_type = FrameType::kKeyframe; - include_dependency_structure = true; } - absl::optional dependency_structure; - if (include_dependency_structure) { - dependency_structure = svc_controller_->DependencyStructure(); - } + struct FrameOut : public VideoEncoderInterface::FrameOutput { + rtc::ArrayView GetBitstreamOutputBuffer(DataSize size) override { + bitstream.resize(size.bytes()); + return bitstream; + } - settings.result_callback = - [cb = callback, ds = std::move(dependency_structure), - info = std::move(frame_infos[settings.spatial_id])]( - const VideoEncoderInterface::EncodeResult& result) mutable { - auto* data = - absl::get_if(&result); + void EncodeComplete( + const VideoEncoderInterface::EncodeResult& result) override { + auto* data = absl::get_if(&result); - EncodeResult res; - if (!data) { - res.oh_no = true; - cb(res); - return; - } + SimpleEncoderWrapper::EncodeResult res; + if (!data) { + res.oh_no = true; + callback(res); + return; + } - res.frame_type = data->frame_type; - res.bitstream_data = std::move(data->bitstream_data); - res.generic_frame_info = info; - if (res.frame_type == FrameType::kKeyframe) { - res.dependency_structure = ds; - } - cb(res); - }; + res.frame_type = data->frame_type; + res.bitstream_data = std::move(bitstream); + res.generic_frame_info = frame_info; + if (res.frame_type == FrameType::kKeyframe) { + res.dependency_structure = svc_controller->DependencyStructure(); + } + callback(res); + } + std::vector bitstream; + EncodeResultCallback callback; + GenericFrameInfo frame_info; + ScalableVideoController* svc_controller; + }; + + auto out = std::make_unique(); + + out->callback = callback; + out->frame_info = std::move(frame_infos[settings.spatial_id]); + out->svc_controller = svc_controller_.get(); + + settings.frame_output = std::move(out); } encoder_->Encode(std::move(frame_buffer), diff --git a/api/video_codecs/simple_encoder_wrapper.h b/api/video_codecs/simple_encoder_wrapper.h index cc1b94743e..c823c9370d 100644 --- a/api/video_codecs/simple_encoder_wrapper.h +++ b/api/video_codecs/simple_encoder_wrapper.h @@ -27,7 +27,7 @@ class SimpleEncoderWrapper { public: struct EncodeResult { bool oh_no = false; - rtc::scoped_refptr bitstream_data; + std::vector bitstream_data; FrameType frame_type; GenericFrameInfo generic_frame_info; absl::optional dependency_structure; diff --git a/api/video_codecs/simple_encoder_wrapper_unittests.cc b/api/video_codecs/simple_encoder_wrapper_unittests.cc index 3aa12b32b2..184a7e8e7a 100644 --- a/api/video_codecs/simple_encoder_wrapper_unittests.cc +++ b/api/video_codecs/simple_encoder_wrapper_unittests.cc @@ -152,7 +152,7 @@ TEST(SimpleEncoderWrapper, EncodeL1T1) { ++num_callbacks; ASSERT_THAT(result.oh_no, Eq(false)); EXPECT_THAT(result.dependency_structure, Ne(absl::nullopt)); - EXPECT_THAT(result.bitstream_data, NotNull()); + EXPECT_THAT(result.bitstream_data, Not(IsEmpty())); EXPECT_THAT(result.frame_type, Eq(FrameType::kKeyframe)); EXPECT_THAT(result.generic_frame_info.spatial_id, Eq(0)); EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0)); @@ -164,7 +164,7 @@ TEST(SimpleEncoderWrapper, EncodeL1T1) { ++num_callbacks; ASSERT_THAT(result.oh_no, Eq(false)); EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt)); - EXPECT_THAT(result.bitstream_data, NotNull()); + EXPECT_THAT(result.bitstream_data, Not(IsEmpty())); EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame)); EXPECT_THAT(result.generic_frame_info.spatial_id, Eq(0)); EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0)); @@ -197,13 +197,13 @@ TEST(SimpleEncoderWrapper, EncodeL2T2_KEY) { if (result.generic_frame_info.spatial_id == 0) { ++num_callbacks; EXPECT_THAT(result.dependency_structure, Ne(absl::nullopt)); - EXPECT_THAT(result.bitstream_data, NotNull()); + EXPECT_THAT(result.bitstream_data, Not(IsEmpty())); EXPECT_THAT(result.frame_type, Eq(FrameType::kKeyframe)); EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0)); } else if (result.generic_frame_info.spatial_id == 1) { ++num_callbacks; EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt)); - EXPECT_THAT(result.bitstream_data, NotNull()); + EXPECT_THAT(result.bitstream_data, Not(IsEmpty())); EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame)); EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(0)); } @@ -216,13 +216,13 @@ TEST(SimpleEncoderWrapper, EncodeL2T2_KEY) { if (result.generic_frame_info.spatial_id == 0) { ++num_callbacks; EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt)); - EXPECT_THAT(result.bitstream_data, NotNull()); + EXPECT_THAT(result.bitstream_data, Not(IsEmpty())); EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame)); EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(1)); } else if (result.generic_frame_info.spatial_id == 1) { ++num_callbacks; EXPECT_THAT(result.dependency_structure, Eq(absl::nullopt)); - EXPECT_THAT(result.bitstream_data, NotNull()); + EXPECT_THAT(result.bitstream_data, Not(IsEmpty())); EXPECT_THAT(result.frame_type, Eq(FrameType::kDeltaFrame)); EXPECT_THAT(result.generic_frame_info.temporal_id, Eq(1)); } diff --git a/api/video_codecs/video_encoder_interface.h b/api/video_codecs/video_encoder_interface.h index dfac9ef6ed..7cced56801 100644 --- a/api/video_codecs/video_encoder_interface.h +++ b/api/video_codecs/video_encoder_interface.h @@ -21,6 +21,7 @@ #include "absl/types/optional.h" #include "absl/types/variant.h" #include "api/units/data_rate.h" +#include "api/units/data_size.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/encoded_image.h" @@ -37,21 +38,23 @@ class VideoEncoderInterface { virtual ~VideoEncoderInterface() = default; enum class FrameType { kKeyframe, kStartFrame, kDeltaFrame }; - struct TemporalUnitSettings { - VideoCodecMode content_hint = VideoCodecMode::kRealtimeVideo; - Timestamp presentation_timestamp; - }; - - // Results from calling Encode. Called once for each configured frame. struct EncodingError {}; struct EncodedData { - rtc::scoped_refptr bitstream_data; FrameType frame_type; int encoded_qp; }; using EncodeResult = absl::variant; - using EncodeResultCallback = - absl::AnyInvocable; + + struct FrameOutput { + virtual ~FrameOutput() = default; + virtual rtc::ArrayView GetBitstreamOutputBuffer(DataSize size) = 0; + virtual void EncodeComplete(const EncodeResult& encode_result) = 0; + }; + + struct TemporalUnitSettings { + VideoCodecMode content_hint = VideoCodecMode::kRealtimeVideo; + Timestamp presentation_timestamp; + }; struct FrameEncodeSettings { struct Cbr { @@ -73,7 +76,7 @@ class VideoEncoderInterface { absl::optional update_buffer; int effort_level = 0; - EncodeResultCallback result_callback; + std::unique_ptr frame_output; }; virtual void Encode(rtc::scoped_refptr frame_buffer,