From 72b99a112860b60e5a3d62117d43165c08d3c5e0 Mon Sep 17 00:00:00 2001 From: Sergey Silkin Date: Thu, 16 Feb 2023 13:57:22 +0100 Subject: [PATCH] Test Android HW codecs Bug: b/261160916, webrtc:14852 Change-Id: Iebeab244a9ca6ae196735016064ccd056b7c888e Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/293401 Commit-Queue: Sergey Silkin Reviewed-by: Mirko Bonadei Reviewed-by: Rasmus Brandt Cr-Commit-Position: refs/heads/main@{#39326} --- api/test/video_codec_tester.h | 4 + .../codecs/test/video_codec_test.cc | 129 +++++++++++------- .../codecs/test/video_codec_tester_impl.cc | 23 ++-- .../test/video_codec_tester_impl_unittest.cc | 2 + 4 files changed, 98 insertions(+), 60 deletions(-) diff --git a/api/test/video_codec_tester.h b/api/test/video_codec_tester.h index 83408e6c1d..149f0de611 100644 --- a/api/test/video_codec_tester.h +++ b/api/test/video_codec_tester.h @@ -89,6 +89,8 @@ class VideoCodecTester { virtual ~Encoder() = default; virtual void Encode(const VideoFrame& frame, EncodeCallback callback) = 0; + + virtual void Flush() = 0; }; // Interface for a video decoder. @@ -100,6 +102,8 @@ class VideoCodecTester { virtual ~Decoder() = default; virtual void Decode(const EncodedImage& frame, DecodeCallback callback) = 0; + + virtual void Flush() = 0; }; // Pulls coded video frames from `video_source` and passes them to `decoder`. diff --git a/modules/video_coding/codecs/test/video_codec_test.cc b/modules/video_coding/codecs/test/video_codec_test.cc index ce7f4e7955..a932fbc03e 100644 --- a/modules/video_coding/codecs/test/video_codec_test.cc +++ b/modules/video_coding/codecs/test/video_codec_test.cc @@ -32,6 +32,9 @@ #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/video_coding/include/video_error_codes.h" #include "modules/video_coding/svc/scalability_mode_util.h" +#if defined(WEBRTC_ANDROID) +#include "modules/video_coding/codecs/test/android_codec_factory_helper.h" +#endif #include "rtc_base/strings/string_builder.h" #include "test/gtest.h" #include "test/testsupport/file_utils.h" @@ -43,6 +46,7 @@ namespace test { namespace { using ::testing::Combine; using ::testing::Values; +using PacingMode = VideoCodecTester::PacingSettings::PacingMode; struct VideoInfo { std::string name; @@ -92,10 +96,6 @@ struct DecodingTestSettings { std::string name; }; -struct QualityExpectations { - double min_apsnr_y; -}; - struct EncodeDecodeTestParams { CodecInfo codec; VideoInfo video; @@ -103,7 +103,9 @@ struct EncodeDecodeTestParams { VideoCodecTester::DecoderSettings decoder_settings; EncodingTestSettings encoding_settings; DecodingTestSettings decoding_settings; - QualityExpectations quality_expectations; + struct Expectations { + double min_apsnr_y; + } test_expectations; }; const EncodingSettings kQvga64Kbps30Fps = { @@ -118,24 +120,10 @@ const EncodingTestSettings kConstantRateQvga64Kbps30Fps = { .num_frames = 300, .frame_settings = {{/*frame_num=*/0, kQvga64Kbps30Fps}}}; -const QualityExpectations kLowQuality = {.min_apsnr_y = 30}; - const VideoInfo kFourPeople_1280x720_30 = { .name = "FourPeople_1280x720_30", .resolution = {.width = 1280, .height = 720}}; -const CodecInfo kLibvpxVp8 = {.type = "VP8", - .encoder = "libvpx", - .decoder = "libvpx"}; - -const CodecInfo kLibvpxVp9 = {.type = "VP9", - .encoder = "libvpx", - .decoder = "libvpx"}; - -const CodecInfo kOpenH264 = {.type = "H264", - .encoder = "openh264", - .decoder = "ffmpeg"}; - class TestRawVideoSource : public VideoCodecTester::RawVideoSource { public: static constexpr Frequency k90kHz = Frequency::Hertz(90000); @@ -236,11 +224,17 @@ class TestEncoder : public VideoCodecTester::Encoder, } } - int result = encoder_->Encode(frame, nullptr); - RTC_CHECK_EQ(result, WEBRTC_VIDEO_CODEC_OK); + encoder_->Encode(frame, nullptr); ++frame_num_; } + void Flush() override { + // TODO(webrtc:14852): For codecs which buffer frames we need a to + // flush them to get last frames. Add such functionality to VideoEncoder + // API. On Android it will map directly to `MediaCodec.flush()`. + encoder_->Release(); + } + protected: Result OnEncodedImage(const EncodedImage& encoded_image, const CodecSpecificInfo* codec_specific_info) override { @@ -331,6 +325,7 @@ class TestDecoder : public VideoCodecTester::Decoder, : decoder_(std::move(decoder)), codec_info_(codec_info), frame_num_(0) { decoder_->RegisterDecodeCompleteCallback(this); } + void Decode(const EncodedImage& frame, DecodeCallback callback) override { callbacks_[frame.Timestamp()] = std::move(callback); @@ -343,16 +338,24 @@ class TestDecoder : public VideoCodecTester::Decoder, ++frame_num_; } + void Flush() override { + // TODO(webrtc:14852): For codecs which buffer frames we need a to + // flush them to get last frames. Add such functionality to VideoDecoder + // API. On Android it will map directly to `MediaCodec.flush()`. + decoder_->Release(); + } + + protected: void Configure() { VideoDecoder::Settings ds; ds.set_codec_type(PayloadStringToCodecType(codec_info_.type)); ds.set_number_of_cores(1); + ds.set_max_render_resolution({1280, 720}); bool result = decoder_->Configure(ds); RTC_CHECK(result); } - protected: int Decoded(VideoFrame& decoded_frame) override { auto cb = callbacks_.find(decoded_frame.timestamp()); RTC_CHECK(cb != callbacks_.end()); @@ -371,7 +374,18 @@ class TestDecoder : public VideoCodecTester::Decoder, std::unique_ptr CreateEncoder( const CodecInfo& codec_info, const std::map& frame_settings) { - auto factory = CreateBuiltinVideoEncoderFactory(); + std::unique_ptr factory; + if (codec_info.encoder == "libvpx" || codec_info.encoder == "libaom" || + codec_info.encoder == "openh264") { + factory = CreateBuiltinVideoEncoderFactory(); + } else if (codec_info.encoder == "mediacodec") { +#if defined(WEBRTC_ANDROID) + InitializeAndroidObjects(); + factory = CreateAndroidEncoderFactory(); +#endif + } + + RTC_CHECK(factory); auto encoder = factory->CreateVideoEncoder(SdpVideoFormat(codec_info.type)); return std::make_unique(std::move(encoder), codec_info, frame_settings); @@ -379,7 +393,18 @@ std::unique_ptr CreateEncoder( std::unique_ptr CreateDecoder( const CodecInfo& codec_info) { - auto factory = CreateBuiltinVideoDecoderFactory(); + std::unique_ptr factory; + if (codec_info.decoder == "libvpx" || codec_info.decoder == "dav1d" || + codec_info.decoder == "ffmpeg") { + factory = CreateBuiltinVideoDecoderFactory(); + } else if (codec_info.decoder == "mediacodec") { +#if defined(WEBRTC_ANDROID) + InitializeAndroidObjects(); + factory = CreateAndroidDecoderFactory(); +#endif + } + + RTC_CHECK(factory); auto decoder = factory->CreateVideoDecoder(SdpVideoFormat(codec_info.type)); return std::make_unique(std::move(decoder), codec_info); } @@ -410,7 +435,7 @@ class EncodeDecodeTest const ::testing::TestParamInfo& info) { return std::string(info.param.encoding_settings.name + info.param.codec.type + info.param.codec.encoder + - info.param.codec.decoder); + info.param.codec.decoder + info.param.video.name); } protected: @@ -437,35 +462,37 @@ TEST_P(EncodeDecodeTest, DISABLED_TestEncodeDecode) { std::vector frames = stats->Slice(slicer); VideoCodecStats::Stream stream = stats->Aggregate(frames); EXPECT_GE(stream.psnr.y.GetAverage(), - test_params_.quality_expectations.min_apsnr_y); + test_params_.test_expectations.min_apsnr_y); } } -std::list ConstantRateTestParameters() { - std::list test_params; - std::vector codecs = {kLibvpxVp8}; - std::vector videos = {kFourPeople_1280x720_30}; - std::vector> - encoding_settings = {{kConstantRateQvga64Kbps30Fps, kLowQuality}}; - for (const CodecInfo& codec : codecs) { - for (const VideoInfo& video : videos) { - for (const auto& es : encoding_settings) { - EncodeDecodeTestParams p; - p.codec = codec; - p.video = video; - p.encoding_settings = es.first; - p.quality_expectations = es.second; - test_params.push_back(p); - } - } - } - return test_params; -} - -INSTANTIATE_TEST_SUITE_P(ConstantRate, - EncodeDecodeTest, - ::testing::ValuesIn(ConstantRateTestParameters()), - EncodeDecodeTest::TestParametersToStr); +INSTANTIATE_TEST_SUITE_P( + ConstantRate, + EncodeDecodeTest, + ::testing::ValuesIn({ + EncodeDecodeTestParams({ + .codec = {.type = "VP8", .encoder = "libvpx", .decoder = "libvpx"}, + .video = kFourPeople_1280x720_30, + .encoder_settings = {.pacing = {.mode = PacingMode::kNoPacing}}, + .decoder_settings = {.pacing = {.mode = PacingMode::kNoPacing}}, + .encoding_settings = kConstantRateQvga64Kbps30Fps, + .test_expectations = {.min_apsnr_y = 30.0}, + }) +#if defined(WEBRTC_ANDROID) + , + EncodeDecodeTestParams({ + .codec = {.type = "VP8", + .encoder = "mediacodec", + .decoder = "mediacodec"}, + .video = kFourPeople_1280x720_30, + .encoder_settings = {.pacing = {.mode = PacingMode::kRealTime}}, + .decoder_settings = {.pacing = {.mode = PacingMode::kRealTime}}, + .encoding_settings = kConstantRateQvga64Kbps30Fps, + .test_expectations = {.min_apsnr_y = 30.0}, + }) +#endif + }), + EncodeDecodeTest::TestParametersToStr); } // namespace test } // namespace webrtc diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl.cc b/modules/video_coding/codecs/test/video_codec_tester_impl.cc index 9445cf7fbb..3583b59b4b 100644 --- a/modules/video_coding/codecs/test/video_codec_tester_impl.cc +++ b/modules/video_coding/codecs/test/video_codec_tester_impl.cc @@ -142,10 +142,7 @@ class LimitedTaskQueue { } void WaitForPreviouslyPostedTasks() { - while (queue_size_ > 0) { - task_executed_.Wait(rtc::Event::kForever); - task_executed_.Reset(); - } + task_queue_.SendTask([] {}); } TaskQueueForTest task_queue_; @@ -174,13 +171,17 @@ class TesterDecoder { decoder_->Decode( frame, [this, spatial_idx = frame.SpatialIndex().value_or(0)]( const VideoFrame& decoded_frame) { - this->analyzer_->FinishDecode(decoded_frame, spatial_idx); + analyzer_->FinishDecode(decoded_frame, spatial_idx); }); }, pacer_.Schedule(timestamp)); } - void Flush() { task_queue_.WaitForPreviouslyPostedTasks(); } + void Flush() { + Timestamp now = Timestamp::Micros(rtc::TimeMicros()); + task_queue_.PostScheduledTask([this] { decoder_->Flush(); }, now); + task_queue_.WaitForPreviouslyPostedTasks(); + } protected: Decoder* const decoder_; @@ -211,16 +212,20 @@ class TesterEncoder { [this, frame] { analyzer_->StartEncode(frame); encoder_->Encode(frame, [this](const EncodedImage& encoded_frame) { - this->analyzer_->FinishEncode(encoded_frame); + analyzer_->FinishEncode(encoded_frame); if (decoder_ != nullptr) { - this->decoder_->Decode(encoded_frame); + decoder_->Decode(encoded_frame); } }); }, pacer_.Schedule(timestamp)); } - void Flush() { task_queue_.WaitForPreviouslyPostedTasks(); } + void Flush() { + Timestamp now = Timestamp::Micros(rtc::TimeMicros()); + task_queue_.PostScheduledTask([this] { encoder_->Flush(); }, now); + task_queue_.WaitForPreviouslyPostedTasks(); + } protected: Encoder* const encoder_; diff --git a/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc b/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc index 8874f26df6..a54c5cbbaa 100644 --- a/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc +++ b/modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc @@ -121,6 +121,7 @@ class MockDecoder : public Decoder { Decode, (const EncodedImage& frame, DecodeCallback callback), (override)); + MOCK_METHOD(void, Flush, (), (override)); }; class MockEncoder : public Encoder { @@ -129,6 +130,7 @@ class MockEncoder : public Encoder { Encode, (const VideoFrame& frame, EncodeCallback callback), (override)); + MOCK_METHOD(void, Flush, (), (override)); }; class MockTaskQueueFactory : public TaskQueueFactory {