admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Initialize() to Encoder/Decoder API in video codec tester

Browse Source 
Initialization of Android HW codecs takes hundreds milliseconds. Exclude this time from frame processing time of first frame by initializing codecs before starting encoding/decoding.

Bug: b/261160916, webrtc:14852
Change-Id: I9ec84c6b12c1d9821b59965cf521170224066563
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/298304
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39613}
This commit is contained in:
Sergey Silkin 2023-03-20 15:24:38 +01:00 committed by WebRTC LUCI CQ
parent 1a5ff94b05
commit aa17f2f0a9
4 changed files with 128 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/test/video_codec_tester.h
View File

@ -91,6 +91,8 @@ class VideoCodecTester {
virtual ~Encoder() = default;
virtual void Initialize() = 0;
virtual void Encode(const VideoFrame& frame, EncodeCallback callback) = 0;
virtual void Flush() = 0;
@ -104,6 +106,8 @@ class VideoCodecTester {
virtual ~Decoder() = default;
virtual void Initialize() = 0;
virtual void Decode(const EncodedImage& frame, DecodeCallback callback) = 0;
virtual void Flush() = 0;

101
modules/video_coding/codecs/test/video_codec_test.cc
View File

@ -80,6 +80,33 @@ struct EncodingSettings {
DataRate bitrate;
};
std::map<LayerId, LayerSettings> layer_settings;
bool IsSameSettings(const EncodingSettings& other) const {
if (scalability_mode != other.scalability_mode) {
return true;
}
for (auto [layer_id, layer] : layer_settings) {
const auto& other_layer = other.layer_settings.at(layer_id);
if (layer.resolution != other_layer.resolution) {
return true;
}
}
return false;
}
bool IsSameRate(const EncodingSettings& other) const {
for (auto [layer_id, layer] : layer_settings) {
const auto& other_layer = other.layer_settings.at(layer_id);
if (layer.bitrate != other_layer.bitrate ||
layer.framerate != other_layer.framerate) {
return true;
}
}
return false;
}
};
const VideoInfo kFourPeople_1280x720_30 = {
@ -188,17 +215,20 @@ class TestEncoder : public VideoCodecTester::Encoder,
encoder_->RegisterEncodeCompleteCallback(this);
}
void Initialize() override {
const EncodingSettings& first_frame_settings = frame_settings_.at(0);
Configure(first_frame_settings);
SetRates(first_frame_settings);
}
void Encode(const VideoFrame& frame, EncodeCallback callback) override {
callbacks_[frame.timestamp()] = std::move(callback);
if (auto fs = frame_settings_.find(frame_num_);
fs != frame_settings_.end()) {
if (fs == frame_settings_.begin() ||
ConfigChanged(fs->second, std::prev(fs)->second)) {
fs != frame_settings_.begin() && fs != frame_settings_.end()) {
if (!fs->second.IsSameSettings(std::prev(fs)->second)) {
Configure(fs->second);
}
if (fs == frame_settings_.begin() ||
RateChanged(fs->second, std::prev(fs)->second)) {
} else if (!fs->second.IsSameRate(std::prev(fs)->second)) {
SetRates(fs->second);
}
}
@ -261,6 +291,8 @@ class TestEncoder : public VideoCodecTester::Encoder,
int result = encoder_->InitEncode(&vc, ves);
ASSERT_EQ(result, WEBRTC_VIDEO_CODEC_OK);
SetRates(es);
}
void SetRates(const EncodingSettings& es) {
@ -284,34 +316,6 @@ class TestEncoder : public VideoCodecTester::Encoder,
encoder_->SetRates(rc);
}
bool ConfigChanged(const EncodingSettings& es,
const EncodingSettings& prev_es) const {
if (es.scalability_mode != prev_es.scalability_mode) {
return true;
}
for (auto [layer_id, layer_settings] : es.layer_settings) {
const auto& prev_layer_settings = prev_es.layer_settings.at(layer_id);
if (layer_settings.resolution != prev_layer_settings.resolution) {
return true;
}
}
return false;
}
bool RateChanged(const EncodingSettings& es,
const EncodingSettings& prev_es) const {
for (auto [layer_id, layer_settings] : es.layer_settings) {
const auto& prev_layer_settings = prev_es.layer_settings.at(layer_id);
if (layer_settings.bitrate != prev_layer_settings.bitrate ||
layer_settings.framerate != prev_layer_settings.framerate) {
return true;
}
}
return false;
}
std::unique_ptr<VideoEncoder> encoder_;
const std::string codec_type_;
const std::map<int, EncodingSettings>& frame_settings_;
@ -324,20 +328,24 @@ class TestDecoder : public VideoCodecTester::Decoder,
public:
TestDecoder(std::unique_ptr<VideoDecoder> decoder,
const std::string codec_type)
: decoder_(std::move(decoder)), codec_type_(codec_type), frame_num_(0) {
: decoder_(std::move(decoder)), codec_type_(codec_type) {
decoder_->RegisterDecodeCompleteCallback(this);
}
void Initialize() override {
VideoDecoder::Settings ds;
ds.set_codec_type(PayloadStringToCodecType(codec_type_));
ds.set_number_of_cores(1);
ds.set_max_render_resolution({1280, 720});
bool result = decoder_->Configure(ds);
ASSERT_TRUE(result);
}
void Decode(const EncodedImage& frame, DecodeCallback callback) override {
callbacks_[frame.Timestamp()] = std::move(callback);
if (frame_num_ == 0) {
Configure();
}
decoder_->Decode(frame, /*missing_frames=*/false,
/*render_time_ms=*/0);
++frame_num_;
}
void Flush() override {
@ -350,16 +358,6 @@ class TestDecoder : public VideoCodecTester::Decoder,
VideoDecoder* decoder() { return decoder_.get(); }
protected:
void Configure() {
VideoDecoder::Settings ds;
ds.set_codec_type(PayloadStringToCodecType(codec_type_));
ds.set_number_of_cores(1);
ds.set_max_render_resolution({1280, 720});
bool result = decoder_->Configure(ds);
ASSERT_TRUE(result);
}
int Decoded(VideoFrame& decoded_frame) override {
auto cb = callbacks_.find(decoded_frame.timestamp());
RTC_CHECK(cb != callbacks_.end());
@ -371,7 +369,6 @@ class TestDecoder : public VideoCodecTester::Decoder,
std::unique_ptr<VideoDecoder> decoder_;
const std::string codec_type_;
int frame_num_;
std::map<uint32_t, DecodeCallback> callbacks_;
};

127
modules/video_coding/codecs/test/video_codec_tester_impl.cc
View File

@ -187,58 +187,6 @@ class TesterY4mWriter {
TaskQueueForTest task_queue_;
};
class TesterDecoder {
public:
TesterDecoder(Decoder* decoder,
VideoCodecAnalyzer* analyzer,
const DecoderSettings& settings)
: decoder_(decoder),
analyzer_(analyzer),
settings_(settings),
pacer_(settings.pacing) {
RTC_CHECK(analyzer_) << "Analyzer must be provided";
if (settings.decoded_y4m_base_path) {
y4m_writer_ =
std::make_unique<TesterY4mWriter>(*settings.decoded_y4m_base_path);
}
}
void Decode(const EncodedImage& frame) {
Timestamp timestamp = Timestamp::Micros((frame.Timestamp() / k90kHz).us());
task_queue_.PostScheduledTask(
[this, frame] {
analyzer_->StartDecode(frame);
decoder_->Decode(
frame, [this, spatial_idx = frame.SpatialIndex().value_or(0)](
const VideoFrame& decoded_frame) {
analyzer_->FinishDecode(decoded_frame, spatial_idx);
if (y4m_writer_) {
y4m_writer_->Write(decoded_frame, spatial_idx);
}
});
},
pacer_.Schedule(timestamp));
}
void Flush() {
Timestamp now = Timestamp::Micros(rtc::TimeMicros());
task_queue_.PostScheduledTask([this] { decoder_->Flush(); }, now);
task_queue_.WaitForPreviouslyPostedTasks();
}
protected:
Decoder* const decoder_;
VideoCodecAnalyzer* const analyzer_;
const DecoderSettings& settings_;
Pacer pacer_;
LimitedTaskQueue task_queue_;
std::unique_ptr<TesterY4mWriter> y4m_writer_;
};
class TesterIvfWriter {
public:
explicit TesterIvfWriter(absl::string_view base_path)
@ -277,6 +225,64 @@ class TesterIvfWriter {
TaskQueueForTest task_queue_;
};
class TesterDecoder {
public:
TesterDecoder(Decoder* decoder,
VideoCodecAnalyzer* analyzer,
const DecoderSettings& settings)
: decoder_(decoder),
analyzer_(analyzer),
settings_(settings),
pacer_(settings.pacing) {
RTC_CHECK(analyzer_) << "Analyzer must be provided";
if (settings.decoded_y4m_base_path) {
y4m_writer_ =
std::make_unique<TesterY4mWriter>(*settings.decoded_y4m_base_path);
}
}
void Initialize() {
task_queue_.PostScheduledTask([this] { decoder_->Initialize(); },
Timestamp::Zero());
task_queue_.WaitForPreviouslyPostedTasks();
}
void Decode(const EncodedImage& frame) {
Timestamp timestamp = Timestamp::Micros((frame.Timestamp() / k90kHz).us());
task_queue_.PostScheduledTask(
[this, frame] {
analyzer_->StartDecode(frame);
decoder_->Decode(
frame, [this, spatial_idx = frame.SpatialIndex().value_or(0)](
const VideoFrame& decoded_frame) {
analyzer_->FinishDecode(decoded_frame, spatial_idx);
if (y4m_writer_) {
y4m_writer_->Write(decoded_frame, spatial_idx);
}
});
},
pacer_.Schedule(timestamp));
}
void Flush() {
task_queue_.PostScheduledTask([this] { decoder_->Flush(); },
Timestamp::Zero());
task_queue_.WaitForPreviouslyPostedTasks();
}
protected:
Decoder* const decoder_;
VideoCodecAnalyzer* const analyzer_;
const DecoderSettings& settings_;
Pacer pacer_;
LimitedTaskQueue task_queue_;
std::unique_ptr<TesterY4mWriter> y4m_writer_;
};
class TesterEncoder {
public:
TesterEncoder(Encoder* encoder,
@ -295,6 +301,12 @@ class TesterEncoder {
}
}
void Initialize() {
task_queue_.PostScheduledTask([this] { encoder_->Initialize(); },
Timestamp::Zero());
task_queue_.WaitForPreviouslyPostedTasks();
}
void Encode(const VideoFrame& frame) {
Timestamp timestamp = Timestamp::Micros((frame.timestamp() / k90kHz).us());
@ -317,8 +329,8 @@ class TesterEncoder {
}
void Flush() {
Timestamp now = Timestamp::Micros(rtc::TimeMicros());
task_queue_.PostScheduledTask([this] { encoder_->Flush(); }, now);
task_queue_.PostScheduledTask([this] { encoder_->Flush(); },
Timestamp::Zero());
task_queue_.WaitForPreviouslyPostedTasks();
}
@ -341,6 +353,8 @@ std::unique_ptr<VideoCodecStats> VideoCodecTesterImpl::RunDecodeTest(
VideoCodecAnalyzer perf_analyzer;
TesterDecoder tester_decoder(decoder, &perf_analyzer, decoder_settings);
tester_decoder.Initialize();
while (auto frame = video_source->PullFrame()) {
tester_decoder.Decode(*frame);
}
@ -359,6 +373,8 @@ std::unique_ptr<VideoCodecStats> VideoCodecTesterImpl::RunEncodeTest(
TesterEncoder tester_encoder(encoder, /*decoder=*/nullptr, &perf_analyzer,
encoder_settings);
tester_encoder.Initialize();
while (auto frame = sync_source.PullFrame()) {
tester_encoder.Encode(*frame);
}
@ -380,6 +396,9 @@ std::unique_ptr<VideoCodecStats> VideoCodecTesterImpl::RunEncodeDecodeTest(
TesterEncoder tester_encoder(encoder, &tester_decoder, &perf_analyzer,
encoder_settings);
tester_encoder.Initialize();
tester_decoder.Initialize();
while (auto frame = sync_source.PullFrame()) {
tester_encoder.Encode(*frame);
}

2
modules/video_coding/codecs/test/video_codec_tester_impl_unittest.cc
View File

@ -117,6 +117,7 @@ class MockCodedVideoSource : public CodedVideoSource {
class MockDecoder : public Decoder {
public:
MOCK_METHOD(void, Initialize, (), (override));
MOCK_METHOD(void,
Decode,
(const EncodedImage& frame, DecodeCallback callback),
@ -126,6 +127,7 @@ class MockDecoder : public Decoder {
class MockEncoder : public Encoder {
public:
MOCK_METHOD(void, Initialize, (), (override));
MOCK_METHOD(void,
Encode,
(const VideoFrame& frame, EncodeCallback callback),