admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add optional is_qp_trusted property for EncoderInfo.

Browse Source 
Some hardware H.264 encoders does not place average QP delta in
slice_qp_delta field. Adding an optional flag in EncoderInfo to notify
quality scaler about this.

Bug: webrtc:12942
Change-Id: I3ee29c5ae9bd7bb34d26eba7e6bede3798ca44b4
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/226921
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34627}
This commit is contained in:
Qiu Jianlin 2021-07-30 06:48:03 +08:00 committed by WebRTC LUCI CQ
parent f137b75a4d
commit b54cfdebfe
7 changed files with 156 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc
View File

@ -145,6 +145,8 @@ class VideoEncoderSoftwareFallbackWrapperTestBase : public ::testing::Test {
info.scaling_settings = ScalingSettings(kLowThreshold, kHighThreshold);
info.supports_native_handle = supports_native_handle_;
info.implementation_name = implementation_name_;
if (is_qp_trusted_)
info.is_qp_trusted = is_qp_trusted_;
return info;
}
@ -156,6 +158,7 @@ class VideoEncoderSoftwareFallbackWrapperTestBase : public ::testing::Test {
int release_count_ = 0;
mutable int supports_native_handle_count_ = 0;
bool supports_native_handle_ = false;
bool is_qp_trusted_ = false;
std::string implementation_name_ = "fake-encoder";
absl::optional<VideoFrame> last_video_frame_;
};
@ -418,6 +421,16 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest, ReportsImplementationName) {
CheckLastEncoderName("fake-encoder");
}
TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
IsQpTrustedNotForwardedDuringFallback) {
// Fake encoder signals trusted QP, default (libvpx) does not.
fake_encoder_->is_qp_trusted_ = true;
EXPECT_TRUE(fake_encoder_->GetEncoderInfo().is_qp_trusted.value_or(false));
UtilizeFallbackEncoder();
EXPECT_FALSE(fallback_wrapper_->GetEncoderInfo().is_qp_trusted.has_value());
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release());
}
TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
ReportsFallbackImplementationName) {
UtilizeFallbackEncoder();

3
api/video_codecs/video_encoder.cc
View File

@ -187,6 +187,9 @@ std::string VideoEncoder::EncoderInfo::ToString() const {
oss << VideoFrameBufferTypeToString(preferred_pixel_formats.at(i));
}
oss << "]";
if (is_qp_trusted.has_value()) {
oss << ", is_qp_trusted = " << is_qp_trusted.value();
}
oss << "}";
return oss.str();
}

4
api/video_codecs/video_encoder.h
View File

@ -260,6 +260,10 @@ class RTC_EXPORT VideoEncoder {
// preferred pixel format. The order of the formats does not matter.
absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
preferred_pixel_formats;
// Indicates whether or not QP value encoder writes into frame/slice/tile
// header can be interpreted as average frame/slice/tile QP.
absl::optional<bool> is_qp_trusted;
};
struct RTC_EXPORT RateControlParameters {

7
media/engine/simulcast_encoder_adapter.cc
View File

@ -876,6 +876,7 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
encoder_info.is_hardware_accelerated =
encoder_impl_info.is_hardware_accelerated;
encoder_info.has_internal_source = encoder_impl_info.has_internal_source;
encoder_info.is_qp_trusted = encoder_impl_info.is_qp_trusted;
} else {
encoder_info.implementation_name += ", ";
encoder_info.implementation_name += encoder_impl_info.implementation_name;
@ -897,6 +898,12 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
// Has internal source only if all encoders have it.
encoder_info.has_internal_source &= encoder_impl_info.has_internal_source;
// Treat QP from frame/slice/tile header as average QP only if all
// encoders report it as average QP.
encoder_info.is_qp_trusted =
encoder_info.is_qp_trusted.value_or(true) &
encoder_impl_info.is_qp_trusted.value_or(true);
}
encoder_info.fps_allocation[i] = encoder_impl_info.fps_allocation[0];
encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple(

28
media/engine/simulcast_encoder_adapter_unittest.cc
View File

@ -241,6 +241,7 @@ class MockVideoEncoder : public VideoEncoder {
info.has_internal_source = has_internal_source_;
info.fps_allocation[0] = fps_allocation_;
info.supports_simulcast = supports_simulcast_;
info.is_qp_trusted = is_qp_trusted_;
return info;
}
@ -308,6 +309,10 @@ class MockVideoEncoder : public VideoEncoder {
video_format_ = video_format;
}
void set_is_qp_trusted(absl::optional<bool> is_qp_trusted) {
is_qp_trusted_ = is_qp_trusted;
}
bool supports_simulcast() const { return supports_simulcast_; }
SdpVideoFormat video_format() const { return video_format_; }
@ -326,6 +331,7 @@ class MockVideoEncoder : public VideoEncoder {
VideoEncoder::RateControlParameters last_set_rates_;
FramerateFractions fps_allocation_;
bool supports_simulcast_ = false;
absl::optional<bool> is_qp_trusted_;
SdpVideoFormat video_format_;
VideoCodec codec_;
@ -1387,6 +1393,28 @@ TEST_F(TestSimulcastEncoderAdapterFake, ReportsInternalSource) {
EXPECT_FALSE(adapter_->GetEncoderInfo().has_internal_source);
}
TEST_F(TestSimulcastEncoderAdapterFake, ReportsIsQpTrusted) {
SimulcastTestFixtureImpl::DefaultSettings(
&codec_, static_cast<const int*>(kTestTemporalLayerProfile),
kVideoCodecVP8);
codec_.numberOfSimulcastStreams = 3;
adapter_->RegisterEncodeCompleteCallback(this);
EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
ASSERT_EQ(3u, helper_->factory()->encoders().size());
// All encoders have internal source, simulcast adapter reports true.
for (MockVideoEncoder* encoder : helper_->factory()->encoders()) {
encoder->set_is_qp_trusted(true);
}
EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
EXPECT_TRUE(adapter_->GetEncoderInfo().is_qp_trusted.value_or(false));
// One encoder reports QP not trusted, simulcast adapter reports false.
helper_->factory()->encoders()[2]->set_is_qp_trusted(false);
EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
EXPECT_FALSE(adapter_->GetEncoderInfo().is_qp_trusted.value_or(true));
}
TEST_F(TestSimulcastEncoderAdapterFake, ReportsFpsAllocation) {
SimulcastTestFixtureImpl::DefaultSettings(
&codec_, static_cast<const int*>(kTestTemporalLayerProfile),

3
video/adaptation/video_stream_encoder_resource_manager.cc
View File

@ -527,7 +527,8 @@ void VideoStreamEncoderResourceManager::ConfigureQualityScaler(
IsResolutionScalingEnabled(degradation_preference_) &&
(scaling_settings.thresholds.has_value() ||
(encoder_settings_.has_value() &&
encoder_settings_->encoder_config().is_quality_scaling_allowed));
encoder_settings_->encoder_config().is_quality_scaling_allowed)) &&
encoder_info.is_qp_trusted.value_or(true);
// TODO(https://crbug.com/webrtc/11222): Should this move to
// QualityScalerResource?

99
video/video_stream_encoder_unittest.cc
View File

@ -888,6 +888,9 @@ class VideoStreamEncoderTest : public ::testing::Test {
info.apply_alignment_to_all_simulcast_layers =
apply_alignment_to_all_simulcast_layers_;
info.preferred_pixel_formats = preferred_pixel_formats_;
if (is_qp_trusted_.has_value()) {
info.is_qp_trusted = is_qp_trusted_;
}
return info;
}
@ -1036,6 +1039,11 @@ class VideoStreamEncoderTest : public ::testing::Test {
preferred_pixel_formats_ = std::move(pixel_formats);
}
void SetIsQpTrusted(absl::optional<bool> trusted) {
MutexLock lock(&local_mutex_);
is_qp_trusted_ = trusted;
}
private:
int32_t Encode(const VideoFrame& input_image,
const std::vector<VideoFrameType>* frame_types) override {
@ -1185,6 +1193,7 @@ class VideoStreamEncoderTest : public ::testing::Test {
RTC_GUARDED_BY(local_mutex_);
absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
};
class TestSink : public VideoStreamEncoder::EncoderSink {
@ -8191,6 +8200,66 @@ TEST_F(VideoStreamEncoderTest,
video_stream_encoder_->Stop();
}
TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
// Disable scaling settings in encoder info.
fake_encoder_.SetQualityScaling(false);
// Set QP not trusted in encoder info.
fake_encoder_.SetIsQpTrusted(false);
// Enable quality scaling in encoder config.
video_encoder_config.is_quality_scaling_allowed = true;
ConfigureEncoder(std::move(video_encoder_config));
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
DataRate::BitsPerSec(kTargetBitrateBps),
DataRate::BitsPerSec(kTargetBitrateBps),
DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
test::FrameForwarder source;
video_stream_encoder_->SetSource(
&source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
WaitForEncodedFrame(1);
video_stream_encoder_->TriggerQualityLow();
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
video_stream_encoder_->Stop();
}
TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
// Disable scaling settings in encoder info.
fake_encoder_.SetQualityScaling(false);
// Set QP trusted in encoder info.
fake_encoder_.SetIsQpTrusted(true);
// Enable quality scaling in encoder config.
video_encoder_config.is_quality_scaling_allowed = false;
ConfigureEncoder(std::move(video_encoder_config));
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
DataRate::BitsPerSec(kTargetBitrateBps),
DataRate::BitsPerSec(kTargetBitrateBps),
DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
test::FrameForwarder source;
video_stream_encoder_->SetSource(
&source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
WaitForEncodedFrame(1);
video_stream_encoder_->TriggerQualityLow();
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
video_stream_encoder_->Stop();
}
#if !defined(WEBRTC_IOS)
// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
// disabled by default on iOS.
@ -8221,6 +8290,36 @@ TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
video_stream_encoder_->Stop();
}
TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
// Disable scaling settings in encoder info.
fake_encoder_.SetQualityScaling(false);
// Set QP trusted in encoder info.
fake_encoder_.SetIsQpTrusted(true);
// Enable quality scaling in encoder config.
video_encoder_config.is_quality_scaling_allowed = true;
ConfigureEncoder(std::move(video_encoder_config));
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
DataRate::BitsPerSec(kTargetBitrateBps),
DataRate::BitsPerSec(kTargetBitrateBps),
DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
test::FrameForwarder source;
video_stream_encoder_->SetSource(
&source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
WaitForEncodedFrame(1);
video_stream_encoder_->TriggerQualityLow();
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
video_stream_encoder_->Stop();
}
#endif
// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)