SimulcastEncoderAdapter: support per layer fallback and single encoder proxying

This CL adds an optional second encoder factory to SimulcastEncoderAdapter,
that can be used to create software fallback adapter per simulcast layer.

It also adds logic to check if the encoder supports simulcast natively, if so
it only allocates a single instance and delegates the simulcast logic to that
encoder instead. This means we will be able to remove EncoderSimulcastProxy.

Bug: webrtc:11000
Change-Id: Ifd5f029cc281ee2cedf9d18efa5e7e460884d6ff
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/155171
Commit-Queue: Erik Språng <sprang@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29364}
This commit is contained in:
Erik Språng 2019-10-01 18:50:03 +02:00 committed by Commit Bot
parent fddbe6c632
commit f4e0c29ed1
8 changed files with 263 additions and 49 deletions

View File

@ -90,7 +90,8 @@ VideoEncoder::EncoderInfo::EncoderInfo()
has_internal_source(false),
fps_allocation{absl::InlinedVector<uint8_t, kMaxTemporalStreams>(
1,
kMaxFramerateFraction)} {}
kMaxFramerateFraction)},
supports_simulcast(false) {}
VideoEncoder::EncoderInfo::EncoderInfo(const EncoderInfo&) = default;

View File

@ -216,6 +216,13 @@ class RTC_EXPORT VideoEncoder {
// Recommended bitrate limits for different resolutions.
std::vector<ResolutionBitrateLimits> resolution_bitrate_limits;
// If true, this encoder has internal support for generating simulcast
// streams. Otherwise, an adapter class will be needed.
// Even if true, the config provided to InitEncode() might not be supported,
// in such case the encoder should return
// WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED.
bool supports_simulcast;
};
struct RateControlParameters {

View File

@ -169,6 +169,7 @@ rtc_static_library("rtc_simulcast_encoder_adapter") {
"../api/video:video_frame",
"../api/video:video_frame_i420",
"../api/video:video_rtp_headers",
"../api/video_codecs:rtc_software_fallback_wrappers",
"../api/video_codecs:video_codecs_api",
"../modules/video_coding:video_codec_interface",
"../modules/video_coding:video_coding_utility",

View File

@ -25,6 +25,7 @@
#include "api/video/video_rotation.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "api/video_codecs/video_encoder_software_fallback_wrapper.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "modules/video_coding/utility/simulcast_rate_allocator.h"
#include "rtc_base/atomic_ops.h"
@ -70,6 +71,17 @@ int NumberOfStreams(const webrtc::VideoCodec& codec) {
return streams;
}
int NumActiveStreams(const webrtc::VideoCodec& codec) {
int num_configured_streams = NumberOfStreams(codec);
int num_active_streams = 0;
for (int i = 0; i < num_configured_streams; ++i) {
if (codec.simulcastStream[i].active) {
++num_active_streams;
}
}
return num_active_streams;
}
int VerifyCodec(const webrtc::VideoCodec* inst) {
if (inst == nullptr) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
@ -124,14 +136,21 @@ namespace webrtc {
SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory,
const SdpVideoFormat& format)
: SimulcastEncoderAdapter(factory, nullptr, format) {}
SimulcastEncoderAdapter::SimulcastEncoderAdapter(
VideoEncoderFactory* primary_factory,
VideoEncoderFactory* fallback_factory,
const SdpVideoFormat& format)
: inited_(0),
factory_(factory),
primary_encoder_factory_(primary_factory),
fallback_encoder_factory_(fallback_factory),
video_format_(format),
encoded_complete_callback_(nullptr),
experimental_boosted_screenshare_qp_(GetScreenshareBoostedQpValue()),
boost_base_layer_quality_(RateControlSettings::ParseFromFieldTrials()
.Vp8BoostBaseLayerQuality()) {
RTC_DCHECK(factory_);
RTC_DCHECK(primary_factory);
encoder_info_.implementation_name = "SimulcastEncoderAdapter";
// The adapter is typically created on the worker thread, but operated on
@ -196,7 +215,8 @@ int SimulcastEncoderAdapter::InitEncode(
int number_of_streams = NumberOfStreams(*inst);
RTC_DCHECK_LE(number_of_streams, kMaxSimulcastStreams);
const bool doing_simulcast = (number_of_streams > 1);
bool doing_simulcast_using_adapter = (number_of_streams > 1);
int num_active_streams = NumActiveStreams(*inst);
codec_ = *inst;
SimulcastRateAllocator rate_allocator(codec_);
@ -225,14 +245,48 @@ int SimulcastEncoderAdapter::InitEncode(
RTC_DCHECK_LT(lowest_resolution_stream_index, number_of_streams);
RTC_DCHECK_LT(highest_resolution_stream_index, number_of_streams);
const SdpVideoFormat format(
codec_.codecType == webrtc::kVideoCodecVP8 ? "VP8" : "H264");
for (int i = 0; i < number_of_streams; ++i) {
// If an existing encoder instance exists, reuse it.
// TODO(brandtr): Set initial RTP state (e.g., picture_id/tl0_pic_idx) here,
// when we start storing that state outside the encoder wrappers.
std::unique_ptr<VideoEncoder> encoder;
if (!stored_encoders_.empty()) {
encoder = std::move(stored_encoders_.top());
stored_encoders_.pop();
} else {
encoder = primary_encoder_factory_->CreateVideoEncoder(format);
if (fallback_encoder_factory_ != nullptr) {
encoder = CreateVideoEncoderSoftwareFallbackWrapper(
fallback_encoder_factory_->CreateVideoEncoder(format),
std::move(encoder));
}
}
bool encoder_initialized = false;
if (doing_simulcast_using_adapter && i == 0 &&
encoder->GetEncoderInfo().supports_simulcast) {
ret = encoder->InitEncode(&codec_, settings);
if (ret < 0) {
encoder->Release();
} else {
doing_simulcast_using_adapter = false;
number_of_streams = 1;
encoder_initialized = true;
}
}
VideoCodec stream_codec;
uint32_t start_bitrate_kbps = start_bitrates[i];
const bool send_stream = start_bitrate_kbps > 0;
if (!doing_simulcast) {
const bool send_stream = doing_simulcast_using_adapter
? start_bitrate_kbps > 0
: num_active_streams > 0;
if (!doing_simulcast_using_adapter) {
stream_codec = codec_;
stream_codec.numberOfSimulcastStreams = 1;
stream_codec.numberOfSimulcastStreams =
std::max<uint8_t>(1, stream_codec.numberOfSimulcastStreams);
} else {
// Cap start bitrate to the min bitrate in order to avoid strange codec
// behavior. Since sending will be false, this should not matter.
@ -253,39 +307,32 @@ int SimulcastEncoderAdapter::InitEncode(
stream_codec.qpMax = kDefaultMaxQp;
}
// If an existing encoder instance exists, reuse it.
// TODO(brandtr): Set initial RTP state (e.g., picture_id/tl0_pic_idx) here,
// when we start storing that state outside the encoder wrappers.
std::unique_ptr<VideoEncoder> encoder;
if (!stored_encoders_.empty()) {
encoder = std::move(stored_encoders_.top());
stored_encoders_.pop();
} else {
encoder = factory_->CreateVideoEncoder(SdpVideoFormat(
codec_.codecType == webrtc::kVideoCodecVP8 ? "VP8" : "H264"));
if (!encoder_initialized) {
ret = encoder->InitEncode(&stream_codec, settings);
if (ret < 0) {
// Explicitly destroy the current encoder; because we haven't registered
// a StreamInfo for it yet, Release won't do anything about it.
encoder.reset();
Release();
return ret;
}
}
ret = encoder->InitEncode(&stream_codec, settings);
if (ret < 0) {
// Explicitly destroy the current encoder; because we haven't registered a
// StreamInfo for it yet, Release won't do anything about it.
encoder.reset();
Release();
return ret;
}
std::unique_ptr<EncodedImageCallback> callback(
new AdapterEncodedImageCallback(this, i));
encoder->RegisterEncodeCompleteCallback(callback.get());
streaminfos_.emplace_back(std::move(encoder), std::move(callback),
stream_codec.width, stream_codec.height,
send_stream);
if (!doing_simulcast) {
if (!doing_simulcast_using_adapter) {
// Without simulcast, just pass through the encoder info from the one
// active encoder.
encoder_info_ = streaminfos_[0].encoder->GetEncoderInfo();
encoder_info_ = encoder->GetEncoderInfo();
encoder->RegisterEncodeCompleteCallback(encoded_complete_callback_);
streaminfos_.emplace_back(std::move(encoder), nullptr, stream_codec.width,
stream_codec.height, send_stream);
} else {
std::unique_ptr<EncodedImageCallback> callback(
new AdapterEncodedImageCallback(this, i));
encoder->RegisterEncodeCompleteCallback(callback.get());
streaminfos_.emplace_back(std::move(encoder), std::move(callback),
stream_codec.width, stream_codec.height,
send_stream);
const EncoderInfo encoder_impl_info =
streaminfos_[i].encoder->GetEncoderInfo();
@ -334,7 +381,7 @@ int SimulcastEncoderAdapter::InitEncode(
}
}
if (doing_simulcast) {
if (doing_simulcast_using_adapter) {
encoder_info_.implementation_name += ")";
}
@ -449,6 +496,9 @@ int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) {
RTC_DCHECK_RUN_ON(&encoder_queue_);
encoded_complete_callback_ = callback;
if (streaminfos_.size() == 1) {
streaminfos_[0].encoder->RegisterEncodeCompleteCallback(callback);
}
return WEBRTC_VIDEO_CODEC_OK;
}
@ -468,6 +518,12 @@ void SimulcastEncoderAdapter::SetRates(
codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
if (streaminfos_.size() == 1) {
// Not doing simulcast.
streaminfos_[0].encoder->SetRates(parameters);
return;
}
for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
uint32_t stream_bitrate_kbps =
parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000;

View File

@ -38,8 +38,15 @@ class VideoEncoderFactory;
// interfaces should be called from the encoder task queue.
class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
public:
explicit SimulcastEncoderAdapter(VideoEncoderFactory* factory,
const SdpVideoFormat& format);
// TODO(bugs.webrtc.org/11000): Remove when downstream usage is gone.
SimulcastEncoderAdapter(VideoEncoderFactory* primarty_factory,
const SdpVideoFormat& format);
// |primary_factory| produces the first-choice encoders to use.
// |fallback_factory|, if non-null, is used to create fallback encoder that
// will be used if InitEncode() fails for the primary encoder.
SimulcastEncoderAdapter(VideoEncoderFactory* primary_factory,
VideoEncoderFactory* fallback_factory,
const SdpVideoFormat& format);
virtual ~SimulcastEncoderAdapter();
// Implements VideoEncoder.
@ -106,7 +113,8 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
void DestroyStoredEncoders();
volatile int inited_; // Accessed atomically.
VideoEncoderFactory* const factory_;
VideoEncoderFactory* const primary_encoder_factory_;
VideoEncoderFactory* const fallback_encoder_factory_;
const SdpVideoFormat video_format_;
VideoCodec codec_;
std::vector<StreamInfo> streaminfos_;

View File

@ -171,6 +171,9 @@ class MockVideoEncoderFactory : public VideoEncoderFactory {
const std::vector<MockVideoEncoder*>& encoders() const;
void SetEncoderNames(const std::vector<const char*>& encoder_names);
void set_init_encode_return_value(int32_t value);
void set_supports_simulcast(bool supports_simulcast) {
supports_simulcast_ = supports_simulcast;
}
void DestroyVideoEncoder(VideoEncoder* encoder);
@ -178,6 +181,7 @@ class MockVideoEncoderFactory : public VideoEncoderFactory {
int32_t init_encode_return_value_ = 0;
std::vector<MockVideoEncoder*> encoders_;
std::vector<const char*> encoder_names_;
bool supports_simulcast_ = false;
};
class MockVideoEncoder : public VideoEncoder {
@ -226,6 +230,7 @@ class MockVideoEncoder : public VideoEncoder {
info.is_hardware_accelerated = is_hardware_accelerated_;
info.has_internal_source = has_internal_source_;
info.fps_allocation[0] = fps_allocation_;
info.supports_simulcast = supports_simulcast_;
return info;
}
@ -277,6 +282,12 @@ class MockVideoEncoder : public VideoEncoder {
RateControlParameters last_set_rates() const { return last_set_rates_; }
void set_supports_simulcast(bool supports_simulcast) {
supports_simulcast_ = supports_simulcast;
}
bool supports_simulcast() const { return supports_simulcast_; }
private:
MockVideoEncoderFactory* const factory_;
bool supports_native_handle_ = false;
@ -288,6 +299,7 @@ class MockVideoEncoder : public VideoEncoder {
int32_t init_encode_return_value_ = 0;
VideoEncoder::RateControlParameters last_set_rates_;
FramerateFractions fps_allocation_;
bool supports_simulcast_ = false;
VideoCodec codec_;
EncodedImageCallback* callback_;
@ -308,6 +320,7 @@ std::unique_ptr<VideoEncoder> MockVideoEncoderFactory::CreateVideoEncoder(
? "codec_implementation_name"
: encoder_names_[encoders_.size()];
encoder->set_implementation_name(encoder_name);
encoder->set_supports_simulcast(supports_simulcast_);
encoders_.push_back(encoder.get());
return encoder;
}
@ -340,19 +353,26 @@ void MockVideoEncoderFactory::set_init_encode_return_value(int32_t value) {
class TestSimulcastEncoderAdapterFakeHelper {
public:
TestSimulcastEncoderAdapterFakeHelper()
: factory_(new MockVideoEncoderFactory()) {}
explicit TestSimulcastEncoderAdapterFakeHelper(bool use_fallback_factory)
: primary_factory_(new MockVideoEncoderFactory()),
fallback_factory_(use_fallback_factory ? new MockVideoEncoderFactory()
: nullptr) {}
// Can only be called once as the SimulcastEncoderAdapter will take the
// ownership of |factory_|.
VideoEncoder* CreateMockEncoderAdapter() {
return new SimulcastEncoderAdapter(factory_.get(), SdpVideoFormat("VP8"));
return new SimulcastEncoderAdapter(
primary_factory_.get(), fallback_factory_.get(), SdpVideoFormat("VP8"));
}
MockVideoEncoderFactory* factory() { return factory_.get(); }
MockVideoEncoderFactory* factory() { return primary_factory_.get(); }
MockVideoEncoderFactory* fallback_factory() {
return fallback_factory_.get();
}
private:
std::unique_ptr<MockVideoEncoderFactory> factory_;
std::unique_ptr<MockVideoEncoderFactory> primary_factory_;
std::unique_ptr<MockVideoEncoderFactory> fallback_factory_;
};
static const int kTestTemporalLayerProfile[3] = {3, 2, 1};
@ -361,17 +381,26 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test,
public EncodedImageCallback {
public:
TestSimulcastEncoderAdapterFake()
: helper_(new TestSimulcastEncoderAdapterFakeHelper()),
adapter_(helper_->CreateMockEncoderAdapter()),
last_encoded_image_width_(-1),
: last_encoded_image_width_(-1),
last_encoded_image_height_(-1),
last_encoded_image_simulcast_index_(-1) {}
last_encoded_image_simulcast_index_(-1),
use_fallback_factory_(false) {}
virtual ~TestSimulcastEncoderAdapterFake() {
if (adapter_) {
adapter_->Release();
}
}
void SetUp() override {
helper_ = std::make_unique<TestSimulcastEncoderAdapterFakeHelper>(
use_fallback_factory_);
adapter_.reset(helper_->CreateMockEncoderAdapter());
last_encoded_image_width_ = -1;
last_encoded_image_height_ = -1;
last_encoded_image_simulcast_index_ = -1;
}
Result OnEncodedImage(const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info,
const RTPFragmentationHeader* fragmentation) override {
@ -482,6 +511,7 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test,
int last_encoded_image_height_;
int last_encoded_image_simulcast_index_;
std::unique_ptr<SimulcastRateAllocator> rate_allocator_;
bool use_fallback_factory_;
};
TEST_F(TestSimulcastEncoderAdapterFake, InitEncode) {
@ -1218,5 +1248,114 @@ TEST_F(TestSimulcastEncoderAdapterFake, SetRateDistributesBandwithAllocation) {
}
}
TEST_F(TestSimulcastEncoderAdapterFake, SupportsSimulcast) {
SimulcastTestFixtureImpl::DefaultSettings(
&codec_, static_cast<const int*>(kTestTemporalLayerProfile),
kVideoCodecVP8);
codec_.numberOfSimulcastStreams = 3;
// Indicate that mock encoders internally support simulcast.
helper_->factory()->set_supports_simulcast(true);
adapter_->RegisterEncodeCompleteCallback(this);
EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
// Only one encoder should have been produced.
ASSERT_EQ(1u, helper_->factory()->encoders().size());
rtc::scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720));
VideoFrame input_frame = VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_timestamp_rtp(100)
.set_timestamp_ms(1000)
.set_rotation(kVideoRotation_180)
.build();
EXPECT_CALL(*helper_->factory()->encoders()[0], Encode)
.WillOnce(Return(WEBRTC_VIDEO_CODEC_OK));
std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey);
EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types));
}
TEST_F(TestSimulcastEncoderAdapterFake, SupportsFallback) {
// Enable support for fallback encoder factory and re-setup.
use_fallback_factory_ = true;
SetUp();
SetupCodec();
// Make sure we have bitrate for all layers.
DataRate max_bitrate = DataRate::Zero();
for (int i = 0; i < 3; ++i) {
max_bitrate += DataRate::kbps(codec_.simulcastStream[i].maxBitrate);
}
const auto rate_settings = VideoEncoder::RateControlParameters(
rate_allocator_->Allocate(
VideoBitrateAllocationParameters(max_bitrate.bps(), 30)),
30.0, max_bitrate);
adapter_->SetRates(rate_settings);
std::vector<MockVideoEncoder*> primary_encoders =
helper_->factory()->encoders();
std::vector<MockVideoEncoder*> fallback_encoders =
helper_->fallback_factory()->encoders();
ASSERT_EQ(3u, primary_encoders.size());
ASSERT_EQ(3u, fallback_encoders.size());
// Create frame to test with.
rtc::scoped_refptr<VideoFrameBuffer> buffer(I420Buffer::Create(1280, 720));
VideoFrame input_frame = VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_timestamp_rtp(100)
.set_timestamp_ms(1000)
.set_rotation(kVideoRotation_180)
.build();
std::vector<VideoFrameType> frame_types(3, VideoFrameType::kVideoFrameKey);
// All primary encoders used.
for (auto codec : primary_encoders) {
EXPECT_CALL(*codec, Encode).WillOnce(Return(WEBRTC_VIDEO_CODEC_OK));
}
EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types));
// Trigger fallback on first encoder.
primary_encoders[0]->set_init_encode_return_value(
WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE);
EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
adapter_->SetRates(rate_settings);
EXPECT_CALL(*fallback_encoders[0], Encode)
.WillOnce(Return(WEBRTC_VIDEO_CODEC_OK));
EXPECT_CALL(*primary_encoders[1], Encode)
.WillOnce(Return(WEBRTC_VIDEO_CODEC_OK));
EXPECT_CALL(*primary_encoders[2], Encode)
.WillOnce(Return(WEBRTC_VIDEO_CODEC_OK));
EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types));
// Trigger fallback on all encoder.
primary_encoders[1]->set_init_encode_return_value(
WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE);
primary_encoders[2]->set_init_encode_return_value(
WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE);
EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
adapter_->SetRates(rate_settings);
EXPECT_CALL(*fallback_encoders[0], Encode)
.WillOnce(Return(WEBRTC_VIDEO_CODEC_OK));
EXPECT_CALL(*fallback_encoders[1], Encode)
.WillOnce(Return(WEBRTC_VIDEO_CODEC_OK));
EXPECT_CALL(*fallback_encoders[2], Encode)
.WillOnce(Return(WEBRTC_VIDEO_CODEC_OK));
EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types));
// Return to primary encoders on all streams.
for (int i = 0; i < 3; ++i) {
primary_encoders[i]->set_init_encode_return_value(WEBRTC_VIDEO_CODEC_OK);
}
EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
adapter_->SetRates(rate_settings);
for (auto codec : primary_encoders) {
EXPECT_CALL(*codec, Encode).WillOnce(Return(WEBRTC_VIDEO_CODEC_OK));
}
EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types));
}
} // namespace test
} // namespace webrtc

View File

@ -623,6 +623,7 @@ VideoEncoder::EncoderInfo H264EncoderImpl::GetEncoderInfo() const {
VideoEncoder::ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold);
info.is_hardware_accelerated = false;
info.has_internal_source = false;
info.supports_simulcast = true;
return info;
}

View File

@ -1212,6 +1212,7 @@ VideoEncoder::EncoderInfo LibvpxVp8Encoder::GetEncoderInfo() const {
rate_control_settings_.LibvpxVp8TrustedRateController();
info.is_hardware_accelerated = false;
info.has_internal_source = false;
info.supports_simulcast = true;
const bool enable_scaling = encoders_.size() == 1 &&
vpx_configs_[0].rc_dropframe_thresh > 0 &&