admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

VideoStreamEncoder report VideoLayersAllocation for simulcast

Browse Source 
Adds support for Vp8 simulcast.

Bug: webrtc:12000
Change-Id: Ib24fd0542642b023ec35f7a7bdc4880d72365edf
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/187341
Commit-Queue: Per Kjellander <perkj@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32416}
This commit is contained in:
Per Kjellander 2020-10-15 17:53:22 +02:00 committed by Commit Bot
parent 4244b5f6b4
commit a94348440b
8 changed files with 229 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
api/video/BUILD.gn
View File

@ -21,7 +21,6 @@ rtc_library("video_rtp_headers") {
"hdr_metadata.h",
"video_content_type.cc",
"video_content_type.h",
"video_layers_allocation.h",
"video_rotation.h",
"video_timing.cc",
"video_timing.h",
@ -182,6 +181,13 @@ rtc_library("video_bitrate_allocation") {
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_source_set("video_layers_allocation") {
visibility = [ "*" ]
sources = [ "video_layers_allocation.h" ]
deps = [ "../units:data_rate" ]
absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ]
}
rtc_library("video_bitrate_allocator") {
visibility = [ "*" ]
sources = [
@ -264,6 +270,7 @@ rtc_source_set("video_stream_encoder") {
":video_bitrate_allocator_factory",
":video_codec_constants",
":video_frame",
":video_layers_allocation",
"..:rtp_parameters",
"..:scoped_refptr",
"../:fec_controller_api",

4
api/video/video_stream_encoder_interface.h
View File

@ -19,6 +19,7 @@
#include "api/scoped_refptr.h"
#include "api/units/data_rate.h"
#include "api/video/video_bitrate_allocator.h"
#include "api/video/video_layers_allocation.h"
#include "api/video/video_sink_interface.h"
#include "api/video/video_source_interface.h"
#include "api/video_codecs/video_encoder.h"
@ -52,6 +53,9 @@ class VideoStreamEncoderInterface : public rtc::VideoSinkInterface<VideoFrame> {
virtual void OnBitrateAllocationUpdated(
const VideoBitrateAllocation& allocation) = 0;
virtual void OnVideoLayersAllocationUpdated(
VideoLayersAllocation allocation) = 0;
};
// If the resource is overusing, the VideoStreamEncoder will try to reduce

3
modules/rtp_rtcp/BUILD.gn
View File

@ -110,6 +110,7 @@ rtc_library("rtp_rtcp_format") {
"../../api/transport/rtp:dependency_descriptor",
"../../api/units:time_delta",
"../../api/video:video_frame",
"../../api/video:video_layers_allocation",
"../../api/video:video_rtp_headers",
"../../common_video",
"../../rtc_base:checks",
@ -278,6 +279,7 @@ rtc_library("rtp_rtcp") {
"../../api/video:video_codec_constants",
"../../api/video:video_frame",
"../../api/video:video_frame_type",
"../../api/video:video_layers_allocation",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../call:rtp_interfaces",
@ -537,6 +539,7 @@ if (rtc_include_tests) {
"../../api/video:video_bitrate_allocator",
"../../api/video:video_codec_constants",
"../../api/video:video_frame",
"../../api/video:video_layers_allocation",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../call:rtp_receiver",

1
video/BUILD.gn
View File

@ -226,6 +226,7 @@ rtc_library("video_stream_encoder_impl") {
"../api/video:video_bitrate_allocator_factory",
"../api/video:video_codec_constants",
"../api/video:video_frame",
"../api/video:video_layers_allocation",
"../api/video:video_rtp_headers",
"../api/video:video_stream_encoder",
"../api/video_codecs:video_codecs_api",

14
video/video_send_stream_impl.cc
View File

@ -472,6 +472,20 @@ void VideoSendStreamImpl::OnBitrateAllocationUpdated(
}
}
void VideoSendStreamImpl::OnVideoLayersAllocationUpdated(
VideoLayersAllocation allocation) {
if (!worker_queue_->IsCurrent()) {
auto ptr = weak_ptr_;
worker_queue_->PostTask([allocation = std::move(allocation), ptr] {
if (!ptr.get())
return;
ptr->OnVideoLayersAllocationUpdated(allocation);
});
return;
}
// TODO(bugs.webrtc.org/12000): Implement
}
void VideoSendStreamImpl::SignalEncoderActive() {
RTC_DCHECK_RUN_ON(worker_queue_);
if (rtp_video_sender_->IsActive()) {

2
video/video_send_stream_impl.h
View File

@ -121,6 +121,8 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
void OnBitrateAllocationUpdated(
const VideoBitrateAllocation& allocation) override;
void OnVideoLayersAllocationUpdated(
VideoLayersAllocation allocation) override;
// Implements EncodedImageCallback. The implementation routes encoded frames
// to the |payload_router_| and |config.pre_encode_callback| if set.

73
video/video_stream_encoder.cc
View File

@ -198,6 +198,73 @@ VideoBitrateAllocation UpdateAllocationFromEncoderInfo(
return new_allocation;
}
// Converts a VideoBitrateAllocation that contains allocated bitrate per layer,
// and an EncoderInfo that contains information about the actual encoder
// structure used by a codec. Stream structures can be Ksvc, Full SVC, Simulcast
// etc.
VideoLayersAllocation CreateVideoLayersAllocation(
const VideoCodec& encoder_config,
const VideoEncoder::RateControlParameters& current_rate,
const VideoEncoder::EncoderInfo& encoder_info) {
const VideoBitrateAllocation& target_bitrate = current_rate.target_bitrate;
VideoLayersAllocation layers_allocation;
if (target_bitrate.get_sum_bps() == 0) {
return layers_allocation;
}
if (encoder_config.numberOfSimulcastStreams > 0) {
layers_allocation.resolution_and_frame_rate_is_valid = true;
for (int si = 0; si < encoder_config.numberOfSimulcastStreams; ++si) {
if (!target_bitrate.IsSpatialLayerUsed(si) ||
target_bitrate.GetSpatialLayerSum(si) == 0) {
break;
}
layers_allocation.active_spatial_layers.emplace_back();
VideoLayersAllocation::SpatialLayer& spatial_layer =
layers_allocation.active_spatial_layers.back();
spatial_layer.width = encoder_config.simulcastStream[si].width;
spatial_layer.height = encoder_config.simulcastStream[si].height;
spatial_layer.rtp_stream_index = si;
spatial_layer.spatial_id = 0;
auto frame_rate_fraction =
VideoEncoder::EncoderInfo::kMaxFramerateFraction;
if (encoder_info.fps_allocation[si].size() == 1) {
// One TL is signalled to be used by the encoder. Do not distribute
// bitrate allocation across TLs (use sum at tl:0).
spatial_layer.target_bitrate_per_temporal_layer.push_back(
DataRate::BitsPerSec(target_bitrate.GetSpatialLayerSum(si)));
frame_rate_fraction = encoder_info.fps_allocation[si][0];
} else { // Temporal layers are supported.
uint32_t temporal_layer_bitrate_bps = 0;
for (size_t ti = 0;
ti < encoder_config.simulcastStream[si].numberOfTemporalLayers;
++ti) {
if (!target_bitrate.HasBitrate(si, ti)) {
break;
}
if (ti < encoder_info.fps_allocation[si].size()) {
// Use frame rate of the top used temporal layer.
frame_rate_fraction = encoder_info.fps_allocation[si][ti];
}
temporal_layer_bitrate_bps += target_bitrate.GetBitrate(si, ti);
spatial_layer.target_bitrate_per_temporal_layer.push_back(
DataRate::BitsPerSec(temporal_layer_bitrate_bps));
}
}
// Encoder may drop frames internally if `maxFramerate` is set.
spatial_layer.frame_rate_fps = std::min(
static_cast<uint8_t>(encoder_config.simulcastStream[si].maxFramerate),
static_cast<uint8_t>(
(current_rate.framerate_fps * frame_rate_fraction) /
VideoEncoder::EncoderInfo::kMaxFramerateFraction));
}
} else {
// TODO(bugs.webrtc.org/12000): Implement support for kSVC and full SVC.
}
return layers_allocation;
}
} // namespace
VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings()
@ -1124,6 +1191,12 @@ void VideoStreamEncoder::SetEncoderRates(
rate_settings.rate_control.bitrate,
static_cast<uint32_t>(rate_settings.rate_control.framerate_fps + 0.5));
stream_resource_manager_.SetEncoderRates(rate_settings.rate_control);
if (settings_.allocation_cb_type ==
VideoStreamEncoderSettings::BitrateAllocationCallbackType::
kVideoLayersAllocation) {
sink_->OnVideoLayersAllocationUpdated(CreateVideoLayersAllocation(
send_codec_, rate_settings.rate_control, encoder_->GetEncoderInfo()));
}
}
if ((settings_.allocation_cb_type ==
VideoStreamEncoderSettings::BitrateAllocationCallbackType::

124
video/video_stream_encoder_unittest.cc
View File

@ -1254,6 +1254,16 @@ class VideoStreamEncoderTest : public ::testing::Test {
return number_of_bitrate_allocations_;
}
VideoLayersAllocation GetLastVideoLayersAllocation() {
MutexLock lock(&mutex_);
return last_layers_allocation_;
}
int number_of_layers_allocations() const {
MutexLock lock(&mutex_);
return number_of_layers_allocations_;
}
private:
Result OnEncodedImage(
const EncodedImage& encoded_image,
@ -1296,6 +1306,24 @@ class VideoStreamEncoderTest : public ::testing::Test {
last_bitrate_allocation_ = allocation;
}
void OnVideoLayersAllocationUpdated(
VideoLayersAllocation allocation) override {
MutexLock lock(&mutex_);
++number_of_layers_allocations_;
last_layers_allocation_ = allocation;
rtc::StringBuilder log;
for (const auto& layer : allocation.active_spatial_layers) {
log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
<< "[";
for (const auto target_bitrate :
layer.target_bitrate_per_temporal_layer) {
log << target_bitrate.kbps() << ",";
}
log << "]";
}
RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
}
TimeController* const time_controller_;
mutable Mutex mutex_;
TestEncoder* test_encoder_;
@ -1313,6 +1341,8 @@ class VideoStreamEncoderTest : public ::testing::Test {
int min_transmit_bitrate_bps_ = 0;
VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
};
class VideoBitrateAllocatorProxyFactory
@ -4013,6 +4043,100 @@ TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
video_stream_encoder_->Stop();
}
TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForV8Simulcast) {
ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
VideoStreamEncoderSettings::BitrateAllocationCallbackType::
kVideoLayersAllocation);
const int kDefaultFps = 30;
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
DataRate::BitsPerSec(kLowTargetBitrateBps),
DataRate::BitsPerSec(kLowTargetBitrateBps),
DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
video_source_.IncomingCapturedFrame(
CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
WaitForEncodedFrame(CurrentTimeMs());
EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
VideoLayersAllocation last_layer_allocation =
sink_.GetLastVideoLayersAllocation();
// kLowTargetBitrateBps is only enough for one spatial layer.
ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
VideoBitrateAllocation bitrate_allocation =
fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
// Check that encoder has been updated too, not just allocation observer.
EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
// VideoLayersAllocation might be updated if frame rate change.
int number_of_layers_allocation = 1;
const int64_t start_time_ms = CurrentTimeMs();
while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
video_source_.IncomingCapturedFrame(
CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
WaitForEncodedFrame(CurrentTimeMs());
AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
number_of_layers_allocation = sink_.number_of_layers_allocations();
VideoLayersAllocation new_allocation =
sink_.GetLastVideoLayersAllocation();
ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
EXPECT_EQ(new_allocation.active_spatial_layers[0]
.target_bitrate_per_temporal_layer,
last_layer_allocation.active_spatial_layers[0]
.target_bitrate_per_temporal_layer);
last_layer_allocation = new_allocation;
}
}
EXPECT_LE(sink_.number_of_layers_allocations(), 3);
video_stream_encoder_->Stop();
}
TEST_F(VideoStreamEncoderTest,
ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
VideoStreamEncoderSettings::BitrateAllocationCallbackType::
kVideoLayersAllocation);
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
DataRate::BitsPerSec(kLowTargetBitrateBps),
DataRate::BitsPerSec(kLowTargetBitrateBps),
DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
video_source_.IncomingCapturedFrame(
CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
WaitForEncodedFrame(CurrentTimeMs());
EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
VideoLayersAllocation last_layer_allocation =
sink_.GetLastVideoLayersAllocation();
// kLowTargetBitrateBps is only enough for one spatial layer.
ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
.target_bitrate_per_temporal_layer[0],
DataRate::BitsPerSec(kLowTargetBitrateBps));
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
video_source_.IncomingCapturedFrame(
CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
WaitForEncodedFrame(CurrentTimeMs());
EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
last_layer_allocation = sink_.GetLastVideoLayersAllocation();
ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
.target_bitrate_per_temporal_layer[0],
DataRate::Zero());
video_stream_encoder_->Stop();
}
TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
// 2 TLs configured, temporal layers supported by encoder.
const int kNumTemporalLayers = 2;