diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index d75ec8f533..b0245482af 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -21,7 +21,6 @@ rtc_library("video_rtp_headers") { "hdr_metadata.h", "video_content_type.cc", "video_content_type.h", - "video_layers_allocation.h", "video_rotation.h", "video_timing.cc", "video_timing.h", @@ -182,6 +181,13 @@ rtc_library("video_bitrate_allocation") { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } +rtc_source_set("video_layers_allocation") { + visibility = [ "*" ] + sources = [ "video_layers_allocation.h" ] + deps = [ "../units:data_rate" ] + absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ] +} + rtc_library("video_bitrate_allocator") { visibility = [ "*" ] sources = [ @@ -264,6 +270,7 @@ rtc_source_set("video_stream_encoder") { ":video_bitrate_allocator_factory", ":video_codec_constants", ":video_frame", + ":video_layers_allocation", "..:rtp_parameters", "..:scoped_refptr", "../:fec_controller_api", diff --git a/api/video/video_stream_encoder_interface.h b/api/video/video_stream_encoder_interface.h index ed9989c37e..34fa6421c4 100644 --- a/api/video/video_stream_encoder_interface.h +++ b/api/video/video_stream_encoder_interface.h @@ -19,6 +19,7 @@ #include "api/scoped_refptr.h" #include "api/units/data_rate.h" #include "api/video/video_bitrate_allocator.h" +#include "api/video/video_layers_allocation.h" #include "api/video/video_sink_interface.h" #include "api/video/video_source_interface.h" #include "api/video_codecs/video_encoder.h" @@ -52,6 +53,9 @@ class VideoStreamEncoderInterface : public rtc::VideoSinkInterface { virtual void OnBitrateAllocationUpdated( const VideoBitrateAllocation& allocation) = 0; + + virtual void OnVideoLayersAllocationUpdated( + VideoLayersAllocation allocation) = 0; }; // If the resource is overusing, the VideoStreamEncoder will try to reduce diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index 9a9752d73b..9761790410 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -110,6 +110,7 @@ rtc_library("rtp_rtcp_format") { "../../api/transport/rtp:dependency_descriptor", "../../api/units:time_delta", "../../api/video:video_frame", + "../../api/video:video_layers_allocation", "../../api/video:video_rtp_headers", "../../common_video", "../../rtc_base:checks", @@ -278,6 +279,7 @@ rtc_library("rtp_rtcp") { "../../api/video:video_codec_constants", "../../api/video:video_frame", "../../api/video:video_frame_type", + "../../api/video:video_layers_allocation", "../../api/video:video_rtp_headers", "../../api/video_codecs:video_codecs_api", "../../call:rtp_interfaces", @@ -537,6 +539,7 @@ if (rtc_include_tests) { "../../api/video:video_bitrate_allocator", "../../api/video:video_codec_constants", "../../api/video:video_frame", + "../../api/video:video_layers_allocation", "../../api/video:video_rtp_headers", "../../api/video_codecs:video_codecs_api", "../../call:rtp_receiver", diff --git a/video/BUILD.gn b/video/BUILD.gn index 115c71a005..c6774dc799 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -226,6 +226,7 @@ rtc_library("video_stream_encoder_impl") { "../api/video:video_bitrate_allocator_factory", "../api/video:video_codec_constants", "../api/video:video_frame", + "../api/video:video_layers_allocation", "../api/video:video_rtp_headers", "../api/video:video_stream_encoder", "../api/video_codecs:video_codecs_api", diff --git a/video/video_send_stream_impl.cc b/video/video_send_stream_impl.cc index 82f8aa8942..4673811e1b 100644 --- a/video/video_send_stream_impl.cc +++ b/video/video_send_stream_impl.cc @@ -472,6 +472,20 @@ void VideoSendStreamImpl::OnBitrateAllocationUpdated( } } +void VideoSendStreamImpl::OnVideoLayersAllocationUpdated( + VideoLayersAllocation allocation) { + if (!worker_queue_->IsCurrent()) { + auto ptr = weak_ptr_; + worker_queue_->PostTask([allocation = std::move(allocation), ptr] { + if (!ptr.get()) + return; + ptr->OnVideoLayersAllocationUpdated(allocation); + }); + return; + } + // TODO(bugs.webrtc.org/12000): Implement +} + void VideoSendStreamImpl::SignalEncoderActive() { RTC_DCHECK_RUN_ON(worker_queue_); if (rtp_video_sender_->IsActive()) { diff --git a/video/video_send_stream_impl.h b/video/video_send_stream_impl.h index d3f50584f6..41a7859a77 100644 --- a/video/video_send_stream_impl.h +++ b/video/video_send_stream_impl.h @@ -121,6 +121,8 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, void OnBitrateAllocationUpdated( const VideoBitrateAllocation& allocation) override; + void OnVideoLayersAllocationUpdated( + VideoLayersAllocation allocation) override; // Implements EncodedImageCallback. The implementation routes encoded frames // to the |payload_router_| and |config.pre_encode_callback| if set. diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index f4283bd249..d261ae075f 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -198,6 +198,73 @@ VideoBitrateAllocation UpdateAllocationFromEncoderInfo( return new_allocation; } +// Converts a VideoBitrateAllocation that contains allocated bitrate per layer, +// and an EncoderInfo that contains information about the actual encoder +// structure used by a codec. Stream structures can be Ksvc, Full SVC, Simulcast +// etc. +VideoLayersAllocation CreateVideoLayersAllocation( + const VideoCodec& encoder_config, + const VideoEncoder::RateControlParameters& current_rate, + const VideoEncoder::EncoderInfo& encoder_info) { + const VideoBitrateAllocation& target_bitrate = current_rate.target_bitrate; + VideoLayersAllocation layers_allocation; + if (target_bitrate.get_sum_bps() == 0) { + return layers_allocation; + } + + if (encoder_config.numberOfSimulcastStreams > 0) { + layers_allocation.resolution_and_frame_rate_is_valid = true; + for (int si = 0; si < encoder_config.numberOfSimulcastStreams; ++si) { + if (!target_bitrate.IsSpatialLayerUsed(si) || + target_bitrate.GetSpatialLayerSum(si) == 0) { + break; + } + layers_allocation.active_spatial_layers.emplace_back(); + VideoLayersAllocation::SpatialLayer& spatial_layer = + layers_allocation.active_spatial_layers.back(); + spatial_layer.width = encoder_config.simulcastStream[si].width; + spatial_layer.height = encoder_config.simulcastStream[si].height; + spatial_layer.rtp_stream_index = si; + spatial_layer.spatial_id = 0; + auto frame_rate_fraction = + VideoEncoder::EncoderInfo::kMaxFramerateFraction; + if (encoder_info.fps_allocation[si].size() == 1) { + // One TL is signalled to be used by the encoder. Do not distribute + // bitrate allocation across TLs (use sum at tl:0). + spatial_layer.target_bitrate_per_temporal_layer.push_back( + DataRate::BitsPerSec(target_bitrate.GetSpatialLayerSum(si))); + frame_rate_fraction = encoder_info.fps_allocation[si][0]; + } else { // Temporal layers are supported. + uint32_t temporal_layer_bitrate_bps = 0; + for (size_t ti = 0; + ti < encoder_config.simulcastStream[si].numberOfTemporalLayers; + ++ti) { + if (!target_bitrate.HasBitrate(si, ti)) { + break; + } + if (ti < encoder_info.fps_allocation[si].size()) { + // Use frame rate of the top used temporal layer. + frame_rate_fraction = encoder_info.fps_allocation[si][ti]; + } + temporal_layer_bitrate_bps += target_bitrate.GetBitrate(si, ti); + spatial_layer.target_bitrate_per_temporal_layer.push_back( + DataRate::BitsPerSec(temporal_layer_bitrate_bps)); + } + } + // Encoder may drop frames internally if `maxFramerate` is set. + spatial_layer.frame_rate_fps = std::min( + static_cast(encoder_config.simulcastStream[si].maxFramerate), + static_cast( + (current_rate.framerate_fps * frame_rate_fraction) / + VideoEncoder::EncoderInfo::kMaxFramerateFraction)); + } + } else { + // TODO(bugs.webrtc.org/12000): Implement support for kSVC and full SVC. + } + + return layers_allocation; +} + } // namespace VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings() @@ -1124,6 +1191,12 @@ void VideoStreamEncoder::SetEncoderRates( rate_settings.rate_control.bitrate, static_cast(rate_settings.rate_control.framerate_fps + 0.5)); stream_resource_manager_.SetEncoderRates(rate_settings.rate_control); + if (settings_.allocation_cb_type == + VideoStreamEncoderSettings::BitrateAllocationCallbackType:: + kVideoLayersAllocation) { + sink_->OnVideoLayersAllocationUpdated(CreateVideoLayersAllocation( + send_codec_, rate_settings.rate_control, encoder_->GetEncoderInfo())); + } } if ((settings_.allocation_cb_type == VideoStreamEncoderSettings::BitrateAllocationCallbackType:: diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index b8c69c6b97..0d34c01080 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -1254,6 +1254,16 @@ class VideoStreamEncoderTest : public ::testing::Test { return number_of_bitrate_allocations_; } + VideoLayersAllocation GetLastVideoLayersAllocation() { + MutexLock lock(&mutex_); + return last_layers_allocation_; + } + + int number_of_layers_allocations() const { + MutexLock lock(&mutex_); + return number_of_layers_allocations_; + } + private: Result OnEncodedImage( const EncodedImage& encoded_image, @@ -1296,6 +1306,24 @@ class VideoStreamEncoderTest : public ::testing::Test { last_bitrate_allocation_ = allocation; } + void OnVideoLayersAllocationUpdated( + VideoLayersAllocation allocation) override { + MutexLock lock(&mutex_); + ++number_of_layers_allocations_; + last_layers_allocation_ = allocation; + rtc::StringBuilder log; + for (const auto& layer : allocation.active_spatial_layers) { + log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps + << "["; + for (const auto target_bitrate : + layer.target_bitrate_per_temporal_layer) { + log << target_bitrate.kbps() << ","; + } + log << "]"; + } + RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str(); + } + TimeController* const time_controller_; mutable Mutex mutex_; TestEncoder* test_encoder_; @@ -1313,6 +1341,8 @@ class VideoStreamEncoderTest : public ::testing::Test { int min_transmit_bitrate_bps_ = 0; VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_); int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0; + VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_); + int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0; }; class VideoBitrateAllocatorProxyFactory @@ -4013,6 +4043,100 @@ TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForV8Simulcast) { + ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false, + VideoStreamEncoderSettings::BitrateAllocationCallbackType:: + kVideoLayersAllocation); + + const int kDefaultFps = 30; + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kLowTargetBitrateBps), + DataRate::BitsPerSec(kLowTargetBitrateBps), + DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0); + + video_source_.IncomingCapturedFrame( + CreateFrame(CurrentTimeMs(), codec_width_, codec_height_)); + WaitForEncodedFrame(CurrentTimeMs()); + EXPECT_EQ(sink_.number_of_layers_allocations(), 1); + VideoLayersAllocation last_layer_allocation = + sink_.GetLastVideoLayersAllocation(); + // kLowTargetBitrateBps is only enough for one spatial layer. + ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u); + + VideoBitrateAllocation bitrate_allocation = + fake_encoder_.GetAndResetLastRateControlSettings()->bitrate; + // Check that encoder has been updated too, not just allocation observer. + EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps); + AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps); + + // VideoLayersAllocation might be updated if frame rate change. + int number_of_layers_allocation = 1; + const int64_t start_time_ms = CurrentTimeMs(); + while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) { + video_source_.IncomingCapturedFrame( + CreateFrame(CurrentTimeMs(), codec_width_, codec_height_)); + WaitForEncodedFrame(CurrentTimeMs()); + AdvanceTime(TimeDelta::Millis(1) / kDefaultFps); + if (number_of_layers_allocation != sink_.number_of_layers_allocations()) { + number_of_layers_allocation = sink_.number_of_layers_allocations(); + VideoLayersAllocation new_allocation = + sink_.GetLastVideoLayersAllocation(); + ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u); + EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps, + last_layer_allocation.active_spatial_layers[0].frame_rate_fps); + EXPECT_EQ(new_allocation.active_spatial_layers[0] + .target_bitrate_per_temporal_layer, + last_layer_allocation.active_spatial_layers[0] + .target_bitrate_per_temporal_layer); + last_layer_allocation = new_allocation; + } + } + EXPECT_LE(sink_.number_of_layers_allocations(), 3); + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + ReportsUpdatedVideoLayersAllocationWhenBweChanges) { + ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false, + VideoStreamEncoderSettings::BitrateAllocationCallbackType:: + kVideoLayersAllocation); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kLowTargetBitrateBps), + DataRate::BitsPerSec(kLowTargetBitrateBps), + DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0); + + video_source_.IncomingCapturedFrame( + CreateFrame(CurrentTimeMs(), codec_width_, codec_height_)); + WaitForEncodedFrame(CurrentTimeMs()); + EXPECT_EQ(sink_.number_of_layers_allocations(), 1); + VideoLayersAllocation last_layer_allocation = + sink_.GetLastVideoLayersAllocation(); + // kLowTargetBitrateBps is only enough for one spatial layer. + ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u); + EXPECT_EQ(last_layer_allocation.active_spatial_layers[0] + .target_bitrate_per_temporal_layer[0], + DataRate::BitsPerSec(kLowTargetBitrateBps)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kSimulcastTargetBitrateBps), + DataRate::BitsPerSec(kSimulcastTargetBitrateBps), + DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0); + video_source_.IncomingCapturedFrame( + CreateFrame(CurrentTimeMs(), codec_width_, codec_height_)); + WaitForEncodedFrame(CurrentTimeMs()); + + EXPECT_EQ(sink_.number_of_layers_allocations(), 2); + last_layer_allocation = sink_.GetLastVideoLayersAllocation(); + ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u); + EXPECT_GT(last_layer_allocation.active_spatial_layers[1] + .target_bitrate_per_temporal_layer[0], + DataRate::Zero()); + + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) { // 2 TLs configured, temporal layers supported by encoder. const int kNumTemporalLayers = 2;