From 1a646ee522d614dc19b3022b8857cc1a37ec47b4 Mon Sep 17 00:00:00 2001 From: sprang Date: Thu, 1 Dec 2016 06:34:11 -0800 Subject: [PATCH] Wire up BitrateAllocation to be sent as RTCP TargetBitrate This is the video parts of https://codereview.webrtc.org/2531383002/ Wire up BitrateAllocation to be sent as RTCP TargetBitrate BUG=webrtc:6301 Review-Url: https://codereview.webrtc.org/2541303003 Cr-Commit-Position: refs/heads/master@{#15359} --- .../include/video_bitrate_allocator.h | 9 + .../modules/video_coding/video_coding_impl.cc | 2 +- .../modules/video_coding/video_coding_impl.h | 26 ++- webrtc/modules/video_coding/video_sender.cc | 31 ++- .../video_coding/video_sender_unittest.cc | 18 +- webrtc/video/end_to_end_tests.cc | 179 +++++++++--------- webrtc/video/payload_router.cc | 24 ++- webrtc/video/payload_router.h | 6 +- webrtc/video/payload_router_unittest.cc | 58 +++++- webrtc/video/video_send_stream.cc | 29 ++- webrtc/video/vie_encoder.cc | 21 +- webrtc/video/vie_encoder.h | 5 + 12 files changed, 270 insertions(+), 138 deletions(-) diff --git a/webrtc/common_video/include/video_bitrate_allocator.h b/webrtc/common_video/include/video_bitrate_allocator.h index 66ff077be5..b85879db4b 100644 --- a/webrtc/common_video/include/video_bitrate_allocator.h +++ b/webrtc/common_video/include/video_bitrate_allocator.h @@ -25,6 +25,15 @@ class VideoBitrateAllocator { virtual uint32_t GetPreferredBitrateBps(uint32_t framerate) = 0; }; +class VideoBitrateAllocationObserver { + public: + VideoBitrateAllocationObserver() {} + virtual ~VideoBitrateAllocationObserver() {} + + virtual void OnBitrateAllocationUpdated( + const BitrateAllocation& allocation) = 0; +}; + } // namespace webrtc #endif // WEBRTC_COMMON_VIDEO_INCLUDE_VIDEO_BITRATE_ALLOCATOR_H_ diff --git a/webrtc/modules/video_coding/video_coding_impl.cc b/webrtc/modules/video_coding/video_coding_impl.cc index 021281bffa..85511bc9d4 100644 --- a/webrtc/modules/video_coding/video_coding_impl.cc +++ b/webrtc/modules/video_coding/video_coding_impl.cc @@ -146,7 +146,7 @@ class VideoCodingModuleImpl : public VideoCodingModule { uint8_t lossRate, int64_t rtt) override { return sender_.SetChannelParameters(target_bitrate, lossRate, rtt, - rate_allocator_.get()); + rate_allocator_.get(), nullptr); } int32_t RegisterProtectionCallback( diff --git a/webrtc/modules/video_coding/video_coding_impl.h b/webrtc/modules/video_coding/video_coding_impl.h index f6e53063fb..b5d04c4a68 100644 --- a/webrtc/modules/video_coding/video_coding_impl.h +++ b/webrtc/modules/video_coding/video_coding_impl.h @@ -35,6 +35,7 @@ namespace webrtc { class VideoBitrateAllocator; +class VideoBitrateAllocationObserver; namespace vcm { @@ -79,14 +80,23 @@ class VideoSender : public Module { int Bitrate(unsigned int* bitrate) const; int FrameRate(unsigned int* framerate) const; - int32_t SetChannelParameters(uint32_t target_bitrate_bps, - uint8_t lossRate, - int64_t rtt, - VideoBitrateAllocator* bitrate_allocator); - // Updates the channel parameters, with a reallocated bitrate based on a - // presumably updated codec configuration, but does not update the encoder - // itself (it will be updated on the next frame). - void UpdateChannelParemeters(VideoBitrateAllocator* bitrate_allocator); + // Update the channel parameters based on new rates and rtt. This will also + // cause an immediate call to VideoEncoder::SetRateAllocation(). + int32_t SetChannelParameters( + uint32_t target_bitrate_bps, + uint8_t loss_rate, + int64_t rtt, + VideoBitrateAllocator* bitrate_allocator, + VideoBitrateAllocationObserver* bitrate_updated_callback); + + // Updates the channel parameters with a new bitrate allocation, but using the + // current targit_bitrate, loss rate and rtt. That is, the distribution or + // caps may be updated to a change to a new VideoCodec or allocation mode. + // The new parameters will be stored as pending EncoderParameters, and the + // encoder will only be updated on the next frame. + void UpdateChannelParemeters( + VideoBitrateAllocator* bitrate_allocator, + VideoBitrateAllocationObserver* bitrate_updated_callback); // Deprecated: // TODO(perkj): Remove once no projects use it. diff --git a/webrtc/modules/video_coding/video_sender.cc b/webrtc/modules/video_coding/video_sender.cc index d754c56550..536e33ad9a 100644 --- a/webrtc/modules/video_coding/video_sender.cc +++ b/webrtc/modules/video_coding/video_sender.cc @@ -63,7 +63,6 @@ void VideoSender::Process() { send_stats_callback_->SendStatistics(bitRate, frameRate); } } - { rtc::CritScope cs(¶ms_crit_); // Force an encoder parameters update, so that incoming frame rate is @@ -122,6 +121,7 @@ int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec, } else if (frame_dropper_enabled_) { _mediaOpt.EnableFrameDropper(true); } + { rtc::CritScope cs(¶ms_crit_); next_frame_types_.clear(); @@ -212,30 +212,41 @@ EncoderParameters VideoSender::UpdateEncoderParameters( bitrate_allocation = default_allocator.GetAllocation(video_target_rate_bps, input_frame_rate); } - EncoderParameters new_encoder_params = {bitrate_allocation, params.loss_rate, params.rtt, input_frame_rate}; return new_encoder_params; } void VideoSender::UpdateChannelParemeters( - VideoBitrateAllocator* bitrate_allocator) { - rtc::CritScope cs(¶ms_crit_); - encoder_params_ = - UpdateEncoderParameters(encoder_params_, bitrate_allocator, - encoder_params_.target_bitrate.get_sum_bps()); + VideoBitrateAllocator* bitrate_allocator, + VideoBitrateAllocationObserver* bitrate_updated_callback) { + BitrateAllocation target_rate; + { + rtc::CritScope cs(¶ms_crit_); + encoder_params_ = + UpdateEncoderParameters(encoder_params_, bitrate_allocator, + encoder_params_.target_bitrate.get_sum_bps()); + target_rate = encoder_params_.target_bitrate; + } + if (bitrate_updated_callback) + bitrate_updated_callback->OnBitrateAllocationUpdated(target_rate); } int32_t VideoSender::SetChannelParameters( uint32_t target_bitrate_bps, - uint8_t lossRate, + uint8_t loss_rate, int64_t rtt, - VideoBitrateAllocator* bitrate_allocator) { + VideoBitrateAllocator* bitrate_allocator, + VideoBitrateAllocationObserver* bitrate_updated_callback) { EncoderParameters encoder_params; - encoder_params.loss_rate = lossRate; + encoder_params.loss_rate = loss_rate; encoder_params.rtt = rtt; encoder_params = UpdateEncoderParameters(encoder_params, bitrate_allocator, target_bitrate_bps); + if (bitrate_updated_callback) { + bitrate_updated_callback->OnBitrateAllocationUpdated( + encoder_params.target_bitrate); + } bool encoder_has_internal_source; { diff --git a/webrtc/modules/video_coding/video_sender_unittest.cc b/webrtc/modules/video_coding/video_sender_unittest.cc index e24bbd758c..a670d8f48f 100644 --- a/webrtc/modules/video_coding/video_sender_unittest.cc +++ b/webrtc/modules/video_coding/video_sender_unittest.cc @@ -312,7 +312,7 @@ TEST_F(TestVideoSenderWithMockEncoder, TestSetRate) { .Times(1) .WillOnce(Return(0)); sender_->SetChannelParameters(new_bitrate_kbps * 1000, 0, 200, - rate_allocator_.get()); + rate_allocator_.get(), nullptr); AddFrame(); clock_.AdvanceTimeMilliseconds(kFrameIntervalMs); @@ -334,7 +334,7 @@ TEST_F(TestVideoSenderWithMockEncoder, TestSetRate) { // Expect no call to encoder_.SetRates if the new bitrate is zero. EXPECT_CALL(encoder_, SetRateAllocation(_, _)).Times(0); - sender_->SetChannelParameters(0, 0, 200, rate_allocator_.get()); + sender_->SetChannelParameters(0, 0, 200, rate_allocator_.get(), nullptr); AddFrame(); } @@ -372,12 +372,12 @@ TEST_F(TestVideoSenderWithMockEncoder, TestEncoderParametersForInternalSource) { .Times(1) .WillOnce(Return(0)); sender_->SetChannelParameters(new_bitrate_kbps * 1000, 0, 200, - rate_allocator_.get()); + rate_allocator_.get(), nullptr); } TEST_F(TestVideoSenderWithMockEncoder, EncoderFramerateUpdatedViaProcess) { sender_->SetChannelParameters(settings_.startBitrate * 1000, 0, 200, - rate_allocator_.get()); + rate_allocator_.get(), nullptr); const int64_t kRateStatsWindowMs = 2000; const uint32_t kInputFps = 20; int64_t start_time = clock_.TimeInMilliseconds(); @@ -406,7 +406,7 @@ TEST_F(TestVideoSenderWithMockEncoder, .Times(1) .WillOnce(Return(0)); sender_->SetChannelParameters(settings_.startBitrate * 1000, kLossRate, kRtt, - rate_allocator_.get()); + rate_allocator_.get(), nullptr); while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) { AddFrame(); clock_.AdvanceTimeMilliseconds(1000 / kInputFps); @@ -427,7 +427,7 @@ TEST_F(TestVideoSenderWithMockEncoder, .Times(1) .WillOnce(Return(0)); sender_->SetChannelParameters(new_bitrate_bps, kLossRate, kRtt, - rate_allocator_.get()); + rate_allocator_.get(), nullptr); AddFrame(); } @@ -481,9 +481,9 @@ class TestVideoSenderWithVp8 : public TestVideoSender { // Note: SetChannelParameters fails if less than 2 frames are in the // buffer since it will fail to calculate the framerate. if (i != 0) { - EXPECT_EQ(VCM_OK, - sender_->SetChannelParameters(available_bitrate_kbps_ * 1000, - 0, 200, rate_allocator_.get())); + EXPECT_EQ(VCM_OK, sender_->SetChannelParameters( + available_bitrate_kbps_ * 1000, 0, 200, + rate_allocator_.get(), nullptr)); } } } diff --git a/webrtc/video/end_to_end_tests.cc b/webrtc/video/end_to_end_tests.cc index 86c181bdf4..4de6db49cd 100644 --- a/webrtc/video/end_to_end_tests.cc +++ b/webrtc/video/end_to_end_tests.cc @@ -125,7 +125,6 @@ class EndToEndTest : public test::CallTest, void DecodesRetransmittedFrame(bool enable_rtx, bool enable_red); void ReceivesPliAndRecovers(int rtp_history_ms); void RespectsRtcpMode(RtcpMode rtcp_mode); - void TestXrReceiverReferenceTimeReport(bool enable_rrtr); void TestSendsSetSsrcs(size_t num_ssrcs, bool send_single_ssrc_first); void TestRtpStatePreservation(bool use_rtx, bool provoke_rtcpsr_before_rtp); void VerifyHistogramStats(bool use_rtx, bool use_red, bool screenshare); @@ -2435,89 +2434,6 @@ TEST_P(EndToEndTest, VerifyHistogramStatsWithScreenshare) { VerifyHistogramStats(kEnabledRtx, kEnabledRed, kScreenshare); } -void EndToEndTest::TestXrReceiverReferenceTimeReport(bool enable_rrtr) { - static const int kNumRtcpReportPacketsToObserve = 5; - class RtcpXrObserver : public test::EndToEndTest { - public: - explicit RtcpXrObserver(bool enable_rrtr) - : EndToEndTest(kDefaultTimeoutMs), - enable_rrtr_(enable_rrtr), - sent_rtcp_sr_(0), - sent_rtcp_rr_(0), - sent_rtcp_rrtr_(0), - sent_rtcp_dlrr_(0) {} - - private: - // Receive stream should send RR packets (and RRTR packets if enabled). - Action OnReceiveRtcp(const uint8_t* packet, size_t length) override { - rtc::CritScope lock(&crit_); - test::RtcpPacketParser parser; - EXPECT_TRUE(parser.Parse(packet, length)); - - sent_rtcp_rr_ += parser.receiver_report()->num_packets(); - EXPECT_EQ(0, parser.sender_report()->num_packets()); - EXPECT_GE(1, parser.xr()->num_packets()); - if (parser.xr()->num_packets() > 0) { - if (parser.xr()->rrtr()) - ++sent_rtcp_rrtr_; - EXPECT_FALSE(parser.xr()->dlrr()); - } - - return SEND_PACKET; - } - // Send stream should send SR packets (and DLRR packets if enabled). - Action OnSendRtcp(const uint8_t* packet, size_t length) override { - rtc::CritScope lock(&crit_); - test::RtcpPacketParser parser; - EXPECT_TRUE(parser.Parse(packet, length)); - - sent_rtcp_sr_ += parser.sender_report()->num_packets(); - EXPECT_GE(1, parser.xr()->num_packets()); - if (parser.xr()->num_packets() > 0) { - EXPECT_FALSE(parser.xr()->rrtr()); - if (parser.xr()->dlrr()) - ++sent_rtcp_dlrr_; - } - - if (sent_rtcp_sr_ > kNumRtcpReportPacketsToObserve && - sent_rtcp_rr_ > kNumRtcpReportPacketsToObserve) { - if (enable_rrtr_) { - EXPECT_LT(0, sent_rtcp_rrtr_); - EXPECT_LT(0, sent_rtcp_dlrr_); - } else { - EXPECT_EQ(0, sent_rtcp_rrtr_); - EXPECT_EQ(0, sent_rtcp_dlrr_); - } - observation_complete_.Set(); - } - return SEND_PACKET; - } - - void ModifyVideoConfigs( - VideoSendStream::Config* send_config, - std::vector* receive_configs, - VideoEncoderConfig* encoder_config) override { - (*receive_configs)[0].rtp.rtcp_mode = RtcpMode::kReducedSize; - (*receive_configs)[0].rtp.rtcp_xr.receiver_reference_time_report = - enable_rrtr_; - } - - void PerformTest() override { - EXPECT_TRUE(Wait()) - << "Timed out while waiting for RTCP SR/RR packets to be sent."; - } - - rtc::CriticalSection crit_; - bool enable_rrtr_; - int sent_rtcp_sr_; - int sent_rtcp_rr_ GUARDED_BY(&crit_); - int sent_rtcp_rrtr_ GUARDED_BY(&crit_); - int sent_rtcp_dlrr_; - } test(enable_rrtr); - - RunBaseTest(&test); -} - void EndToEndTest::TestSendsSetSsrcs(size_t num_ssrcs, bool send_single_ssrc_first) { class SendsSetSsrcs : public test::EndToEndTest { @@ -3064,12 +2980,99 @@ TEST_P(EndToEndTest, GetStats) { RunBaseTest(&test); } -TEST_P(EndToEndTest, ReceiverReferenceTimeReportEnabled) { - TestXrReceiverReferenceTimeReport(true); +class RtcpXrObserver : public test::EndToEndTest { + public: + explicit RtcpXrObserver(bool enable_rrtr) + : EndToEndTest(test::CallTest::kDefaultTimeoutMs), + enable_rrtr_(enable_rrtr), + sent_rtcp_sr_(0), + sent_rtcp_rr_(0), + sent_rtcp_rrtr_(0), + sent_rtcp_target_bitrate_(false), + sent_rtcp_dlrr_(0) {} + + private: + // Receive stream should send RR packets (and RRTR packets if enabled). + Action OnReceiveRtcp(const uint8_t* packet, size_t length) override { + rtc::CritScope lock(&crit_); + test::RtcpPacketParser parser; + EXPECT_TRUE(parser.Parse(packet, length)); + + sent_rtcp_rr_ += parser.receiver_report()->num_packets(); + EXPECT_EQ(0, parser.sender_report()->num_packets()); + EXPECT_GE(1, parser.xr()->num_packets()); + if (parser.xr()->num_packets() > 0) { + if (parser.xr()->rrtr()) + ++sent_rtcp_rrtr_; + EXPECT_FALSE(parser.xr()->dlrr()); + } + + return SEND_PACKET; + } + // Send stream should send SR packets (and DLRR packets if enabled). + Action OnSendRtcp(const uint8_t* packet, size_t length) override { + rtc::CritScope lock(&crit_); + test::RtcpPacketParser parser; + EXPECT_TRUE(parser.Parse(packet, length)); + + sent_rtcp_sr_ += parser.sender_report()->num_packets(); + EXPECT_LE(parser.xr()->num_packets(), 1); + if (parser.xr()->num_packets() > 0) { + EXPECT_FALSE(parser.xr()->rrtr()); + if (parser.xr()->dlrr()) + ++sent_rtcp_dlrr_; + if (parser.xr()->target_bitrate()) + sent_rtcp_target_bitrate_ = true; + } + + if (sent_rtcp_sr_ > kNumRtcpReportPacketsToObserve && + sent_rtcp_rr_ > kNumRtcpReportPacketsToObserve && + sent_rtcp_target_bitrate_) { + if (enable_rrtr_) { + EXPECT_GT(sent_rtcp_rrtr_, 0); + EXPECT_GT(sent_rtcp_dlrr_, 0); + } else { + EXPECT_EQ(sent_rtcp_rrtr_, 0); + EXPECT_EQ(sent_rtcp_dlrr_, 0); + } + observation_complete_.Set(); + } + return SEND_PACKET; + } + + void ModifyVideoConfigs( + VideoSendStream::Config* send_config, + std::vector* receive_configs, + VideoEncoderConfig* encoder_config) override { + (*receive_configs)[0].rtp.rtcp_mode = RtcpMode::kReducedSize; + (*receive_configs)[0].rtp.rtcp_xr.receiver_reference_time_report = + enable_rrtr_; + } + + void PerformTest() override { + EXPECT_TRUE(Wait()) + << "Timed out while waiting for RTCP SR/RR packets to be sent."; + } + + static const int kNumRtcpReportPacketsToObserve = 5; + + rtc::CriticalSection crit_; + bool enable_rrtr_; + int sent_rtcp_sr_; + int sent_rtcp_rr_ GUARDED_BY(&crit_); + int sent_rtcp_rrtr_ GUARDED_BY(&crit_); + bool sent_rtcp_target_bitrate_ GUARDED_BY(&crit_); + int sent_rtcp_dlrr_; +}; + +TEST_P(EndToEndTest, TestExtendedReportsWithRrtr) { + RtcpXrObserver test(true); + RunBaseTest(&test); } -TEST_P(EndToEndTest, ReceiverReferenceTimeReportDisabled) { - TestXrReceiverReferenceTimeReport(false); +TEST_P(EndToEndTest, TestExtendedReportsWithoutRrtr) { + RtcpXrObserver test(false); + RunBaseTest(&test); } TEST_P(EndToEndTest, TestReceivedRtpPacketStats) { diff --git a/webrtc/video/payload_router.cc b/webrtc/video/payload_router.cc index 33d8f76e86..6c8e36f72f 100644 --- a/webrtc/video/payload_router.cc +++ b/webrtc/video/payload_router.cc @@ -101,7 +101,7 @@ size_t PayloadRouter::DefaultMaxPayloadLength() { return IP_PACKET_SIZE - kIpUdpSrtpLength; } -void PayloadRouter::set_active(bool active) { +void PayloadRouter::SetActive(bool active) { rtc::CritScope lock(&crit_); if (active_ == active) return; @@ -113,7 +113,7 @@ void PayloadRouter::set_active(bool active) { } } -bool PayloadRouter::active() { +bool PayloadRouter::IsActive() { rtc::CritScope lock(&crit_); return active_ && !rtp_modules_.empty(); } @@ -158,4 +158,24 @@ size_t PayloadRouter::MaxPayloadLength() const { return min_payload_length; } +void PayloadRouter::OnBitrateAllocationUpdated( + const BitrateAllocation& bitrate) { + rtc::CritScope lock(&crit_); + if (IsActive()) { + if (rtp_modules_.size() == 1) { + // If spatial scalability is enabled, it is covered by a single stream. + rtp_modules_[0]->SetVideoBitrateAllocation(bitrate); + } else { + // Simulcast is in use, split the BitrateAllocation into one struct per + // rtp stream, moving over the temporal layer allocation. + for (size_t si = 0; si < rtp_modules_.size(); ++si) { + BitrateAllocation layer_bitrate; + for (int tl = 0; tl < kMaxTemporalStreams; ++tl) + layer_bitrate.SetBitrate(0, tl, bitrate.GetBitrate(si, tl)); + rtp_modules_[si]->SetVideoBitrateAllocation(layer_bitrate); + } + } + } +} + } // namespace webrtc diff --git a/webrtc/video/payload_router.h b/webrtc/video/payload_router.h index 6426b00615..3b60ce23c0 100644 --- a/webrtc/video/payload_router.h +++ b/webrtc/video/payload_router.h @@ -40,8 +40,8 @@ class PayloadRouter : public EncodedImageCallback { // PayloadRouter will only route packets if being active, all packets will be // dropped otherwise. - void set_active(bool active); - bool active(); + void SetActive(bool active); + bool IsActive(); // Implements EncodedImageCallback. // Returns 0 if the packet was routed / sent, -1 otherwise. @@ -54,6 +54,8 @@ class PayloadRouter : public EncodedImageCallback { // and RTP headers. size_t MaxPayloadLength() const; + void OnBitrateAllocationUpdated(const BitrateAllocation& bitrate); + private: void UpdateModuleSendingState() EXCLUSIVE_LOCKS_REQUIRED(crit_); diff --git a/webrtc/video/payload_router_unittest.cc b/webrtc/video/payload_router_unittest.cc index adddefe00b..935de5f3e9 100644 --- a/webrtc/video/payload_router_unittest.cc +++ b/webrtc/video/payload_router_unittest.cc @@ -49,7 +49,7 @@ TEST(PayloadRouterTest, SendOnOneModule) { EncodedImageCallback::Result::OK, payload_router.OnEncodedImage(encoded_image, nullptr, nullptr).error); - payload_router.set_active(true); + payload_router.SetActive(true); EXPECT_CALL(rtp, SendOutgoingData(encoded_image._frameType, payload_type, encoded_image._timeStamp, encoded_image.capture_time_ms_, &payload, @@ -60,7 +60,7 @@ TEST(PayloadRouterTest, SendOnOneModule) { EncodedImageCallback::Result::OK, payload_router.OnEncodedImage(encoded_image, nullptr, nullptr).error); - payload_router.set_active(false); + payload_router.SetActive(false); EXPECT_CALL(rtp, SendOutgoingData(encoded_image._frameType, payload_type, encoded_image._timeStamp, encoded_image.capture_time_ms_, &payload, @@ -70,7 +70,7 @@ TEST(PayloadRouterTest, SendOnOneModule) { EncodedImageCallback::Result::OK, payload_router.OnEncodedImage(encoded_image, nullptr, nullptr).error); - payload_router.set_active(true); + payload_router.SetActive(true); EXPECT_CALL(rtp, SendOutgoingData(encoded_image._frameType, payload_type, encoded_image._timeStamp, encoded_image.capture_time_ms_, &payload, @@ -106,7 +106,7 @@ TEST(PayloadRouterTest, SendSimulcast) { codec_info_1.codecType = kVideoCodecVP8; codec_info_1.codecSpecific.VP8.simulcastIdx = 0; - payload_router.set_active(true); + payload_router.SetActive(true); EXPECT_CALL(rtp_1, SendOutgoingData(encoded_image._frameType, payload_type, encoded_image._timeStamp, encoded_image.capture_time_ms_, &payload, @@ -136,7 +136,7 @@ TEST(PayloadRouterTest, SendSimulcast) { .error); // Inactive. - payload_router.set_active(false); + payload_router.SetActive(false); EXPECT_CALL(rtp_1, SendOutgoingData(_, _, _, _, _, _, _, _, _)) .Times(0); EXPECT_CALL(rtp_2, SendOutgoingData(_, _, _, _, _, _, _, _, _)) @@ -182,4 +182,52 @@ TEST(PayloadRouterTest, MaxPayloadLength) { .WillOnce(Return(kTestMinPayloadLength)); EXPECT_EQ(kTestMinPayloadLength, payload_router.MaxPayloadLength()); } + +TEST(PayloadRouterTest, SimulcastTargetBitrate) { + NiceMock rtp_1; + NiceMock rtp_2; + std::vector modules; + modules.push_back(&rtp_1); + modules.push_back(&rtp_2); + PayloadRouter payload_router(modules, 42); + payload_router.SetActive(true); + + BitrateAllocation bitrate; + bitrate.SetBitrate(0, 0, 10000); + bitrate.SetBitrate(0, 1, 20000); + bitrate.SetBitrate(1, 0, 40000); + bitrate.SetBitrate(1, 1, 80000); + + BitrateAllocation layer0_bitrate; + layer0_bitrate.SetBitrate(0, 0, 10000); + layer0_bitrate.SetBitrate(0, 1, 20000); + + BitrateAllocation layer1_bitrate; + layer1_bitrate.SetBitrate(0, 0, 40000); + layer1_bitrate.SetBitrate(0, 1, 80000); + + EXPECT_CALL(rtp_1, SetVideoBitrateAllocation(layer0_bitrate)).Times(1); + EXPECT_CALL(rtp_2, SetVideoBitrateAllocation(layer1_bitrate)).Times(1); + + payload_router.OnBitrateAllocationUpdated(bitrate); +} + +TEST(PayloadRouterTest, SvcTargetBitrate) { + NiceMock rtp_1; + std::vector modules; + modules.push_back(&rtp_1); + PayloadRouter payload_router(modules, 42); + payload_router.SetActive(true); + + BitrateAllocation bitrate; + bitrate.SetBitrate(0, 0, 10000); + bitrate.SetBitrate(0, 1, 20000); + bitrate.SetBitrate(1, 0, 40000); + bitrate.SetBitrate(1, 1, 80000); + + EXPECT_CALL(rtp_1, SetVideoBitrateAllocation(bitrate)).Times(1); + + payload_router.OnBitrateAllocationUpdated(bitrate); +} + } // namespace webrtc diff --git a/webrtc/video/video_send_stream.cc b/webrtc/video/video_send_stream.cc index a1c1a0b079..188854aede 100644 --- a/webrtc/video/video_send_stream.cc +++ b/webrtc/video/video_send_stream.cc @@ -17,6 +17,7 @@ #include #include "webrtc/common_types.h" +#include "webrtc/common_video/include/video_bitrate_allocator.h" #include "webrtc/base/checks.h" #include "webrtc/base/file.h" #include "webrtc/base/logging.h" @@ -289,7 +290,8 @@ namespace internal { class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, public webrtc::OverheadObserver, public webrtc::VCMProtectionCallback, - public ViEEncoder::EncoderSink { + public ViEEncoder::EncoderSink, + public VideoBitrateAllocationObserver { public: VideoSendStreamImpl(SendStatisticsProxy* stats_proxy, rtc::TaskQueue* worker_queue, @@ -357,6 +359,9 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation) override; + // Implements VideoBitrateAllocationObserver. + void OnBitrateAllocationUpdated(const BitrateAllocation& allocation) override; + void ConfigureProtection(); void ConfigureSsrcs(); void SignalEncoderTimedOut(); @@ -598,7 +603,6 @@ VideoSendStream::VideoSendStream( vie_encoder_.reset(new ViEEncoder( num_cpu_cores, &stats_proxy_, config_.encoder_settings, config_.pre_encode_callback, config_.post_encode_callback)); - worker_queue_->PostTask(std::unique_ptr(new ConstructionTask( &send_stream_, &thread_sync_event_, &stats_proxy_, vie_encoder_.get(), module_process_thread, call_stats, congestion_controller, packet_router, @@ -610,7 +614,7 @@ VideoSendStream::VideoSendStream( // it was created on. thread_sync_event_.Wait(rtc::Event::kForever); send_stream_->RegisterProcessThread(module_process_thread); - + vie_encoder_->SetBitrateObserver(send_stream_.get()); vie_encoder_->RegisterProcessThread(module_process_thread); ReconfigureVideoEncoder(std::move(encoder_config)); @@ -849,7 +853,7 @@ void VideoSendStreamImpl::DeRegisterProcessThread() { VideoSendStreamImpl::~VideoSendStreamImpl() { RTC_DCHECK_RUN_ON(worker_queue_); - RTC_DCHECK(!payload_router_.active()) + RTC_DCHECK(!payload_router_.IsActive()) << "VideoSendStreamImpl::Stop not called"; LOG(LS_INFO) << "~VideoSendStreamInternal: " << config_->ToString(); @@ -873,10 +877,10 @@ bool VideoSendStreamImpl::DeliverRtcp(const uint8_t* packet, size_t length) { void VideoSendStreamImpl::Start() { RTC_DCHECK_RUN_ON(worker_queue_); LOG(LS_INFO) << "VideoSendStream::Start"; - if (payload_router_.active()) + if (payload_router_.IsActive()) return; TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Start"); - payload_router_.set_active(true); + payload_router_.SetActive(true); bitrate_allocator_->AddObserver( this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_, @@ -898,10 +902,10 @@ void VideoSendStreamImpl::Start() { void VideoSendStreamImpl::Stop() { RTC_DCHECK_RUN_ON(worker_queue_); LOG(LS_INFO) << "VideoSendStream::Stop"; - if (!payload_router_.active()) + if (!payload_router_.IsActive()) return; TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Stop"); - payload_router_.set_active(false); + payload_router_.SetActive(false); bitrate_allocator_->RemoveObserver(this); { rtc::CritScope lock(&encoder_activity_crit_sect_); @@ -923,6 +927,11 @@ void VideoSendStreamImpl::SignalEncoderTimedOut() { } } +void VideoSendStreamImpl::OnBitrateAllocationUpdated( + const BitrateAllocation& allocation) { + payload_router_.OnBitrateAllocationUpdated(allocation); +} + void VideoSendStreamImpl::SignalEncoderActive() { RTC_DCHECK_RUN_ON(worker_queue_); LOG(LS_INFO) << "SignalEncoderActive, Encoder is active."; @@ -965,7 +974,7 @@ void VideoSendStreamImpl::OnEncoderConfigurationChanged( streams[0].width, streams[0].height, number_of_temporal_layers, config_->rtp.max_packet_size); - if (payload_router_.active()) { + if (payload_router_.IsActive()) { // The send stream is started already. Update the allocator with new bitrate // limits. bitrate_allocator_->AddObserver( @@ -1174,7 +1183,7 @@ uint32_t VideoSendStreamImpl::OnBitrateUpdated(uint32_t bitrate_bps, int64_t rtt, int64_t probing_interval_ms) { RTC_DCHECK_RUN_ON(worker_queue_); - RTC_DCHECK(payload_router_.active()) + RTC_DCHECK(payload_router_.IsActive()) << "VideoSendStream::Start has not been called."; if (webrtc::field_trial::FindFullName("WebRTC-SendSideBwe-WithOverhead") == diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index 343cd4c8f0..de6b9a8a63 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -19,6 +19,7 @@ #include "webrtc/base/logging.h" #include "webrtc/base/trace_event.h" #include "webrtc/base/timeutils.h" +#include "webrtc/common_video/include/video_bitrate_allocator.h" #include "webrtc/modules/pacing/paced_sender.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/include/video_coding.h" @@ -26,7 +27,6 @@ #include "webrtc/video/overuse_frame_detector.h" #include "webrtc/video/send_statistics_proxy.h" #include "webrtc/video_frame.h" - namespace webrtc { namespace { @@ -270,6 +270,7 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, last_frame_log_ms_(clock_->TimeInMilliseconds()), captured_frame_count_(0), dropped_frame_count_(0), + bitrate_observer_(nullptr), encoder_queue_("EncoderQueue") { encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); @@ -292,6 +293,7 @@ void ViEEncoder::Stop() { RTC_DCHECK_RUN_ON(&encoder_queue_); overuse_detector_.StopCheckForOveruse(); rate_allocator_.reset(); + bitrate_observer_ = nullptr; video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, false); quality_scaler_ = nullptr; @@ -314,6 +316,16 @@ void ViEEncoder::DeRegisterProcessThread() { module_process_thread_->DeRegisterModule(&video_sender_); } +void ViEEncoder::SetBitrateObserver( + VideoBitrateAllocationObserver* bitrate_observer) { + RTC_DCHECK_RUN_ON(&thread_checker_); + encoder_queue_.PostTask([this, bitrate_observer] { + RTC_DCHECK_RUN_ON(&encoder_queue_); + RTC_DCHECK(!bitrate_observer_); + bitrate_observer_ = bitrate_observer; + }); +} + void ViEEncoder::SetSource( rtc::VideoSourceInterface* source, const VideoSendStream::DegradationPreference& degradation_preference) { @@ -405,7 +417,9 @@ void ViEEncoder::ReconfigureEncoder() { RTC_DCHECK(success); } - video_sender_.UpdateChannelParemeters(rate_allocator_.get()); + video_sender_.UpdateChannelParemeters(rate_allocator_.get(), + bitrate_observer_); + if (stats_proxy_) { int framerate = stats_proxy_->GetSendFrameRate(); if (framerate == 0) @@ -660,7 +674,8 @@ void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, << " rtt " << round_trip_time_ms; video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, - round_trip_time_ms, rate_allocator_.get()); + round_trip_time_ms, rate_allocator_.get(), + bitrate_observer_); encoder_start_bitrate_bps_ = bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_; diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h index 7884bbe297..5ad1ff7be5 100644 --- a/webrtc/video/vie_encoder.h +++ b/webrtc/video/vie_encoder.h @@ -37,6 +37,7 @@ namespace webrtc { class ProcessThread; class SendStatisticsProxy; +class VideoBitrateAllocationObserver; // VieEncoder represent a video encoder that accepts raw video frames as input // and produces an encoded bit stream. @@ -91,6 +92,8 @@ class ViEEncoder : public rtc::VideoSinkInterface, // TODO(perkj): Can we remove VideoCodec.startBitrate ? void SetStartBitrate(int start_bitrate_bps); + void SetBitrateObserver(VideoBitrateAllocationObserver* bitrate_observer); + void ConfigureEncoder(VideoEncoderConfig config, size_t max_data_payload_length, bool nack_enabled); @@ -231,6 +234,8 @@ class ViEEncoder : public rtc::VideoSinkInterface, int captured_frame_count_ ACCESS_ON(&encoder_queue_); int dropped_frame_count_ ACCESS_ON(&encoder_queue_); + VideoBitrateAllocationObserver* bitrate_observer_ ACCESS_ON(&encoder_queue_); + // All public methods are proxied to |encoder_queue_|. It must must be // destroyed first to make sure no tasks are run that use other members. rtc::TaskQueue encoder_queue_;