From 54d1da13a584680ae80a1f229291e5bb7e76e6e1 Mon Sep 17 00:00:00 2001 From: Alex Narest Date: Tue, 17 Oct 2017 19:49:15 +0200 Subject: [PATCH] BWE allocation strategy allows controlling of bitrate allocation with WEBRTC external logic. This CL implements the main logic and IOS appRTC integration. Unit tests and Android appRTC will be in separate CL. Bug: webrtc:8243 Change-Id: If8e5195294046a47316e9fade1b0dfec211155e1 Reviewed-on: https://webrtc-review.googlesource.com/4860 Commit-Queue: Alex Narest Reviewed-by: Niels Moller Reviewed-by: Stefan Holmer Reviewed-by: Taylor Brandstetter Cr-Commit-Position: refs/heads/master@{#20329} --- api/peerconnectioninterface.h | 7 + api/peerconnectionproxy.h | 3 + audio/audio_send_stream.cc | 2 + call/bitrate_allocator.cc | 31 ++++- call/bitrate_allocator.h | 29 ++-- call/call.cc | 22 +++ call/call.h | 5 + examples/BUILD.gn | 4 + examples/objc/AppRTCMobile/ARDAppClient.m | 11 ++ .../ARDBitrateAllocationStrategy.h | 23 ++++ .../ARDBitrateAllocationStrategy.mm | 40 ++++++ .../objc/AppRTCMobile/ios/ARDAppDelegate.m | 3 +- media/engine/fakewebrtccall.cc | 6 + media/engine/fakewebrtccall.h | 3 + modules/pacing/paced_sender.cc | 12 +- modules/pacing/paced_sender.h | 7 + .../test/bbr_paced_sender.h | 1 + modules/rtp_rtcp/include/rtp_rtcp_defines.h | 8 ++ pc/peerconnection.cc | 18 +++ pc/peerconnection.h | 4 + rtc_base/BUILD.gn | 2 + rtc_base/bitrateallocationstrategy.cc | 125 ++++++++++++++++++ rtc_base/bitrateallocationstrategy.h | 101 ++++++++++++++ sdk/BUILD.gn | 2 + .../RTCBitrateAllocationStrategy.mm | 28 ++++ .../PeerConnection/RTCConfiguration.mm | 1 - .../PeerConnection/RTCPeerConnection.mm | 10 ++ .../WebRTC/RTCBitrateAllocationStrategy.h | 32 +++++ .../Headers/WebRTC/RTCPeerConnection.h | 8 ++ voice_engine/channel.cc | 4 + 30 files changed, 534 insertions(+), 18 deletions(-) create mode 100644 examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.h create mode 100644 examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.mm create mode 100644 rtc_base/bitrateallocationstrategy.cc create mode 100644 rtc_base/bitrateallocationstrategy.h create mode 100644 sdk/objc/Framework/Classes/PeerConnection/RTCBitrateAllocationStrategy.mm create mode 100644 sdk/objc/Framework/Headers/WebRTC/RTCBitrateAllocationStrategy.h diff --git a/api/peerconnectioninterface.h b/api/peerconnectioninterface.h index dd1f87c931..f4ef859cb4 100644 --- a/api/peerconnectioninterface.h +++ b/api/peerconnectioninterface.h @@ -786,6 +786,13 @@ class PeerConnectionInterface : public rtc::RefCountInterface { // to the provided value. virtual RTCError SetBitrate(const BitrateParameters& bitrate) = 0; + // Sets current strategy. If not set default WebRTC allocator will be used. + // May be changed during an active session. The strategy + // ownership is passed with std::unique_ptr + virtual void SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) = 0; + // Returns the current SignalingState. virtual SignalingState signaling_state() = 0; virtual IceConnectionState ice_connection_state() = 0; diff --git a/api/peerconnectionproxy.h b/api/peerconnectionproxy.h index c490b8f69d..a8ea3fa360 100644 --- a/api/peerconnectionproxy.h +++ b/api/peerconnectionproxy.h @@ -102,6 +102,9 @@ BEGIN_SIGNALING_PROXY_MAP(PeerConnection) const std::vector&); PROXY_METHOD1(void, RegisterUMAObserver, UMAObserver*) PROXY_METHOD1(RTCError, SetBitrate, const BitrateParameters&); + PROXY_METHOD1(void, + SetBitrateAllocationStrategy, + std::unique_ptr); PROXY_METHOD0(SignalingState, signaling_state) PROXY_METHOD0(IceConnectionState, ice_connection_state) PROXY_METHOD0(IceGatheringState, ice_gathering_state) diff --git a/audio/audio_send_stream.cc b/audio/audio_send_stream.cc index 1cfc0bad1e..7895af3bb7 100644 --- a/audio/audio_send_stream.cc +++ b/audio/audio_send_stream.cc @@ -236,6 +236,8 @@ void AudioSendStream::ConfigureStream( void AudioSendStream::Start() { RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); if (config_.min_bitrate_bps != -1 && config_.max_bitrate_bps != -1) { + // Audio BWE is enabled. + transport_->packet_sender()->SetAccountForAudioPackets(true); ConfigureBitrateObserver(config_.min_bitrate_bps, config_.max_bitrate_bps); } diff --git a/call/bitrate_allocator.cc b/call/bitrate_allocator.cc index b379f7f086..e186380ee6 100644 --- a/call/bitrate_allocator.cc +++ b/call/bitrate_allocator.cc @@ -12,6 +12,7 @@ #include "call/bitrate_allocator.h" #include +#include #include #include "modules/bitrate_controller/include/bitrate_controller.h" @@ -56,7 +57,8 @@ BitrateAllocator::BitrateAllocator(LimitObserver* limit_observer) clock_(Clock::GetRealTimeClock()), last_bwe_log_time_(0), total_requested_padding_bitrate_(0), - total_requested_min_bitrate_(0) { + total_requested_min_bitrate_(0), + bitrate_allocation_strategy_(nullptr) { sequenced_checker_.Detach(); } @@ -199,6 +201,7 @@ void BitrateAllocator::UpdateAllocationLimits() { void BitrateAllocator::RemoveObserver(BitrateAllocatorObserver* observer) { RTC_DCHECK_CALLED_SEQUENTIALLY(&sequenced_checker_); + auto it = FindObserverConfig(observer); if (it != bitrate_observer_configs_.end()) { bitrate_observer_configs_.erase(it); @@ -224,6 +227,13 @@ int BitrateAllocator::GetStartBitrate(BitrateAllocatorObserver* observer) { } } +void BitrateAllocator::SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&sequenced_checker_); + bitrate_allocation_strategy_ = std::move(bitrate_allocation_strategy); +} + BitrateAllocator::ObserverConfigs::iterator BitrateAllocator::FindObserverConfig(const BitrateAllocatorObserver* observer) { RTC_DCHECK_CALLED_SEQUENTIALLY(&sequenced_checker_); @@ -241,6 +251,25 @@ BitrateAllocator::ObserverAllocation BitrateAllocator::AllocateBitrates( if (bitrate_observer_configs_.empty()) return ObserverAllocation(); + if (bitrate_allocation_strategy_ != nullptr) { + std::vector + track_configs(bitrate_observer_configs_.size()); + int i = 0; + for (const auto& c : bitrate_observer_configs_) { + track_configs[i++] = &c; + } + std::vector track_allocations = + bitrate_allocation_strategy_->AllocateBitrates(bitrate, track_configs); + // The strategy should return allocation for all tracks. + RTC_CHECK(track_allocations.size() == bitrate_observer_configs_.size()); + ObserverAllocation allocation; + auto track_allocations_it = track_allocations.begin(); + for (const auto& observer_config : bitrate_observer_configs_) { + allocation[observer_config.observer] = *track_allocations_it++; + } + return allocation; + } + if (bitrate == 0) return ZeroRateAllocation(); diff --git a/call/bitrate_allocator.h b/call/bitrate_allocator.h index d51740cc1b..cf518f6c3d 100644 --- a/call/bitrate_allocator.h +++ b/call/bitrate_allocator.h @@ -14,10 +14,12 @@ #include #include +#include #include #include #include +#include "rtc_base/bitrateallocationstrategy.h" #include "rtc_base/sequenced_task_checker.h" namespace webrtc { @@ -94,32 +96,35 @@ class BitrateAllocator { // the list of added observers, a best guess is returned. int GetStartBitrate(BitrateAllocatorObserver* observer); + // Sets external allocation strategy. If strategy is not set default WebRTC + // allocation mechanism will be used. The strategy may be changed during call. + // Setting NULL value will restore default WEBRTC allocation strategy. + void SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy); + private: // Note: All bitrates for member variables and methods are in bps. - struct ObserverConfig { + struct ObserverConfig : rtc::BitrateAllocationStrategy::TrackConfig { ObserverConfig(BitrateAllocatorObserver* observer, uint32_t min_bitrate_bps, uint32_t max_bitrate_bps, uint32_t pad_up_bitrate_bps, bool enforce_min_bitrate, std::string track_id) - : observer(observer), - min_bitrate_bps(min_bitrate_bps), - max_bitrate_bps(max_bitrate_bps), + : TrackConfig(min_bitrate_bps, + max_bitrate_bps, + enforce_min_bitrate, + track_id), + observer(observer), pad_up_bitrate_bps(pad_up_bitrate_bps), - enforce_min_bitrate(enforce_min_bitrate), allocated_bitrate_bps(-1), - media_ratio(1.0), - track_id(track_id) {} + media_ratio(1.0) {} BitrateAllocatorObserver* observer; - uint32_t min_bitrate_bps; - uint32_t max_bitrate_bps; uint32_t pad_up_bitrate_bps; - bool enforce_min_bitrate; int64_t allocated_bitrate_bps; double media_ratio; // Part of the total bitrate used for media [0.0, 1.0]. - std::string track_id; }; // Calculates the minimum requested send bitrate and max padding bitrate and @@ -172,6 +177,8 @@ class BitrateAllocator { int64_t last_bwe_log_time_ RTC_GUARDED_BY(&sequenced_checker_); uint32_t total_requested_padding_bitrate_ RTC_GUARDED_BY(&sequenced_checker_); uint32_t total_requested_min_bitrate_ RTC_GUARDED_BY(&sequenced_checker_); + std::unique_ptr bitrate_allocation_strategy_ + RTC_GUARDED_BY(&sequenced_checker_); }; } // namespace webrtc #endif // CALL_BITRATE_ALLOCATOR_H_ diff --git a/call/call.cc b/call/call.cc index 215c1035be..90aaba38ca 100644 --- a/call/call.cc +++ b/call/call.cc @@ -214,6 +214,10 @@ class Call : public webrtc::Call, void SetBitrateConfigMask( const webrtc::Call::Config::BitrateConfigMask& bitrate_config) override; + void SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) override; + void SignalChannelNetworkState(MediaType media, NetworkState state) override; void OnTransportOverheadChanged(MediaType media, @@ -1007,6 +1011,24 @@ void Call::UpdateCurrentBitrateConfig(const rtc::Optional& new_start) { config_.bitrate_config = updated; } +void Call::SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) { + if (!worker_queue_.IsCurrent()) { + rtc::BitrateAllocationStrategy* strategy_raw = + bitrate_allocation_strategy.release(); + auto functor = [this, strategy_raw]() { + SetBitrateAllocationStrategy( + rtc::WrapUnique(strategy_raw)); + }; + worker_queue_.PostTask([functor] { functor(); }); + return; + } + RTC_DCHECK_RUN_ON(&worker_queue_); + bitrate_allocator_->SetBitrateAllocationStrategy( + std::move(bitrate_allocation_strategy)); +} + void Call::SignalChannelNetworkState(MediaType media, NetworkState state) { RTC_DCHECK_CALLED_SEQUENTIALLY(&configuration_sequence_checker_); switch (media) { diff --git a/call/call.h b/call/call.h index f26d9d6540..4de5b55f9d 100644 --- a/call/call.h +++ b/call/call.h @@ -24,6 +24,7 @@ #include "call/video_receive_stream.h" #include "call/video_send_stream.h" #include "common_types.h" // NOLINT(build/include) +#include "rtc_base/bitrateallocationstrategy.h" #include "rtc_base/networkroute.h" #include "rtc_base/platform_file.h" #include "rtc_base/socket.h" @@ -183,6 +184,10 @@ class Call { virtual void SetBitrateConfigMask( const Config::BitrateConfigMask& bitrate_mask) = 0; + virtual void SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) = 0; + // TODO(skvlad): When the unbundled case with multiple streams for the same // media type going over different networks is supported, track the state // for each stream separately. Right now it's global per media type. diff --git a/examples/BUILD.gn b/examples/BUILD.gn index 0041d04876..b7474d6c8d 100644 --- a/examples/BUILD.gn +++ b/examples/BUILD.gn @@ -176,6 +176,7 @@ if (is_ios || (is_mac && target_cpu != "x86")) { if (is_ios) { deps = [ ":AppRTCMobile_ios_frameworks", + "../rtc_base:rtc_base", ] } else { deps = [ @@ -206,6 +207,8 @@ if (is_ios || (is_mac && target_cpu != "x86")) { "objc/AppRTCMobile/ARDAppClient.m", "objc/AppRTCMobile/ARDAppEngineClient.h", "objc/AppRTCMobile/ARDAppEngineClient.m", + "objc/AppRTCMobile/ARDBitrateAllocationStrategy.h", + "objc/AppRTCMobile/ARDBitrateAllocationStrategy.mm", "objc/AppRTCMobile/ARDBitrateTracker.h", "objc/AppRTCMobile/ARDBitrateTracker.m", "objc/AppRTCMobile/ARDCaptureController.h", @@ -249,6 +252,7 @@ if (is_ios || (is_mac && target_cpu != "x86")) { deps = [ ":apprtc_common", ":socketrocket", + "../rtc_base:rtc_base", ] if (is_ios) { deps += [ ":AppRTCMobile_ios_frameworks" ] diff --git a/examples/objc/AppRTCMobile/ARDAppClient.m b/examples/objc/AppRTCMobile/ARDAppClient.m index 8e933b921b..f606de00f7 100644 --- a/examples/objc/AppRTCMobile/ARDAppClient.m +++ b/examples/objc/AppRTCMobile/ARDAppClient.m @@ -25,6 +25,7 @@ #import "WebRTC/RTCVideoTrack.h" #import "ARDAppEngineClient.h" +#import "ARDBitrateAllocationStrategy.h" #import "ARDJoinResponse.h" #import "ARDMessageResponse.h" #import "ARDSettingsModel.h" @@ -50,6 +51,7 @@ static NSString * const kARDMediaStreamId = @"ARDAMS"; static NSString * const kARDAudioTrackId = @"ARDAMSa0"; static NSString * const kARDVideoTrackId = @"ARDAMSv0"; static NSString * const kARDVideoTrackKind = @"video"; +static uint32_t const kSufficientAudioBitrate = 16000; // TODO(tkchin): Add these as UI options. static BOOL const kARDAppClientEnableTracing = NO; @@ -104,6 +106,7 @@ static int const kKbpsMultiplier = 1000; ARDTimerProxy *_statsTimer; ARDSettingsModel *_settings; RTCVideoTrack *_localVideoTrack; + ARDBitrateAllocationStrategy *_bitrateAllocationStrategy; } @synthesize shouldGetStats = _shouldGetStats; @@ -306,12 +309,14 @@ static int const kKbpsMultiplier = 1000; _hasReceivedSdp = NO; _messageQueue = [NSMutableArray array]; _localVideoTrack = nil; + #if defined(WEBRTC_IOS) [_factory stopAecDump]; [_peerConnection stopRtcEventLog]; #endif [_peerConnection close]; _peerConnection = nil; + _bitrateAllocationStrategy = nil; self.state = kARDAppClientStateDisconnected; #if defined(WEBRTC_IOS) if (kARDAppClientEnableTracing) { @@ -533,8 +538,14 @@ static int const kKbpsMultiplier = 1000; _peerConnection = [_factory peerConnectionWithConfiguration:config constraints:constraints delegate:self]; + _bitrateAllocationStrategy = [ARDBitrateAllocationStrategy + createAudioPriorityBitrateAllocationStrategyForPeerConnection:_peerConnection + withAudioTrack:kARDAudioTrackId + sufficientAudioBitrate:kSufficientAudioBitrate]; + // Create AV senders. [self createMediaSenders]; + if (_isInitiator) { // Send offer. __weak ARDAppClient *weakSelf = self; diff --git a/examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.h b/examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.h new file mode 100644 index 0000000000..18f83ea605 --- /dev/null +++ b/examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.h @@ -0,0 +1,23 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import "WebRTC/RTCPeerConnection.h" + +@interface ARDBitrateAllocationStrategy : NSObject + ++ (ARDBitrateAllocationStrategy*) + createAudioPriorityBitrateAllocationStrategyForPeerConnection:(RTCPeerConnection*)peerConnection + withAudioTrack:(NSString*)audioTrackID + sufficientAudioBitrate:(uint32_t)sufficientAudioBitrate; + +- (instancetype)init NS_UNAVAILABLE; + +@end diff --git a/examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.mm b/examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.mm new file mode 100644 index 0000000000..5901f8cfea --- /dev/null +++ b/examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.mm @@ -0,0 +1,40 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "ARDBitrateAllocationStrategy.h" +#import "WebRTC/RTCBitrateAllocationStrategy.h" + +#include "rtc_base/bitrateallocationstrategy.h" + +@implementation ARDBitrateAllocationStrategy + ++ (ARDBitrateAllocationStrategy*) + createAudioPriorityBitrateAllocationStrategyForPeerConnection:(RTCPeerConnection*)peerConnection + withAudioTrack:(NSString*)audioTrackID + sufficientAudioBitrate:(uint32_t)sufficientAudioBitrate { + return [[ARDBitrateAllocationStrategy alloc] initWithPeerCoonnection:peerConnection + withAudioTrack:audioTrackID + sufficientAudioBitrate:sufficientAudioBitrate]; +} + +- (instancetype)initWithPeerCoonnection:(RTCPeerConnection*)peerConnection + withAudioTrack:(NSString*)audioTrackID + sufficientAudioBitrate:(uint32_t)sufficientAudioBitrate { + if (self = [super init]) { + [peerConnection + setBitrateAllocationStrategy:[[RTCBitrateAllocationStrategy alloc] + initWith:new rtc::AudioPriorityBitrateAllocationStrategy( + std::string(audioTrackID.UTF8String), + sufficientAudioBitrate)]]; + } + return self; +} + +@end diff --git a/examples/objc/AppRTCMobile/ios/ARDAppDelegate.m b/examples/objc/AppRTCMobile/ios/ARDAppDelegate.m index 07c83a0826..20ae170b9a 100644 --- a/examples/objc/AppRTCMobile/ios/ARDAppDelegate.m +++ b/examples/objc/AppRTCMobile/ios/ARDAppDelegate.m @@ -26,7 +26,8 @@ - (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { NSDictionary *fieldTrials = @{ - kRTCFieldTrialH264HighProfileKey: kRTCFieldTrialEnabledValue, + kRTCFieldTrialH264HighProfileKey : kRTCFieldTrialEnabledValue, + kRTCFieldTrialAudioSendSideBweKey : kRTCFieldTrialEnabledValue }; RTCInitFieldTrialDictionary(fieldTrials); RTCInitializeSSL(); diff --git a/media/engine/fakewebrtccall.cc b/media/engine/fakewebrtccall.cc index 8556cc7db0..d43c7ba8a3 100644 --- a/media/engine/fakewebrtccall.cc +++ b/media/engine/fakewebrtccall.cc @@ -595,6 +595,12 @@ void FakeCall::SetBitrateConfigMask( // TODO(zstein): not implemented } +void FakeCall::SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy){ + // TODO(alexnarest): not implemented +}; + void FakeCall::SignalChannelNetworkState(webrtc::MediaType media, webrtc::NetworkState state) { switch (media) { diff --git a/media/engine/fakewebrtccall.h b/media/engine/fakewebrtccall.h index 04dc5d9e5d..3d0825d237 100644 --- a/media/engine/fakewebrtccall.h +++ b/media/engine/fakewebrtccall.h @@ -296,6 +296,9 @@ class FakeCall final : public webrtc::Call, public webrtc::PacketReceiver { const webrtc::Call::Config::BitrateConfig& bitrate_config) override; void SetBitrateConfigMask( const webrtc::Call::Config::BitrateConfigMask& mask) override; + void SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) override; void OnNetworkRouteChanged(const std::string& transport_name, const rtc::NetworkRoute& network_route) override {} void SignalChannelNetworkState(webrtc::MediaType media, diff --git a/modules/pacing/paced_sender.cc b/modules/pacing/paced_sender.cc index e6b781ef16..ffc8cf3777 100644 --- a/modules/pacing/paced_sender.cc +++ b/modules/pacing/paced_sender.cc @@ -62,7 +62,8 @@ PacedSender::PacedSender(const Clock* clock, packets_(new PacketQueue(clock)), packet_counter_(0), pacing_factor_(kDefaultPaceMultiplier), - queue_time_limit(kMaxQueueLengthMs) { + queue_time_limit(kMaxQueueLengthMs), + account_for_audio_(false) { UpdateBudgetWithElapsedTime(kMinPacketLimitMs); } @@ -153,6 +154,11 @@ void PacedSender::InsertPacket(RtpPacketSender::Priority priority, retransmission, packet_counter_++)); } +void PacedSender::SetAccountForAudioPackets(bool account_for_audio) { + rtc::CritScope cs(&critsect_); + account_for_audio_ = account_for_audio; +} + int64_t PacedSender::ExpectedQueueTimeMs() const { rtc::CritScope cs(&critsect_); RTC_DCHECK_GT(pacing_bitrate_kbps_, 0); @@ -317,9 +323,7 @@ bool PacedSender::SendPacket(const PacketQueue::Packet& packet, critsect_.Enter(); if (success) { - // TODO(holmer): High priority packets should only be accounted for if we - // are allocating bandwidth for audio. - if (packet.priority != kHighPriority) { + if (packet.priority != kHighPriority || account_for_audio_) { // Update media bytes sent. // TODO(eladalon): TimeToSendPacket() can also return |true| in some // situations where nothing actually ended up being sent to the network, diff --git a/modules/pacing/paced_sender.h b/modules/pacing/paced_sender.h index 458efb4ff2..d8381caadd 100644 --- a/modules/pacing/paced_sender.h +++ b/modules/pacing/paced_sender.h @@ -107,6 +107,12 @@ class PacedSender : public Pacer { size_t bytes, bool retransmission) override; + // Currently audio traffic is not accounted by pacer and passed through. + // With the introduction of audio BWE audio traffic will be accounted for + // the pacer budget calculation. The audio traffic still will be injected + // at high priority. + void SetAccountForAudioPackets(bool account_for_audio) override; + // Returns the time since the oldest queued packet was enqueued. virtual int64_t QueueInMs() const; @@ -190,6 +196,7 @@ class PacedSender : public Pacer { float pacing_factor_ RTC_GUARDED_BY(critsect_); int64_t queue_time_limit RTC_GUARDED_BY(critsect_); + bool account_for_audio_ RTC_GUARDED_BY(critsect_); }; } // namespace webrtc #endif // MODULES_PACING_PACED_SENDER_H_ diff --git a/modules/remote_bitrate_estimator/test/bbr_paced_sender.h b/modules/remote_bitrate_estimator/test/bbr_paced_sender.h index 6cb45e3966..096f9d61ec 100644 --- a/modules/remote_bitrate_estimator/test/bbr_paced_sender.h +++ b/modules/remote_bitrate_estimator/test/bbr_paced_sender.h @@ -68,6 +68,7 @@ class BbrPacedSender : public Pacer { int64_t capture_time_ms, size_t bytes, bool retransmission) override; + void SetAccountForAudioPackets(bool account_for_audio) override {} int64_t TimeUntilNextProcess() override; void OnBytesAcked(size_t bytes) override; void Process() override; diff --git a/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/modules/rtp_rtcp/include/rtp_rtcp_defines.h index 1e25f2880a..9023d68541 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp_defines.h +++ b/modules/rtp_rtcp/include/rtp_rtcp_defines.h @@ -498,6 +498,14 @@ class RtpPacketSender { int64_t capture_time_ms, size_t bytes, bool retransmission) = 0; + + // Currently audio traffic is not accounted by pacer and passed through. + // With the introduction of audio BWE audio traffic will be accounted for + // the pacer budget calculation. The audio traffic still will be injected + // at high priority. + // TODO(alexnarest): Make it pure virtual after rtp_sender_unittest will be + // updated to support it + virtual void SetAccountForAudioPackets(bool account_for_audio) {} }; class TransportSequenceNumberAllocator { diff --git a/pc/peerconnection.cc b/pc/peerconnection.cc index a78c29045f..9eb12d694c 100644 --- a/pc/peerconnection.cc +++ b/pc/peerconnection.cc @@ -1293,6 +1293,24 @@ RTCError PeerConnection::SetBitrate(const BitrateParameters& bitrate) { return RTCError::OK(); } +void PeerConnection::SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) { + rtc::Thread* worker_thread = factory_->worker_thread(); + if (!worker_thread->IsCurrent()) { + rtc::BitrateAllocationStrategy* strategy_raw = + bitrate_allocation_strategy.release(); + auto functor = [this, strategy_raw]() { + call_->SetBitrateAllocationStrategy( + rtc::WrapUnique(strategy_raw)); + }; + worker_thread->Invoke(RTC_FROM_HERE, functor); + return; + } + RTC_DCHECK(call_.get()); + call_->SetBitrateAllocationStrategy(std::move(bitrate_allocation_strategy)); +} + std::unique_ptr PeerConnection::GetRemoteAudioSSLCertificate() { if (!session_) { diff --git a/pc/peerconnection.h b/pc/peerconnection.h index 7d0be4dda3..116b24983a 100644 --- a/pc/peerconnection.h +++ b/pc/peerconnection.h @@ -145,6 +145,10 @@ class PeerConnection : public PeerConnectionInterface, RTCError SetBitrate(const BitrateParameters& bitrate) override; + void SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) override; + RTC_DEPRECATED bool StartRtcEventLog(rtc::PlatformFile file, int64_t max_size_bytes) override; bool StartRtcEventLog(std::unique_ptr output) override; diff --git a/rtc_base/BUILD.gn b/rtc_base/BUILD.gn index 0646fb3d32..65b16e5d80 100644 --- a/rtc_base/BUILD.gn +++ b/rtc_base/BUILD.gn @@ -113,6 +113,8 @@ rtc_source_set("rtc_base_approved_generic") { "bind.h", "bitbuffer.cc", "bitbuffer.h", + "bitrateallocationstrategy.cc", + "bitrateallocationstrategy.h", "buffer.h", "bufferqueue.cc", "bufferqueue.h", diff --git a/rtc_base/bitrateallocationstrategy.cc b/rtc_base/bitrateallocationstrategy.cc new file mode 100644 index 0000000000..66528d75a4 --- /dev/null +++ b/rtc_base/bitrateallocationstrategy.cc @@ -0,0 +1,125 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/bitrateallocationstrategy.h" +#include +#include + +namespace rtc { + +std::vector BitrateAllocationStrategy::SetAllBitratesToMinimum( + const ArrayView track_configs) { + std::vector track_allocations; + for (const auto* track_config : track_configs) { + track_allocations.push_back(track_config->min_bitrate_bps); + } + return track_allocations; +} + +std::vector BitrateAllocationStrategy::DistributeBitratesEvenly( + const ArrayView track_configs, + uint32_t available_bitrate) { + std::vector track_allocations = + SetAllBitratesToMinimum(track_configs); + uint32_t sum_min_bitrates = 0; + uint32_t sum_max_bitrates = 0; + for (const auto* track_config : track_configs) { + sum_min_bitrates += track_config->min_bitrate_bps; + sum_max_bitrates += track_config->max_bitrate_bps; + } + if (sum_min_bitrates >= available_bitrate) { + return track_allocations; + } else if (available_bitrate >= sum_max_bitrates) { + auto track_allocations_it = track_allocations.begin(); + for (const auto* track_config : track_configs) { + *track_allocations_it++ = track_config->max_bitrate_bps; + } + return track_allocations; + } else { + // If sum_min_bitrates < available_bitrate < sum_max_bitrates allocate + // bitrates evenly up to max_bitrate_bps starting from the track with the + // lowest max_bitrate_bps. Remainder of available bitrate split evenly among + // remaining tracks. + std::multimap max_bitrate_sorted_configs; + for (const TrackConfig** track_configs_it = track_configs.begin(); + track_configs_it != track_configs.end(); ++track_configs_it) { + max_bitrate_sorted_configs.insert( + std::make_pair((*track_configs_it)->max_bitrate_bps, + track_configs_it - track_configs.begin())); + } + uint32_t total_available_increase = available_bitrate - sum_min_bitrates; + int processed_configs = 0; + for (const auto& track_config_pair : max_bitrate_sorted_configs) { + uint32_t available_increase = + total_available_increase / + (static_cast(track_configs.size() - processed_configs)); + uint32_t consumed_increase = + std::min(track_configs[track_config_pair.second]->max_bitrate_bps - + track_configs[track_config_pair.second]->min_bitrate_bps, + available_increase); + track_allocations[track_config_pair.second] += consumed_increase; + total_available_increase -= consumed_increase; + ++processed_configs; + } + return track_allocations; + } +} + +AudioPriorityBitrateAllocationStrategy::AudioPriorityBitrateAllocationStrategy( + std::string audio_track_id, + uint32_t sufficient_audio_bitrate) + : audio_track_id_(audio_track_id), + sufficient_audio_bitrate_(sufficient_audio_bitrate) {} + +std::vector AudioPriorityBitrateAllocationStrategy::AllocateBitrates( + uint32_t available_bitrate, + const ArrayView track_configs) { + const TrackConfig* audio_track_config = NULL; + size_t audio_config_index = 0; + uint32_t sum_min_bitrates = 0; + + for (const auto*& track_config : track_configs) { + sum_min_bitrates += track_config->min_bitrate_bps; + if (track_config->track_id == audio_track_id_) { + audio_track_config = track_config; + audio_config_index = &track_config - &track_configs[0]; + } + } + if (audio_track_config == nullptr) { + return DistributeBitratesEvenly(track_configs, available_bitrate); + } + auto safe_sufficient_audio_bitrate = std::min( + std::max(audio_track_config->min_bitrate_bps, sufficient_audio_bitrate_), + audio_track_config->max_bitrate_bps); + if (available_bitrate <= sum_min_bitrates) { + return SetAllBitratesToMinimum(track_configs); + } else { + if (available_bitrate <= sum_min_bitrates + safe_sufficient_audio_bitrate - + audio_track_config->min_bitrate_bps) { + std::vector track_allocations = + SetAllBitratesToMinimum(track_configs); + track_allocations[audio_config_index] += + available_bitrate - sum_min_bitrates; + return track_allocations; + } else { + // Setting audio track minimum to safe_sufficient_audio_bitrate will + // allow using DistributeBitratesEvenly to allocate at least sufficient + // bitrate for audio and the rest evenly. + TrackConfig sufficient_track_config(*track_configs[audio_config_index]); + sufficient_track_config.min_bitrate_bps = safe_sufficient_audio_bitrate; + track_configs[audio_config_index] = &sufficient_track_config; + std::vector track_allocations = + DistributeBitratesEvenly(track_configs, available_bitrate); + return track_allocations; + } + } +} + +} // namespace rtc diff --git a/rtc_base/bitrateallocationstrategy.h b/rtc_base/bitrateallocationstrategy.h new file mode 100644 index 0000000000..f711d1f18f --- /dev/null +++ b/rtc_base/bitrateallocationstrategy.h @@ -0,0 +1,101 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_BITRATEALLOCATIONSTRATEGY_H_ +#define RTC_BASE_BITRATEALLOCATIONSTRATEGY_H_ + +#include +#include +#include +#include +#include "api/array_view.h" +#include "rtc_base/checks.h" + +namespace rtc { + +// Pluggable strategy allows configuration of bitrate allocation per media +// track. +// +// The strategy should provide allocation for every track passed with +// track_configs in AllocateBitrates. The allocations are constrained by +// max_bitrate_bps, min_bitrate_bps defining the track supported range and +// enforce_min_bitrate indicating if the track my be paused by allocating 0 +// bitrate. +class BitrateAllocationStrategy { + public: + struct TrackConfig { + TrackConfig(uint32_t min_bitrate_bps, + uint32_t max_bitrate_bps, + bool enforce_min_bitrate, + std::string track_id) + : min_bitrate_bps(min_bitrate_bps), + max_bitrate_bps(max_bitrate_bps), + enforce_min_bitrate(enforce_min_bitrate), + track_id(track_id) {} + TrackConfig(const TrackConfig& track_config) = default; + virtual ~TrackConfig() = default; + TrackConfig() {} + + // Minimum bitrate supported by track. + uint32_t min_bitrate_bps; + + // Maximum bitrate supported by track. + uint32_t max_bitrate_bps; + + // True means track may not be paused by allocating 0 bitrate. + bool enforce_min_bitrate; + + // MediaStreamTrack ID as defined by application. May be empty. + std::string track_id; + }; + + static std::vector SetAllBitratesToMinimum( + const ArrayView track_configs); + static std::vector DistributeBitratesEvenly( + const ArrayView track_configs, + uint32_t available_bitrate); + + // Strategy is expected to allocate all available_bitrate up to the sum of + // max_bitrate_bps of all tracks. If available_bitrate is less than the sum of + // min_bitrate_bps of all tracks, tracks having enforce_min_bitrate set to + // false may get 0 allocation and are suppoused to pause, tracks with + // enforce_min_bitrate set to true are expecting to get min_bitrate_bps. + // + // If the strategy will allocate more than available_bitrate it may cause + // overuse of the currently available network capacity and may cause increase + // in RTT and packet loss. Allocating less than available bitrate may cause + // available_bitrate decrease. + virtual std::vector AllocateBitrates( + uint32_t available_bitrate, + const ArrayView track_configs) = 0; + + virtual ~BitrateAllocationStrategy() = default; +}; + +// Simple allocation strategy giving priority to audio until +// sufficient_audio_bitrate is reached. Bitrate is distributed evenly between +// the tracks after sufficient_audio_bitrate is reached. This implementation +// does not pause tracks even if enforce_min_bitrate is false. +class AudioPriorityBitrateAllocationStrategy + : public BitrateAllocationStrategy { + public: + AudioPriorityBitrateAllocationStrategy(std::string audio_track_id, + uint32_t sufficient_audio_bitrate); + std::vector AllocateBitrates( + uint32_t available_bitrate, + const ArrayView track_configs) override; + + private: + std::string audio_track_id_; + uint32_t sufficient_audio_bitrate_; +}; +} // namespace rtc + +#endif // RTC_BASE_BITRATEALLOCATIONSTRATEGY_H_ diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index 14011325e7..d4151580ee 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -386,6 +386,7 @@ if (is_ios || is_mac) { "objc/Framework/Classes/PeerConnection/RTCAudioSource.mm", "objc/Framework/Classes/PeerConnection/RTCAudioTrack+Private.h", "objc/Framework/Classes/PeerConnection/RTCAudioTrack.mm", + "objc/Framework/Classes/PeerConnection/RTCBitrateAllocationStrategy.mm", "objc/Framework/Classes/PeerConnection/RTCConfiguration+Private.h", "objc/Framework/Classes/PeerConnection/RTCConfiguration.mm", "objc/Framework/Classes/PeerConnection/RTCDataChannel+Private.h", @@ -453,6 +454,7 @@ if (is_ios || is_mac) { "objc/Framework/Headers/WebRTC/RTCAVFoundationVideoSource.h", "objc/Framework/Headers/WebRTC/RTCAudioSource.h", "objc/Framework/Headers/WebRTC/RTCAudioTrack.h", + "objc/Framework/Headers/WebRTC/RTCBitrateAllocationStrategy.h", "objc/Framework/Headers/WebRTC/RTCConfiguration.h", "objc/Framework/Headers/WebRTC/RTCDataChannel.h", "objc/Framework/Headers/WebRTC/RTCDataChannelConfiguration.h", diff --git a/sdk/objc/Framework/Classes/PeerConnection/RTCBitrateAllocationStrategy.mm b/sdk/objc/Framework/Classes/PeerConnection/RTCBitrateAllocationStrategy.mm new file mode 100644 index 0000000000..bc997676dc --- /dev/null +++ b/sdk/objc/Framework/Classes/PeerConnection/RTCBitrateAllocationStrategy.mm @@ -0,0 +1,28 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "WebRTC/RTCBitrateAllocationStrategy.h" + +#include "rtc_base/bitrateallocationstrategy.h" +#include "rtc_base/checks.h" + +@implementation RTCBitrateAllocationStrategy + +@synthesize strategy = _strategy; + +- (instancetype)initWith:(rtc::BitrateAllocationStrategy*)strategy { + RTC_DCHECK(strategy); + if (self = [super init]) { + _strategy = strategy; + } + return self; +} + +@end diff --git a/sdk/objc/Framework/Classes/PeerConnection/RTCConfiguration.mm b/sdk/objc/Framework/Classes/PeerConnection/RTCConfiguration.mm index b9a9e4b137..17aed858aa 100644 --- a/sdk/objc/Framework/Classes/PeerConnection/RTCConfiguration.mm +++ b/sdk/objc/Framework/Classes/PeerConnection/RTCConfiguration.mm @@ -178,7 +178,6 @@ nativeConfig->ice_regather_interval_range = rtc::Optional(*nativeIntervalRange); } - return nativeConfig.release(); } diff --git a/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnection.mm b/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnection.mm index e443e850da..4fcc634f49 100644 --- a/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnection.mm +++ b/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnection.mm @@ -21,6 +21,7 @@ #import "RTCRtpReceiver+Private.h" #import "RTCRtpSender+Private.h" #import "RTCSessionDescription+Private.h" +#import "WebRTC/RTCBitrateAllocationStrategy.h" #import "WebRTC/RTCLogging.h" #include @@ -385,6 +386,15 @@ void PeerConnectionDelegateAdapter::OnIceCandidatesRemoved( return _peerConnection->SetBitrate(params).ok(); } +- (void)setBitrateAllocationStrategy: + (RTCBitrateAllocationStrategy *_Nullable)bitrateAllocationStrategy { + if (bitrateAllocationStrategy) + _peerConnection->SetBitrateAllocationStrategy( + std::unique_ptr(bitrateAllocationStrategy.strategy)); + else + _peerConnection->SetBitrateAllocationStrategy(nullptr); +} + - (BOOL)startRtcEventLogWithFilePath:(NSString *)filePath maxSizeInBytes:(int64_t)maxSizeInBytes { RTC_DCHECK(filePath.length); diff --git a/sdk/objc/Framework/Headers/WebRTC/RTCBitrateAllocationStrategy.h b/sdk/objc/Framework/Headers/WebRTC/RTCBitrateAllocationStrategy.h new file mode 100644 index 0000000000..c510008962 --- /dev/null +++ b/sdk/objc/Framework/Headers/WebRTC/RTCBitrateAllocationStrategy.h @@ -0,0 +1,32 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +namespace rtc { + +class BitrateAllocationStrategy; +} + +RTC_EXPORT +@interface RTCBitrateAllocationStrategy : NSObject + +- (instancetype)init NS_UNAVAILABLE; +- (instancetype)initWith:(rtc::BitrateAllocationStrategy*)strategy; + +/** Native bitrate allocation strategy. */ +@property(nonatomic, readonly) rtc::BitrateAllocationStrategy* strategy; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnection.h b/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnection.h index 7b0c4492f4..c74af1303f 100644 --- a/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnection.h +++ b/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnection.h @@ -24,6 +24,7 @@ @class RTCRtpSender; @class RTCSessionDescription; @class RTCLegacyStatsReport; +@class RTCBitrateAllocationStrategy; NS_ASSUME_NONNULL_BEGIN @@ -193,6 +194,13 @@ RTC_EXPORT currentBitrateBps:(nullable NSNumber *)currentBitrateBps maxBitrateBps:(nullable NSNumber *)maxBitrateBps; +/** Sets current strategy. If not set default WebRTC allocator will be used. + * May be changed during an active session. The strategy + * ownership is passed with std::unique_ptr + */ +- (void)setBitrateAllocationStrategy: + (RTCBitrateAllocationStrategy *_Nullable)bitrateAllocationStrategy; + /** Start or stop recording an Rtc EventLog. */ - (BOOL)startRtcEventLogWithFilePath:(NSString *)filePath maxSizeInBytes:(int64_t)maxSizeInBytes; diff --git a/voice_engine/channel.cc b/voice_engine/channel.cc index 542ea2d831..0130d4271d 100644 --- a/voice_engine/channel.cc +++ b/voice_engine/channel.cc @@ -215,6 +215,10 @@ class RtpPacketSenderProxy : public RtpPacketSender { } } + void SetAccountForAudioPackets(bool account_for_audio) override { + RTC_NOTREACHED(); + } + private: rtc::ThreadChecker thread_checker_; rtc::CriticalSection crit_;