diff --git a/api/peerconnectioninterface.h b/api/peerconnectioninterface.h index dd1f87c931..839b0f4baa 100644 --- a/api/peerconnectioninterface.h +++ b/api/peerconnectioninterface.h @@ -786,6 +786,14 @@ class PeerConnectionInterface : public rtc::RefCountInterface { // to the provided value. virtual RTCError SetBitrate(const BitrateParameters& bitrate) = 0; + // Sets current strategy. If not set default WebRTC allocator will be used. + // May be changed during an active session. The strategy + // ownership is passed with std::unique_ptr + // TODO(alexnarest): Make this pure virtual when tests will be updated + virtual void SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) {} + // Returns the current SignalingState. virtual SignalingState signaling_state() = 0; virtual IceConnectionState ice_connection_state() = 0; diff --git a/api/peerconnectionproxy.h b/api/peerconnectionproxy.h index c490b8f69d..a8ea3fa360 100644 --- a/api/peerconnectionproxy.h +++ b/api/peerconnectionproxy.h @@ -102,6 +102,9 @@ BEGIN_SIGNALING_PROXY_MAP(PeerConnection) const std::vector&); PROXY_METHOD1(void, RegisterUMAObserver, UMAObserver*) PROXY_METHOD1(RTCError, SetBitrate, const BitrateParameters&); + PROXY_METHOD1(void, + SetBitrateAllocationStrategy, + std::unique_ptr); PROXY_METHOD0(SignalingState, signaling_state) PROXY_METHOD0(IceConnectionState, ice_connection_state) PROXY_METHOD0(IceGatheringState, ice_gathering_state) diff --git a/audio/audio_send_stream.cc b/audio/audio_send_stream.cc index 1cfc0bad1e..7895af3bb7 100644 --- a/audio/audio_send_stream.cc +++ b/audio/audio_send_stream.cc @@ -236,6 +236,8 @@ void AudioSendStream::ConfigureStream( void AudioSendStream::Start() { RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); if (config_.min_bitrate_bps != -1 && config_.max_bitrate_bps != -1) { + // Audio BWE is enabled. + transport_->packet_sender()->SetAccountForAudioPackets(true); ConfigureBitrateObserver(config_.min_bitrate_bps, config_.max_bitrate_bps); } diff --git a/call/bitrate_allocator.cc b/call/bitrate_allocator.cc index b379f7f086..e186380ee6 100644 --- a/call/bitrate_allocator.cc +++ b/call/bitrate_allocator.cc @@ -12,6 +12,7 @@ #include "call/bitrate_allocator.h" #include +#include #include #include "modules/bitrate_controller/include/bitrate_controller.h" @@ -56,7 +57,8 @@ BitrateAllocator::BitrateAllocator(LimitObserver* limit_observer) clock_(Clock::GetRealTimeClock()), last_bwe_log_time_(0), total_requested_padding_bitrate_(0), - total_requested_min_bitrate_(0) { + total_requested_min_bitrate_(0), + bitrate_allocation_strategy_(nullptr) { sequenced_checker_.Detach(); } @@ -199,6 +201,7 @@ void BitrateAllocator::UpdateAllocationLimits() { void BitrateAllocator::RemoveObserver(BitrateAllocatorObserver* observer) { RTC_DCHECK_CALLED_SEQUENTIALLY(&sequenced_checker_); + auto it = FindObserverConfig(observer); if (it != bitrate_observer_configs_.end()) { bitrate_observer_configs_.erase(it); @@ -224,6 +227,13 @@ int BitrateAllocator::GetStartBitrate(BitrateAllocatorObserver* observer) { } } +void BitrateAllocator::SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&sequenced_checker_); + bitrate_allocation_strategy_ = std::move(bitrate_allocation_strategy); +} + BitrateAllocator::ObserverConfigs::iterator BitrateAllocator::FindObserverConfig(const BitrateAllocatorObserver* observer) { RTC_DCHECK_CALLED_SEQUENTIALLY(&sequenced_checker_); @@ -241,6 +251,25 @@ BitrateAllocator::ObserverAllocation BitrateAllocator::AllocateBitrates( if (bitrate_observer_configs_.empty()) return ObserverAllocation(); + if (bitrate_allocation_strategy_ != nullptr) { + std::vector + track_configs(bitrate_observer_configs_.size()); + int i = 0; + for (const auto& c : bitrate_observer_configs_) { + track_configs[i++] = &c; + } + std::vector track_allocations = + bitrate_allocation_strategy_->AllocateBitrates(bitrate, track_configs); + // The strategy should return allocation for all tracks. + RTC_CHECK(track_allocations.size() == bitrate_observer_configs_.size()); + ObserverAllocation allocation; + auto track_allocations_it = track_allocations.begin(); + for (const auto& observer_config : bitrate_observer_configs_) { + allocation[observer_config.observer] = *track_allocations_it++; + } + return allocation; + } + if (bitrate == 0) return ZeroRateAllocation(); diff --git a/call/bitrate_allocator.h b/call/bitrate_allocator.h index d51740cc1b..cf518f6c3d 100644 --- a/call/bitrate_allocator.h +++ b/call/bitrate_allocator.h @@ -14,10 +14,12 @@ #include #include +#include #include #include #include +#include "rtc_base/bitrateallocationstrategy.h" #include "rtc_base/sequenced_task_checker.h" namespace webrtc { @@ -94,32 +96,35 @@ class BitrateAllocator { // the list of added observers, a best guess is returned. int GetStartBitrate(BitrateAllocatorObserver* observer); + // Sets external allocation strategy. If strategy is not set default WebRTC + // allocation mechanism will be used. The strategy may be changed during call. + // Setting NULL value will restore default WEBRTC allocation strategy. + void SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy); + private: // Note: All bitrates for member variables and methods are in bps. - struct ObserverConfig { + struct ObserverConfig : rtc::BitrateAllocationStrategy::TrackConfig { ObserverConfig(BitrateAllocatorObserver* observer, uint32_t min_bitrate_bps, uint32_t max_bitrate_bps, uint32_t pad_up_bitrate_bps, bool enforce_min_bitrate, std::string track_id) - : observer(observer), - min_bitrate_bps(min_bitrate_bps), - max_bitrate_bps(max_bitrate_bps), + : TrackConfig(min_bitrate_bps, + max_bitrate_bps, + enforce_min_bitrate, + track_id), + observer(observer), pad_up_bitrate_bps(pad_up_bitrate_bps), - enforce_min_bitrate(enforce_min_bitrate), allocated_bitrate_bps(-1), - media_ratio(1.0), - track_id(track_id) {} + media_ratio(1.0) {} BitrateAllocatorObserver* observer; - uint32_t min_bitrate_bps; - uint32_t max_bitrate_bps; uint32_t pad_up_bitrate_bps; - bool enforce_min_bitrate; int64_t allocated_bitrate_bps; double media_ratio; // Part of the total bitrate used for media [0.0, 1.0]. - std::string track_id; }; // Calculates the minimum requested send bitrate and max padding bitrate and @@ -172,6 +177,8 @@ class BitrateAllocator { int64_t last_bwe_log_time_ RTC_GUARDED_BY(&sequenced_checker_); uint32_t total_requested_padding_bitrate_ RTC_GUARDED_BY(&sequenced_checker_); uint32_t total_requested_min_bitrate_ RTC_GUARDED_BY(&sequenced_checker_); + std::unique_ptr bitrate_allocation_strategy_ + RTC_GUARDED_BY(&sequenced_checker_); }; } // namespace webrtc #endif // CALL_BITRATE_ALLOCATOR_H_ diff --git a/call/call.cc b/call/call.cc index 215c1035be..90aaba38ca 100644 --- a/call/call.cc +++ b/call/call.cc @@ -214,6 +214,10 @@ class Call : public webrtc::Call, void SetBitrateConfigMask( const webrtc::Call::Config::BitrateConfigMask& bitrate_config) override; + void SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) override; + void SignalChannelNetworkState(MediaType media, NetworkState state) override; void OnTransportOverheadChanged(MediaType media, @@ -1007,6 +1011,24 @@ void Call::UpdateCurrentBitrateConfig(const rtc::Optional& new_start) { config_.bitrate_config = updated; } +void Call::SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) { + if (!worker_queue_.IsCurrent()) { + rtc::BitrateAllocationStrategy* strategy_raw = + bitrate_allocation_strategy.release(); + auto functor = [this, strategy_raw]() { + SetBitrateAllocationStrategy( + rtc::WrapUnique(strategy_raw)); + }; + worker_queue_.PostTask([functor] { functor(); }); + return; + } + RTC_DCHECK_RUN_ON(&worker_queue_); + bitrate_allocator_->SetBitrateAllocationStrategy( + std::move(bitrate_allocation_strategy)); +} + void Call::SignalChannelNetworkState(MediaType media, NetworkState state) { RTC_DCHECK_CALLED_SEQUENTIALLY(&configuration_sequence_checker_); switch (media) { diff --git a/call/call.h b/call/call.h index f26d9d6540..4de5b55f9d 100644 --- a/call/call.h +++ b/call/call.h @@ -24,6 +24,7 @@ #include "call/video_receive_stream.h" #include "call/video_send_stream.h" #include "common_types.h" // NOLINT(build/include) +#include "rtc_base/bitrateallocationstrategy.h" #include "rtc_base/networkroute.h" #include "rtc_base/platform_file.h" #include "rtc_base/socket.h" @@ -183,6 +184,10 @@ class Call { virtual void SetBitrateConfigMask( const Config::BitrateConfigMask& bitrate_mask) = 0; + virtual void SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) = 0; + // TODO(skvlad): When the unbundled case with multiple streams for the same // media type going over different networks is supported, track the state // for each stream separately. Right now it's global per media type. diff --git a/call/rampup_tests.cc b/call/rampup_tests.cc index 85346b7479..1d584195d2 100644 --- a/call/rampup_tests.cc +++ b/call/rampup_tests.cc @@ -418,19 +418,21 @@ RampUpDownUpTester::~RampUpDownUpTester() {} void RampUpDownUpTester::PollStats() { do { - if (send_stream_) { + int transmit_bitrate_bps = 0; + bool suspended = false; + if (num_video_streams_ > 0) { webrtc::VideoSendStream::Stats stats = send_stream_->GetStats(); - int transmit_bitrate_bps = 0; for (auto it : stats.substreams) { transmit_bitrate_bps += it.second.total_bitrate_bps; } - EvolveTestState(transmit_bitrate_bps, stats.suspended); - } else if (num_audio_streams_ > 0 && sender_call_ != nullptr) { + suspended = stats.suspended; + } + if (num_audio_streams_ > 0 && sender_call_ != nullptr) { // An audio send stream doesn't have bitrate stats, so the call send BW is // currently used instead. - int transmit_bitrate_bps = sender_call_->GetStats().send_bandwidth_bps; - EvolveTestState(transmit_bitrate_bps, false); + transmit_bitrate_bps = sender_call_->GetStats().send_bandwidth_bps; } + EvolveTestState(transmit_bitrate_bps, suspended); } while (!stop_event_.Wait(kPollIntervalMs)); } diff --git a/examples/BUILD.gn b/examples/BUILD.gn index 0041d04876..b7474d6c8d 100644 --- a/examples/BUILD.gn +++ b/examples/BUILD.gn @@ -176,6 +176,7 @@ if (is_ios || (is_mac && target_cpu != "x86")) { if (is_ios) { deps = [ ":AppRTCMobile_ios_frameworks", + "../rtc_base:rtc_base", ] } else { deps = [ @@ -206,6 +207,8 @@ if (is_ios || (is_mac && target_cpu != "x86")) { "objc/AppRTCMobile/ARDAppClient.m", "objc/AppRTCMobile/ARDAppEngineClient.h", "objc/AppRTCMobile/ARDAppEngineClient.m", + "objc/AppRTCMobile/ARDBitrateAllocationStrategy.h", + "objc/AppRTCMobile/ARDBitrateAllocationStrategy.mm", "objc/AppRTCMobile/ARDBitrateTracker.h", "objc/AppRTCMobile/ARDBitrateTracker.m", "objc/AppRTCMobile/ARDCaptureController.h", @@ -249,6 +252,7 @@ if (is_ios || (is_mac && target_cpu != "x86")) { deps = [ ":apprtc_common", ":socketrocket", + "../rtc_base:rtc_base", ] if (is_ios) { deps += [ ":AppRTCMobile_ios_frameworks" ] diff --git a/examples/objc/AppRTCMobile/ARDAppClient.m b/examples/objc/AppRTCMobile/ARDAppClient.m index 8e933b921b..f606de00f7 100644 --- a/examples/objc/AppRTCMobile/ARDAppClient.m +++ b/examples/objc/AppRTCMobile/ARDAppClient.m @@ -25,6 +25,7 @@ #import "WebRTC/RTCVideoTrack.h" #import "ARDAppEngineClient.h" +#import "ARDBitrateAllocationStrategy.h" #import "ARDJoinResponse.h" #import "ARDMessageResponse.h" #import "ARDSettingsModel.h" @@ -50,6 +51,7 @@ static NSString * const kARDMediaStreamId = @"ARDAMS"; static NSString * const kARDAudioTrackId = @"ARDAMSa0"; static NSString * const kARDVideoTrackId = @"ARDAMSv0"; static NSString * const kARDVideoTrackKind = @"video"; +static uint32_t const kSufficientAudioBitrate = 16000; // TODO(tkchin): Add these as UI options. static BOOL const kARDAppClientEnableTracing = NO; @@ -104,6 +106,7 @@ static int const kKbpsMultiplier = 1000; ARDTimerProxy *_statsTimer; ARDSettingsModel *_settings; RTCVideoTrack *_localVideoTrack; + ARDBitrateAllocationStrategy *_bitrateAllocationStrategy; } @synthesize shouldGetStats = _shouldGetStats; @@ -306,12 +309,14 @@ static int const kKbpsMultiplier = 1000; _hasReceivedSdp = NO; _messageQueue = [NSMutableArray array]; _localVideoTrack = nil; + #if defined(WEBRTC_IOS) [_factory stopAecDump]; [_peerConnection stopRtcEventLog]; #endif [_peerConnection close]; _peerConnection = nil; + _bitrateAllocationStrategy = nil; self.state = kARDAppClientStateDisconnected; #if defined(WEBRTC_IOS) if (kARDAppClientEnableTracing) { @@ -533,8 +538,14 @@ static int const kKbpsMultiplier = 1000; _peerConnection = [_factory peerConnectionWithConfiguration:config constraints:constraints delegate:self]; + _bitrateAllocationStrategy = [ARDBitrateAllocationStrategy + createAudioPriorityBitrateAllocationStrategyForPeerConnection:_peerConnection + withAudioTrack:kARDAudioTrackId + sufficientAudioBitrate:kSufficientAudioBitrate]; + // Create AV senders. [self createMediaSenders]; + if (_isInitiator) { // Send offer. __weak ARDAppClient *weakSelf = self; diff --git a/examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.h b/examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.h new file mode 100644 index 0000000000..18f83ea605 --- /dev/null +++ b/examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.h @@ -0,0 +1,23 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import "WebRTC/RTCPeerConnection.h" + +@interface ARDBitrateAllocationStrategy : NSObject + ++ (ARDBitrateAllocationStrategy*) + createAudioPriorityBitrateAllocationStrategyForPeerConnection:(RTCPeerConnection*)peerConnection + withAudioTrack:(NSString*)audioTrackID + sufficientAudioBitrate:(uint32_t)sufficientAudioBitrate; + +- (instancetype)init NS_UNAVAILABLE; + +@end diff --git a/examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.mm b/examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.mm new file mode 100644 index 0000000000..5901f8cfea --- /dev/null +++ b/examples/objc/AppRTCMobile/ARDBitrateAllocationStrategy.mm @@ -0,0 +1,40 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "ARDBitrateAllocationStrategy.h" +#import "WebRTC/RTCBitrateAllocationStrategy.h" + +#include "rtc_base/bitrateallocationstrategy.h" + +@implementation ARDBitrateAllocationStrategy + ++ (ARDBitrateAllocationStrategy*) + createAudioPriorityBitrateAllocationStrategyForPeerConnection:(RTCPeerConnection*)peerConnection + withAudioTrack:(NSString*)audioTrackID + sufficientAudioBitrate:(uint32_t)sufficientAudioBitrate { + return [[ARDBitrateAllocationStrategy alloc] initWithPeerCoonnection:peerConnection + withAudioTrack:audioTrackID + sufficientAudioBitrate:sufficientAudioBitrate]; +} + +- (instancetype)initWithPeerCoonnection:(RTCPeerConnection*)peerConnection + withAudioTrack:(NSString*)audioTrackID + sufficientAudioBitrate:(uint32_t)sufficientAudioBitrate { + if (self = [super init]) { + [peerConnection + setBitrateAllocationStrategy:[[RTCBitrateAllocationStrategy alloc] + initWith:new rtc::AudioPriorityBitrateAllocationStrategy( + std::string(audioTrackID.UTF8String), + sufficientAudioBitrate)]]; + } + return self; +} + +@end diff --git a/examples/objc/AppRTCMobile/ios/ARDAppDelegate.m b/examples/objc/AppRTCMobile/ios/ARDAppDelegate.m index 07c83a0826..20ae170b9a 100644 --- a/examples/objc/AppRTCMobile/ios/ARDAppDelegate.m +++ b/examples/objc/AppRTCMobile/ios/ARDAppDelegate.m @@ -26,7 +26,8 @@ - (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { NSDictionary *fieldTrials = @{ - kRTCFieldTrialH264HighProfileKey: kRTCFieldTrialEnabledValue, + kRTCFieldTrialH264HighProfileKey : kRTCFieldTrialEnabledValue, + kRTCFieldTrialAudioSendSideBweKey : kRTCFieldTrialEnabledValue }; RTCInitFieldTrialDictionary(fieldTrials); RTCInitializeSSL(); diff --git a/media/engine/fakewebrtccall.cc b/media/engine/fakewebrtccall.cc index 8556cc7db0..d43c7ba8a3 100644 --- a/media/engine/fakewebrtccall.cc +++ b/media/engine/fakewebrtccall.cc @@ -595,6 +595,12 @@ void FakeCall::SetBitrateConfigMask( // TODO(zstein): not implemented } +void FakeCall::SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy){ + // TODO(alexnarest): not implemented +}; + void FakeCall::SignalChannelNetworkState(webrtc::MediaType media, webrtc::NetworkState state) { switch (media) { diff --git a/media/engine/fakewebrtccall.h b/media/engine/fakewebrtccall.h index 04dc5d9e5d..3d0825d237 100644 --- a/media/engine/fakewebrtccall.h +++ b/media/engine/fakewebrtccall.h @@ -296,6 +296,9 @@ class FakeCall final : public webrtc::Call, public webrtc::PacketReceiver { const webrtc::Call::Config::BitrateConfig& bitrate_config) override; void SetBitrateConfigMask( const webrtc::Call::Config::BitrateConfigMask& mask) override; + void SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) override; void OnNetworkRouteChanged(const std::string& transport_name, const rtc::NetworkRoute& network_route) override {} void SignalChannelNetworkState(webrtc::MediaType media, diff --git a/modules/pacing/paced_sender.cc b/modules/pacing/paced_sender.cc index c122f865f1..38896313e1 100644 --- a/modules/pacing/paced_sender.cc +++ b/modules/pacing/paced_sender.cc @@ -63,7 +63,8 @@ PacedSender::PacedSender(const Clock* clock, packets_(rtc::MakeUnique(clock)), packet_counter_(0), pacing_factor_(kDefaultPaceMultiplier), - queue_time_limit(kMaxQueueLengthMs) { + queue_time_limit(kMaxQueueLengthMs), + account_for_audio_(false) { UpdateBudgetWithElapsedTime(kMinPacketLimitMs); } @@ -154,6 +155,11 @@ void PacedSender::InsertPacket(RtpPacketSender::Priority priority, retransmission, packet_counter_++)); } +void PacedSender::SetAccountForAudioPackets(bool account_for_audio) { + rtc::CritScope cs(&critsect_); + account_for_audio_ = account_for_audio; +} + int64_t PacedSender::ExpectedQueueTimeMs() const { rtc::CritScope cs(&critsect_); RTC_DCHECK_GT(pacing_bitrate_kbps_, 0); @@ -318,9 +324,7 @@ bool PacedSender::SendPacket(const PacketQueue::Packet& packet, critsect_.Enter(); if (success) { - // TODO(holmer): High priority packets should only be accounted for if we - // are allocating bandwidth for audio. - if (packet.priority != kHighPriority) { + if (packet.priority != kHighPriority || account_for_audio_) { // Update media bytes sent. // TODO(eladalon): TimeToSendPacket() can also return |true| in some // situations where nothing actually ended up being sent to the network, diff --git a/modules/pacing/paced_sender.h b/modules/pacing/paced_sender.h index 3c5ad6033f..8e4716126b 100644 --- a/modules/pacing/paced_sender.h +++ b/modules/pacing/paced_sender.h @@ -107,6 +107,12 @@ class PacedSender : public Pacer { size_t bytes, bool retransmission) override; + // Currently audio traffic is not accounted by pacer and passed through. + // With the introduction of audio BWE audio traffic will be accounted for + // the pacer budget calculation. The audio traffic still will be injected + // at high priority. + void SetAccountForAudioPackets(bool account_for_audio) override; + // Returns the time since the oldest queued packet was enqueued. virtual int64_t QueueInMs() const; @@ -192,6 +198,7 @@ class PacedSender : public Pacer { float pacing_factor_ RTC_GUARDED_BY(critsect_); int64_t queue_time_limit RTC_GUARDED_BY(critsect_); + bool account_for_audio_ RTC_GUARDED_BY(critsect_); }; } // namespace webrtc #endif // MODULES_PACING_PACED_SENDER_H_ diff --git a/modules/remote_bitrate_estimator/test/bbr_paced_sender.h b/modules/remote_bitrate_estimator/test/bbr_paced_sender.h index 6cb45e3966..096f9d61ec 100644 --- a/modules/remote_bitrate_estimator/test/bbr_paced_sender.h +++ b/modules/remote_bitrate_estimator/test/bbr_paced_sender.h @@ -68,6 +68,7 @@ class BbrPacedSender : public Pacer { int64_t capture_time_ms, size_t bytes, bool retransmission) override; + void SetAccountForAudioPackets(bool account_for_audio) override {} int64_t TimeUntilNextProcess() override; void OnBytesAcked(size_t bytes) override; void Process() override; diff --git a/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/modules/rtp_rtcp/include/rtp_rtcp_defines.h index 1e25f2880a..9023d68541 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp_defines.h +++ b/modules/rtp_rtcp/include/rtp_rtcp_defines.h @@ -498,6 +498,14 @@ class RtpPacketSender { int64_t capture_time_ms, size_t bytes, bool retransmission) = 0; + + // Currently audio traffic is not accounted by pacer and passed through. + // With the introduction of audio BWE audio traffic will be accounted for + // the pacer budget calculation. The audio traffic still will be injected + // at high priority. + // TODO(alexnarest): Make it pure virtual after rtp_sender_unittest will be + // updated to support it + virtual void SetAccountForAudioPackets(bool account_for_audio) {} }; class TransportSequenceNumberAllocator { diff --git a/pc/peerconnection.cc b/pc/peerconnection.cc index bec13a5886..7cad0e1121 100644 --- a/pc/peerconnection.cc +++ b/pc/peerconnection.cc @@ -1331,6 +1331,24 @@ RTCError PeerConnection::SetBitrate(const BitrateParameters& bitrate) { return RTCError::OK(); } +void PeerConnection::SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) { + rtc::Thread* worker_thread = factory_->worker_thread(); + if (!worker_thread->IsCurrent()) { + rtc::BitrateAllocationStrategy* strategy_raw = + bitrate_allocation_strategy.release(); + auto functor = [this, strategy_raw]() { + call_->SetBitrateAllocationStrategy( + rtc::WrapUnique(strategy_raw)); + }; + worker_thread->Invoke(RTC_FROM_HERE, functor); + return; + } + RTC_DCHECK(call_.get()); + call_->SetBitrateAllocationStrategy(std::move(bitrate_allocation_strategy)); +} + std::unique_ptr PeerConnection::GetRemoteAudioSSLCertificate() { if (!session_) { diff --git a/pc/peerconnection.h b/pc/peerconnection.h index 7d0be4dda3..116b24983a 100644 --- a/pc/peerconnection.h +++ b/pc/peerconnection.h @@ -145,6 +145,10 @@ class PeerConnection : public PeerConnectionInterface, RTCError SetBitrate(const BitrateParameters& bitrate) override; + void SetBitrateAllocationStrategy( + std::unique_ptr + bitrate_allocation_strategy) override; + RTC_DEPRECATED bool StartRtcEventLog(rtc::PlatformFile file, int64_t max_size_bytes) override; bool StartRtcEventLog(std::unique_ptr output) override; diff --git a/rtc_base/BUILD.gn b/rtc_base/BUILD.gn index eddef51aa1..443d32ef80 100644 --- a/rtc_base/BUILD.gn +++ b/rtc_base/BUILD.gn @@ -113,6 +113,8 @@ rtc_source_set("rtc_base_approved_generic") { "bind.h", "bitbuffer.cc", "bitbuffer.h", + "bitrateallocationstrategy.cc", + "bitrateallocationstrategy.h", "buffer.h", "bufferqueue.cc", "bufferqueue.h", diff --git a/rtc_base/bitrateallocationstrategy.cc b/rtc_base/bitrateallocationstrategy.cc new file mode 100644 index 0000000000..66528d75a4 --- /dev/null +++ b/rtc_base/bitrateallocationstrategy.cc @@ -0,0 +1,125 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/bitrateallocationstrategy.h" +#include +#include + +namespace rtc { + +std::vector BitrateAllocationStrategy::SetAllBitratesToMinimum( + const ArrayView track_configs) { + std::vector track_allocations; + for (const auto* track_config : track_configs) { + track_allocations.push_back(track_config->min_bitrate_bps); + } + return track_allocations; +} + +std::vector BitrateAllocationStrategy::DistributeBitratesEvenly( + const ArrayView track_configs, + uint32_t available_bitrate) { + std::vector track_allocations = + SetAllBitratesToMinimum(track_configs); + uint32_t sum_min_bitrates = 0; + uint32_t sum_max_bitrates = 0; + for (const auto* track_config : track_configs) { + sum_min_bitrates += track_config->min_bitrate_bps; + sum_max_bitrates += track_config->max_bitrate_bps; + } + if (sum_min_bitrates >= available_bitrate) { + return track_allocations; + } else if (available_bitrate >= sum_max_bitrates) { + auto track_allocations_it = track_allocations.begin(); + for (const auto* track_config : track_configs) { + *track_allocations_it++ = track_config->max_bitrate_bps; + } + return track_allocations; + } else { + // If sum_min_bitrates < available_bitrate < sum_max_bitrates allocate + // bitrates evenly up to max_bitrate_bps starting from the track with the + // lowest max_bitrate_bps. Remainder of available bitrate split evenly among + // remaining tracks. + std::multimap max_bitrate_sorted_configs; + for (const TrackConfig** track_configs_it = track_configs.begin(); + track_configs_it != track_configs.end(); ++track_configs_it) { + max_bitrate_sorted_configs.insert( + std::make_pair((*track_configs_it)->max_bitrate_bps, + track_configs_it - track_configs.begin())); + } + uint32_t total_available_increase = available_bitrate - sum_min_bitrates; + int processed_configs = 0; + for (const auto& track_config_pair : max_bitrate_sorted_configs) { + uint32_t available_increase = + total_available_increase / + (static_cast(track_configs.size() - processed_configs)); + uint32_t consumed_increase = + std::min(track_configs[track_config_pair.second]->max_bitrate_bps - + track_configs[track_config_pair.second]->min_bitrate_bps, + available_increase); + track_allocations[track_config_pair.second] += consumed_increase; + total_available_increase -= consumed_increase; + ++processed_configs; + } + return track_allocations; + } +} + +AudioPriorityBitrateAllocationStrategy::AudioPriorityBitrateAllocationStrategy( + std::string audio_track_id, + uint32_t sufficient_audio_bitrate) + : audio_track_id_(audio_track_id), + sufficient_audio_bitrate_(sufficient_audio_bitrate) {} + +std::vector AudioPriorityBitrateAllocationStrategy::AllocateBitrates( + uint32_t available_bitrate, + const ArrayView track_configs) { + const TrackConfig* audio_track_config = NULL; + size_t audio_config_index = 0; + uint32_t sum_min_bitrates = 0; + + for (const auto*& track_config : track_configs) { + sum_min_bitrates += track_config->min_bitrate_bps; + if (track_config->track_id == audio_track_id_) { + audio_track_config = track_config; + audio_config_index = &track_config - &track_configs[0]; + } + } + if (audio_track_config == nullptr) { + return DistributeBitratesEvenly(track_configs, available_bitrate); + } + auto safe_sufficient_audio_bitrate = std::min( + std::max(audio_track_config->min_bitrate_bps, sufficient_audio_bitrate_), + audio_track_config->max_bitrate_bps); + if (available_bitrate <= sum_min_bitrates) { + return SetAllBitratesToMinimum(track_configs); + } else { + if (available_bitrate <= sum_min_bitrates + safe_sufficient_audio_bitrate - + audio_track_config->min_bitrate_bps) { + std::vector track_allocations = + SetAllBitratesToMinimum(track_configs); + track_allocations[audio_config_index] += + available_bitrate - sum_min_bitrates; + return track_allocations; + } else { + // Setting audio track minimum to safe_sufficient_audio_bitrate will + // allow using DistributeBitratesEvenly to allocate at least sufficient + // bitrate for audio and the rest evenly. + TrackConfig sufficient_track_config(*track_configs[audio_config_index]); + sufficient_track_config.min_bitrate_bps = safe_sufficient_audio_bitrate; + track_configs[audio_config_index] = &sufficient_track_config; + std::vector track_allocations = + DistributeBitratesEvenly(track_configs, available_bitrate); + return track_allocations; + } + } +} + +} // namespace rtc diff --git a/rtc_base/bitrateallocationstrategy.h b/rtc_base/bitrateallocationstrategy.h new file mode 100644 index 0000000000..f711d1f18f --- /dev/null +++ b/rtc_base/bitrateallocationstrategy.h @@ -0,0 +1,101 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_BITRATEALLOCATIONSTRATEGY_H_ +#define RTC_BASE_BITRATEALLOCATIONSTRATEGY_H_ + +#include +#include +#include +#include +#include "api/array_view.h" +#include "rtc_base/checks.h" + +namespace rtc { + +// Pluggable strategy allows configuration of bitrate allocation per media +// track. +// +// The strategy should provide allocation for every track passed with +// track_configs in AllocateBitrates. The allocations are constrained by +// max_bitrate_bps, min_bitrate_bps defining the track supported range and +// enforce_min_bitrate indicating if the track my be paused by allocating 0 +// bitrate. +class BitrateAllocationStrategy { + public: + struct TrackConfig { + TrackConfig(uint32_t min_bitrate_bps, + uint32_t max_bitrate_bps, + bool enforce_min_bitrate, + std::string track_id) + : min_bitrate_bps(min_bitrate_bps), + max_bitrate_bps(max_bitrate_bps), + enforce_min_bitrate(enforce_min_bitrate), + track_id(track_id) {} + TrackConfig(const TrackConfig& track_config) = default; + virtual ~TrackConfig() = default; + TrackConfig() {} + + // Minimum bitrate supported by track. + uint32_t min_bitrate_bps; + + // Maximum bitrate supported by track. + uint32_t max_bitrate_bps; + + // True means track may not be paused by allocating 0 bitrate. + bool enforce_min_bitrate; + + // MediaStreamTrack ID as defined by application. May be empty. + std::string track_id; + }; + + static std::vector SetAllBitratesToMinimum( + const ArrayView track_configs); + static std::vector DistributeBitratesEvenly( + const ArrayView track_configs, + uint32_t available_bitrate); + + // Strategy is expected to allocate all available_bitrate up to the sum of + // max_bitrate_bps of all tracks. If available_bitrate is less than the sum of + // min_bitrate_bps of all tracks, tracks having enforce_min_bitrate set to + // false may get 0 allocation and are suppoused to pause, tracks with + // enforce_min_bitrate set to true are expecting to get min_bitrate_bps. + // + // If the strategy will allocate more than available_bitrate it may cause + // overuse of the currently available network capacity and may cause increase + // in RTT and packet loss. Allocating less than available bitrate may cause + // available_bitrate decrease. + virtual std::vector AllocateBitrates( + uint32_t available_bitrate, + const ArrayView track_configs) = 0; + + virtual ~BitrateAllocationStrategy() = default; +}; + +// Simple allocation strategy giving priority to audio until +// sufficient_audio_bitrate is reached. Bitrate is distributed evenly between +// the tracks after sufficient_audio_bitrate is reached. This implementation +// does not pause tracks even if enforce_min_bitrate is false. +class AudioPriorityBitrateAllocationStrategy + : public BitrateAllocationStrategy { + public: + AudioPriorityBitrateAllocationStrategy(std::string audio_track_id, + uint32_t sufficient_audio_bitrate); + std::vector AllocateBitrates( + uint32_t available_bitrate, + const ArrayView track_configs) override; + + private: + std::string audio_track_id_; + uint32_t sufficient_audio_bitrate_; +}; +} // namespace rtc + +#endif // RTC_BASE_BITRATEALLOCATIONSTRATEGY_H_ diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index 14011325e7..d4151580ee 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -386,6 +386,7 @@ if (is_ios || is_mac) { "objc/Framework/Classes/PeerConnection/RTCAudioSource.mm", "objc/Framework/Classes/PeerConnection/RTCAudioTrack+Private.h", "objc/Framework/Classes/PeerConnection/RTCAudioTrack.mm", + "objc/Framework/Classes/PeerConnection/RTCBitrateAllocationStrategy.mm", "objc/Framework/Classes/PeerConnection/RTCConfiguration+Private.h", "objc/Framework/Classes/PeerConnection/RTCConfiguration.mm", "objc/Framework/Classes/PeerConnection/RTCDataChannel+Private.h", @@ -453,6 +454,7 @@ if (is_ios || is_mac) { "objc/Framework/Headers/WebRTC/RTCAVFoundationVideoSource.h", "objc/Framework/Headers/WebRTC/RTCAudioSource.h", "objc/Framework/Headers/WebRTC/RTCAudioTrack.h", + "objc/Framework/Headers/WebRTC/RTCBitrateAllocationStrategy.h", "objc/Framework/Headers/WebRTC/RTCConfiguration.h", "objc/Framework/Headers/WebRTC/RTCDataChannel.h", "objc/Framework/Headers/WebRTC/RTCDataChannelConfiguration.h", diff --git a/sdk/objc/Framework/Classes/PeerConnection/RTCBitrateAllocationStrategy.mm b/sdk/objc/Framework/Classes/PeerConnection/RTCBitrateAllocationStrategy.mm new file mode 100644 index 0000000000..bc997676dc --- /dev/null +++ b/sdk/objc/Framework/Classes/PeerConnection/RTCBitrateAllocationStrategy.mm @@ -0,0 +1,28 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "WebRTC/RTCBitrateAllocationStrategy.h" + +#include "rtc_base/bitrateallocationstrategy.h" +#include "rtc_base/checks.h" + +@implementation RTCBitrateAllocationStrategy + +@synthesize strategy = _strategy; + +- (instancetype)initWith:(rtc::BitrateAllocationStrategy*)strategy { + RTC_DCHECK(strategy); + if (self = [super init]) { + _strategy = strategy; + } + return self; +} + +@end diff --git a/sdk/objc/Framework/Classes/PeerConnection/RTCConfiguration.mm b/sdk/objc/Framework/Classes/PeerConnection/RTCConfiguration.mm index b9a9e4b137..17aed858aa 100644 --- a/sdk/objc/Framework/Classes/PeerConnection/RTCConfiguration.mm +++ b/sdk/objc/Framework/Classes/PeerConnection/RTCConfiguration.mm @@ -178,7 +178,6 @@ nativeConfig->ice_regather_interval_range = rtc::Optional(*nativeIntervalRange); } - return nativeConfig.release(); } diff --git a/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnection.mm b/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnection.mm index e443e850da..4fcc634f49 100644 --- a/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnection.mm +++ b/sdk/objc/Framework/Classes/PeerConnection/RTCPeerConnection.mm @@ -21,6 +21,7 @@ #import "RTCRtpReceiver+Private.h" #import "RTCRtpSender+Private.h" #import "RTCSessionDescription+Private.h" +#import "WebRTC/RTCBitrateAllocationStrategy.h" #import "WebRTC/RTCLogging.h" #include @@ -385,6 +386,15 @@ void PeerConnectionDelegateAdapter::OnIceCandidatesRemoved( return _peerConnection->SetBitrate(params).ok(); } +- (void)setBitrateAllocationStrategy: + (RTCBitrateAllocationStrategy *_Nullable)bitrateAllocationStrategy { + if (bitrateAllocationStrategy) + _peerConnection->SetBitrateAllocationStrategy( + std::unique_ptr(bitrateAllocationStrategy.strategy)); + else + _peerConnection->SetBitrateAllocationStrategy(nullptr); +} + - (BOOL)startRtcEventLogWithFilePath:(NSString *)filePath maxSizeInBytes:(int64_t)maxSizeInBytes { RTC_DCHECK(filePath.length); diff --git a/sdk/objc/Framework/Headers/WebRTC/RTCBitrateAllocationStrategy.h b/sdk/objc/Framework/Headers/WebRTC/RTCBitrateAllocationStrategy.h new file mode 100644 index 0000000000..c510008962 --- /dev/null +++ b/sdk/objc/Framework/Headers/WebRTC/RTCBitrateAllocationStrategy.h @@ -0,0 +1,32 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +namespace rtc { + +class BitrateAllocationStrategy; +} + +RTC_EXPORT +@interface RTCBitrateAllocationStrategy : NSObject + +- (instancetype)init NS_UNAVAILABLE; +- (instancetype)initWith:(rtc::BitrateAllocationStrategy*)strategy; + +/** Native bitrate allocation strategy. */ +@property(nonatomic, readonly) rtc::BitrateAllocationStrategy* strategy; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnection.h b/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnection.h index 7b0c4492f4..c74af1303f 100644 --- a/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnection.h +++ b/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnection.h @@ -24,6 +24,7 @@ @class RTCRtpSender; @class RTCSessionDescription; @class RTCLegacyStatsReport; +@class RTCBitrateAllocationStrategy; NS_ASSUME_NONNULL_BEGIN @@ -193,6 +194,13 @@ RTC_EXPORT currentBitrateBps:(nullable NSNumber *)currentBitrateBps maxBitrateBps:(nullable NSNumber *)maxBitrateBps; +/** Sets current strategy. If not set default WebRTC allocator will be used. + * May be changed during an active session. The strategy + * ownership is passed with std::unique_ptr + */ +- (void)setBitrateAllocationStrategy: + (RTCBitrateAllocationStrategy *_Nullable)bitrateAllocationStrategy; + /** Start or stop recording an Rtc EventLog. */ - (BOOL)startRtcEventLogWithFilePath:(NSString *)filePath maxSizeInBytes:(int64_t)maxSizeInBytes; diff --git a/voice_engine/channel.cc b/voice_engine/channel.cc index b9d785d293..01aa96525c 100644 --- a/voice_engine/channel.cc +++ b/voice_engine/channel.cc @@ -215,6 +215,10 @@ class RtpPacketSenderProxy : public RtpPacketSender { } } + void SetAccountForAudioPackets(bool account_for_audio) override { + RTC_NOTREACHED(); + } + private: rtc::ThreadChecker thread_checker_; rtc::CriticalSection crit_;