From b3944f021dcb42e47fe64667e5470d36fe28c30a Mon Sep 17 00:00:00 2001 From: Alex Narest Date: Fri, 13 Oct 2017 14:56:18 +0200 Subject: [PATCH] Media track ID visibility at BWE level Track IDs are assigned by application during track creation. Track IDs are used by custom bitrate allocation strategies to identify tracks. Track ID can be empty, in that case bitrate allocation strategies will not be able to handle these tracks specifically and will handle them as a default. Bug: webrtc:8243 Change-Id: I89987e33328320bfd0539ad532342df6da144c98 Reviewed-on: https://webrtc-review.googlesource.com/4820 Commit-Queue: Alex Narest Reviewed-by: Taylor Brandstetter Reviewed-by: Niels Moller Reviewed-by: Stefan Holmer Cr-Commit-Position: refs/heads/master@{#20285} --- audio/audio_send_stream.cc | 2 +- call/audio_send_stream.h | 3 ++ call/bitrate_allocator.cc | 5 ++- call/bitrate_allocator.h | 11 ++++-- call/bitrate_allocator_unittest.cc | 47 +++++++++++------------ call/video_send_stream.h | 3 ++ examples/objc/AppRTCMobile/ARDAppClient.m | 30 +++++---------- media/engine/webrtcvideoengine.cc | 1 + media/engine/webrtcvoiceengine.cc | 4 +- video/video_send_stream.cc | 9 +++-- 10 files changed, 60 insertions(+), 55 deletions(-) diff --git a/audio/audio_send_stream.cc b/audio/audio_send_stream.cc index 62620a44f2..1cfc0bad1e 100644 --- a/audio/audio_send_stream.cc +++ b/audio/audio_send_stream.cc @@ -627,7 +627,7 @@ void AudioSendStream::ConfigureBitrateObserver(int min_bitrate_bps, config_.min_bitrate_bps = min_bitrate_bps; config_.max_bitrate_bps = max_bitrate_bps; bitrate_allocator_->AddObserver(this, min_bitrate_bps, max_bitrate_bps, 0, - true); + true, config_.track_id); thread_sync_event.Set(); }); thread_sync_event.Wait(rtc::Event::kForever); diff --git a/call/audio_send_stream.h b/call/audio_send_stream.h index 9b0f0815bb..9ed0d1e63f 100644 --- a/call/audio_send_stream.h +++ b/call/audio_send_stream.h @@ -131,6 +131,9 @@ class AudioSendStream { rtc::Optional send_codec_spec; rtc::scoped_refptr encoder_factory; + + // Track ID as specified during track creation. + std::string track_id; }; virtual ~AudioSendStream() = default; diff --git a/call/bitrate_allocator.cc b/call/bitrate_allocator.cc index 8570ae378d..b379f7f086 100644 --- a/call/bitrate_allocator.cc +++ b/call/bitrate_allocator.cc @@ -124,7 +124,8 @@ void BitrateAllocator::AddObserver(BitrateAllocatorObserver* observer, uint32_t min_bitrate_bps, uint32_t max_bitrate_bps, uint32_t pad_up_bitrate_bps, - bool enforce_min_bitrate) { + bool enforce_min_bitrate, + std::string track_id) { RTC_DCHECK_CALLED_SEQUENTIALLY(&sequenced_checker_); auto it = FindObserverConfig(observer); @@ -137,7 +138,7 @@ void BitrateAllocator::AddObserver(BitrateAllocatorObserver* observer, } else { bitrate_observer_configs_.push_back( ObserverConfig(observer, min_bitrate_bps, max_bitrate_bps, - pad_up_bitrate_bps, enforce_min_bitrate)); + pad_up_bitrate_bps, enforce_min_bitrate, track_id)); } ObserverAllocation allocation; diff --git a/call/bitrate_allocator.h b/call/bitrate_allocator.h index 1ac348770d..d51740cc1b 100644 --- a/call/bitrate_allocator.h +++ b/call/bitrate_allocator.h @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -82,7 +83,8 @@ class BitrateAllocator { uint32_t min_bitrate_bps, uint32_t max_bitrate_bps, uint32_t pad_up_bitrate_bps, - bool enforce_min_bitrate); + bool enforce_min_bitrate, + std::string track_id); // Removes a previously added observer, but will not trigger a new bitrate // allocation. @@ -99,14 +101,16 @@ class BitrateAllocator { uint32_t min_bitrate_bps, uint32_t max_bitrate_bps, uint32_t pad_up_bitrate_bps, - bool enforce_min_bitrate) + bool enforce_min_bitrate, + std::string track_id) : observer(observer), min_bitrate_bps(min_bitrate_bps), max_bitrate_bps(max_bitrate_bps), pad_up_bitrate_bps(pad_up_bitrate_bps), enforce_min_bitrate(enforce_min_bitrate), allocated_bitrate_bps(-1), - media_ratio(1.0) {} + media_ratio(1.0), + track_id(track_id) {} BitrateAllocatorObserver* observer; uint32_t min_bitrate_bps; @@ -115,6 +119,7 @@ class BitrateAllocator { bool enforce_min_bitrate; int64_t allocated_bitrate_bps; double media_ratio; // Part of the total bitrate used for media [0.0, 1.0]. + std::string track_id; }; // Calculates the minimum requested send bitrate and max padding bitrate and diff --git a/call/bitrate_allocator_unittest.cc b/call/bitrate_allocator_unittest.cc index 03ad78cdfe..31828b6cc1 100644 --- a/call/bitrate_allocator_unittest.cc +++ b/call/bitrate_allocator_unittest.cc @@ -81,7 +81,7 @@ TEST_F(BitrateAllocatorTest, UpdatingBitrateObserver) { EXPECT_CALL(limit_observer_, OnAllocationLimitsChanged(kMinSendBitrateBps, kPadUpToBitrateBps)); allocator_->AddObserver(&bitrate_observer, kMinSendBitrateBps, 1500000, - kPadUpToBitrateBps, true); + kPadUpToBitrateBps, true, ""); EXPECT_EQ(300000, allocator_->GetStartBitrate(&bitrate_observer)); allocator_->OnNetworkChanged(200000, 0, 0, kDefaultProbingIntervalMs); EXPECT_EQ(200000, allocator_->GetStartBitrate(&bitrate_observer)); @@ -95,11 +95,11 @@ TEST_F(BitrateAllocatorTest, UpdatingBitrateObserver) { EXPECT_CALL(limit_observer_, OnAllocationLimitsChanged(kMinSendBitrateBps, 0)); allocator_->AddObserver(&bitrate_observer, kMinSendBitrateBps, 4000000, 0, - true); + true, ""); EXPECT_EQ(4000000, allocator_->GetStartBitrate(&bitrate_observer)); allocator_->AddObserver(&bitrate_observer, kMinSendBitrateBps, 1500000, 0, - true); + true, ""); EXPECT_EQ(3000000, allocator_->GetStartBitrate(&bitrate_observer)); EXPECT_EQ(3000000u, bitrate_observer.last_bitrate_bps_); allocator_->OnNetworkChanged(1500000, 0, 0, kDefaultProbingIntervalMs); @@ -110,11 +110,11 @@ TEST_F(BitrateAllocatorTest, TwoBitrateObserversOneRtcpObserver) { TestBitrateObserver bitrate_observer_1; TestBitrateObserver bitrate_observer_2; EXPECT_CALL(limit_observer_, OnAllocationLimitsChanged(100000, 0)); - allocator_->AddObserver(&bitrate_observer_1, 100000, 300000, 0, true); + allocator_->AddObserver(&bitrate_observer_1, 100000, 300000, 0, true, ""); EXPECT_EQ(300000, allocator_->GetStartBitrate(&bitrate_observer_1)); EXPECT_CALL(limit_observer_, OnAllocationLimitsChanged(100000 + 200000, 0)); - allocator_->AddObserver(&bitrate_observer_2, 200000, 300000, 0, true); + allocator_->AddObserver(&bitrate_observer_2, 200000, 300000, 0, true, ""); EXPECT_EQ(200000, allocator_->GetStartBitrate(&bitrate_observer_2)); // Test too low start bitrate, hence lower than sum of min. Min bitrates @@ -158,7 +158,7 @@ TEST_F(BitrateAllocatorTest, RemoveObserverTriggersLimitObserver) { EXPECT_CALL(limit_observer_, OnAllocationLimitsChanged(kMinSendBitrateBps, kPadUpToBitrateBps)); allocator_->AddObserver(&bitrate_observer, kMinSendBitrateBps, 1500000, - kPadUpToBitrateBps, true); + kPadUpToBitrateBps, true, ""); EXPECT_CALL(limit_observer_, OnAllocationLimitsChanged(0, 0)); allocator_->RemoveObserver(&bitrate_observer); } @@ -182,7 +182,7 @@ TEST_F(BitrateAllocatorTestNoEnforceMin, OneBitrateObserver) { // Expect OnAllocationLimitsChanged with |min_send_bitrate_bps| = 0 since // AddObserver is called with |enforce_min_bitrate| = false. EXPECT_CALL(limit_observer_, OnAllocationLimitsChanged(0, 120000)); - allocator_->AddObserver(&bitrate_observer_1, 100000, 400000, 0, false); + allocator_->AddObserver(&bitrate_observer_1, 100000, 400000, 0, false, ""); EXPECT_EQ(300000, allocator_->GetStartBitrate(&bitrate_observer_1)); // High BWE. @@ -202,14 +202,14 @@ TEST_F(BitrateAllocatorTestNoEnforceMin, ThreeBitrateObservers) { TestBitrateObserver bitrate_observer_2; TestBitrateObserver bitrate_observer_3; // Set up the observers with min bitrates at 100000, 200000, and 300000. - allocator_->AddObserver(&bitrate_observer_1, 100000, 400000, 0, false); + allocator_->AddObserver(&bitrate_observer_1, 100000, 400000, 0, false, ""); EXPECT_EQ(300000, allocator_->GetStartBitrate(&bitrate_observer_1)); - allocator_->AddObserver(&bitrate_observer_2, 200000, 400000, 0, false); + allocator_->AddObserver(&bitrate_observer_2, 200000, 400000, 0, false, ""); EXPECT_EQ(200000, allocator_->GetStartBitrate(&bitrate_observer_2)); EXPECT_EQ(100000u, bitrate_observer_1.last_bitrate_bps_); - allocator_->AddObserver(&bitrate_observer_3, 300000, 400000, 0, false); + allocator_->AddObserver(&bitrate_observer_3, 300000, 400000, 0, false, ""); EXPECT_EQ(0, allocator_->GetStartBitrate(&bitrate_observer_3)); EXPECT_EQ(100000u, bitrate_observer_1.last_bitrate_bps_); EXPECT_EQ(200000u, bitrate_observer_2.last_bitrate_bps_); @@ -263,8 +263,7 @@ TEST_F(BitrateAllocatorTestNoEnforceMin, OneBitrateObserverWithPacketLoss) { // Expect OnAllocationLimitsChanged with |min_send_bitrate_bps| = 0 since // AddObserver is called with |enforce_min_bitrate| = false. EXPECT_CALL(limit_observer_, OnAllocationLimitsChanged(0, 168000)); - allocator_->AddObserver( - &bitrate_observer, 100000, 400000, 0, false); + allocator_->AddObserver(&bitrate_observer, 100000, 400000, 0, false, ""); EXPECT_EQ(300000, allocator_->GetStartBitrate(&bitrate_observer)); // High BWE. @@ -310,9 +309,9 @@ TEST_F(BitrateAllocatorTestNoEnforceMin, TwoBitrateObserverWithPacketLoss) { TestBitrateObserver bitrate_observer_1; TestBitrateObserver bitrate_observer_2; - allocator_->AddObserver(&bitrate_observer_1, 100000, 400000, 0, false); + allocator_->AddObserver(&bitrate_observer_1, 100000, 400000, 0, false, ""); EXPECT_EQ(300000, allocator_->GetStartBitrate(&bitrate_observer_1)); - allocator_->AddObserver(&bitrate_observer_2, 200000, 400000, 0, false); + allocator_->AddObserver(&bitrate_observer_2, 200000, 400000, 0, false, ""); EXPECT_EQ(200000, allocator_->GetStartBitrate(&bitrate_observer_2)); EXPECT_EQ(100000u, bitrate_observer_1.last_bitrate_bps_); @@ -361,14 +360,14 @@ TEST_F(BitrateAllocatorTest, ThreeBitrateObserversLowBweEnforceMin) { TestBitrateObserver bitrate_observer_2; TestBitrateObserver bitrate_observer_3; - allocator_->AddObserver(&bitrate_observer_1, 100000, 400000, 0, true); + allocator_->AddObserver(&bitrate_observer_1, 100000, 400000, 0, true, ""); EXPECT_EQ(300000, allocator_->GetStartBitrate(&bitrate_observer_1)); - allocator_->AddObserver(&bitrate_observer_2, 200000, 400000, 0, true); + allocator_->AddObserver(&bitrate_observer_2, 200000, 400000, 0, true, ""); EXPECT_EQ(200000, allocator_->GetStartBitrate(&bitrate_observer_2)); EXPECT_EQ(100000u, bitrate_observer_1.last_bitrate_bps_); - allocator_->AddObserver(&bitrate_observer_3, 300000, 400000, 0, true); + allocator_->AddObserver(&bitrate_observer_3, 300000, 400000, 0, true, ""); EXPECT_EQ(300000, allocator_->GetStartBitrate(&bitrate_observer_3)); EXPECT_EQ(100000, static_cast(bitrate_observer_1.last_bitrate_bps_)); EXPECT_EQ(200000, static_cast(bitrate_observer_2.last_bitrate_bps_)); @@ -389,7 +388,7 @@ TEST_F(BitrateAllocatorTest, AddObserverWhileNetworkDown) { TestBitrateObserver bitrate_observer_1; EXPECT_CALL(limit_observer_, OnAllocationLimitsChanged(50000, 0)); - allocator_->AddObserver(&bitrate_observer_1, 50000, 400000, 0, true); + allocator_->AddObserver(&bitrate_observer_1, 50000, 400000, 0, true, ""); EXPECT_EQ(300000, allocator_->GetStartBitrate(&bitrate_observer_1)); // Set network down, ie, no available bitrate. @@ -400,7 +399,7 @@ TEST_F(BitrateAllocatorTest, AddObserverWhileNetworkDown) { TestBitrateObserver bitrate_observer_2; // Adding an observer while the network is down should not affect the limits. EXPECT_CALL(limit_observer_, OnAllocationLimitsChanged(50000 + 50000, 0)); - allocator_->AddObserver(&bitrate_observer_2, 50000, 400000, 0, true); + allocator_->AddObserver(&bitrate_observer_2, 50000, 400000, 0, true, ""); // Expect the start_bitrate to be set as if the network was still up but that // the new observer have been notified that the network is down. @@ -416,11 +415,11 @@ TEST_F(BitrateAllocatorTest, AddObserverWhileNetworkDown) { TEST_F(BitrateAllocatorTest, MixedEnforecedConfigs) { TestBitrateObserver enforced_observer; - allocator_->AddObserver(&enforced_observer, 6000, 30000, 0, true); + allocator_->AddObserver(&enforced_observer, 6000, 30000, 0, true, ""); EXPECT_EQ(60000, allocator_->GetStartBitrate(&enforced_observer)); TestBitrateObserver not_enforced_observer; - allocator_->AddObserver(¬_enforced_observer, 30000, 2500000, 0, false); + allocator_->AddObserver(¬_enforced_observer, 30000, 2500000, 0, false, ""); EXPECT_EQ(270000, allocator_->GetStartBitrate(¬_enforced_observer)); EXPECT_EQ(30000u, enforced_observer.last_bitrate_bps_); @@ -458,7 +457,7 @@ TEST_F(BitrateAllocatorTest, MixedEnforecedConfigs) { TEST_F(BitrateAllocatorTest, AvoidToggleAbsolute) { TestBitrateObserver observer; - allocator_->AddObserver(&observer, 30000, 300000, 0, false); + allocator_->AddObserver(&observer, 30000, 300000, 0, false, ""); EXPECT_EQ(300000, allocator_->GetStartBitrate(&observer)); allocator_->OnNetworkChanged(30000, 0, 50, kDefaultProbingIntervalMs); @@ -484,7 +483,7 @@ TEST_F(BitrateAllocatorTest, AvoidToggleAbsolute) { TEST_F(BitrateAllocatorTest, AvoidTogglePercent) { TestBitrateObserver observer; - allocator_->AddObserver(&observer, 300000, 600000, 0, false); + allocator_->AddObserver(&observer, 300000, 600000, 0, false, ""); EXPECT_EQ(300000, allocator_->GetStartBitrate(&observer)); allocator_->OnNetworkChanged(300000, 0, 50, kDefaultProbingIntervalMs); @@ -510,7 +509,7 @@ TEST_F(BitrateAllocatorTest, AvoidTogglePercent) { TEST_F(BitrateAllocatorTest, PassProbingInterval) { TestBitrateObserver observer; - allocator_->AddObserver(&observer, 300000, 600000, 0, false); + allocator_->AddObserver(&observer, 300000, 600000, 0, false, ""); EXPECT_EQ(300000, allocator_->GetStartBitrate(&observer)); allocator_->OnNetworkChanged(300000, 0, 50, 5000); diff --git a/call/video_send_stream.h b/call/video_send_stream.h index c8f41fbd34..1bc08cb21b 100644 --- a/call/video_send_stream.h +++ b/call/video_send_stream.h @@ -220,6 +220,9 @@ class VideoSendStream { // Enables periodic bandwidth probing in application-limited region. bool periodic_alr_bandwidth_probing = false; + // Track ID as specified during track creation. + std::string track_id; + private: // Access to the copy constructor is private to force use of the Copy() // method for those exceptional cases where we do use it. diff --git a/examples/objc/AppRTCMobile/ARDAppClient.m b/examples/objc/AppRTCMobile/ARDAppClient.m index 2dfc617637..c6d69c8f9f 100644 --- a/examples/objc/AppRTCMobile/ARDAppClient.m +++ b/examples/objc/AppRTCMobile/ARDAppClient.m @@ -538,8 +538,7 @@ static int const kKbpsMultiplier = 1000; constraints:constraints delegate:self]; // Create AV senders. - [self createAudioSender]; - [self createVideoSender]; + [self createMediaSenders]; if (_isInitiator) { // Send offer. __weak ARDAppClient *weakSelf = self; @@ -663,19 +662,6 @@ static int const kKbpsMultiplier = 1000; } } -- (RTCRtpSender *)createVideoSender { - RTCRtpSender *sender = - [_peerConnection senderWithKind:kRTCMediaStreamTrackKindVideo - streamId:kARDMediaStreamId]; - _localVideoTrack = [self createLocalVideoTrack]; - if (_localVideoTrack) { - sender.track = _localVideoTrack; - [_delegate appClient:self didReceiveLocalVideoTrack:_localVideoTrack]; - } - - return sender; -} - - (void)setMaxBitrateForPeerConnectionVideoSender { for (RTCRtpSender *sender in _peerConnection.senders) { if (sender.track != nil) { @@ -698,16 +684,18 @@ static int const kKbpsMultiplier = 1000; [sender setParameters:parametersToModify]; } -- (RTCRtpSender *)createAudioSender { +- (void)createMediaSenders { RTCMediaConstraints *constraints = [self defaultMediaAudioConstraints]; RTCAudioSource *source = [_factory audioSourceWithConstraints:constraints]; RTCAudioTrack *track = [_factory audioTrackWithSource:source trackId:kARDAudioTrackId]; - RTCRtpSender *sender = - [_peerConnection senderWithKind:kRTCMediaStreamTrackKindAudio - streamId:kARDMediaStreamId]; - sender.track = track; - return sender; + RTCMediaStream *stream = [_factory mediaStreamWithStreamId:kARDMediaStreamId]; + [stream addAudioTrack:track]; + _localVideoTrack = [self createLocalVideoTrack]; + if (_localVideoTrack) { + [stream addVideoTrack:_localVideoTrack]; + } + [_peerConnection addStream:stream]; } - (RTCVideoTrack *)createLocalVideoTrack { diff --git a/media/engine/webrtcvideoengine.cc b/media/engine/webrtcvideoengine.cc index 5db4d207a2..80b450789e 100644 --- a/media/engine/webrtcvideoengine.cc +++ b/media/engine/webrtcvideoengine.cc @@ -1597,6 +1597,7 @@ WebRtcVideoChannel::WebRtcVideoSendStream::WebRtcVideoSendStream( } parameters_.config.rtp.c_name = sp.cname; + parameters_.config.track_id = sp.id; if (rtp_extensions) { parameters_.config.rtp.extensions = *rtp_extensions; } diff --git a/media/engine/webrtcvoiceengine.cc b/media/engine/webrtcvoiceengine.cc index a6875f8198..c2e310a311 100644 --- a/media/engine/webrtcvoiceengine.cc +++ b/media/engine/webrtcvoiceengine.cc @@ -809,6 +809,7 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream webrtc::AudioTransport* voe_audio_transport, uint32_t ssrc, const std::string& c_name, + const std::string track_id, const rtc::Optional& send_codec_spec, const std::vector& extensions, @@ -835,6 +836,7 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream config_.rtp.extensions = extensions; config_.audio_network_adaptor_config = audio_network_adaptor_config; config_.encoder_factory = encoder_factory; + config_.track_id = track_id; rtp_parameters_.encodings[0].ssrc = rtc::Optional(ssrc); if (send_codec_spec) { @@ -1836,7 +1838,7 @@ bool WebRtcVoiceMediaChannel::AddSendStream(const StreamParams& sp) { rtc::Optional audio_network_adaptor_config = GetAudioNetworkAdaptorConfig(options_); WebRtcAudioSendStream* stream = new WebRtcAudioSendStream( - channel, audio_transport, ssrc, sp.cname, send_codec_spec_, + channel, audio_transport, ssrc, sp.cname, sp.id, send_codec_spec_, send_rtp_extensions_, max_send_bitrate_bps_, audio_network_adaptor_config, call_, this, engine()->encoder_factory_); send_streams_.insert(std::make_pair(ssrc, stream)); diff --git a/video/video_send_stream.cc b/video/video_send_stream.cc index f0380b2f3c..53e76cf15f 100644 --- a/video/video_send_stream.cc +++ b/video/video_send_stream.cc @@ -842,7 +842,8 @@ void VideoSendStreamImpl::Start() { bitrate_allocator_->AddObserver( this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_, - max_padding_bitrate_, !config_->suspend_below_min_bitrate); + max_padding_bitrate_, !config_->suspend_below_min_bitrate, + config_->track_id); // Start monitoring encoder activity. { @@ -895,7 +896,8 @@ void VideoSendStreamImpl::SignalEncoderActive() { LOG(LS_INFO) << "SignalEncoderActive, Encoder is active."; bitrate_allocator_->AddObserver( this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_, - max_padding_bitrate_, !config_->suspend_below_min_bitrate); + max_padding_bitrate_, !config_->suspend_below_min_bitrate, + config_->track_id); } void VideoSendStreamImpl::OnEncoderConfigurationChanged( @@ -937,7 +939,8 @@ void VideoSendStreamImpl::OnEncoderConfigurationChanged( // limits. bitrate_allocator_->AddObserver( this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_, - max_padding_bitrate_, !config_->suspend_below_min_bitrate); + max_padding_bitrate_, !config_->suspend_below_min_bitrate, + config_->track_id); } }