Prepare the AudioSendStream to be hooked up to send-side BWE.

This CL contains three changes as a preparation for adding audio send streams
to the send-side BWE:
1. Audio packets are passed through the pacer with high priority. This
is needed to be able to set transport sequence numbers on the packets.
2. A feedback observer is passed to the audio stream's rtcp receiver so
that the BWE can get notified of any BWE feedback being received on the
audio feedback channel.
3. Support for the transport sequence number header extension is added
to audio send streams.

BUG=webrtc:5263,webrtc:5307
R=mflodman@webrtc.org, solenberg@webrtc.org

Review URL: https://codereview.webrtc.org/1479023002 .

Cr-Commit-Position: refs/heads/master@{#10909}

talk/media/webrtc/webrtcvoiceengine.cc

@@ -530,6 +530,7 @@ void WebRtcVoiceEngine::Construct() {
         kRtpTransportSequenceNumberHeaderExtensionDefaultId));
   }
   options_ = GetDefaultEngineOptions();
+  voe_config_.Set<webrtc::VoicePacing>(new webrtc::VoicePacing(true));
 }
 
 WebRtcVoiceEngine::~WebRtcVoiceEngine() {

webrtc/audio/audio_send_stream.cc

@@ -17,6 +17,9 @@
 #include "webrtc/audio/scoped_voe_interface.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
+#include "webrtc/call/congestion_controller.h"
+#include "webrtc/modules/pacing/paced_sender.h"
+#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
 #include "webrtc/voice_engine/channel_proxy.h"
 #include "webrtc/voice_engine/include/voe_audio_processing.h"
 #include "webrtc/voice_engine/include/voe_codec.h"
@@ -55,22 +58,31 @@ std::string AudioSendStream::Config::ToString() const {
 namespace internal {
 AudioSendStream::AudioSendStream(
     const webrtc::AudioSendStream::Config& config,
-    const rtc::scoped_refptr<webrtc::AudioState>& audio_state)
+    const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
+    CongestionController* congestion_controller)
     : config_(config), audio_state_(audio_state) {
   LOG(LS_INFO) << "AudioSendStream: " << config_.ToString();
   RTC_DCHECK_NE(config_.voe_channel_id, -1);
   RTC_DCHECK(audio_state_.get());
+  RTC_DCHECK(congestion_controller);
 
   VoiceEngineImpl* voe_impl = static_cast<VoiceEngineImpl*>(voice_engine());
   channel_proxy_ = voe_impl->GetChannelProxy(config_.voe_channel_id);
+  channel_proxy_->SetCongestionControlObjects(
+      congestion_controller->pacer(),
+      congestion_controller->GetTransportFeedbackObserver(),
+      congestion_controller->packet_router());
   channel_proxy_->SetRTCPStatus(true);
   channel_proxy_->SetLocalSSRC(config.rtp.ssrc);
   channel_proxy_->SetRTCP_CNAME(config.rtp.c_name);
+
   for (const auto& extension : config.rtp.extensions) {
     if (extension.name == RtpExtension::kAbsSendTime) {
       channel_proxy_->SetSendAbsoluteSenderTimeStatus(true, extension.id);
     } else if (extension.name == RtpExtension::kAudioLevel) {
       channel_proxy_->SetSendAudioLevelIndicationStatus(true, extension.id);
+    } else if (extension.name == RtpExtension::kTransportSequenceNumber) {
+      channel_proxy_->EnableSendTransportSequenceNumber(extension.id);
     } else {
       RTC_NOTREACHED() << "Registering unsupported RTP extension.";
     }
@@ -80,6 +92,7 @@ AudioSendStream::AudioSendStream(
 AudioSendStream::~AudioSendStream() {
   RTC_DCHECK(thread_checker_.CalledOnValidThread());
   LOG(LS_INFO) << "~AudioSendStream: " << config_.ToString();
+  channel_proxy_->SetCongestionControlObjects(nullptr, nullptr, nullptr);
 }
 
 void AudioSendStream::Start() {

webrtc/audio/audio_send_stream.h

@@ -17,6 +17,7 @@
 #include "webrtc/base/scoped_ptr.h"
 
 namespace webrtc {
+class CongestionController;
 class VoiceEngine;
 
 namespace voe {
@@ -27,7 +28,8 @@ namespace internal {
 class AudioSendStream final : public webrtc::AudioSendStream {
  public:
   AudioSendStream(const webrtc::AudioSendStream::Config& config,
-                  const rtc::scoped_refptr<webrtc::AudioState>& audio_state);
+                  const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
+                  CongestionController* congestion_controller);
   ~AudioSendStream() override;
 
   // webrtc::SendStream implementation.

webrtc/audio/audio_send_stream_unittest.cc

@@ -16,8 +16,12 @@
 #include "webrtc/audio/audio_send_stream.h"
 #include "webrtc/audio/audio_state.h"
 #include "webrtc/audio/conversion.h"
+#include "webrtc/call/congestion_controller.h"
+#include "webrtc/modules/bitrate_controller/include/bitrate_controller.h"
+#include "webrtc/modules/pacing/paced_sender.h"
 #include "webrtc/test/mock_voe_channel_proxy.h"
 #include "webrtc/test/mock_voice_engine.h"
+#include "webrtc/video_engine/call_stats.h"
 
 namespace webrtc {
 namespace test {
@@ -31,6 +35,7 @@ const uint32_t kSsrc = 1234;
 const char* kCName = "foo_name";
 const int kAudioLevelId = 2;
 const int kAbsSendTimeId = 3;
+const int kTransportSequenceNumberId = 4;
 const int kEchoDelayMedian = 254;
 const int kEchoDelayStdDev = -3;
 const int kEchoReturnLoss = -65;
@@ -45,7 +50,12 @@ const uint8_t kTelephoneEventCode = 45;
 const uint32_t kTelephoneEventDuration = 6789;
 
 struct ConfigHelper {
-  ConfigHelper() : stream_config_(nullptr) {
+  ConfigHelper()
+      : stream_config_(nullptr),
+        process_thread_(ProcessThread::Create("AudioTestThread")),
+        congestion_controller_(process_thread_.get(),
+                               &call_stats_,
+                               &bitrate_observer_) {
     using testing::Invoke;
     using testing::StrEq;
 
@@ -68,6 +78,18 @@ struct ConfigHelper {
               SetSendAbsoluteSenderTimeStatus(true, kAbsSendTimeId)).Times(1);
           EXPECT_CALL(*channel_proxy_,
               SetSendAudioLevelIndicationStatus(true, kAudioLevelId)).Times(1);
+          EXPECT_CALL(*channel_proxy_, EnableSendTransportSequenceNumber(
+                                           kTransportSequenceNumberId))
+              .Times(1);
+          EXPECT_CALL(*channel_proxy_,
+                      SetCongestionControlObjects(
+                          congestion_controller_.pacer(),
+                          congestion_controller_.GetTransportFeedbackObserver(),
+                          congestion_controller_.packet_router()))
+              .Times(1);
+          EXPECT_CALL(*channel_proxy_,
+                      SetCongestionControlObjects(nullptr, nullptr, nullptr))
+              .Times(1);
           return channel_proxy_;
         }));
     stream_config_.voe_channel_id = kChannelId;
@@ -77,10 +99,15 @@ struct ConfigHelper {
         RtpExtension(RtpExtension::kAudioLevel, kAudioLevelId));
     stream_config_.rtp.extensions.push_back(
         RtpExtension(RtpExtension::kAbsSendTime, kAbsSendTimeId));
+    stream_config_.rtp.extensions.push_back(RtpExtension(
+        RtpExtension::kTransportSequenceNumber, kTransportSequenceNumberId));
   }
 
   AudioSendStream::Config& config() { return stream_config_; }
   rtc::scoped_refptr<AudioState> audio_state() { return audio_state_; }
+  CongestionController* congestion_controller() {
+    return &congestion_controller_;
+  }
 
   void SetupMockForSendTelephoneEvent() {
     EXPECT_TRUE(channel_proxy_);
@@ -126,10 +153,21 @@ struct ConfigHelper {
   }
 
  private:
+  class NullBitrateObserver : public BitrateObserver {
+   public:
+    virtual void OnNetworkChanged(uint32_t bitrate_bps,
+                                  uint8_t fraction_loss,
+                                  int64_t rtt_ms) {}
+  };
+
   testing::StrictMock<MockVoiceEngine> voice_engine_;
   rtc::scoped_refptr<AudioState> audio_state_;
   AudioSendStream::Config stream_config_;
   testing::StrictMock<MockVoEChannelProxy>* channel_proxy_ = nullptr;
+  CallStats call_stats_;
+  NullBitrateObserver bitrate_observer_;
+  rtc::scoped_ptr<ProcessThread> process_thread_;
+  CongestionController congestion_controller_;
 };
 }  // namespace
 
@@ -152,12 +190,14 @@ TEST(AudioSendStreamTest, ConfigToString) {
 
 TEST(AudioSendStreamTest, ConstructDestruct) {
   ConfigHelper helper;
-  internal::AudioSendStream send_stream(helper.config(), helper.audio_state());
+  internal::AudioSendStream send_stream(helper.config(), helper.audio_state(),
+                                        helper.congestion_controller());
 }
 
 TEST(AudioSendStreamTest, SendTelephoneEvent) {
   ConfigHelper helper;
-  internal::AudioSendStream send_stream(helper.config(), helper.audio_state());
+  internal::AudioSendStream send_stream(helper.config(), helper.audio_state(),
+                                        helper.congestion_controller());
   helper.SetupMockForSendTelephoneEvent();
   EXPECT_TRUE(send_stream.SendTelephoneEvent(kTelephoneEventPayloadType,
       kTelephoneEventCode, kTelephoneEventDuration));
@@ -165,7 +205,8 @@ TEST(AudioSendStreamTest, SendTelephoneEvent) {
 
 TEST(AudioSendStreamTest, GetStats) {
   ConfigHelper helper;
-  internal::AudioSendStream send_stream(helper.config(), helper.audio_state());
+  internal::AudioSendStream send_stream(helper.config(), helper.audio_state(),
+                                        helper.congestion_controller());
   helper.SetupMockForGetStats();
   AudioSendStream::Stats stats = send_stream.GetStats();
   EXPECT_EQ(kSsrc, stats.local_ssrc);
@@ -192,7 +233,8 @@ TEST(AudioSendStreamTest, GetStats) {
 
 TEST(AudioSendStreamTest, GetStatsTypingNoiseDetected) {
   ConfigHelper helper;
-  internal::AudioSendStream send_stream(helper.config(), helper.audio_state());
+  internal::AudioSendStream send_stream(helper.config(), helper.audio_state(),
+                                        helper.congestion_controller());
   helper.SetupMockForGetStats();
   EXPECT_FALSE(send_stream.GetStats().typing_noise_detected);
 

webrtc/audio_send_stream.h

@@ -64,7 +64,7 @@ class AudioSendStream : public SendStream {
       // Sender SSRC.
       uint32_t ssrc = 0;
 
-      // RTP header extensions used for the received stream.
+      // RTP header extensions used for the sent stream.
       std::vector<RtpExtension> extensions;
 
       // RTCP CNAME, see RFC 3550.

webrtc/call/call.cc

@@ -300,8 +300,8 @@ webrtc::AudioSendStream* Call::CreateAudioSendStream(
     const webrtc::AudioSendStream::Config& config) {
   TRACE_EVENT0("webrtc", "Call::CreateAudioSendStream");
   RTC_DCHECK(configuration_thread_checker_.CalledOnValidThread());
-  AudioSendStream* send_stream =
+  AudioSendStream* send_stream = new AudioSendStream(
-      new AudioSendStream(config, config_.audio_state);
+      config, config_.audio_state, congestion_controller_.get());
   if (!network_enabled_)
     send_stream->SignalNetworkState(kNetworkDown);
   {

webrtc/call/call_perf_tests.cc

@@ -18,6 +18,8 @@
 #include "webrtc/base/thread_annotations.h"
 #include "webrtc/call.h"
 #include "webrtc/call/transport_adapter.h"
+#include "webrtc/common.h"
+#include "webrtc/config.h"
 #include "webrtc/modules/audio_coding/include/audio_coding_module.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
 #include "webrtc/modules/rtp_rtcp/source/rtcp_utility.h"
@@ -189,6 +191,8 @@ class VideoRtcpAndSyncObserver : public SyncRtcpObserver, public VideoRenderer {
 
 void CallPerfTest::TestAudioVideoSync(bool fec, bool create_audio_first) {
   const char* kSyncGroup = "av_sync";
+  const uint32_t kAudioSendSsrc = 1234;
+  const uint32_t kAudioRecvSsrc = 5678;
   class AudioPacketReceiver : public PacketReceiver {
    public:
     AudioPacketReceiver(int channel, VoENetwork* voe_network)
@@ -228,37 +232,45 @@ void CallPerfTest::TestAudioVideoSync(bool fec, bool create_audio_first) {
   test::FakeAudioDevice fake_audio_device(Clock::GetRealTimeClock(),
                                           audio_filename);
   EXPECT_EQ(0, voe_base->Init(&fake_audio_device, nullptr));
-  int channel = voe_base->CreateChannel();
+  Config voe_config;
+  voe_config.Set<VoicePacing>(new VoicePacing(true));
+  int send_channel_id = voe_base->CreateChannel(voe_config);
+  int recv_channel_id = voe_base->CreateChannel();
 
   SyncRtcpObserver audio_observer;
 
-  AudioState::Config audio_state_config;
+  AudioState::Config send_audio_state_config;
-  audio_state_config.voice_engine = voice_engine;
+  send_audio_state_config.voice_engine = voice_engine;
+  Call::Config sender_config;
+  sender_config.audio_state = AudioState::Create(send_audio_state_config);
   Call::Config receiver_config;
-  receiver_config.audio_state = AudioState::Create(audio_state_config);
+  receiver_config.audio_state = sender_config.audio_state;
-  CreateCalls(Call::Config(), receiver_config);
+  CreateCalls(sender_config, receiver_config);
 
-  CodecInst isac = {103, "ISAC", 16000, 480, 1, 32000};
+  AudioPacketReceiver voe_send_packet_receiver(send_channel_id, voe_network);
-  EXPECT_EQ(0, voe_codec->SetSendCodec(channel, isac));
+  AudioPacketReceiver voe_recv_packet_receiver(recv_channel_id, voe_network);
-
-  AudioPacketReceiver voe_packet_receiver(channel, voe_network);
 
   FakeNetworkPipe::Config net_config;
   net_config.queue_delay_ms = 500;
   net_config.loss_percent = 5;
   test::PacketTransport audio_send_transport(
       nullptr, &audio_observer, test::PacketTransport::kSender, net_config);
-  audio_send_transport.SetReceiver(&voe_packet_receiver);
+  audio_send_transport.SetReceiver(&voe_recv_packet_receiver);
   test::PacketTransport audio_receive_transport(
       nullptr, &audio_observer, test::PacketTransport::kReceiver, net_config);
-  audio_receive_transport.SetReceiver(&voe_packet_receiver);
+  audio_receive_transport.SetReceiver(&voe_send_packet_receiver);
 
-  internal::TransportAdapter transport_adapter(&audio_send_transport);
+  internal::TransportAdapter send_transport_adapter(&audio_send_transport);
-  transport_adapter.Enable();
+  send_transport_adapter.Enable();
-  EXPECT_EQ(0,
+  EXPECT_EQ(0, voe_network->RegisterExternalTransport(send_channel_id,
-            voe_network->RegisterExternalTransport(channel, transport_adapter));
+                                                      send_transport_adapter));
 
-  VideoRtcpAndSyncObserver observer(Clock::GetRealTimeClock(), channel,
+  internal::TransportAdapter recv_transport_adapter(&audio_receive_transport);
+  recv_transport_adapter.Enable();
+  EXPECT_EQ(0, voe_network->RegisterExternalTransport(recv_channel_id,
+                                                      recv_transport_adapter));
+
+  VideoRtcpAndSyncObserver observer(Clock::GetRealTimeClock(), recv_channel_id,
                                     voe_sync, &audio_observer);
 
   test::PacketTransport sync_send_transport(sender_call_.get(), &observer,
@@ -275,6 +287,15 @@ void CallPerfTest::TestAudioVideoSync(bool fec, bool create_audio_first) {
   CreateSendConfig(1, &sync_send_transport);
   CreateMatchingReceiveConfigs(&sync_receive_transport);
 
+  AudioSendStream::Config audio_send_config(&audio_send_transport);
+  audio_send_config.voe_channel_id = send_channel_id;
+  audio_send_config.rtp.ssrc = kAudioSendSsrc;
+  AudioSendStream* audio_send_stream =
+      sender_call_->CreateAudioSendStream(audio_send_config);
+
+  CodecInst isac = {103, "ISAC", 16000, 480, 1, 32000};
+  EXPECT_EQ(0, voe_codec->SetSendCodec(send_channel_id, isac));
+
   send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
   if (fec) {
     send_config_.rtp.fec.red_payload_type = kRedPayloadType;
@@ -286,20 +307,22 @@ void CallPerfTest::TestAudioVideoSync(bool fec, bool create_audio_first) {
   receive_configs_[0].renderer = &observer;
   receive_configs_[0].sync_group = kSyncGroup;
 
-  AudioReceiveStream::Config audio_config;
+  AudioReceiveStream::Config audio_recv_config;
-  audio_config.voe_channel_id = channel;
+  audio_recv_config.rtp.remote_ssrc = kAudioSendSsrc;
-  audio_config.sync_group = kSyncGroup;
+  audio_recv_config.rtp.local_ssrc = kAudioRecvSsrc;
+  audio_recv_config.voe_channel_id = recv_channel_id;
+  audio_recv_config.sync_group = kSyncGroup;
 
-  AudioReceiveStream* audio_receive_stream = nullptr;
+  AudioReceiveStream* audio_receive_stream;
 
   if (create_audio_first) {
     audio_receive_stream =
-        receiver_call_->CreateAudioReceiveStream(audio_config);
+        receiver_call_->CreateAudioReceiveStream(audio_recv_config);
     CreateStreams();
   } else {
     CreateStreams();
     audio_receive_stream =
-        receiver_call_->CreateAudioReceiveStream(audio_config);
+        receiver_call_->CreateAudioReceiveStream(audio_recv_config);
   }
 
   CreateFrameGeneratorCapturer();
@@ -307,16 +330,16 @@ void CallPerfTest::TestAudioVideoSync(bool fec, bool create_audio_first) {
   Start();
 
   fake_audio_device.Start();
-  EXPECT_EQ(0, voe_base->StartPlayout(channel));
+  EXPECT_EQ(0, voe_base->StartPlayout(recv_channel_id));
-  EXPECT_EQ(0, voe_base->StartReceive(channel));
+  EXPECT_EQ(0, voe_base->StartReceive(recv_channel_id));
-  EXPECT_EQ(0, voe_base->StartSend(channel));
+  EXPECT_EQ(0, voe_base->StartSend(send_channel_id));
 
   EXPECT_EQ(kEventSignaled, observer.Wait())
       << "Timed out while waiting for audio and video to be synchronized.";
 
-  EXPECT_EQ(0, voe_base->StopSend(channel));
+  EXPECT_EQ(0, voe_base->StopSend(send_channel_id));
-  EXPECT_EQ(0, voe_base->StopReceive(channel));
+  EXPECT_EQ(0, voe_base->StopReceive(recv_channel_id));
-  EXPECT_EQ(0, voe_base->StopPlayout(channel));
+  EXPECT_EQ(0, voe_base->StopPlayout(recv_channel_id));
   fake_audio_device.Stop();
 
   Stop();
@@ -325,16 +348,18 @@ void CallPerfTest::TestAudioVideoSync(bool fec, bool create_audio_first) {
   audio_send_transport.StopSending();
   audio_receive_transport.StopSending();
 
-  voe_base->DeleteChannel(channel);
+  DestroyStreams();
+
+  sender_call_->DestroyAudioSendStream(audio_send_stream);
+  receiver_call_->DestroyAudioReceiveStream(audio_receive_stream);
+
+  voe_base->DeleteChannel(send_channel_id);
+  voe_base->DeleteChannel(recv_channel_id);
   voe_base->Release();
   voe_codec->Release();
   voe_network->Release();
   voe_sync->Release();
 
-  DestroyStreams();
-
-  receiver_call_->DestroyAudioReceiveStream(audio_receive_stream);
-
   DestroyCalls();
 
   VoiceEngine::Delete(voice_engine);

webrtc/config.h

@@ -138,6 +138,12 @@ struct NetEqFastAccelerate {
   bool enabled;
 };
 
+struct VoicePacing {
+  VoicePacing() : enabled(false) {}
+  explicit VoicePacing(bool value) : enabled(value) {}
+  bool enabled;
+};
+
 }  // namespace webrtc
 
 #endif  // WEBRTC_CONFIG_H_

webrtc/modules/pacing/paced_sender.cc

@@ -358,10 +358,9 @@ int32_t PacedSender::Process() {
   CriticalSectionScoped cs(critsect_.get());
   int64_t elapsed_time_ms = (now_us - time_last_update_us_ + 500) / 1000;
   time_last_update_us_ = now_us;
-  if (paused_)
-    return 0;
   int target_bitrate_kbps = max_bitrate_kbps_;
-  if (elapsed_time_ms > 0) {
+  // TODO(holmer): Remove the !paused_ check when issue 5307 has been fixed.
+  if (!paused_ && elapsed_time_ms > 0) {
     size_t queue_size_bytes = packets_->SizeInBytes();
     if (queue_size_bytes > 0) {
       // Assuming equal size packets and input/output rate, the average packet
@@ -389,7 +388,11 @@ int32_t PacedSender::Process() {
     // element from the priority queue but keep it in storage, so that we can
     // reinsert it if send fails.
     const paced_sender::Packet& packet = packets_->BeginPop();
-    if (SendPacket(packet)) {
+
+    // TODO(holmer): Because of this bug issue 5307 we have to send audio
+    // packets even when the pacer is paused. Here we assume audio packets are
+    // always high priority and that they are the only high priority packets.
+    if ((!paused_ || packet.priority == kHighPriority) && SendPacket(packet)) {
       // Send succeeded, remove it from the queue.
       packets_->FinalizePop(packet);
       if (prober_->IsProbing())
@@ -401,7 +404,8 @@ int32_t PacedSender::Process() {
     }
   }
 
-  if (!packets_->Empty())
+  // TODO(holmer): Remove the paused_ check when issue 5307 has been fixed.
+  if (paused_ || !packets_->Empty())
     return 0;
 
   size_t padding_needed;

webrtc/modules/rtp_rtcp/source/rtp_sender.cc

@@ -469,7 +469,8 @@ int32_t RTPSender::CheckPayloadType(int8_t payload_type,
   std::map<int8_t, RtpUtility::Payload*>::iterator it =
       payload_type_map_.find(payload_type);
   if (it == payload_type_map_.end()) {
-    LOG(LS_WARNING) << "Payload type " << payload_type << " not registered.";
+    LOG(LS_WARNING) << "Payload type " << static_cast<int>(payload_type)
+                    << " not registered.";
     return -1;
   }
   SetSendPayloadType(payload_type);
@@ -512,7 +513,8 @@ int32_t RTPSender::SendOutgoingData(FrameType frame_type,
   }
   RtpVideoCodecTypes video_type = kRtpVideoGeneric;
   if (CheckPayloadType(payload_type, &video_type) != 0) {
-    LOG(LS_ERROR) << "Don't send data with unknown payload type.";
+    LOG(LS_ERROR) << "Don't send data with unknown payload type: "
+                  << static_cast<int>(payload_type) << ".";
     return -1;
   }
 
@@ -725,7 +727,7 @@ int32_t RTPSender::ReSendPacket(uint16_t packet_id, int64_t min_resend_time) {
     // TickTime.
     int64_t corrected_capture_tims_ms = capture_time_ms + clock_delta_ms_;
     paced_sender_->InsertPacket(
-        RtpPacketSender::kHighPriority, header.ssrc, header.sequenceNumber,
+        RtpPacketSender::kNormalPriority, header.ssrc, header.sequenceNumber,
         corrected_capture_tims_ms, length - header.headerLength, true);
 
     return length;
@@ -1003,7 +1005,7 @@ bool RTPSender::IsFecPacket(const uint8_t* buffer,
 }
 
 size_t RTPSender::TimeToSendPadding(size_t bytes) {
-  if (bytes == 0)
+  if (audio_configured_ || bytes == 0)
     return 0;
   {
     CriticalSectionScoped cs(send_critsect_.get());

webrtc/modules/rtp_rtcp/source/rtp_sender_audio.cc

@@ -16,6 +16,7 @@
 #include "webrtc/base/trace_event.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
 #include "webrtc/modules/rtp_rtcp/source/byte_io.h"
+#include "webrtc/system_wrappers/include/tick_util.h"
 
 namespace webrtc {
 
@@ -368,7 +369,8 @@ int32_t RTPSenderAudio::SendAudio(
                            _rtpSender->Timestamp(), "seqnum",
                            _rtpSender->SequenceNumber());
     return _rtpSender->SendToNetwork(dataBuffer, payloadSize, rtpHeaderLength,
-                                     -1, kAllowRetransmission,
+                                     TickTime::MillisecondTimestamp(),
+                                     kAllowRetransmission,
                                      RtpPacketSender::kHighPriority);
   }
 
@@ -476,9 +478,9 @@ RTPSenderAudio::SendTelephoneEventPacket(bool ended,
                              "Audio::SendTelephoneEvent", "timestamp",
                              dtmfTimeStamp, "seqnum",
                              _rtpSender->SequenceNumber());
-        retVal = _rtpSender->SendToNetwork(dtmfbuffer, 4, 12, -1,
+        retVal = _rtpSender->SendToNetwork(
-                                           kAllowRetransmission,
+            dtmfbuffer, 4, 12, TickTime::MillisecondTimestamp(),
-                                           RtpPacketSender::kHighPriority);
+            kAllowRetransmission, RtpPacketSender::kHighPriority);
         sendCount--;
 
     }while (sendCount > 0 && retVal == 0);

webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc

@@ -104,7 +104,7 @@ void RTPSenderVideo::SendVideoPacket(uint8_t* data_buffer,
                                      StorageType storage) {
   if (_rtpSender.SendToNetwork(data_buffer, payload_length, rtp_header_length,
                                capture_time_ms, storage,
-                               RtpPacketSender::kNormalPriority) == 0) {
+                               RtpPacketSender::kLowPriority) == 0) {
     _videoBitrate.Update(payload_length + rtp_header_length);
     TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
                          "Video::PacketNormal", "timestamp", capture_timestamp,
@@ -150,7 +150,7 @@ void RTPSenderVideo::SendVideoPacketAsRed(uint8_t* data_buffer,
   if (_rtpSender.SendToNetwork(
           red_packet->data(), red_packet->length() - rtp_header_length,
           rtp_header_length, capture_time_ms, media_packet_storage,
-          RtpPacketSender::kNormalPriority) == 0) {
+          RtpPacketSender::kLowPriority) == 0) {
     _videoBitrate.Update(red_packet->length());
     TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
                          "Video::PacketRed", "timestamp", capture_timestamp,
@@ -162,7 +162,7 @@ void RTPSenderVideo::SendVideoPacketAsRed(uint8_t* data_buffer,
     if (_rtpSender.SendToNetwork(
             fec_packet->data(), fec_packet->length() - rtp_header_length,
             rtp_header_length, capture_time_ms, fec_storage,
-            RtpPacketSender::kNormalPriority) == 0) {
+            RtpPacketSender::kLowPriority) == 0) {
       _fecOverheadRate.Update(fec_packet->length());
       TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
                            "Video::PacketFec", "timestamp", capture_timestamp,

webrtc/test/mock_voe_channel_proxy.h

@@ -25,8 +25,13 @@ class MockVoEChannelProxy : public voe::ChannelProxy {
   MOCK_METHOD1(SetRTCP_CNAME, void(const std::string& c_name));
   MOCK_METHOD2(SetSendAbsoluteSenderTimeStatus, void(bool enable, int id));
   MOCK_METHOD2(SetSendAudioLevelIndicationStatus, void(bool enable, int id));
+  MOCK_METHOD1(EnableSendTransportSequenceNumber, void(int id));
   MOCK_METHOD2(SetReceiveAbsoluteSenderTimeStatus, void(bool enable, int id));
   MOCK_METHOD2(SetReceiveAudioLevelIndicationStatus, void(bool enable, int id));
+  MOCK_METHOD3(SetCongestionControlObjects,
+               void(RtpPacketSender* rtp_packet_sender,
+                    TransportFeedbackObserver* transport_feedback_observer,
+                    PacketRouter* seq_num_allocator));
   MOCK_CONST_METHOD0(GetRTCPStatistics, CallStatistics());
   MOCK_CONST_METHOD0(GetRemoteRTCPReportBlocks, std::vector<ReportBlock>());
   MOCK_CONST_METHOD0(GetNetworkStatistics, NetworkStatistics());

webrtc/voice_engine/BUILD.gn

@@ -106,6 +106,7 @@ source_set("voice_engine") {
     "../modules/audio_processing",
     "../modules/bitrate_controller",
     "../modules/media_file",
+    "../modules/pacing",
     "../modules/rtp_rtcp",
     "../modules/utility",
     "../system_wrappers",

webrtc/voice_engine/channel.cc

@@ -15,12 +15,14 @@
 #include "webrtc/base/checks.h"
 #include "webrtc/base/format_macros.h"
 #include "webrtc/base/logging.h"
+#include "webrtc/base/thread_checker.h"
 #include "webrtc/base/timeutils.h"
 #include "webrtc/common.h"
 #include "webrtc/config.h"
 #include "webrtc/modules/audio_device/include/audio_device.h"
 #include "webrtc/modules/audio_processing/include/audio_processing.h"
 #include "webrtc/modules/include/module_common_types.h"
+#include "webrtc/modules/pacing/packet_router.h"
 #include "webrtc/modules/rtp_rtcp/include/receive_statistics.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_payload_registry.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
@@ -44,6 +46,104 @@
 namespace webrtc {
 namespace voe {
 
+class TransportFeedbackProxy : public TransportFeedbackObserver {
+ public:
+  TransportFeedbackProxy() : feedback_observer_(nullptr) {
+    pacer_thread_.DetachFromThread();
+    network_thread_.DetachFromThread();
+  }
+
+  void SetTransportFeedbackObserver(
+      TransportFeedbackObserver* feedback_observer) {
+    RTC_DCHECK(thread_checker_.CalledOnValidThread());
+    rtc::CritScope lock(&crit_);
+    feedback_observer_ = feedback_observer;
+  }
+
+  // Implements TransportFeedbackObserver.
+  void AddPacket(uint16_t sequence_number,
+                 size_t length,
+                 bool was_paced) override {
+    RTC_DCHECK(pacer_thread_.CalledOnValidThread());
+    rtc::CritScope lock(&crit_);
+    if (feedback_observer_)
+      feedback_observer_->AddPacket(sequence_number, length, was_paced);
+  }
+  void OnTransportFeedback(const rtcp::TransportFeedback& feedback) override {
+    RTC_DCHECK(network_thread_.CalledOnValidThread());
+    rtc::CritScope lock(&crit_);
+    if (feedback_observer_)
+      feedback_observer_->OnTransportFeedback(feedback);
+  }
+
+ private:
+  rtc::CriticalSection crit_;
+  rtc::ThreadChecker thread_checker_;
+  rtc::ThreadChecker pacer_thread_;
+  rtc::ThreadChecker network_thread_;
+  TransportFeedbackObserver* feedback_observer_ GUARDED_BY(&crit_);
+};
+
+class TransportSequenceNumberProxy : public TransportSequenceNumberAllocator {
+ public:
+  TransportSequenceNumberProxy() : seq_num_allocator_(nullptr) {
+    pacer_thread_.DetachFromThread();
+  }
+
+  void SetSequenceNumberAllocator(
+      TransportSequenceNumberAllocator* seq_num_allocator) {
+    RTC_DCHECK(thread_checker_.CalledOnValidThread());
+    rtc::CritScope lock(&crit_);
+    seq_num_allocator_ = seq_num_allocator;
+  }
+
+  // Implements TransportSequenceNumberAllocator.
+  uint16_t AllocateSequenceNumber() override {
+    RTC_DCHECK(pacer_thread_.CalledOnValidThread());
+    rtc::CritScope lock(&crit_);
+    if (!seq_num_allocator_)
+      return 0;
+    return seq_num_allocator_->AllocateSequenceNumber();
+  }
+
+ private:
+  rtc::CriticalSection crit_;
+  rtc::ThreadChecker thread_checker_;
+  rtc::ThreadChecker pacer_thread_;
+  TransportSequenceNumberAllocator* seq_num_allocator_ GUARDED_BY(&crit_);
+};
+
+class RtpPacketSenderProxy : public RtpPacketSender {
+ public:
+  RtpPacketSenderProxy() : rtp_packet_sender_(nullptr) {
+  }
+
+  void SetPacketSender(RtpPacketSender* rtp_packet_sender) {
+    RTC_DCHECK(thread_checker_.CalledOnValidThread());
+    rtc::CritScope lock(&crit_);
+    rtp_packet_sender_ = rtp_packet_sender;
+  }
+
+  // Implements RtpPacketSender.
+  void InsertPacket(Priority priority,
+                    uint32_t ssrc,
+                    uint16_t sequence_number,
+                    int64_t capture_time_ms,
+                    size_t bytes,
+                    bool retransmission) override {
+    rtc::CritScope lock(&crit_);
+    if (rtp_packet_sender_) {
+      rtp_packet_sender_->InsertPacket(priority, ssrc, sequence_number,
+                                       capture_time_ms, bytes, retransmission);
+    }
+  }
+
+ private:
+  rtc::ThreadChecker thread_checker_;
+  rtc::CriticalSection crit_;
+  RtpPacketSender* rtp_packet_sender_ GUARDED_BY(&crit_);
+};
+
 // Extend the default RTCP statistics struct with max_jitter, defined as the
 // maximum jitter value seen in an RTCP report block.
 struct ChannelStatistics : public RtcpStatistics {
@@ -692,7 +792,8 @@ Channel::Channel(int32_t channelId,
                  const Config& config)
     : _fileCritSect(*CriticalSectionWrapper::CreateCriticalSection()),
       _callbackCritSect(*CriticalSectionWrapper::CreateCriticalSection()),
-    volume_settings_critsect_(*CriticalSectionWrapper::CreateCriticalSection()),
+      volume_settings_critsect_(
+          *CriticalSectionWrapper::CreateCriticalSection()),
       _instanceId(instanceId),
       _channelId(channelId),
       event_log_(event_log),
@@ -772,7 +873,14 @@ Channel::Channel(int32_t channelId,
       rtcp_observer_(new VoERtcpObserver(this)),
       network_predictor_(new NetworkPredictor(Clock::GetRealTimeClock())),
       assoc_send_channel_lock_(CriticalSectionWrapper::CreateCriticalSection()),
-    associate_send_channel_(ChannelOwner(nullptr)) {
+      associate_send_channel_(ChannelOwner(nullptr)),
+      pacing_enabled_(config.Get<VoicePacing>().enabled),
+      feedback_observer_proxy_(pacing_enabled_ ? new TransportFeedbackProxy()
+                                               : nullptr),
+      seq_num_allocator_proxy_(
+          pacing_enabled_ ? new TransportSequenceNumberProxy() : nullptr),
+      rtp_packet_sender_proxy_(pacing_enabled_ ? new RtpPacketSenderProxy()
+                                               : nullptr) {
     WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(_instanceId,_channelId),
                  "Channel::Channel() - ctor");
     AudioCodingModule::Config acm_config;
@@ -797,6 +905,10 @@ Channel::Channel(int32_t channelId,
     configuration.audio_messages = this;
     configuration.receive_statistics = rtp_receive_statistics_.get();
     configuration.bandwidth_callback = rtcp_observer_.get();
+    configuration.paced_sender = rtp_packet_sender_proxy_.get();
+    configuration.transport_sequence_number_allocator =
+        seq_num_allocator_proxy_.get();
+    configuration.transport_feedback_callback = feedback_observer_proxy_.get();
 
     _rtpRtcpModule.reset(RtpRtcp::CreateRtpRtcp(configuration));
 
@@ -2787,6 +2899,33 @@ int Channel::SetReceiveAbsoluteSenderTimeStatus(bool enable, unsigned char id) {
   return 0;
 }
 
+void Channel::EnableSendTransportSequenceNumber(int id) {
+  int ret =
+      SetSendRtpHeaderExtension(true, kRtpExtensionTransportSequenceNumber, id);
+  RTC_DCHECK_EQ(0, ret);
+}
+
+void Channel::SetCongestionControlObjects(
+    RtpPacketSender* rtp_packet_sender,
+    TransportFeedbackObserver* transport_feedback_observer,
+    PacketRouter* packet_router) {
+  RTC_DCHECK(feedback_observer_proxy_.get());
+  RTC_DCHECK(seq_num_allocator_proxy_.get());
+  RTC_DCHECK(rtp_packet_sender_proxy_.get());
+  RTC_DCHECK(packet_router != nullptr || packet_router_ != nullptr);
+  feedback_observer_proxy_->SetTransportFeedbackObserver(
+      transport_feedback_observer);
+  seq_num_allocator_proxy_->SetSequenceNumberAllocator(packet_router);
+  rtp_packet_sender_proxy_->SetPacketSender(rtp_packet_sender);
+  _rtpRtcpModule->SetStorePacketsStatus(rtp_packet_sender != nullptr, 600);
+  if (packet_router != nullptr) {
+    packet_router->AddRtpModule(_rtpRtcpModule.get());
+  } else {
+    packet_router_->RemoveRtpModule(_rtpRtcpModule.get());
+  }
+  packet_router_ = packet_router;
+}
+
 void Channel::SetRTCPStatus(bool enable) {
   WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId),
                "Channel::SetRTCPStatus()");
@@ -3165,6 +3304,8 @@ bool Channel::GetCodecFECStatus() {
 
 void Channel::SetNACKStatus(bool enable, int maxNumberOfPackets) {
   // None of these functions can fail.
+  // If pacing is enabled we always store packets.
+  if (!pacing_enabled_)
     _rtpRtcpModule->SetStorePacketsStatus(enable, maxNumberOfPackets);
   rtp_receive_statistics_->SetMaxReorderingThreshold(maxNumberOfPackets);
   rtp_receiver_->SetNACKStatus(enable ? kNackRtcp : kNackOff);

webrtc/voice_engine/channel.h

@@ -11,13 +11,13 @@
 #ifndef WEBRTC_VOICE_ENGINE_CHANNEL_H_
 #define WEBRTC_VOICE_ENGINE_CHANNEL_H_
 
+#include "webrtc/base/criticalsection.h"
 #include "webrtc/base/scoped_ptr.h"
 #include "webrtc/common_audio/resampler/include/push_resampler.h"
 #include "webrtc/common_types.h"
 #include "webrtc/modules/audio_coding/include/audio_coding_module.h"
 #include "webrtc/modules/audio_conference_mixer/include/audio_conference_mixer_defines.h"
 #include "webrtc/modules/audio_processing/rms_level.h"
-#include "webrtc/modules/bitrate_controller/include/bitrate_controller.h"
 #include "webrtc/modules/rtp_rtcp/include/remote_ntp_time_estimator.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
@@ -48,6 +48,7 @@ class AudioDeviceModule;
 class Config;
 class CriticalSectionWrapper;
 class FileWrapper;
+class PacketRouter;
 class ProcessThread;
 class ReceiveStatistics;
 class RemoteNtpTimeEstimator;
@@ -68,9 +69,12 @@ struct SenderInfo;
 namespace voe {
 
 class OutputMixer;
+class RtpPacketSenderProxy;
 class Statistics;
 class StatisticsProxy;
+class TransportFeedbackProxy;
 class TransmitMixer;
+class TransportSequenceNumberProxy;
 class VoERtcpObserver;
 
 // Helper class to simplify locking scheme for members that are accessed from
@@ -321,6 +325,13 @@ public:
     int SetReceiveAudioLevelIndicationStatus(bool enable, unsigned char id);
     int SetSendAbsoluteSenderTimeStatus(bool enable, unsigned char id);
     int SetReceiveAbsoluteSenderTimeStatus(bool enable, unsigned char id);
+    void EnableSendTransportSequenceNumber(int id);
+
+    void SetCongestionControlObjects(
+        RtpPacketSender* rtp_packet_sender,
+        TransportFeedbackObserver* transport_feedback_observer,
+        PacketRouter* packet_router);
+
     void SetRTCPStatus(bool enable);
     int GetRTCPStatus(bool& enabled);
     int SetRTCP_CNAME(const char cName[256]);
@@ -584,6 +595,12 @@ private:
     // An associated send channel.
     rtc::scoped_ptr<CriticalSectionWrapper> assoc_send_channel_lock_;
     ChannelOwner associate_send_channel_ GUARDED_BY(assoc_send_channel_lock_);
+
+    bool pacing_enabled_;
+    PacketRouter* packet_router_ = nullptr;
+    rtc::scoped_ptr<TransportFeedbackProxy> feedback_observer_proxy_;
+    rtc::scoped_ptr<TransportSequenceNumberProxy> seq_num_allocator_proxy_;
+    rtc::scoped_ptr<RtpPacketSenderProxy> rtp_packet_sender_proxy_;
 };
 
 }  // namespace voe

webrtc/voice_engine/channel_proxy.cc

@@ -52,6 +52,11 @@ void ChannelProxy::SetSendAudioLevelIndicationStatus(bool enable, int id) {
   RTC_DCHECK_EQ(0, error);
 }
 
+void ChannelProxy::EnableSendTransportSequenceNumber(int id) {
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  channel()->EnableSendTransportSequenceNumber(id);
+}
+
 void ChannelProxy::SetReceiveAbsoluteSenderTimeStatus(bool enable, int id) {
   RTC_DCHECK(thread_checker_.CalledOnValidThread());
   int error = channel()->SetReceiveAbsoluteSenderTimeStatus(enable, id);
@@ -64,6 +69,15 @@ void ChannelProxy::SetReceiveAudioLevelIndicationStatus(bool enable, int id) {
   RTC_DCHECK_EQ(0, error);
 }
 
+void ChannelProxy::SetCongestionControlObjects(
+    RtpPacketSender* rtp_packet_sender,
+    TransportFeedbackObserver* transport_feedback_observer,
+    PacketRouter* packet_router) {
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  channel()->SetCongestionControlObjects(
+      rtp_packet_sender, transport_feedback_observer, packet_router);
+}
+
 CallStatistics ChannelProxy::GetRTCPStatistics() const {
   RTC_DCHECK(thread_checker_.CalledOnValidThread());
   CallStatistics stats = {0};
@@ -124,5 +138,6 @@ Channel* ChannelProxy::channel() const {
   RTC_DCHECK(channel_owner_.channel());
   return channel_owner_.channel();
 }
+
 }  // namespace voe
 }  // namespace webrtc

webrtc/voice_engine/channel_proxy.h

@@ -19,6 +19,11 @@
 #include <vector>
 
 namespace webrtc {
+
+class PacketRouter;
+class RtpPacketSender;
+class TransportFeedbackObserver;
+
 namespace voe {
 
 class Channel;
@@ -41,8 +46,13 @@ class ChannelProxy {
   virtual void SetRTCP_CNAME(const std::string& c_name);
   virtual void SetSendAbsoluteSenderTimeStatus(bool enable, int id);
   virtual void SetSendAudioLevelIndicationStatus(bool enable, int id);
+  virtual void EnableSendTransportSequenceNumber(int id);
   virtual void SetReceiveAbsoluteSenderTimeStatus(bool enable, int id);
   virtual void SetReceiveAudioLevelIndicationStatus(bool enable, int id);
+  virtual void SetCongestionControlObjects(
+      RtpPacketSender* rtp_packet_sender,
+      TransportFeedbackObserver* transport_feedback_observer,
+      PacketRouter* packet_router);
 
   virtual CallStatistics GetRTCPStatistics() const;
   virtual std::vector<ReportBlock> GetRemoteRTCPReportBlocks() const;

webrtc/voice_engine/voice_engine.gyp

@@ -23,6 +23,7 @@
         '<(webrtc_root)/modules/modules.gyp:audio_processing',
         '<(webrtc_root)/modules/modules.gyp:bitrate_controller',
         '<(webrtc_root)/modules/modules.gyp:media_file',
+        '<(webrtc_root)/modules/modules.gyp:paced_sender',
         '<(webrtc_root)/modules/modules.gyp:rtp_rtcp',
         '<(webrtc_root)/modules/modules.gyp:webrtc_utility',
         '<(webrtc_root)/system_wrappers/system_wrappers.gyp:system_wrappers',