From 41b9c801c2b8153e43485a583ae512e7ad486123 Mon Sep 17 00:00:00 2001 From: minyue Date: Thu, 6 Oct 2016 07:13:54 -0700 Subject: [PATCH] Adding audio network adaptor to AudioEncoderOpus. BUG=webrtc:6303 Review-Url: https://codereview.webrtc.org/2362703002 Cr-Commit-Position: refs/heads/master@{#14555} --- webrtc/modules/audio_coding/BUILD.gn | 8 +- .../audio_network_adaptor.gypi | 4 + .../mock/mock_audio_network_adaptor.h | 45 +++ .../mock/mock_controller_manager.h | 1 - .../audio_coding/codecs/audio_encoder.cc | 19 ++ .../audio_coding/codecs/audio_encoder.h | 27 ++ .../codecs/opus/audio_encoder_opus.cc | 146 +++++++++- .../codecs/opus/audio_encoder_opus.h | 36 ++- .../opus/audio_encoder_opus_unittest.cc | 271 ++++++++++++++---- .../audio_coding/codecs/opus/opus.gypi | 1 + .../audio_coding/codecs/opus/opus_interface.c | 2 +- .../audio_coding/codecs/opus/opus_interface.h | 2 +- 12 files changed, 489 insertions(+), 73 deletions(-) create mode 100644 webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h diff --git a/webrtc/modules/audio_coding/BUILD.gn b/webrtc/modules/audio_coding/BUILD.gn index d1dc1edad8..ea633ae91b 100644 --- a/webrtc/modules/audio_coding/BUILD.gn +++ b/webrtc/modules/audio_coding/BUILD.gn @@ -682,6 +682,7 @@ rtc_static_library("webrtc_opus") { deps = [ ":audio_decoder_interface", ":audio_encoder_interface", + ":audio_network_adaptor", "../../base:rtc_base_approved", ] @@ -737,8 +738,13 @@ source_set("audio_network_adaptor") { configs += [ "../..:common_config" ] public_configs = [ "../..:common_inherited_config" ] + deps = [ + "../..:webrtc_common", + "../../system_wrappers", + ] + if (rtc_enable_protobuf) { - deps = [ + deps += [ ":ana_config_proto", ":ana_debug_dump_proto", ] diff --git a/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor.gypi b/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor.gypi index 7750276563..d696f84b3f 100644 --- a/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor.gypi +++ b/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor.gypi @@ -34,6 +34,10 @@ 'smoothing_filter.h', 'smoothing_filter.cc', ], # sources + 'dependencies': [ + '<(webrtc_root)/common.gyp:webrtc_common', + '<(webrtc_root)/system_wrappers/system_wrappers.gyp:system_wrappers', + ], 'conditions': [ ['enable_protobuf==1', { 'dependencies': [ diff --git a/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h b/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h new file mode 100644 index 0000000000..7d5b4e568d --- /dev/null +++ b/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_AUDIO_NETWORK_ADAPTOR_H_ +#define WEBRTC_MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_AUDIO_NETWORK_ADAPTOR_H_ + +#include "webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" +#include "webrtc/test/gmock.h" + +namespace webrtc { + +class MockAudioNetworkAdaptor : public AudioNetworkAdaptor { + public: + virtual ~MockAudioNetworkAdaptor() { Die(); } + MOCK_METHOD0(Die, void()); + + MOCK_METHOD1(SetUplinkBandwidth, void(int uplink_bandwidth_bps)); + + MOCK_METHOD1(SetUplinkPacketLossFraction, + void(float uplink_packet_loss_fraction)); + + MOCK_METHOD1(SetRtt, void(int rtt_ms)); + + MOCK_METHOD1(SetTargetAudioBitrate, void(int target_audio_bitrate_bps)); + + MOCK_METHOD2(SetReceiverFrameLengthRange, + void(int min_frame_length_ms, int max_frame_length_ms)); + + MOCK_METHOD0(GetEncoderRuntimeConfig, EncoderRuntimeConfig()); + + MOCK_METHOD1(StartDebugDump, void(FILE* file_handle)); + + MOCK_METHOD0(StopDebugDump, void()); +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_AUDIO_NETWORK_ADAPTOR_H_ diff --git a/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h b/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h index 5a5808f15e..4976fd8d87 100644 --- a/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h +++ b/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h @@ -11,7 +11,6 @@ #ifndef WEBRTC_MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_MANAGER_H_ #define WEBRTC_MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_MOCK_MOCK_CONTROLLER_MANAGER_H_ -#include #include #include "webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h" diff --git a/webrtc/modules/audio_coding/codecs/audio_encoder.cc b/webrtc/modules/audio_coding/codecs/audio_encoder.cc index c433dcdd0f..1216484b4d 100644 --- a/webrtc/modules/audio_coding/codecs/audio_encoder.cc +++ b/webrtc/modules/audio_coding/codecs/audio_encoder.cc @@ -67,4 +67,23 @@ void AudioEncoder::SetTargetBitrate(int target_bps) {} rtc::ArrayView> AudioEncoder::ReclaimContainedEncoders() { return nullptr; } +bool AudioEncoder::EnableAudioNetworkAdaptor(const std::string& config_string, + const Clock* clock) { + return false; +} + +void AudioEncoder::DisableAudioNetworkAdaptor() {} + +void AudioEncoder::OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) {} + +void AudioEncoder::OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) {} + +void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) {} + +void AudioEncoder::OnReceivedRtt(int rtt_ms) {} + +void AudioEncoder::SetReceiverFrameLengthRange(int min_frame_length_ms, + int max_frame_length_ms) {} + } // namespace webrtc diff --git a/webrtc/modules/audio_coding/codecs/audio_encoder.h b/webrtc/modules/audio_coding/codecs/audio_encoder.h index f09525f145..19dc332757 100644 --- a/webrtc/modules/audio_coding/codecs/audio_encoder.h +++ b/webrtc/modules/audio_coding/codecs/audio_encoder.h @@ -21,6 +21,8 @@ namespace webrtc { +class Clock; + // This is the interface class for encoders in AudioCoding module. Each codec // type must have an implementation of this class. class AudioEncoder { @@ -162,6 +164,31 @@ class AudioEncoder { virtual rtc::ArrayView> ReclaimContainedEncoders(); + // Enables audio network adaptor. Returns true if successful. + virtual bool EnableAudioNetworkAdaptor(const std::string& config_string, + const Clock* clock); + + // Disables audio network adaptor. + virtual void DisableAudioNetworkAdaptor(); + + // Provides uplink bandwidth to this encoder to allow it to adapt. + virtual void OnReceivedUplinkBandwidth(int uplink_bandwidth_bps); + + // Provides uplink packet loss fraction to this encoder to allow it to adapt. + virtual void OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction); + + // Provides target audio bitrate to this encoder to allow it to adapt. + virtual void OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps); + + // Provides RTT to this encoder to allow it to adapt. + virtual void OnReceivedRtt(int rtt_ms); + + // To allow encoder to adapt its frame length, it must be provided the frame + // length range that receives can accept. + virtual void SetReceiverFrameLengthRange(int min_frame_length_ms, + int max_frame_length_ms); + protected: // Subclasses implement this to perform the actual encoding. Called by // Encode(). diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc index d03f2d3cc2..ae9dae2fbe 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc @@ -15,7 +15,10 @@ #include "webrtc/base/checks.h" #include "webrtc/base/safe_conversions.h" #include "webrtc/common_types.h" +#include "webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h" +#include "webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h" #include "webrtc/modules/audio_coding/codecs/opus/opus_interface.h" +#include "webrtc/system_wrappers/include/clock.h" namespace webrtc { @@ -24,6 +27,7 @@ namespace { const int kSampleRateHz = 48000; const int kMinBitrateBps = 500; const int kMaxBitrateBps = 512000; +constexpr int kSupportedFrameLengths[] = {20, 60}; AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) { AudioEncoderOpus::Config config; @@ -104,13 +108,23 @@ int AudioEncoderOpus::Config::GetBitrateBps() const { return num_channels == 1 ? 32000 : 64000; // Default value. } -AudioEncoderOpus::AudioEncoderOpus(const Config& config) - : packet_loss_rate_(0.0), inst_(nullptr) { +AudioEncoderOpus::AudioEncoderOpus( + const Config& config, + AudioNetworkAdaptorCreator&& audio_network_adaptor_creator) + : packet_loss_rate_(0.0), + inst_(nullptr), + audio_network_adaptor_creator_( + audio_network_adaptor_creator + ? audio_network_adaptor_creator + : [this](const std::string& config_string, const Clock* clock) { + return DefaultAudioNetworkAdaptorCreator(config_string, + clock); + }) { RTC_CHECK(RecreateEncoderInstance(config)); } AudioEncoderOpus::AudioEncoderOpus(const CodecInst& codec_inst) - : AudioEncoderOpus(CreateConfig(codec_inst)) {} + : AudioEncoderOpus(CreateConfig(codec_inst), nullptr) {} AudioEncoderOpus::~AudioEncoderOpus() { RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_)); @@ -141,15 +155,23 @@ void AudioEncoderOpus::Reset() { } bool AudioEncoderOpus::SetFec(bool enable) { - auto conf = config_; - conf.fec_enabled = enable; - return RecreateEncoderInstance(conf); + if (enable) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_)); + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_)); + } + config_.fec_enabled = enable; + return true; } bool AudioEncoderOpus::SetDtx(bool enable) { - auto conf = config_; - conf.dtx_enabled = enable; - return RecreateEncoderInstance(conf); + if (enable) { + RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_)); + } else { + RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_)); + } + config_.dtx_enabled = enable; + return true; } bool AudioEncoderOpus::GetDtx() const { @@ -192,6 +214,57 @@ void AudioEncoderOpus::SetTargetBitrate(int bits_per_second) { RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, config_.GetBitrateBps())); } +bool AudioEncoderOpus::EnableAudioNetworkAdaptor( + const std::string& config_string, + const Clock* clock) { + audio_network_adaptor_ = audio_network_adaptor_creator_(config_string, clock); + return audio_network_adaptor_.get() != nullptr; +} + +void AudioEncoderOpus::DisableAudioNetworkAdaptor() { + audio_network_adaptor_.reset(nullptr); +} + +void AudioEncoderOpus::OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) { + if (!audio_network_adaptor_) + return; + audio_network_adaptor_->SetUplinkBandwidth(uplink_bandwidth_bps); + ApplyAudioNetworkAdaptor(); +} + +void AudioEncoderOpus::OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) { + if (!audio_network_adaptor_) + return; + audio_network_adaptor_->SetUplinkPacketLossFraction( + uplink_packet_loss_fraction); + ApplyAudioNetworkAdaptor(); +} + +void AudioEncoderOpus::OnReceivedTargetAudioBitrate( + int target_audio_bitrate_bps) { + if (!audio_network_adaptor_) + return; + audio_network_adaptor_->SetTargetAudioBitrate(target_audio_bitrate_bps); + ApplyAudioNetworkAdaptor(); +} + +void AudioEncoderOpus::OnReceivedRtt(int rtt_ms) { + if (!audio_network_adaptor_) + return; + audio_network_adaptor_->SetRtt(rtt_ms); + ApplyAudioNetworkAdaptor(); +} + +void AudioEncoderOpus::SetReceiverFrameLengthRange(int min_frame_length_ms, + int max_frame_length_ms) { + if (!audio_network_adaptor_) + return; + audio_network_adaptor_->SetReceiverFrameLengthRange(min_frame_length_ms, + max_frame_length_ms); + ApplyAudioNetworkAdaptor(); +} + AudioEncoder::EncodedInfo AudioEncoderOpus::EncodeImpl( uint32_t rtp_timestamp, rtc::ArrayView audio, @@ -226,6 +299,9 @@ AudioEncoder::EncodedInfo AudioEncoderOpus::EncodeImpl( }); input_buffer_.clear(); + // Will use new packet size for next encoding. + config_.frame_size_ms = next_frame_length_ms_; + info.encoded_timestamp = first_timestamp_in_buffer_; info.payload_type = config_.payload_type; info.send_even_if_empty = true; // Allows Opus to send empty packets. @@ -282,7 +358,59 @@ bool AudioEncoderOpus::RecreateEncoderInstance(const Config& config) { WebRtcOpus_SetPacketLossRate( inst_, static_cast(packet_loss_rate_ * 100 + .5))); config_ = config; + + num_channels_to_encode_ = NumChannels(); + next_frame_length_ms_ = config_.frame_size_ms; return true; } +void AudioEncoderOpus::SetFrameLength(int frame_length_ms) { + next_frame_length_ms_ = frame_length_ms; +} + +void AudioEncoderOpus::SetNumChannelsToEncode(size_t num_channels_to_encode) { + RTC_DCHECK_GT(num_channels_to_encode, 0u); + RTC_DCHECK_LE(num_channels_to_encode, config_.num_channels); + + if (num_channels_to_encode_ == num_channels_to_encode) + return; + + RTC_CHECK_EQ(0, WebRtcOpus_SetForceChannels(inst_, num_channels_to_encode)); + num_channels_to_encode_ = num_channels_to_encode; +} + +void AudioEncoderOpus::ApplyAudioNetworkAdaptor() { + auto config = audio_network_adaptor_->GetEncoderRuntimeConfig(); + // |audio_network_adaptor_| is supposed to be configured to output all + // following parameters. + RTC_DCHECK(config.bitrate_bps); + RTC_DCHECK(config.frame_length_ms); + RTC_DCHECK(config.uplink_packet_loss_fraction); + RTC_DCHECK(config.enable_fec); + RTC_DCHECK(config.enable_dtx); + RTC_DCHECK(config.num_channels); + + RTC_DCHECK(*config.frame_length_ms == 20 || *config.frame_length_ms == 60); + + SetTargetBitrate(*config.bitrate_bps); + SetFrameLength(*config.frame_length_ms); + SetFec(*config.enable_fec); + SetProjectedPacketLossRate(*config.uplink_packet_loss_fraction); + SetDtx(*config.enable_dtx); + SetNumChannelsToEncode(*config.num_channels); +} + +std::unique_ptr +AudioEncoderOpus::DefaultAudioNetworkAdaptorCreator( + const std::string& config_string, + const Clock* clock) const { + AudioNetworkAdaptorImpl::Config config; + config.clock = clock; + return std::unique_ptr(new AudioNetworkAdaptorImpl( + config, ControllerManagerImpl::Create( + config_string, NumChannels(), kSupportedFrameLengths, + num_channels_to_encode_, next_frame_length_ms_, + GetTargetBitrate(), config_.fec_enabled, GetDtx(), clock))); +} + } // namespace webrtc diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h index 48fb494dbe..150a84162c 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h @@ -11,10 +11,12 @@ #ifndef WEBRTC_MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_OPUS_H_ #define WEBRTC_MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_OPUS_H_ +#include #include #include "webrtc/base/constructormagic.h" #include "webrtc/base/optional.h" +#include "webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" #include "webrtc/modules/audio_coding/codecs/opus/opus_interface.h" #include "webrtc/modules/audio_coding/codecs/audio_encoder.h" @@ -58,8 +60,15 @@ class AudioEncoderOpus final : public AudioEncoder { #endif }; - explicit AudioEncoderOpus(const Config& config); + using AudioNetworkAdaptorCreator = + std::function(const std::string&, + const Clock*)>; + AudioEncoderOpus( + const Config& config, + AudioNetworkAdaptorCreator&& audio_network_adaptor_creator = nullptr); + explicit AudioEncoderOpus(const CodecInst& codec_inst); + ~AudioEncoderOpus() override; int SampleRateHz() const override; @@ -82,9 +91,23 @@ class AudioEncoderOpus final : public AudioEncoder { void SetProjectedPacketLossRate(double fraction) override; void SetTargetBitrate(int target_bps) override; + bool EnableAudioNetworkAdaptor(const std::string& config_string, + const Clock* clock) override; + void DisableAudioNetworkAdaptor() override; + void OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) override; + void OnReceivedUplinkPacketLossFraction( + float uplink_packet_loss_fraction) override; + void OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) override; + void OnReceivedRtt(int rtt_ms) override; + void SetReceiverFrameLengthRange(int min_frame_length_ms, + int max_frame_length_ms) override; + // Getters for testing. double packet_loss_rate() const { return packet_loss_rate_; } ApplicationMode application() const { return config_.application; } + bool fec_enabled() const { return config_.fec_enabled; } + size_t num_channels_to_encode() const { return num_channels_to_encode_; } + int next_frame_length_ms() const { return next_frame_length_ms_; } protected: EncodedInfo EncodeImpl(uint32_t rtp_timestamp, @@ -96,12 +119,23 @@ class AudioEncoderOpus final : public AudioEncoder { size_t SamplesPer10msFrame() const; size_t SufficientOutputBufferSize() const; bool RecreateEncoderInstance(const Config& config); + void SetFrameLength(int frame_length_ms); + void SetNumChannelsToEncode(size_t num_channels_to_encode); + void ApplyAudioNetworkAdaptor(); + std::unique_ptr DefaultAudioNetworkAdaptorCreator( + const std::string& config_string, + const Clock* clock) const; Config config_; double packet_loss_rate_; std::vector input_buffer_; OpusEncInst* inst_; uint32_t first_timestamp_in_buffer_; + size_t num_channels_to_encode_; + int next_frame_length_ms_; + AudioNetworkAdaptorCreator audio_network_adaptor_creator_; + std::unique_ptr audio_network_adaptor_; + RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderOpus); }; diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc index 1b836f3664..3e0e1865ef 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc @@ -12,92 +12,158 @@ #include "webrtc/base/checks.h" #include "webrtc/common_types.h" +#include "webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h" #include "webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h" #include "webrtc/test/gtest.h" namespace webrtc { +using ::testing::NiceMock; +using ::testing::Return; namespace { -const CodecInst kOpusSettings = {105, "opus", 48000, 960, 1, 32000}; -} // namespace -class AudioEncoderOpusTest : public ::testing::Test { - protected: - void CreateCodec(int num_channels) { - codec_inst_.channels = num_channels; - encoder_.reset(new AudioEncoderOpus(codec_inst_)); - auto expected_app = - num_channels == 1 ? AudioEncoderOpus::kVoip : AudioEncoderOpus::kAudio; - EXPECT_EQ(expected_app, encoder_->application()); - } +const CodecInst kDefaultOpusSettings = {105, "opus", 48000, 960, 1, 32000}; - CodecInst codec_inst_ = kOpusSettings; - std::unique_ptr encoder_; +AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) { + AudioEncoderOpus::Config config; + config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48); + config.num_channels = codec_inst.channels; + config.bitrate_bps = rtc::Optional(codec_inst.rate); + config.payload_type = codec_inst.pltype; + config.application = config.num_channels == 1 ? AudioEncoderOpus::kVoip + : AudioEncoderOpus::kAudio; + return config; +} + +struct AudioEncoderOpusStates { + std::shared_ptr mock_audio_network_adaptor; + std::unique_ptr encoder; }; -TEST_F(AudioEncoderOpusTest, DefaultApplicationModeMono) { - CreateCodec(1); +AudioEncoderOpusStates CreateCodec(size_t num_channels) { + AudioEncoderOpusStates states; + states.mock_audio_network_adaptor = + std::make_shared(nullptr); + + std::weak_ptr mock_ptr( + states.mock_audio_network_adaptor); + AudioEncoderOpus::AudioNetworkAdaptorCreator creator = [mock_ptr]( + const std::string&, const Clock*) { + std::unique_ptr adaptor( + new NiceMock()); + EXPECT_CALL(*adaptor, Die()); + if (auto sp = mock_ptr.lock()) { + *sp = adaptor.get(); + } else { + RTC_NOTREACHED(); + } + return adaptor; + }; + + CodecInst codec_inst = kDefaultOpusSettings; + codec_inst.channels = num_channels; + auto config = CreateConfig(codec_inst); + states.encoder.reset(new AudioEncoderOpus(config, std::move(creator))); + return states; } -TEST_F(AudioEncoderOpusTest, DefaultApplicationModeStereo) { - CreateCodec(2); +AudioNetworkAdaptor::EncoderRuntimeConfig CreateEncoderRuntimeConfig() { + constexpr int kBitrate = 40000; + constexpr int kFrameLength = 60; + constexpr bool kEnableFec = true; + constexpr bool kEnableDtx = false; + constexpr size_t kNumChannels = 1; + constexpr float kPacketLossFraction = 0.1f; + AudioNetworkAdaptor::EncoderRuntimeConfig config; + config.bitrate_bps = rtc::Optional(kBitrate); + config.frame_length_ms = rtc::Optional(kFrameLength); + config.enable_fec = rtc::Optional(kEnableFec); + config.enable_dtx = rtc::Optional(kEnableDtx); + config.num_channels = rtc::Optional(kNumChannels); + config.uplink_packet_loss_fraction = + rtc::Optional(kPacketLossFraction); + return config; } -TEST_F(AudioEncoderOpusTest, ChangeApplicationMode) { - CreateCodec(2); - EXPECT_TRUE(encoder_->SetApplication(AudioEncoder::Application::kSpeech)); - EXPECT_EQ(AudioEncoderOpus::kVoip, encoder_->application()); +void CheckEncoderRuntimeConfig( + const AudioEncoderOpus* encoder, + const AudioNetworkAdaptor::EncoderRuntimeConfig& config) { + EXPECT_EQ(*config.bitrate_bps, encoder->GetTargetBitrate()); + EXPECT_EQ(*config.frame_length_ms, encoder->next_frame_length_ms()); + EXPECT_EQ(*config.enable_fec, encoder->fec_enabled()); + EXPECT_EQ(*config.enable_dtx, encoder->GetDtx()); + EXPECT_EQ(*config.num_channels, encoder->num_channels_to_encode()); } -TEST_F(AudioEncoderOpusTest, ResetWontChangeApplicationMode) { - CreateCodec(2); +} // namespace + +TEST(AudioEncoderOpusTest, DefaultApplicationModeMono) { + auto states = CreateCodec(1); + EXPECT_EQ(AudioEncoderOpus::kVoip, states.encoder->application()); +} + +TEST(AudioEncoderOpusTest, DefaultApplicationModeStereo) { + auto states = CreateCodec(2); + EXPECT_EQ(AudioEncoderOpus::kAudio, states.encoder->application()); +} + +TEST(AudioEncoderOpusTest, ChangeApplicationMode) { + auto states = CreateCodec(2); + EXPECT_TRUE( + states.encoder->SetApplication(AudioEncoder::Application::kSpeech)); + EXPECT_EQ(AudioEncoderOpus::kVoip, states.encoder->application()); +} + +TEST(AudioEncoderOpusTest, ResetWontChangeApplicationMode) { + auto states = CreateCodec(2); // Trigger a reset. - encoder_->Reset(); + states.encoder->Reset(); // Verify that the mode is still kAudio. - EXPECT_EQ(AudioEncoderOpus::kAudio, encoder_->application()); + EXPECT_EQ(AudioEncoderOpus::kAudio, states.encoder->application()); // Now change to kVoip. - EXPECT_TRUE(encoder_->SetApplication(AudioEncoder::Application::kSpeech)); - EXPECT_EQ(AudioEncoderOpus::kVoip, encoder_->application()); + EXPECT_TRUE( + states.encoder->SetApplication(AudioEncoder::Application::kSpeech)); + EXPECT_EQ(AudioEncoderOpus::kVoip, states.encoder->application()); // Trigger a reset again. - encoder_->Reset(); + states.encoder->Reset(); // Verify that the mode is still kVoip. - EXPECT_EQ(AudioEncoderOpus::kVoip, encoder_->application()); + EXPECT_EQ(AudioEncoderOpus::kVoip, states.encoder->application()); } -TEST_F(AudioEncoderOpusTest, ToggleDtx) { - CreateCodec(2); +TEST(AudioEncoderOpusTest, ToggleDtx) { + auto states = CreateCodec(2); // Enable DTX - EXPECT_TRUE(encoder_->SetDtx(true)); + EXPECT_TRUE(states.encoder->SetDtx(true)); // Verify that the mode is still kAudio. - EXPECT_EQ(AudioEncoderOpus::kAudio, encoder_->application()); + EXPECT_EQ(AudioEncoderOpus::kAudio, states.encoder->application()); // Turn off DTX. - EXPECT_TRUE(encoder_->SetDtx(false)); + EXPECT_TRUE(states.encoder->SetDtx(false)); } -TEST_F(AudioEncoderOpusTest, SetBitrate) { - CreateCodec(1); - // Constants are replicated from audio_encoder_opus.cc. +TEST(AudioEncoderOpusTest, SetBitrate) { + auto states = CreateCodec(1); + // Constants are replicated from audio_states.encoderopus.cc. const int kMinBitrateBps = 500; const int kMaxBitrateBps = 512000; // Set a too low bitrate. - encoder_->SetTargetBitrate(kMinBitrateBps - 1); - EXPECT_EQ(kMinBitrateBps, encoder_->GetTargetBitrate()); + states.encoder->SetTargetBitrate(kMinBitrateBps - 1); + EXPECT_EQ(kMinBitrateBps, states.encoder->GetTargetBitrate()); // Set a too high bitrate. - encoder_->SetTargetBitrate(kMaxBitrateBps + 1); - EXPECT_EQ(kMaxBitrateBps, encoder_->GetTargetBitrate()); + states.encoder->SetTargetBitrate(kMaxBitrateBps + 1); + EXPECT_EQ(kMaxBitrateBps, states.encoder->GetTargetBitrate()); // Set the minimum rate. - encoder_->SetTargetBitrate(kMinBitrateBps); - EXPECT_EQ(kMinBitrateBps, encoder_->GetTargetBitrate()); + states.encoder->SetTargetBitrate(kMinBitrateBps); + EXPECT_EQ(kMinBitrateBps, states.encoder->GetTargetBitrate()); // Set the maximum rate. - encoder_->SetTargetBitrate(kMaxBitrateBps); - EXPECT_EQ(kMaxBitrateBps, encoder_->GetTargetBitrate()); + states.encoder->SetTargetBitrate(kMaxBitrateBps); + EXPECT_EQ(kMaxBitrateBps, states.encoder->GetTargetBitrate()); // Set rates from 1000 up to 32000 bps. for (int rate = 1000; rate <= 32000; rate += 1000) { - encoder_->SetTargetBitrate(rate); - EXPECT_EQ(rate, encoder_->GetTargetBitrate()); + states.encoder->SetTargetBitrate(rate); + EXPECT_EQ(rate, states.encoder->GetTargetBitrate()); } } @@ -128,26 +194,113 @@ void TestSetPacketLossRate(AudioEncoderOpus* encoder, } // namespace -TEST_F(AudioEncoderOpusTest, PacketLossRateOptimized) { - CreateCodec(1); +TEST(AudioEncoderOpusTest, PacketLossRateOptimized) { + auto states = CreateCodec(1); auto I = [](double a, double b) { return IntervalSteps(a, b, 10); }; const double eps = 1e-15; // Note that the order of the following calls is critical. // clang-format off - TestSetPacketLossRate(encoder_.get(), I(0.00 , 0.01 - eps), 0.00); - TestSetPacketLossRate(encoder_.get(), I(0.01 + eps, 0.06 - eps), 0.01); - TestSetPacketLossRate(encoder_.get(), I(0.06 + eps, 0.11 - eps), 0.05); - TestSetPacketLossRate(encoder_.get(), I(0.11 + eps, 0.22 - eps), 0.10); - TestSetPacketLossRate(encoder_.get(), I(0.22 + eps, 1.00 ), 0.20); + TestSetPacketLossRate(states.encoder.get(), I(0.00 , 0.01 - eps), 0.00); + TestSetPacketLossRate(states.encoder.get(), I(0.01 + eps, 0.06 - eps), 0.01); + TestSetPacketLossRate(states.encoder.get(), I(0.06 + eps, 0.11 - eps), 0.05); + TestSetPacketLossRate(states.encoder.get(), I(0.11 + eps, 0.22 - eps), 0.10); + TestSetPacketLossRate(states.encoder.get(), I(0.22 + eps, 1.00 ), 0.20); - TestSetPacketLossRate(encoder_.get(), I(1.00 , 0.18 + eps), 0.20); - TestSetPacketLossRate(encoder_.get(), I(0.18 - eps, 0.09 + eps), 0.10); - TestSetPacketLossRate(encoder_.get(), I(0.09 - eps, 0.04 + eps), 0.05); - TestSetPacketLossRate(encoder_.get(), I(0.04 - eps, 0.01 + eps), 0.01); - TestSetPacketLossRate(encoder_.get(), I(0.01 - eps, 0.00 ), 0.00); + TestSetPacketLossRate(states.encoder.get(), I(1.00 , 0.18 + eps), 0.20); + TestSetPacketLossRate(states.encoder.get(), I(0.18 - eps, 0.09 + eps), 0.10); + TestSetPacketLossRate(states.encoder.get(), I(0.09 - eps, 0.04 + eps), 0.05); + TestSetPacketLossRate(states.encoder.get(), I(0.04 - eps, 0.01 + eps), 0.01); + TestSetPacketLossRate(states.encoder.get(), I(0.01 - eps, 0.00 ), 0.00); // clang-format on } +TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnSetUplinkBandwidth) { + auto states = CreateCodec(2); + printf("passed!\n"); + states.encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + // Since using mock audio network adaptor, any bandwidth value is fine. + constexpr int kUplinkBandwidth = 50000; + EXPECT_CALL(**states.mock_audio_network_adaptor, + SetUplinkBandwidth(kUplinkBandwidth)); + states.encoder->OnReceivedUplinkBandwidth(kUplinkBandwidth); + + CheckEncoderRuntimeConfig(states.encoder.get(), config); +} + +TEST(AudioEncoderOpusTest, + InvokeAudioNetworkAdaptorOnSetUplinkPacketLossFraction) { + auto states = CreateCodec(2); + states.encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + // Since using mock audio network adaptor, any packet loss fraction is fine. + constexpr float kUplinkPacketLoss = 0.1f; + EXPECT_CALL(**states.mock_audio_network_adaptor, + SetUplinkPacketLossFraction(kUplinkPacketLoss)); + states.encoder->OnReceivedUplinkPacketLossFraction(kUplinkPacketLoss); + + CheckEncoderRuntimeConfig(states.encoder.get(), config); +} + +TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnSetTargetAudioBitrate) { + auto states = CreateCodec(2); + states.encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + // Since using mock audio network adaptor, any target audio bitrate is fine. + constexpr int kTargetAudioBitrate = 30000; + EXPECT_CALL(**states.mock_audio_network_adaptor, + SetTargetAudioBitrate(kTargetAudioBitrate)); + states.encoder->OnReceivedTargetAudioBitrate(kTargetAudioBitrate); + + CheckEncoderRuntimeConfig(states.encoder.get(), config); +} + +TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnSetRtt) { + auto states = CreateCodec(2); + states.encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + // Since using mock audio network adaptor, any rtt is fine. + constexpr int kRtt = 30; + EXPECT_CALL(**states.mock_audio_network_adaptor, SetRtt(kRtt)); + states.encoder->OnReceivedRtt(kRtt); + + CheckEncoderRuntimeConfig(states.encoder.get(), config); +} + +TEST(AudioEncoderOpusTest, + InvokeAudioNetworkAdaptorOnSetReceiverFrameLengthRange) { + auto states = CreateCodec(2); + states.encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + constexpr int kMinFrameLength = 10; + constexpr int kMaxFrameLength = 60; + EXPECT_CALL(**states.mock_audio_network_adaptor, + SetReceiverFrameLengthRange(kMinFrameLength, kMaxFrameLength)); + states.encoder->SetReceiverFrameLengthRange(kMinFrameLength, kMaxFrameLength); + + CheckEncoderRuntimeConfig(states.encoder.get(), config); +} + } // namespace webrtc diff --git a/webrtc/modules/audio_coding/codecs/opus/opus.gypi b/webrtc/modules/audio_coding/codecs/opus/opus.gypi index 9f6ef3e4b4..36391d2656 100644 --- a/webrtc/modules/audio_coding/codecs/opus/opus.gypi +++ b/webrtc/modules/audio_coding/codecs/opus/opus.gypi @@ -38,6 +38,7 @@ ], 'dependencies': [ 'audio_encoder_interface', + 'audio_network_adaptor', ], 'sources': [ 'audio_decoder_opus.cc', diff --git a/webrtc/modules/audio_coding/codecs/opus/opus_interface.c b/webrtc/modules/audio_coding/codecs/opus/opus_interface.c index aff089f092..e2b4fdbeee 100644 --- a/webrtc/modules/audio_coding/codecs/opus/opus_interface.c +++ b/webrtc/modules/audio_coding/codecs/opus/opus_interface.c @@ -208,7 +208,7 @@ int16_t WebRtcOpus_SetComplexity(OpusEncInst* inst, int32_t complexity) { } } -int16_t WebRtcOpus_SetForceChannels(OpusEncInst* inst, int32_t num_channels) { +int16_t WebRtcOpus_SetForceChannels(OpusEncInst* inst, size_t num_channels) { if (!inst) return -1; if (num_channels == 0) { diff --git a/webrtc/modules/audio_coding/codecs/opus/opus_interface.h b/webrtc/modules/audio_coding/codecs/opus/opus_interface.h index d3c314e25a..3db5152e47 100644 --- a/webrtc/modules/audio_coding/codecs/opus/opus_interface.h +++ b/webrtc/modules/audio_coding/codecs/opus/opus_interface.h @@ -215,7 +215,7 @@ int16_t WebRtcOpus_SetComplexity(OpusEncInst* inst, int32_t complexity); * Return value : 0 - Success * -1 - Error */ -int16_t WebRtcOpus_SetForceChannels(OpusEncInst* inst, int32_t num_channels); +int16_t WebRtcOpus_SetForceChannels(OpusEncInst* inst, size_t num_channels); int16_t WebRtcOpus_DecoderCreate(OpusDecInst** inst, size_t channels); int16_t WebRtcOpus_DecoderFree(OpusDecInst* inst);