diff --git a/webrtc/BUILD.gn b/webrtc/BUILD.gn index 17e3c2ac88..242d8ea981 100644 --- a/webrtc/BUILD.gn +++ b/webrtc/BUILD.gn @@ -623,6 +623,7 @@ if (rtc_include_tests) { "call/call_perf_tests.cc", "call/rampup_tests.cc", "call/rampup_tests.h", + "modules/audio_coding/codecs/opus/opus_complexity_unittest.cc", "modules/audio_coding/neteq/test/neteq_performance_unittest.cc", "modules/audio_processing/audio_processing_performance_unittest.cc", "modules/audio_processing/level_controller/level_controller_complexity_unittest.cc", diff --git a/webrtc/build/webrtc.gni b/webrtc/build/webrtc.gni index ced2c1281a..5ab5d35973 100644 --- a/webrtc/build/webrtc.gni +++ b/webrtc/build/webrtc.gni @@ -17,6 +17,9 @@ declare_args() { # Disable this to avoid building the Opus audio codec. rtc_include_opus = true + # Enable this to let the Opus audio codec change complexity on the fly. + rtc_opus_variable_complexity = false + # Disable to use absolute header paths for some libraries. rtc_relative_path = true diff --git a/webrtc/modules/audio_coding/BUILD.gn b/webrtc/modules/audio_coding/BUILD.gn index f6188a0425..0e78abfea6 100644 --- a/webrtc/modules/audio_coding/BUILD.gn +++ b/webrtc/modules/audio_coding/BUILD.gn @@ -711,10 +711,17 @@ rtc_static_library("webrtc_opus") { "../../base:rtc_base_approved", ] + defines = [] + if (rtc_build_opus) { public_deps = [ rtc_opus_dir, ] + if (rtc_opus_variable_complexity) { + defines += [ "WEBRTC_OPUS_VARIABLE_COMPLEXITY=1" ] + } else { + defines += [ "WEBRTC_OPUS_VARIABLE_COMPLEXITY=0" ] + } } else if (build_with_mozilla) { include_dirs = [ getenv("DIST") + "/include/opus" ] } diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc index f466926f19..0f0958c2dc 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc @@ -44,6 +44,9 @@ AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) { config.application = config.num_channels == 1 ? AudioEncoderOpus::kVoip : AudioEncoderOpus::kAudio; config.supported_frame_lengths_ms.push_back(config.frame_size_ms); +#if WEBRTC_OPUS_VARIABLE_COMPLEXITY + config.low_rate_complexity = 9; +#endif return config; } @@ -118,7 +121,11 @@ class AudioEncoderOpus::PacketLossFractionSmoother { rtc::ExpFilter smoother_; }; -AudioEncoderOpus::Config::Config() = default; +AudioEncoderOpus::Config::Config() { +#if WEBRTC_OPUS_VARIABLE_COMPLEXITY + low_rate_complexity = 9; +#endif +} AudioEncoderOpus::Config::Config(const Config&) = default; AudioEncoderOpus::Config::~Config() = default; auto AudioEncoderOpus::Config::operator=(const Config&) -> Config& = default; @@ -133,6 +140,8 @@ bool AudioEncoderOpus::Config::IsOk() const { return false; if (complexity < 0 || complexity > 10) return false; + if (low_rate_complexity < 0 || low_rate_complexity > 10) + return false; return true; } @@ -144,6 +153,21 @@ int AudioEncoderOpus::Config::GetBitrateBps() const { return num_channels == 1 ? 32000 : 64000; // Default value. } +rtc::Optional AudioEncoderOpus::Config::GetNewComplexity() const { + RTC_DCHECK(IsOk()); + const int bitrate_bps = GetBitrateBps(); + if (bitrate_bps >= + complexity_threshold_bps - complexity_threshold_window_bps && + bitrate_bps <= + complexity_threshold_bps + complexity_threshold_window_bps) { + // Within the hysteresis window; make no change. + return rtc::Optional(); + } + return bitrate_bps <= complexity_threshold_bps + ? rtc::Optional(low_rate_complexity) + : rtc::Optional(complexity); +} + AudioEncoderOpus::AudioEncoderOpus( const Config& config, AudioNetworkAdaptorCreator&& audio_network_adaptor_creator) @@ -250,6 +274,11 @@ void AudioEncoderOpus::SetTargetBitrate(int bits_per_second) { std::max(std::min(bits_per_second, kMaxBitrateBps), kMinBitrateBps)); RTC_DCHECK(config_.IsOk()); RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, config_.GetBitrateBps())); + const auto new_complexity = config_.GetNewComplexity(); + if (new_complexity && complexity_ != *new_complexity) { + complexity_ = *new_complexity; + RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_)); + } } bool AudioEncoderOpus::EnableAudioNetworkAdaptor( @@ -399,7 +428,10 @@ bool AudioEncoderOpus::RecreateEncoderInstance(const Config& config) { } RTC_CHECK_EQ( 0, WebRtcOpus_SetMaxPlaybackRate(inst_, config.max_playback_rate_hz)); - RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, config.complexity)); + // Use the default complexity if the start bitrate is within the hysteresis + // window. + complexity_ = config.GetNewComplexity().value_or(config.complexity); + RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_)); if (config.dtx_enabled) { RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_)); } else { diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h index 81ca17f7e0..f30d657d13 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h @@ -12,6 +12,8 @@ #define WEBRTC_MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_OPUS_H_ #include +#include +#include #include #include "webrtc/base/constructormagic.h" @@ -39,6 +41,11 @@ class AudioEncoderOpus final : public AudioEncoder { bool IsOk() const; int GetBitrateBps() const; + // Returns empty if the current bitrate falls within the hysteresis window, + // defined by complexity_threshold_bps +/- complexity_threshold_window_bps. + // Otherwise, returns the current complexity depending on whether the + // current bitrate is above or below complexity_threshold_bps. + rtc::Optional GetNewComplexity() const; int frame_size_ms = 20; size_t num_channels = 1; @@ -48,6 +55,11 @@ class AudioEncoderOpus final : public AudioEncoder { bool fec_enabled = false; int max_playback_rate_hz = 48000; int complexity = kDefaultComplexity; + // This value may change in the struct's constructor. + int low_rate_complexity = kDefaultComplexity; + // low_rate_complexity is used when the bitrate is below this threshold. + int complexity_threshold_bps = 12500; + int complexity_threshold_window_bps = 1500; bool dtx_enabled = false; std::vector supported_frame_lengths_ms; const Clock* clock = nullptr; @@ -140,6 +152,7 @@ class AudioEncoderOpus final : public AudioEncoder { uint32_t first_timestamp_in_buffer_; size_t num_channels_to_encode_; int next_frame_length_ms_; + int complexity_; std::unique_ptr packet_loss_fraction_smoother_; AudioNetworkAdaptorCreator audio_network_adaptor_creator_; std::unique_ptr audio_network_adaptor_; diff --git a/webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc b/webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc new file mode 100644 index 0000000000..83a4539d0c --- /dev/null +++ b/webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/base/format_macros.h" +#include "webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h" +#include "webrtc/modules/audio_coding/neteq/tools/audio_loop.h" +#include "webrtc/test/gtest.h" +#include "webrtc/test/testsupport/fileutils.h" +#include "webrtc/test/testsupport/perf_test.h" +#include "webrtc/system_wrappers/include/clock.h" + +namespace webrtc { + +namespace { +int64_t RunComplexityTest(const AudioEncoderOpus::Config& config) { + // Create encoder. + AudioEncoderOpus encoder(config); + // Open speech file. + const std::string kInputFileName = + webrtc::test::ResourcePath("audio_coding/speech_mono_32_48kHz", "pcm"); + test::AudioLoop audio_loop; + constexpr int kSampleRateHz = 48000; + EXPECT_EQ(kSampleRateHz, encoder.SampleRateHz()); + constexpr size_t kMaxLoopLengthSamples = + kSampleRateHz * 10; // 10 second loop. + constexpr size_t kInputBlockSizeSamples = + 10 * kSampleRateHz / 1000; // 60 ms. + EXPECT_TRUE(audio_loop.Init(kInputFileName, kMaxLoopLengthSamples, + kInputBlockSizeSamples)); + // Encode. + webrtc::Clock* clock = webrtc::Clock::GetRealTimeClock(); + const int64_t start_time_ms = clock->TimeInMilliseconds(); + AudioEncoder::EncodedInfo info; + rtc::Buffer encoded(500); + uint32_t rtp_timestamp = 0u; + for (size_t i = 0; i < 10000; ++i) { + encoded.Clear(); + info = encoder.Encode(rtp_timestamp, audio_loop.GetNextBlock(), &encoded); + rtp_timestamp += kInputBlockSizeSamples; + } + return clock->TimeInMilliseconds() - start_time_ms; +} +} // namespace + +// This test encodes an audio file using Opus twice with different bitrates +// (12.5 kbps and 15.5 kbps). The runtime for each is measured, and the ratio +// between the two is calculated and tracked. This test explicitly sets the +// low_rate_complexity to 9. When running on desktop platforms, this is the same +// as the regular complexity, and the expectation is that the resulting ratio +// should be less than 100% (since the encoder runs faster at lower bitrates, +// given a fixed complexity setting). On the other hand, when running on +// mobiles, the regular complexity is 5, and we expect the resulting ratio to +// be higher, since we have explicitly asked for a higher complexity setting at +// the lower rate. +TEST(AudioEncoderOpusComplexityAdaptationTest, AdaptationOn) { + // Create config. + AudioEncoderOpus::Config config; + config.bitrate_bps = rtc::Optional(12500); + config.low_rate_complexity = 9; + int64_t runtime_12500bps = RunComplexityTest(config); + + config.bitrate_bps = rtc::Optional(15500); + int64_t runtime_15500bps = RunComplexityTest(config); + + test::PrintResult("opus_encoding_complexity_ratio", "", "adaptation_on", + 100.0 * runtime_12500bps / runtime_15500bps, "percent", + true); +} + +// This test is identical to the one above, but without the complexity +// adaptation enabled (neither on desktop, nor on mobile). The expectation is +// that the resulting ratio is less than 100% at all times. +TEST(AudioEncoderOpusComplexityAdaptationTest, AdaptationOff) { + // Create config. + AudioEncoderOpus::Config config; + config.bitrate_bps = rtc::Optional(12500); + int64_t runtime_12500bps = RunComplexityTest(config); + + config.bitrate_bps = rtc::Optional(15500); + int64_t runtime_15500bps = RunComplexityTest(config); + + test::PrintResult("opus_encoding_complexity_ratio", "", "adaptation_off", + 100.0 * runtime_12500bps / runtime_15500bps, "", true); +} +} // namespace webrtc