From 875862ca86b5aec6445e02723676d4a8fcbb42a6 Mon Sep 17 00:00:00 2001 From: "henrik.lundin" Date: Tue, 22 Nov 2016 02:07:54 -0800 Subject: [PATCH] Let Opus increase complexity for low bitrates This change adds code that lets Opus increase the complexity setting at low bitrates (only relevant for mobile where the default complexity is not already maximum). The feature is default off. Also adding a performance test to make sure the complexity adaptation has desired effect. BUG=webrtc:6708 Review-Url: https://codereview.webrtc.org/2503443002 Cr-Commit-Position: refs/heads/master@{#15182} --- webrtc/BUILD.gn | 1 + webrtc/build/webrtc.gni | 3 + webrtc/modules/audio_coding/BUILD.gn | 7 ++ .../codecs/opus/audio_encoder_opus.cc | 36 +++++++- .../codecs/opus/audio_encoder_opus.h | 13 +++ .../codecs/opus/opus_complexity_unittest.cc | 92 +++++++++++++++++++ 6 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc diff --git a/webrtc/BUILD.gn b/webrtc/BUILD.gn index 17e3c2ac88..242d8ea981 100644 --- a/webrtc/BUILD.gn +++ b/webrtc/BUILD.gn @@ -623,6 +623,7 @@ if (rtc_include_tests) { "call/call_perf_tests.cc", "call/rampup_tests.cc", "call/rampup_tests.h", + "modules/audio_coding/codecs/opus/opus_complexity_unittest.cc", "modules/audio_coding/neteq/test/neteq_performance_unittest.cc", "modules/audio_processing/audio_processing_performance_unittest.cc", "modules/audio_processing/level_controller/level_controller_complexity_unittest.cc", diff --git a/webrtc/build/webrtc.gni b/webrtc/build/webrtc.gni index ced2c1281a..5ab5d35973 100644 --- a/webrtc/build/webrtc.gni +++ b/webrtc/build/webrtc.gni @@ -17,6 +17,9 @@ declare_args() { # Disable this to avoid building the Opus audio codec. rtc_include_opus = true + # Enable this to let the Opus audio codec change complexity on the fly. + rtc_opus_variable_complexity = false + # Disable to use absolute header paths for some libraries. rtc_relative_path = true diff --git a/webrtc/modules/audio_coding/BUILD.gn b/webrtc/modules/audio_coding/BUILD.gn index f6188a0425..0e78abfea6 100644 --- a/webrtc/modules/audio_coding/BUILD.gn +++ b/webrtc/modules/audio_coding/BUILD.gn @@ -711,10 +711,17 @@ rtc_static_library("webrtc_opus") { "../../base:rtc_base_approved", ] + defines = [] + if (rtc_build_opus) { public_deps = [ rtc_opus_dir, ] + if (rtc_opus_variable_complexity) { + defines += [ "WEBRTC_OPUS_VARIABLE_COMPLEXITY=1" ] + } else { + defines += [ "WEBRTC_OPUS_VARIABLE_COMPLEXITY=0" ] + } } else if (build_with_mozilla) { include_dirs = [ getenv("DIST") + "/include/opus" ] } diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc index f466926f19..0f0958c2dc 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc @@ -44,6 +44,9 @@ AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) { config.application = config.num_channels == 1 ? AudioEncoderOpus::kVoip : AudioEncoderOpus::kAudio; config.supported_frame_lengths_ms.push_back(config.frame_size_ms); +#if WEBRTC_OPUS_VARIABLE_COMPLEXITY + config.low_rate_complexity = 9; +#endif return config; } @@ -118,7 +121,11 @@ class AudioEncoderOpus::PacketLossFractionSmoother { rtc::ExpFilter smoother_; }; -AudioEncoderOpus::Config::Config() = default; +AudioEncoderOpus::Config::Config() { +#if WEBRTC_OPUS_VARIABLE_COMPLEXITY + low_rate_complexity = 9; +#endif +} AudioEncoderOpus::Config::Config(const Config&) = default; AudioEncoderOpus::Config::~Config() = default; auto AudioEncoderOpus::Config::operator=(const Config&) -> Config& = default; @@ -133,6 +140,8 @@ bool AudioEncoderOpus::Config::IsOk() const { return false; if (complexity < 0 || complexity > 10) return false; + if (low_rate_complexity < 0 || low_rate_complexity > 10) + return false; return true; } @@ -144,6 +153,21 @@ int AudioEncoderOpus::Config::GetBitrateBps() const { return num_channels == 1 ? 32000 : 64000; // Default value. } +rtc::Optional AudioEncoderOpus::Config::GetNewComplexity() const { + RTC_DCHECK(IsOk()); + const int bitrate_bps = GetBitrateBps(); + if (bitrate_bps >= + complexity_threshold_bps - complexity_threshold_window_bps && + bitrate_bps <= + complexity_threshold_bps + complexity_threshold_window_bps) { + // Within the hysteresis window; make no change. + return rtc::Optional(); + } + return bitrate_bps <= complexity_threshold_bps + ? rtc::Optional(low_rate_complexity) + : rtc::Optional(complexity); +} + AudioEncoderOpus::AudioEncoderOpus( const Config& config, AudioNetworkAdaptorCreator&& audio_network_adaptor_creator) @@ -250,6 +274,11 @@ void AudioEncoderOpus::SetTargetBitrate(int bits_per_second) { std::max(std::min(bits_per_second, kMaxBitrateBps), kMinBitrateBps)); RTC_DCHECK(config_.IsOk()); RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, config_.GetBitrateBps())); + const auto new_complexity = config_.GetNewComplexity(); + if (new_complexity && complexity_ != *new_complexity) { + complexity_ = *new_complexity; + RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_)); + } } bool AudioEncoderOpus::EnableAudioNetworkAdaptor( @@ -399,7 +428,10 @@ bool AudioEncoderOpus::RecreateEncoderInstance(const Config& config) { } RTC_CHECK_EQ( 0, WebRtcOpus_SetMaxPlaybackRate(inst_, config.max_playback_rate_hz)); - RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, config.complexity)); + // Use the default complexity if the start bitrate is within the hysteresis + // window. + complexity_ = config.GetNewComplexity().value_or(config.complexity); + RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_)); if (config.dtx_enabled) { RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_)); } else { diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h index 81ca17f7e0..f30d657d13 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h @@ -12,6 +12,8 @@ #define WEBRTC_MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_OPUS_H_ #include +#include +#include #include #include "webrtc/base/constructormagic.h" @@ -39,6 +41,11 @@ class AudioEncoderOpus final : public AudioEncoder { bool IsOk() const; int GetBitrateBps() const; + // Returns empty if the current bitrate falls within the hysteresis window, + // defined by complexity_threshold_bps +/- complexity_threshold_window_bps. + // Otherwise, returns the current complexity depending on whether the + // current bitrate is above or below complexity_threshold_bps. + rtc::Optional GetNewComplexity() const; int frame_size_ms = 20; size_t num_channels = 1; @@ -48,6 +55,11 @@ class AudioEncoderOpus final : public AudioEncoder { bool fec_enabled = false; int max_playback_rate_hz = 48000; int complexity = kDefaultComplexity; + // This value may change in the struct's constructor. + int low_rate_complexity = kDefaultComplexity; + // low_rate_complexity is used when the bitrate is below this threshold. + int complexity_threshold_bps = 12500; + int complexity_threshold_window_bps = 1500; bool dtx_enabled = false; std::vector supported_frame_lengths_ms; const Clock* clock = nullptr; @@ -140,6 +152,7 @@ class AudioEncoderOpus final : public AudioEncoder { uint32_t first_timestamp_in_buffer_; size_t num_channels_to_encode_; int next_frame_length_ms_; + int complexity_; std::unique_ptr packet_loss_fraction_smoother_; AudioNetworkAdaptorCreator audio_network_adaptor_creator_; std::unique_ptr audio_network_adaptor_; diff --git a/webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc b/webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc new file mode 100644 index 0000000000..83a4539d0c --- /dev/null +++ b/webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/base/format_macros.h" +#include "webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h" +#include "webrtc/modules/audio_coding/neteq/tools/audio_loop.h" +#include "webrtc/test/gtest.h" +#include "webrtc/test/testsupport/fileutils.h" +#include "webrtc/test/testsupport/perf_test.h" +#include "webrtc/system_wrappers/include/clock.h" + +namespace webrtc { + +namespace { +int64_t RunComplexityTest(const AudioEncoderOpus::Config& config) { + // Create encoder. + AudioEncoderOpus encoder(config); + // Open speech file. + const std::string kInputFileName = + webrtc::test::ResourcePath("audio_coding/speech_mono_32_48kHz", "pcm"); + test::AudioLoop audio_loop; + constexpr int kSampleRateHz = 48000; + EXPECT_EQ(kSampleRateHz, encoder.SampleRateHz()); + constexpr size_t kMaxLoopLengthSamples = + kSampleRateHz * 10; // 10 second loop. + constexpr size_t kInputBlockSizeSamples = + 10 * kSampleRateHz / 1000; // 60 ms. + EXPECT_TRUE(audio_loop.Init(kInputFileName, kMaxLoopLengthSamples, + kInputBlockSizeSamples)); + // Encode. + webrtc::Clock* clock = webrtc::Clock::GetRealTimeClock(); + const int64_t start_time_ms = clock->TimeInMilliseconds(); + AudioEncoder::EncodedInfo info; + rtc::Buffer encoded(500); + uint32_t rtp_timestamp = 0u; + for (size_t i = 0; i < 10000; ++i) { + encoded.Clear(); + info = encoder.Encode(rtp_timestamp, audio_loop.GetNextBlock(), &encoded); + rtp_timestamp += kInputBlockSizeSamples; + } + return clock->TimeInMilliseconds() - start_time_ms; +} +} // namespace + +// This test encodes an audio file using Opus twice with different bitrates +// (12.5 kbps and 15.5 kbps). The runtime for each is measured, and the ratio +// between the two is calculated and tracked. This test explicitly sets the +// low_rate_complexity to 9. When running on desktop platforms, this is the same +// as the regular complexity, and the expectation is that the resulting ratio +// should be less than 100% (since the encoder runs faster at lower bitrates, +// given a fixed complexity setting). On the other hand, when running on +// mobiles, the regular complexity is 5, and we expect the resulting ratio to +// be higher, since we have explicitly asked for a higher complexity setting at +// the lower rate. +TEST(AudioEncoderOpusComplexityAdaptationTest, AdaptationOn) { + // Create config. + AudioEncoderOpus::Config config; + config.bitrate_bps = rtc::Optional(12500); + config.low_rate_complexity = 9; + int64_t runtime_12500bps = RunComplexityTest(config); + + config.bitrate_bps = rtc::Optional(15500); + int64_t runtime_15500bps = RunComplexityTest(config); + + test::PrintResult("opus_encoding_complexity_ratio", "", "adaptation_on", + 100.0 * runtime_12500bps / runtime_15500bps, "percent", + true); +} + +// This test is identical to the one above, but without the complexity +// adaptation enabled (neither on desktop, nor on mobile). The expectation is +// that the resulting ratio is less than 100% at all times. +TEST(AudioEncoderOpusComplexityAdaptationTest, AdaptationOff) { + // Create config. + AudioEncoderOpus::Config config; + config.bitrate_bps = rtc::Optional(12500); + int64_t runtime_12500bps = RunComplexityTest(config); + + config.bitrate_bps = rtc::Optional(15500); + int64_t runtime_15500bps = RunComplexityTest(config); + + test::PrintResult("opus_encoding_complexity_ratio", "", "adaptation_off", + 100.0 * runtime_12500bps / runtime_15500bps, "", true); +} +} // namespace webrtc