admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Injectable audio encoders: WebRtcVoiceEngine and company

Browse Source 
These are the changes made to WebRtcVoiceEngine and surrounding
code. It still contains some things that are inelegant, like how
AudioCodecSpec and AudioFormatInfo is ferried around in
SendCodecSpec. This should probably be resolved before landing.

There are also a few test still that are disabled. They should be
removed or fixed, as the case may be.

I've put this CL up to get a better overview of the changes made and
how reviewable they are.

BUG=webrtc:5806

Review-Url: https://codereview.webrtc.org/2705093002
Cr-Commit-Position: refs/heads/master@{#17904}
This commit is contained in:
ossu 2017-04-27 02:08:52 -07:00 committed by Commit bot
parent 66753c34ad
commit 20a4b3fb2a
27 changed files with 1020 additions and 1548 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
webrtc/audio/BUILD.gn
View File

@ -39,6 +39,9 @@ rtc_static_library("audio") {
"../base:rtc_task_queue",
"../call:call_interfaces",
"../common_audio",
"../modules/audio_coding:audio_encoder_factory_interface",
"../modules/audio_coding:builtin_audio_encoder_factory",
"../modules/audio_coding:cng",
"../modules/audio_device",
"../modules/audio_processing",
"../modules/bitrate_controller:bitrate_controller",

12
webrtc/audio/DEPS
View File

@ -15,3 +15,15 @@ include_rules = [
"+webrtc/system_wrappers",
"+webrtc/voice_engine",
]
specific_include_rules = {
"audio_send_stream.cc": [
"+webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h",
],
# TODO(ossu): Remove this exception when builtin_audio_encoder_factory.h
# has moved to api/, or when the proper mocks have been made.
"audio_send_stream_unittest.cc": [
"+webrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory.h",
],
}

461
webrtc/audio/audio_send_stream.cc
View File

@ -11,16 +11,20 @@
#include "webrtc/audio/audio_send_stream.h"
#include <string>
#include <utility>
#include <vector>
#include "webrtc/audio/audio_state.h"
#include "webrtc/audio/conversion.h"
#include "webrtc/audio/scoped_voe_interface.h"
#include "webrtc/base/checks.h"
#include "webrtc/base/event.h"
#include "webrtc/base/function_view.h"
#include "webrtc/base/logging.h"
#include "webrtc/base/task_queue.h"
#include "webrtc/base/timeutils.h"
#include "webrtc/call/rtp_transport_controller_send_interface.h"
#include "webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h"
#include "webrtc/modules/bitrate_controller/include/bitrate_controller.h"
#include "webrtc/modules/congestion_controller/include/send_side_congestion_controller.h"
#include "webrtc/modules/pacing/paced_sender.h"
@ -32,21 +36,22 @@
namespace webrtc {
namespace {
constexpr char kOpusCodecName[] = "opus";
bool IsCodec(const webrtc::CodecInst& codec, const char* ref_name) {
return (STR_CASE_CMP(codec.plname, ref_name) == 0);
}
} // namespace
namespace internal {
// TODO(elad.alon): Subsequent CL will make these values experiment-dependent.
constexpr size_t kPacketLossTrackerMaxWindowSizeMs = 15000;
constexpr size_t kPacketLossRateMinNumAckedPackets = 50;
constexpr size_t kRecoverablePacketLossRateMinNumAckedPairs = 40;
namespace {
void CallEncoder(const std::unique_ptr<voe::ChannelProxy>& channel_proxy,
rtc::FunctionView<void(AudioEncoder*)> lambda) {
channel_proxy->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder_ptr) {
RTC_DCHECK(encoder_ptr);
lambda(encoder_ptr->get());
});
}
} // namespace
AudioSendStream::AudioSendStream(
const webrtc::AudioSendStream::Config& config,
const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
@ -56,52 +61,28 @@ AudioSendStream::AudioSendStream(
RtcEventLog* event_log,
RtcpRttStats* rtcp_rtt_stats)
: worker_queue_(worker_queue),
config_(config),
config_(Config(nullptr)),
audio_state_(audio_state),
event_log_(event_log),
bitrate_allocator_(bitrate_allocator),
transport_(transport),
packet_loss_tracker_(kPacketLossTrackerMaxWindowSizeMs,
kPacketLossRateMinNumAckedPackets,
kRecoverablePacketLossRateMinNumAckedPairs) {
LOG(LS_INFO) << "AudioSendStream: " << config_.ToString();
RTC_DCHECK_NE(config_.voe_channel_id, -1);
LOG(LS_INFO) << "AudioSendStream: " << config.ToString();
RTC_DCHECK_NE(config.voe_channel_id, -1);
RTC_DCHECK(audio_state_.get());
RTC_DCHECK(transport);
RTC_DCHECK(transport->send_side_cc());
VoiceEngineImpl* voe_impl = static_cast<VoiceEngineImpl*>(voice_engine());
channel_proxy_ = voe_impl->GetChannelProxy(config_.voe_channel_id);
channel_proxy_->SetRtcEventLog(event_log);
channel_proxy_ = voe_impl->GetChannelProxy(config.voe_channel_id);
channel_proxy_->SetRtcEventLog(event_log_);
channel_proxy_->SetRtcpRttStats(rtcp_rtt_stats);
channel_proxy_->SetRTCPStatus(true);
channel_proxy_->SetLocalSSRC(config.rtp.ssrc);
channel_proxy_->SetRTCP_CNAME(config.rtp.c_name);
// TODO(solenberg): Config NACK history window (which is a packet count),
// using the actual packet size for the configured codec.
channel_proxy_->SetNACKStatus(config_.rtp.nack.rtp_history_ms != 0,
config_.rtp.nack.rtp_history_ms / 20);
channel_proxy_->RegisterExternalTransport(config.send_transport);
transport_->send_side_cc()->RegisterPacketFeedbackObserver(this);
for (const auto& extension : config.rtp.extensions) {
if (extension.uri == RtpExtension::kAudioLevelUri) {
channel_proxy_->SetSendAudioLevelIndicationStatus(true, extension.id);
} else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) {
channel_proxy_->EnableSendTransportSequenceNumber(extension.id);
transport->send_side_cc()->EnablePeriodicAlrProbing(true);
bandwidth_observer_.reset(transport->send_side_cc()
->GetBitrateController()
->CreateRtcpBandwidthObserver());
} else {
RTC_NOTREACHED() << "Registering unsupported RTP extension.";
}
}
channel_proxy_->RegisterSenderCongestionControlObjects(
transport, bandwidth_observer_.get());
if (!SetupSendCodec()) {
LOG(LS_ERROR) << "Failed to set up send codec state.";
}
ConfigureStream(this, config, true);
pacer_thread_checker_.DetachFromThread();
}
@ -116,17 +97,102 @@ AudioSendStream::~AudioSendStream() {
channel_proxy_->SetRtcpRttStats(nullptr);
}
void AudioSendStream::Reconfigure(
const webrtc::AudioSendStream::Config& new_config) {
ConfigureStream(this, new_config, false);
}
void AudioSendStream::ConfigureStream(
webrtc::internal::AudioSendStream* stream,
const webrtc::AudioSendStream::Config& new_config,
bool first_time) {
LOG(LS_INFO) << "AudioSendStream::Configuring: " << new_config.ToString();
const auto& channel_proxy = stream->channel_proxy_;
const auto& old_config = stream->config_;
if (first_time || old_config.rtp.ssrc != new_config.rtp.ssrc) {
channel_proxy->SetLocalSSRC(new_config.rtp.ssrc);
}
if (first_time || old_config.rtp.c_name != new_config.rtp.c_name) {
channel_proxy->SetRTCP_CNAME(new_config.rtp.c_name);
}
// TODO(solenberg): Config NACK history window (which is a packet count),
// using the actual packet size for the configured codec.
if (first_time || old_config.rtp.nack.rtp_history_ms !=
new_config.rtp.nack.rtp_history_ms) {
channel_proxy->SetNACKStatus(new_config.rtp.nack.rtp_history_ms != 0,
new_config.rtp.nack.rtp_history_ms / 20);
}
if (first_time ||
new_config.send_transport != old_config.send_transport) {
if (old_config.send_transport) {
channel_proxy->DeRegisterExternalTransport();
}
channel_proxy->RegisterExternalTransport(new_config.send_transport);
}
// RFC 5285: Each distinct extension MUST have a unique ID. The value 0 is
// reserved for padding and MUST NOT be used as a local identifier.
// So it should be safe to use 0 here to indicate "not configured".
struct ExtensionIds {
int audio_level = 0;
int transport_sequence_number = 0;
};
auto find_extension_ids = [](const std::vector<RtpExtension>& extensions) {
ExtensionIds ids;
for (const auto& extension : extensions) {
if (extension.uri == RtpExtension::kAudioLevelUri) {
ids.audio_level = extension.id;
} else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) {
ids.transport_sequence_number = extension.id;
}
}
return ids;
};
const ExtensionIds old_ids = find_extension_ids(old_config.rtp.extensions);
const ExtensionIds new_ids = find_extension_ids(new_config.rtp.extensions);
// Audio level indication
if (first_time || new_ids.audio_level != old_ids.audio_level) {
channel_proxy->SetSendAudioLevelIndicationStatus(new_ids.audio_level != 0,
new_ids.audio_level);
}
// Transport sequence number
if (first_time ||
new_ids.transport_sequence_number != old_ids.transport_sequence_number) {
if (old_ids.transport_sequence_number) {
channel_proxy->ResetSenderCongestionControlObjects();
stream->bandwidth_observer_.reset();
}
if (new_ids.transport_sequence_number != 0) {
channel_proxy->EnableSendTransportSequenceNumber(
new_ids.transport_sequence_number);
stream->transport_->send_side_cc()->EnablePeriodicAlrProbing(true);
stream->bandwidth_observer_.reset(stream->transport_->send_side_cc()
->GetBitrateController()
->CreateRtcpBandwidthObserver());
}
channel_proxy->RegisterSenderCongestionControlObjects(
stream->transport_, stream->bandwidth_observer_.get());
}
if (!ReconfigureSendCodec(stream, new_config)) {
LOG(LS_ERROR) << "Failed to set up send codec state.";
}
ReconfigureBitrateObserver(stream, new_config);
stream->config_ = new_config;
}
void AudioSendStream::Start() {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
if (config_.min_bitrate_bps != -1 && config_.max_bitrate_bps != -1) {
RTC_DCHECK_GE(config_.max_bitrate_bps, config_.min_bitrate_bps);
rtc::Event thread_sync_event(false /* manual_reset */, false);
worker_queue_->PostTask([this, &thread_sync_event] {
bitrate_allocator_->AddObserver(this, config_.min_bitrate_bps,
config_.max_bitrate_bps, 0, true);
thread_sync_event.Set();
});
thread_sync_event.Wait(rtc::Event::kForever);
ConfigureBitrateObserver(config_.min_bitrate_bps, config_.max_bitrate_bps);
}
ScopedVoEInterface<VoEBase> base(voice_engine());
@ -138,12 +204,7 @@ void AudioSendStream::Start() {
void AudioSendStream::Stop() {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
rtc::Event thread_sync_event(false /* manual_reset */, false);
worker_queue_->PostTask([this, &thread_sync_event] {
bitrate_allocator_->RemoveObserver(this);
thread_sync_event.Set();
});
thread_sync_event.Wait(rtc::Event::kForever);
RemoveBitrateObserver();
ScopedVoEInterface<VoEBase> base(voice_engine());
int error = base->StopSend(config_.voe_channel_id);
@ -183,11 +244,10 @@ webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const {
// implementation.
stats.aec_quality_min = -1;
webrtc::CodecInst codec_inst = {0};
if (channel_proxy_->GetSendCodec(&codec_inst)) {
RTC_DCHECK_NE(codec_inst.pltype, -1);
stats.codec_name = codec_inst.plname;
stats.codec_payload_type = rtc::Optional<int>(codec_inst.pltype);
if (config_.send_codec_spec) {
const auto& spec = *config_.send_codec_spec;
stats.codec_name = spec.format.name;
stats.codec_payload_type = rtc::Optional<int>(spec.payload_type);
// Get data from the last remote RTCP report.
for (const auto& block : channel_proxy_->GetRemoteRTCPReportBlocks()) {
@ -196,10 +256,10 @@ webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const {
stats.packets_lost = block.cumulative_num_packets_lost;
stats.fraction_lost = Q8ToFloat(block.fraction_lost);
stats.ext_seqnum = block.extended_highest_sequence_number;
// Convert samples to milliseconds.
if (codec_inst.plfreq / 1000 > 0) {
// Convert timestamps to milliseconds.
if (spec.format.clockrate_hz / 1000 > 0) {
stats.jitter_ms =
block.interarrival_jitter / (codec_inst.plfreq / 1000);
block.interarrival_jitter / (spec.format.clockrate_hz / 1000);
}
break;
}
@ -324,116 +384,193 @@ VoiceEngine* AudioSendStream::voice_engine() const {
}
// Apply current codec settings to a single voe::Channel used for sending.
bool AudioSendStream::SetupSendCodec() {
// Disable VAD and FEC unless we know the other side wants them.
channel_proxy_->SetVADStatus(false);
channel_proxy_->SetCodecFECStatus(false);
bool AudioSendStream::SetupSendCodec(AudioSendStream* stream,
const Config& new_config) {
RTC_DCHECK(new_config.send_codec_spec);
const auto& spec = *new_config.send_codec_spec;
std::unique_ptr<AudioEncoder> encoder =
new_config.encoder_factory->MakeAudioEncoder(spec.payload_type,
spec.format);
// We disable audio network adaptor here. This will on one hand make sure that
// audio network adaptor is disabled by default, and on the other allow audio
// network adaptor to be reconfigured, since SetReceiverFrameLengthRange can
// be only called when audio network adaptor is disabled.
channel_proxy_->DisableAudioNetworkAdaptor();
const auto& send_codec_spec = config_.send_codec_spec;
// We set the codec first, since the below extra configuration is only applied
// to the "current" codec.
// If codec is already configured, we do not it again.
// TODO(minyue): check if this check is really needed, or can we move it into
// |codec->SetSendCodec|.
webrtc::CodecInst current_codec = {0};
if (!channel_proxy_->GetSendCodec(&current_codec) ||
(send_codec_spec.codec_inst != current_codec)) {
if (!channel_proxy_->SetSendCodec(send_codec_spec.codec_inst)) {
LOG(LS_WARNING) << "SetSendCodec() failed.";
return false;
}
if (!encoder) {
LOG(LS_ERROR) << "Unable to create encoder for " << spec.format;
return false;
}
// If a bitrate has been specified for the codec, use it over the
// codec's default.
if (spec.target_bitrate_bps) {
encoder->OnReceivedTargetAudioBitrate(*spec.target_bitrate_bps);
}
// Codec internal FEC. Treat any failure as fatal internal error.
if (send_codec_spec.enable_codec_fec) {
if (!channel_proxy_->SetCodecFECStatus(true)) {
LOG(LS_WARNING) << "SetCodecFECStatus() failed.";
return false;
}
}
// DTX and maxplaybackrate are only set if current codec is Opus.
if (IsCodec(send_codec_spec.codec_inst, kOpusCodecName)) {
if (!channel_proxy_->SetOpusDtx(send_codec_spec.enable_opus_dtx)) {
LOG(LS_WARNING) << "SetOpusDtx() failed.";
return false;
}
// If opus_max_playback_rate <= 0, the default maximum playback rate
// (48 kHz) will be used.
if (send_codec_spec.opus_max_playback_rate > 0) {
if (!channel_proxy_->SetOpusMaxPlaybackRate(
send_codec_spec.opus_max_playback_rate)) {
LOG(LS_WARNING) << "SetOpusMaxPlaybackRate() failed.";
return false;
}
}
if (config_.audio_network_adaptor_config) {
// Audio network adaptor is only allowed for Opus currently.
// |SetReceiverFrameLengthRange| needs to be called before
// |EnableAudioNetworkAdaptor|.
channel_proxy_->SetReceiverFrameLengthRange(send_codec_spec.min_ptime_ms,
send_codec_spec.max_ptime_ms);
channel_proxy_->EnableAudioNetworkAdaptor(
*config_.audio_network_adaptor_config);
// Enable ANA if configured (currently only used by Opus).
if (new_config.audio_network_adaptor_config) {
if (encoder->EnableAudioNetworkAdaptor(
*new_config.audio_network_adaptor_config, stream->event_log_)) {
LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
<< config_.rtp.ssrc;
<< new_config.rtp.ssrc;
} else {
RTC_NOTREACHED();
}
}
// Set the CN payloadtype and the VAD status.
if (send_codec_spec.cng_payload_type != -1) {
// The CN payload type for 8000 Hz clockrate is fixed at 13.
if (send_codec_spec.cng_plfreq != 8000) {
webrtc::PayloadFrequencies cn_freq;
switch (send_codec_spec.cng_plfreq) {
case 16000:
cn_freq = webrtc::kFreq16000Hz;
break;
case 32000:
cn_freq = webrtc::kFreq32000Hz;
break;
default:
RTC_NOTREACHED();
return false;
}
if (!channel_proxy_->SetSendCNPayloadType(
send_codec_spec.cng_payload_type, cn_freq)) {
LOG(LS_WARNING) << "SetSendCNPayloadType() failed.";
// TODO(ajm): This failure condition will be removed from VoE.
// Restore the return here when we update to a new enough webrtc.
//
// Not returning false because the SetSendCNPayloadType will fail if
// the channel is already sending.
// This can happen if the remote description is applied twice, for
// example in the case of ROAP on top of JSEP, where both side will
// send the offer.
}
}
// Only turn on VAD if we have a CN payload type that matches the
// clockrate for the codec we are going to use.
if (send_codec_spec.cng_plfreq == send_codec_spec.codec_inst.plfreq &&
send_codec_spec.codec_inst.channels == 1) {
// TODO(minyue): If CN frequency == 48000 Hz is allowed, consider the
// interaction between VAD and Opus FEC.
if (!channel_proxy_->SetVADStatus(true)) {
LOG(LS_WARNING) << "SetVADStatus() failed.";
return false;
}
}
// Wrap the encoder in a an AudioEncoderCNG, if VAD is enabled.
if (spec.cng_payload_type) {
AudioEncoderCng::Config cng_config;
cng_config.num_channels = encoder->NumChannels();
cng_config.payload_type = *spec.cng_payload_type;
cng_config.speech_encoder = std::move(encoder);
cng_config.vad_mode = Vad::kVadNormal;
encoder.reset(new AudioEncoderCng(std::move(cng_config)));
}
stream->channel_proxy_->SetEncoder(new_config.send_codec_spec->payload_type,
std::move(encoder));
return true;
}
bool AudioSendStream::ReconfigureSendCodec(AudioSendStream* stream,
const Config& new_config) {
const auto& old_config = stream->config_;
if (new_config.send_codec_spec == old_config.send_codec_spec) {
return true;
}
// If we have no encoder, or the format or payload type's changed, create a
// new encoder.
if (!old_config.send_codec_spec ||
new_config.send_codec_spec->format !=
old_config.send_codec_spec->format ||
new_config.send_codec_spec->payload_type !=
old_config.send_codec_spec->payload_type) {
return SetupSendCodec(stream, new_config);
}
// Should never move a stream from fully configured to unconfigured.
RTC_CHECK(new_config.send_codec_spec);
const rtc::Optional<int>& new_target_bitrate_bps =
new_config.send_codec_spec->target_bitrate_bps;
// If a bitrate has been specified for the codec, use it over the
// codec's default.
if (new_target_bitrate_bps &&
new_target_bitrate_bps !=
old_config.send_codec_spec->target_bitrate_bps) {
CallEncoder(stream->channel_proxy_, [&](AudioEncoder* encoder) {
encoder->OnReceivedTargetAudioBitrate(*new_target_bitrate_bps);
});
}
ReconfigureANA(stream, new_config);
ReconfigureCNG(stream, new_config);
return true;
}
void AudioSendStream::ReconfigureANA(AudioSendStream* stream,
const Config& new_config) {
if (new_config.audio_network_adaptor_config ==
stream->config_.audio_network_adaptor_config) {
return;
}
if (new_config.audio_network_adaptor_config) {
CallEncoder(stream->channel_proxy_, [&](AudioEncoder* encoder) {
if (encoder->EnableAudioNetworkAdaptor(
*new_config.audio_network_adaptor_config, stream->event_log_)) {
LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
<< new_config.rtp.ssrc;
} else {
RTC_NOTREACHED();
}
});
} else {
CallEncoder(stream->channel_proxy_, [&](AudioEncoder* encoder) {
encoder->DisableAudioNetworkAdaptor();
});
LOG(LS_INFO) << "Audio network adaptor disabled on SSRC "
<< new_config.rtp.ssrc;
}
}
void AudioSendStream::ReconfigureCNG(AudioSendStream* stream,
const Config& new_config) {
if (new_config.send_codec_spec->cng_payload_type ==
stream->config_.send_codec_spec->cng_payload_type) {
return;
}
// Wrap or unwrap the encoder in an AudioEncoderCNG.
stream->channel_proxy_->ModifyEncoder(
[&](std::unique_ptr<AudioEncoder>* encoder_ptr) {
std::unique_ptr<AudioEncoder> old_encoder(std::move(*encoder_ptr));
auto sub_encoders = old_encoder->ReclaimContainedEncoders();
if (!sub_encoders.empty()) {
// Replace enc with its sub encoder. We need to put the sub
// encoder in a temporary first, since otherwise the old value
// of enc would be destroyed before the new value got assigned,
// which would be bad since the new value is a part of the old
// value.
auto tmp = std::move(sub_encoders[0]);
old_encoder = std::move(tmp);
}
if (new_config.send_codec_spec->cng_payload_type) {
AudioEncoderCng::Config config;
config.speech_encoder = std::move(old_encoder);
config.num_channels = config.speech_encoder->NumChannels();
config.payload_type = *new_config.send_codec_spec->cng_payload_type;
config.vad_mode = Vad::kVadNormal;
encoder_ptr->reset(new AudioEncoderCng(std::move(config)));
} else {
*encoder_ptr = std::move(old_encoder);
}
});
}
void AudioSendStream::ReconfigureBitrateObserver(
AudioSendStream* stream,
const webrtc::AudioSendStream::Config& new_config) {
// Since the Config's default is for both of these to be -1, this test will
// allow us to configure the bitrate observer if the new config has bitrate
// limits set, but would only have us call RemoveBitrateObserver if we were
// previously configured with bitrate limits.
if (stream->config_.min_bitrate_bps == new_config.min_bitrate_bps &&
stream->config_.max_bitrate_bps == new_config.max_bitrate_bps) {
return;
}
if (new_config.min_bitrate_bps != -1 && new_config.max_bitrate_bps != -1) {
stream->ConfigureBitrateObserver(new_config.min_bitrate_bps,
new_config.max_bitrate_bps);
} else {
stream->RemoveBitrateObserver();
}
}
void AudioSendStream::ConfigureBitrateObserver(int min_bitrate_bps,
int max_bitrate_bps) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
RTC_DCHECK_GE(max_bitrate_bps, min_bitrate_bps);
rtc::Event thread_sync_event(false /* manual_reset */, false);
worker_queue_->PostTask([&] {
// We may get a callback immediately as the observer is registered, so make
// sure the bitrate limits in config_ are up-to-date.
config_.min_bitrate_bps = min_bitrate_bps;
config_.max_bitrate_bps = max_bitrate_bps;
bitrate_allocator_->AddObserver(this, min_bitrate_bps, max_bitrate_bps, 0,
true);
thread_sync_event.Set();
});
thread_sync_event.Wait(rtc::Event::kForever);
}
void AudioSendStream::RemoveBitrateObserver() {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
rtc::Event thread_sync_event(false /* manual_reset */, false);
worker_queue_->PostTask([this, &thread_sync_event] {
bitrate_allocator_->RemoveObserver(this);
thread_sync_event.Set();
});
thread_sync_event.Wait(rtc::Event::kForever);
}
} // namespace internal
} // namespace webrtc

21
webrtc/audio/audio_send_stream.h
View File

@ -48,6 +48,8 @@ class AudioSendStream final : public webrtc::AudioSendStream,
~AudioSendStream() override;
// webrtc::AudioSendStream implementation.
void Reconfigure(const webrtc::AudioSendStream::Config& config) override;
void Start() override;
void Stop() override;
bool SendTelephoneEvent(int payload_type, int payload_frequency, int event,
@ -75,14 +77,29 @@ class AudioSendStream final : public webrtc::AudioSendStream,
private:
VoiceEngine* voice_engine() const;
bool SetupSendCodec();
// These are all static to make it less likely that (the old) config_ is
// accessed unintentionally.
static void ConfigureStream(AudioSendStream* stream,
const Config& new_config,
bool first_time);
static bool SetupSendCodec(AudioSendStream* stream, const Config& new_config);
static bool ReconfigureSendCodec(AudioSendStream* stream,
const Config& new_config);
static void ReconfigureANA(AudioSendStream* stream, const Config& new_config);
static void ReconfigureCNG(AudioSendStream* stream, const Config& new_config);
static void ReconfigureBitrateObserver(AudioSendStream* stream,
const Config& new_config);
void ConfigureBitrateObserver(int min_bitrate_bps, int max_bitrate_bps);
void RemoveBitrateObserver();
rtc::ThreadChecker worker_thread_checker_;
rtc::ThreadChecker pacer_thread_checker_;
rtc::TaskQueue* worker_queue_;
const webrtc::AudioSendStream::Config config_;
webrtc::AudioSendStream::Config config_;
rtc::scoped_refptr<webrtc::AudioState> audio_state_;
std::unique_ptr<voe::ChannelProxy> channel_proxy_;
RtcEventLog* const event_log_;
BitrateAllocator* const bitrate_allocator_;
RtpTransportControllerSendInterface* const transport_;

233
webrtc/audio/audio_send_stream_unittest.cc
View File

@ -9,6 +9,7 @@
*/
#include <string>
#include <utility>
#include <vector>
#include "webrtc/audio/audio_send_stream.h"
@ -17,6 +18,8 @@
#include "webrtc/base/task_queue.h"
#include "webrtc/call/rtp_transport_controller_send_interface.h"
#include "webrtc/logging/rtc_event_log/mock/mock_rtc_event_log.h"
#include "webrtc/modules/audio_coding/codecs/mock/mock_audio_encoder.h"
#include "webrtc/modules/audio_coding/codecs/mock/mock_audio_encoder_factory.h"
#include "webrtc/modules/audio_mixer/audio_mixer_impl.h"
#include "webrtc/modules/audio_processing/include/mock_audio_processing.h"
#include "webrtc/modules/congestion_controller/include/mock/mock_congestion_observer.h"
@ -35,6 +38,7 @@ namespace {
using testing::_;
using testing::Eq;
using testing::Ne;
using testing::Invoke;
using testing::Return;
const int kChannelId = 1;
@ -56,6 +60,14 @@ const int kTelephoneEventPayloadFrequency = 65432;
const int kTelephoneEventCode = 45;
const int kTelephoneEventDuration = 6789;
const CodecInst kIsacCodec = {103, "isac", 16000, 320, 1, 32000};
constexpr int kIsacPayloadType = 103;
const SdpAudioFormat kIsacFormat = {"isac", 16000, 1};
const SdpAudioFormat kOpusFormat = {"opus", 48000, 2};
const SdpAudioFormat kG722Format = {"g722", 8000, 1};
const AudioCodecSpec kCodecSpecs[] = {
{kIsacFormat, {16000, 1, 32000, 10000, 32000}},
{kOpusFormat, {48000, 1, 32000, 6000, 510000}},
{kG722Format, {16000, 1, 64000}}};
class MockLimitObserver : public BitrateAllocator::LimitObserver {
public:
@ -69,6 +81,47 @@ class MockTransmitMixer : public voe::TransmitMixer {
MOCK_CONST_METHOD0(AudioLevelFullRange, int16_t());
};
std::unique_ptr<MockAudioEncoder> SetupAudioEncoderMock(
int payload_type,
const SdpAudioFormat& format) {
for (const auto& spec : kCodecSpecs) {
if (format == spec.format) {
std::unique_ptr<MockAudioEncoder> encoder(new MockAudioEncoder);
ON_CALL(*encoder.get(), SampleRateHz())
.WillByDefault(Return(spec.info.sample_rate_hz));
ON_CALL(*encoder.get(), NumChannels())
.WillByDefault(Return(spec.info.num_channels));
ON_CALL(*encoder.get(), RtpTimestampRateHz())
.WillByDefault(Return(spec.format.clockrate_hz));
return encoder;
}
}
return nullptr;
}
rtc::scoped_refptr<MockAudioEncoderFactory> SetupEncoderFactoryMock() {
rtc::scoped_refptr<MockAudioEncoderFactory> factory =
new rtc::RefCountedObject<MockAudioEncoderFactory>();
ON_CALL(*factory.get(), GetSupportedEncoders())
.WillByDefault(Return(std::vector<AudioCodecSpec>(
std::begin(kCodecSpecs), std::end(kCodecSpecs))));
ON_CALL(*factory.get(), QueryAudioEncoder(_))
.WillByDefault(Invoke([](const SdpAudioFormat& format) {
for (const auto& spec : kCodecSpecs) {
if (format == spec.format) {
return rtc::Optional<AudioCodecInfo>(spec.info);
}
}
return rtc::Optional<AudioCodecInfo>();
}));
ON_CALL(*factory.get(), MakeAudioEncoderMock(_, _, _))
.WillByDefault(Invoke([](int payload_type, const SdpAudioFormat& format,
std::unique_ptr<AudioEncoder>* return_value) {
*return_value = SetupAudioEncoderMock(payload_type, format);
}));
return factory;
}
struct ConfigHelper {
class FakeRtpTransportController
: public RtpTransportControllerSendInterface {
@ -97,7 +150,7 @@ struct ConfigHelper {
SendSideCongestionController send_side_cc_;
};
explicit ConfigHelper(bool audio_bwe_enabled)
ConfigHelper(bool audio_bwe_enabled, bool expect_set_encoder_call)
: stream_config_(nullptr),
fake_transport_(&event_log_),
bitrate_allocator_(&limit_observer_),
@ -124,8 +177,13 @@ struct ConfigHelper {
return channel_proxy_;
}));
SetupMockForSetupSendCodec();
SetupMockForSetupSendCodec(expect_set_encoder_call);
// Use ISAC as default codec so as to prevent unnecessary |voice_engine_|
// calls from the default ctor behavior.
stream_config_.send_codec_spec =
rtc::Optional<AudioSendStream::Config::SendCodecSpec>(
{kIsacPayloadType, kIsacFormat});
stream_config_.voe_channel_id = kChannelId;
stream_config_.rtp.ssrc = kSsrc;
stream_config_.rtp.nack.rtp_history_ms = 200;
@ -136,16 +194,18 @@ struct ConfigHelper {
stream_config_.rtp.extensions.push_back(
RtpExtension(RtpExtension::kTransportSequenceNumberUri,
kTransportSequenceNumberId));
stream_config_.send_codec_spec.transport_cc_enabled = true;
stream_config_.send_codec_spec->transport_cc_enabled = true;
}
// Use ISAC as default codec so as to prevent unnecessary |voice_engine_|
// calls from the default ctor behavior.
stream_config_.send_codec_spec.codec_inst = kIsacCodec;
stream_config_.encoder_factory = SetupEncoderFactoryMock();
stream_config_.min_bitrate_bps = 10000;
stream_config_.max_bitrate_bps = 65000;
}
AudioSendStream::Config& config() { return stream_config_; }
MockAudioEncoderFactory& mock_encoder_factory() {
return *static_cast<MockAudioEncoderFactory*>(
stream_config_.encoder_factory.get());
}
rtc::scoped_refptr<AudioState> audio_state() { return audio_state_; }
MockVoEChannelProxy* channel_proxy() { return channel_proxy_; }
RtpTransportControllerSendInterface* transport() { return &fake_transport_; }
@ -176,6 +236,8 @@ struct ConfigHelper {
&fake_transport_, Eq(nullptr)))
.Times(1);
}
EXPECT_CALL(*channel_proxy_, SetBitrate(_, _))
.Times(1);
EXPECT_CALL(*channel_proxy_, ResetSenderCongestionControlObjects())
.Times(1);
EXPECT_CALL(*channel_proxy_, RegisterExternalTransport(nullptr)).Times(1);
@ -188,17 +250,13 @@ struct ConfigHelper {
.Times(1); // Destructor resets the rtt stats.
}
void SetupMockForSetupSendCodec() {
EXPECT_CALL(*channel_proxy_, SetVADStatus(false))
.WillOnce(Return(true));
EXPECT_CALL(*channel_proxy_, SetCodecFECStatus(false))
.WillOnce(Return(true));
EXPECT_CALL(*channel_proxy_, DisableAudioNetworkAdaptor());
// Let |GetSendCodec| return false for the first time to indicate that no
// send codec has been set.
EXPECT_CALL(*channel_proxy_, GetSendCodec(_)).WillOnce(Return(false));
EXPECT_CALL(*channel_proxy_, SetSendCodec(_)).WillOnce(Return(true));
void SetupMockForSetupSendCodec(bool expect_set_encoder_call) {
if (expect_set_encoder_call) {
EXPECT_CALL(*channel_proxy_, SetEncoderForMock(_, _))
.WillOnce(Return(true));
}
}
RtcpRttStats* rtcp_rtt_stats() { return &rtcp_rtt_stats_; }
void SetupMockForSendTelephoneEvent() {
@ -230,8 +288,6 @@ struct ConfigHelper {
.WillRepeatedly(Return(kCallStats));
EXPECT_CALL(*channel_proxy_, GetRemoteRTCPReportBlocks())
.WillRepeatedly(Return(report_blocks));
EXPECT_CALL(*channel_proxy_, GetSendCodec(_))
.WillRepeatedly(DoAll(SetArgPointee<0>(kIsacCodec), Return(true)));
EXPECT_CALL(voice_engine_, transmit_mixer())
.WillRepeatedly(Return(&transmit_mixer_));
EXPECT_CALL(voice_engine_, audio_processing())
@ -281,16 +337,13 @@ TEST(AudioSendStreamTest, ConfigToString) {
config.voe_channel_id = kChannelId;
config.min_bitrate_bps = 12000;
config.max_bitrate_bps = 34000;
config.send_codec_spec.nack_enabled = true;
config.send_codec_spec.transport_cc_enabled = false;
config.send_codec_spec.enable_codec_fec = true;
config.send_codec_spec.enable_opus_dtx = false;
config.send_codec_spec.opus_max_playback_rate = 32000;
config.send_codec_spec.cng_payload_type = 42;
config.send_codec_spec.cng_plfreq = 56;
config.send_codec_spec.min_ptime_ms = 20;
config.send_codec_spec.max_ptime_ms = 60;
config.send_codec_spec.codec_inst = kIsacCodec;
config.send_codec_spec =
rtc::Optional<AudioSendStream::Config::SendCodecSpec>(
{kIsacPayloadType, kIsacFormat});
config.send_codec_spec->nack_enabled = true;
config.send_codec_spec->transport_cc_enabled = false;
config.send_codec_spec->cng_payload_type = rtc::Optional<int>(42);
config.encoder_factory = MockAudioEncoderFactory::CreateUnusedFactory();
config.rtp.extensions.push_back(
RtpExtension(RtpExtension::kAudioLevelUri, kAudioLevelId));
EXPECT_EQ(
@ -299,15 +352,14 @@ TEST(AudioSendStreamTest, ConfigToString) {
"{rtp_history_ms: 0}, c_name: foo_name}, send_transport: null, "
"voe_channel_id: 1, min_bitrate_bps: 12000, max_bitrate_bps: 34000, "
"send_codec_spec: {nack_enabled: true, transport_cc_enabled: false, "
"enable_codec_fec: true, enable_opus_dtx: false, opus_max_playback_rate: "
"32000, cng_payload_type: 42, cng_plfreq: 56, min_ptime: 20, max_ptime: "
"60, codec_inst: {pltype: 103, plname: \"isac\", plfreq: 16000, pacsize: "
"320, channels: 1, rate: 32000}}}",
"cng_payload_type: 42, payload_type: 103, "
"format: {name: isac, clockrate_hz: 16000, num_channels: 1, "
"parameters: {}}}}",
config.ToString());
}
TEST(AudioSendStreamTest, ConstructDestruct) {
ConfigHelper helper(false);
ConfigHelper helper(false, true);
internal::AudioSendStream send_stream(
helper.config(), helper.audio_state(), helper.worker_queue(),
helper.transport(), helper.bitrate_allocator(), helper.event_log(),
@ -315,7 +367,7 @@ TEST(AudioSendStreamTest, ConstructDestruct) {
}
TEST(AudioSendStreamTest, SendTelephoneEvent) {
ConfigHelper helper(false);
ConfigHelper helper(false, true);
internal::AudioSendStream send_stream(
helper.config(), helper.audio_state(), helper.worker_queue(),
helper.transport(), helper.bitrate_allocator(), helper.event_log(),
@ -327,7 +379,7 @@ TEST(AudioSendStreamTest, SendTelephoneEvent) {
}
TEST(AudioSendStreamTest, SetMuted) {
ConfigHelper helper(false);
ConfigHelper helper(false, true);
internal::AudioSendStream send_stream(
helper.config(), helper.audio_state(), helper.worker_queue(),
helper.transport(), helper.bitrate_allocator(), helper.event_log(),
@ -337,7 +389,7 @@ TEST(AudioSendStreamTest, SetMuted) {
}
TEST(AudioSendStreamTest, AudioBweCorrectObjectsOnChannelProxy) {
ConfigHelper helper(true);
ConfigHelper helper(true, true);
internal::AudioSendStream send_stream(
helper.config(), helper.audio_state(), helper.worker_queue(),
helper.transport(), helper.bitrate_allocator(), helper.event_log(),
@ -345,7 +397,7 @@ TEST(AudioSendStreamTest, AudioBweCorrectObjectsOnChannelProxy) {
}
TEST(AudioSendStreamTest, NoAudioBweCorrectObjectsOnChannelProxy) {
ConfigHelper helper(false);
ConfigHelper helper(false, true);
internal::AudioSendStream send_stream(
helper.config(), helper.audio_state(), helper.worker_queue(),
helper.transport(), helper.bitrate_allocator(), helper.event_log(),
@ -353,7 +405,7 @@ TEST(AudioSendStreamTest, NoAudioBweCorrectObjectsOnChannelProxy) {
}
TEST(AudioSendStreamTest, GetStats) {
ConfigHelper helper(false);
ConfigHelper helper(false, true);
internal::AudioSendStream send_stream(
helper.config(), helper.audio_state(), helper.worker_queue(),
helper.transport(), helper.bitrate_allocator(), helper.event_log(),
@ -384,7 +436,7 @@ TEST(AudioSendStreamTest, GetStats) {
}
TEST(AudioSendStreamTest, GetStatsTypingNoiseDetected) {
ConfigHelper helper(false);
ConfigHelper helper(false, true);
internal::AudioSendStream send_stream(
helper.config(), helper.audio_state(), helper.worker_queue(),
helper.transport(), helper.bitrate_allocator(), helper.event_log(),
@ -402,43 +454,23 @@ TEST(AudioSendStreamTest, GetStatsTypingNoiseDetected) {
EXPECT_FALSE(send_stream.GetStats().typing_noise_detected);
}
TEST(AudioSendStreamTest, SendCodecAppliesConfigParams) {
ConfigHelper helper(false);
TEST(AudioSendStreamTest, SendCodecAppliesNetworkAdaptor) {
ConfigHelper helper(false, true);
auto stream_config = helper.config();
const CodecInst kOpusCodec = {111, "opus", 48000, 960, 2, 64000};
stream_config.send_codec_spec.codec_inst = kOpusCodec;
stream_config.send_codec_spec.enable_codec_fec = true;
stream_config.send_codec_spec.enable_opus_dtx = true;
stream_config.send_codec_spec.opus_max_playback_rate = 12345;
stream_config.send_codec_spec.cng_plfreq = 16000;
stream_config.send_codec_spec.cng_payload_type = 105;
stream_config.send_codec_spec.min_ptime_ms = 10;
stream_config.send_codec_spec.max_ptime_ms = 60;
stream_config.send_codec_spec =
rtc::Optional<AudioSendStream::Config::SendCodecSpec>({0, kOpusFormat});
stream_config.audio_network_adaptor_config =
rtc::Optional<std::string>("abced");
EXPECT_CALL(*helper.channel_proxy(), SetCodecFECStatus(true))
.WillOnce(Return(true));
EXPECT_CALL(
*helper.channel_proxy(),
SetOpusDtx(stream_config.send_codec_spec.enable_opus_dtx))
.WillOnce(Return(true));
EXPECT_CALL(
*helper.channel_proxy(),
SetOpusMaxPlaybackRate(
stream_config.send_codec_spec.opus_max_playback_rate))
.WillOnce(Return(true));
EXPECT_CALL(*helper.channel_proxy(),
SetSendCNPayloadType(
stream_config.send_codec_spec.cng_payload_type,
webrtc::kFreq16000Hz))
.WillOnce(Return(true));
EXPECT_CALL(
*helper.channel_proxy(),
SetReceiverFrameLengthRange(stream_config.send_codec_spec.min_ptime_ms,
stream_config.send_codec_spec.max_ptime_ms));
EXPECT_CALL(
*helper.channel_proxy(),
EnableAudioNetworkAdaptor(*stream_config.audio_network_adaptor_config));
EXPECT_CALL(helper.mock_encoder_factory(), MakeAudioEncoderMock(_, _, _))
.WillOnce(Invoke([](int payload_type, const SdpAudioFormat& format,
std::unique_ptr<AudioEncoder>* return_value) {
auto mock_encoder = SetupAudioEncoderMock(payload_type, format);
EXPECT_CALL(*mock_encoder.get(), EnableAudioNetworkAdaptor(_, _))
.WillOnce(Return(true));
*return_value = std::move(mock_encoder);
}));
internal::AudioSendStream send_stream(
stream_config, helper.audio_state(), helper.worker_queue(),
helper.transport(), helper.bitrate_allocator(), helper.event_log(),
@ -446,24 +478,37 @@ TEST(AudioSendStreamTest, SendCodecAppliesConfigParams) {
}
// VAD is applied when codec is mono and the CNG frequency matches the codec
// sample rate.
// clock rate.
TEST(AudioSendStreamTest, SendCodecCanApplyVad) {
ConfigHelper helper(false);
ConfigHelper helper(false, false);
auto stream_config = helper.config();
const CodecInst kG722Codec = {9, "g722", 8000, 160, 1, 16000};
stream_config.send_codec_spec.codec_inst = kG722Codec;
stream_config.send_codec_spec.cng_plfreq = 8000;
stream_config.send_codec_spec.cng_payload_type = 105;
EXPECT_CALL(*helper.channel_proxy(), SetVADStatus(true))
.WillOnce(Return(true));
stream_config.send_codec_spec =
rtc::Optional<AudioSendStream::Config::SendCodecSpec>({9, kG722Format});
stream_config.send_codec_spec->cng_payload_type = rtc::Optional<int>(105);
using ::testing::Invoke;
std::unique_ptr<AudioEncoder> stolen_encoder;
EXPECT_CALL(*helper.channel_proxy(), SetEncoderForMock(_, _))
.WillOnce(
Invoke([&stolen_encoder](int payload_type,
std::unique_ptr<AudioEncoder>* encoder) {
stolen_encoder = std::move(*encoder);
return true;
}));
internal::AudioSendStream send_stream(
stream_config, helper.audio_state(), helper.worker_queue(),
helper.transport(), helper.bitrate_allocator(), helper.event_log(),
helper.rtcp_rtt_stats());
// We cannot truly determine if the encoder created is an AudioEncoderCng. It
// is the only reasonable implementation that will return something from
// ReclaimContainedEncoders, though.
ASSERT_TRUE(stolen_encoder);
EXPECT_FALSE(stolen_encoder->ReclaimContainedEncoders().empty());
}
TEST(AudioSendStreamTest, DoesNotPassHigherBitrateThanMaxBitrate) {
ConfigHelper helper(false);
ConfigHelper helper(false, true);
internal::AudioSendStream send_stream(
helper.config(), helper.audio_state(), helper.worker_queue(),
helper.transport(), helper.bitrate_allocator(), helper.event_log(),
@ -475,7 +520,7 @@ TEST(AudioSendStreamTest, DoesNotPassHigherBitrateThanMaxBitrate) {
}
TEST(AudioSendStreamTest, ProbingIntervalOnBitrateUpdated) {
ConfigHelper helper(false);
ConfigHelper helper(false, true);
internal::AudioSendStream send_stream(
helper.config(), helper.audio_state(), helper.worker_queue(),
helper.transport(), helper.bitrate_allocator(), helper.event_log(),
@ -484,5 +529,27 @@ TEST(AudioSendStreamTest, ProbingIntervalOnBitrateUpdated) {
send_stream.OnBitrateUpdated(50000, 0.0, 50, 5000);
}
// Test that AudioSendStream doesn't recreate the encoder unnecessarily.
TEST(AudioSendStreamTest, DontRecreateEncoder) {
ConfigHelper helper(false, false);
// WillOnce is (currently) the default used by ConfigHelper if asked to set an
// expectation for SetEncoder. Since this behavior is essential for this test
// to be correct, it's instead set-up manually here. Otherwise a simple change
// to ConfigHelper (say to WillRepeatedly) would silently make this test
// useless.
EXPECT_CALL(*helper.channel_proxy(), SetEncoderForMock(_, _))
.WillOnce(Return(true));
auto stream_config = helper.config();
stream_config.send_codec_spec =
rtc::Optional<AudioSendStream::Config::SendCodecSpec>({9, kG722Format});
stream_config.send_codec_spec->cng_payload_type = rtc::Optional<int>(105);
internal::AudioSendStream send_stream(
stream_config, helper.audio_state(), helper.worker_queue(),
helper.transport(), helper.bitrate_allocator(), helper.event_log(),
helper.rtcp_rtt_stats());
send_stream.Reconfigure(stream_config);
}
} // namespace test
} // namespace webrtc

25
webrtc/audio/test/low_bandwidth_audio_test.cc
View File

@ -97,8 +97,12 @@ test::PacketTransport* AudioQualityTest::CreateReceiveTransport() {
void AudioQualityTest::ModifyAudioConfigs(
AudioSendStream::Config* send_config,
std::vector<AudioReceiveStream::Config>* receive_configs) {
send_config->send_codec_spec.codec_inst = webrtc::CodecInst{
test::CallTest::kAudioSendPayloadType, "OPUS", 48000, 960, 2, 64000};
// Large bitrate by default.
const webrtc::SdpAudioFormat kDefaultFormat("OPUS", 48000, 2,
{{"stereo", "1"}});
send_config->send_codec_spec =
rtc::Optional<AudioSendStream::Config::SendCodecSpec>(
{test::CallTest::kAudioSendPayloadType, kDefaultFormat});
}
void AudioQualityTest::PerformTest() {
@ -130,14 +134,15 @@ TEST_F(LowBandwidthAudioTest, GoodNetworkHighBitrate) {
class Mobile2GNetworkTest : public AudioQualityTest {
void ModifyAudioConfigs(AudioSendStream::Config* send_config,
std::vector<AudioReceiveStream::Config>* receive_configs) override {
send_config->send_codec_spec.codec_inst = CodecInst{
test::CallTest::kAudioSendPayloadType, // pltype
"OPUS", // plname
48000, // plfreq
2880, // pacsize
1, // channels
6000 // rate bits/sec
};
send_config->send_codec_spec =
rtc::Optional<AudioSendStream::Config::SendCodecSpec>(
{test::CallTest::kAudioSendPayloadType,
{"OPUS",
48000,
2,
{{"maxaveragebitrate", "6000"},
{"ptime", "60"},
{"stereo", "1"}}}});
}
FakeNetworkPipe::Config GetNetworkPipeConfig() override {

2
webrtc/call/BUILD.gn
View File

@ -29,6 +29,7 @@ rtc_source_set("call_interfaces") {
"../api/audio_codecs:audio_codecs_api",
"../base:rtc_base",
"../base:rtc_base_approved",
"../modules/audio_coding:audio_encoder_factory_interface",
"../modules/audio_coding:audio_encoder_interface",
]
}
@ -131,6 +132,7 @@ if (rtc_include_tests) {
"../base:rtc_base_approved",
"../logging:rtc_event_log_api",
"../modules/audio_coding",
"../modules/audio_coding:builtin_audio_encoder_factory",
"../modules/audio_mixer:audio_mixer_impl",
"../modules/rtp_rtcp",
"../system_wrappers",

47
webrtc/call/audio_send_stream.cc
View File

@ -12,21 +12,6 @@
#include <string>
namespace {
std::string ToString(const webrtc::CodecInst& codec_inst) {
std::stringstream ss;
ss << "{pltype: " << codec_inst.pltype;
ss << ", plname: \"" << codec_inst.plname << "\"";
ss << ", plfreq: " << codec_inst.plfreq;
ss << ", pacsize: " << codec_inst.pacsize;
ss << ", channels: " << codec_inst.channels;
ss << ", rate: " << codec_inst.rate;
ss << '}';
return ss.str();
}
} // namespace
namespace webrtc {
AudioSendStream::Stats::Stats() = default;
@ -44,7 +29,8 @@ std::string AudioSendStream::Config::ToString() const {
ss << ", voe_channel_id: " << voe_channel_id;
ss << ", min_bitrate_bps: " << min_bitrate_bps;
ss << ", max_bitrate_bps: " << max_bitrate_bps;
ss << ", send_codec_spec: " << send_codec_spec.ToString();
ss << ", send_codec_spec: "
<< (send_codec_spec ? send_codec_spec->ToString() : "<unset>");
ss << '}';
return ss.str();
}
@ -70,24 +56,20 @@ std::string AudioSendStream::Config::Rtp::ToString() const {
return ss.str();
}
AudioSendStream::Config::SendCodecSpec::SendCodecSpec() {
webrtc::CodecInst empty_inst = {0};
codec_inst = empty_inst;
codec_inst.pltype = -1;
}
AudioSendStream::Config::SendCodecSpec::SendCodecSpec(
int payload_type,
const SdpAudioFormat& format)
: payload_type(payload_type), format(format) {}
AudioSendStream::Config::SendCodecSpec::~SendCodecSpec() = default;
std::string AudioSendStream::Config::SendCodecSpec::ToString() const {
std::stringstream ss;
ss << "{nack_enabled: " << (nack_enabled ? "true" : "false");
ss << ", transport_cc_enabled: " << (transport_cc_enabled ? "true" : "false");
ss << ", enable_codec_fec: " << (enable_codec_fec ? "true" : "false");
ss << ", enable_opus_dtx: " << (enable_opus_dtx ? "true" : "false");
ss << ", opus_max_playback_rate: " << opus_max_playback_rate;
ss << ", cng_payload_type: " << cng_payload_type;
ss << ", cng_plfreq: " << cng_plfreq;
ss << ", min_ptime: " << min_ptime_ms;
ss << ", max_ptime: " << max_ptime_ms;
ss << ", codec_inst: " << ::ToString(codec_inst);
ss << ", cng_payload_type: "
<< (cng_payload_type ? std::to_string(*cng_payload_type) : "<unset>");
ss << ", payload_type: " << payload_type;
ss << ", format: " << format;
ss << '}';
return ss.str();
}
@ -96,12 +78,9 @@ bool AudioSendStream::Config::SendCodecSpec::operator==(
const AudioSendStream::Config::SendCodecSpec& rhs) const {
if (nack_enabled == rhs.nack_enabled &&
transport_cc_enabled == rhs.transport_cc_enabled &&
enable_codec_fec == rhs.enable_codec_fec &&
enable_opus_dtx == rhs.enable_opus_dtx &&
opus_max_playback_rate == rhs.opus_max_playback_rate &&
cng_payload_type == rhs.cng_payload_type &&
cng_plfreq == rhs.cng_plfreq && max_ptime_ms == rhs.max_ptime_ms &&
min_ptime_ms == rhs.min_ptime_ms && codec_inst == rhs.codec_inst) {
payload_type == rhs.payload_type && format == rhs.format &&
target_bitrate_bps == rhs.target_bitrate_bps) {
return true;
}
return false;

27
webrtc/call/audio_send_stream.h
View File

@ -15,10 +15,11 @@
#include <string>
#include <vector>
#include "webrtc/api/audio_codecs/audio_format.h"
#include "webrtc/api/call/transport.h"
#include "webrtc/base/optional.h"
#include "webrtc/config.h"
#include "webrtc/modules/audio_coding/codecs/audio_encoder.h"
#include "webrtc/modules/audio_coding/codecs/audio_encoder_factory.h"
#include "webrtc/typedefs.h"
namespace webrtc {
@ -102,7 +103,8 @@ class AudioSendStream {
rtc::Optional<std::string> audio_network_adaptor_config;
struct SendCodecSpec {
SendCodecSpec();
SendCodecSpec(int payload_type, const SdpAudioFormat& format);
~SendCodecSpec();
std::string ToString() const;
bool operator==(const SendCodecSpec& rhs) const;
@ -110,19 +112,22 @@ class AudioSendStream {
return !(*this == rhs);
}
int payload_type;
SdpAudioFormat format;
bool nack_enabled = false;
bool transport_cc_enabled = false;
bool enable_codec_fec = false;
bool enable_opus_dtx = false;
int opus_max_playback_rate = 0;
int cng_payload_type = -1;
int cng_plfreq = -1;
int max_ptime_ms = -1;
int min_ptime_ms = -1;
webrtc::CodecInst codec_inst;
} send_codec_spec;
rtc::Optional<int> cng_payload_type;
// If unset, use the encoder's default target bitrate.
rtc::Optional<int> target_bitrate_bps;
};
rtc::Optional<SendCodecSpec> send_codec_spec;
rtc::scoped_refptr<AudioEncoderFactory> encoder_factory;
};
// Reconfigure the stream according to the Configuration.
virtual void Reconfigure(const Config& config) = 0;
// Starts stream activity.
// When a stream is active, it can receive, process and deliver packets.
virtual void Start() = 0;

7
webrtc/call/call_perf_tests.cc
View File

@ -19,6 +19,7 @@
#include "webrtc/call/call.h"
#include "webrtc/config.h"
#include "webrtc/logging/rtc_event_log/rtc_event_log.h"
#include "webrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory.h"
#include "webrtc/modules/audio_coding/include/audio_coding_module.h"
#include "webrtc/modules/audio_mixer/audio_mixer_impl.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
@ -232,8 +233,10 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec,
AudioSendStream::Config audio_send_config(&audio_send_transport);
audio_send_config.voe_channel_id = send_channel_id;
audio_send_config.rtp.ssrc = kAudioSendSsrc;
audio_send_config.send_codec_spec.codec_inst =
CodecInst{kAudioSendPayloadType, "ISAC", 16000, 480, 1, 32000};
audio_send_config.send_codec_spec =
rtc::Optional<AudioSendStream::Config::SendCodecSpec>(
{kAudioSendPayloadType, {"ISAC", 16000, 1}});
audio_send_config.encoder_factory = CreateBuiltinAudioEncoderFactory();
AudioSendStream* audio_send_stream =
sender_call_->CreateAudioSendStream(audio_send_config);

2
webrtc/media/BUILD.gn
View File

@ -105,6 +105,8 @@ rtc_static_library("rtc_media_base") {
"../base:rtc_base_approved",
"../call:call_interfaces",
"../common_video:common_video",
"../modules/audio_coding:audio_encoder_factory_interface",
"../modules/audio_coding:builtin_audio_encoder_factory",
"../p2p",
]

5
webrtc/media/engine/fakewebrtccall.cc
View File

@ -26,6 +26,11 @@ FakeAudioSendStream::FakeAudioSendStream(
RTC_DCHECK(config.voe_channel_id != -1);
}
void FakeAudioSendStream::Reconfigure(
const webrtc::AudioSendStream::Config& config) {
config_ = config;
}
const webrtc::AudioSendStream::Config&
FakeAudioSendStream::GetConfig() const {
return config_;

2
webrtc/media/engine/fakewebrtccall.h
View File

@ -55,6 +55,8 @@ class FakeAudioSendStream final : public webrtc::AudioSendStream {
private:
// webrtc::AudioSendStream implementation.
void Reconfigure(const webrtc::AudioSendStream::Config& config) override;
void Start() override { sending_ = true; }
void Stop() override { sending_ = false; }

709
webrtc/media/engine/webrtcvoiceengine.cc
View File

@ -37,7 +37,7 @@
#include "webrtc/media/engine/payload_type_mapper.h"
#include "webrtc/media/engine/webrtcmediaengine.h"
#include "webrtc/media/engine/webrtcvoe.h"
#include "webrtc/modules/audio_coding/acm2/rent_a_codec.h"
#include "webrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory.h"
#include "webrtc/modules/audio_mixer/audio_mixer_impl.h"
#include "webrtc/modules/audio_processing/include/audio_processing.h"
#include "webrtc/system_wrappers/include/field_trial.h"
@ -66,27 +66,9 @@ constexpr int kNackRtpHistoryMs = 5000;
#error "Set WEBRTC_INTELLIGIBILITY_ENHANCER to either 0 or 1"
#endif
// Codec parameters for Opus.
// draft-spittka-payload-rtp-opus-03
// Recommended bitrates:
// 8-12 kb/s for NB speech,
// 16-20 kb/s for WB speech,
// 28-40 kb/s for FB speech,
// 48-64 kb/s for FB mono music, and
// 64-128 kb/s for FB stereo music.
// The current implementation applies the following values to mono signals,
// and multiplies them by 2 for stereo.
const int kOpusBitrateNbBps = 12000;
const int kOpusBitrateWbBps = 20000;
const int kOpusBitrateFbBps = 32000;
// Opus bitrate should be in the range between 6000 and 510000.
// For SendSideBwe, Opus bitrate should be in the range between 6000 and 32000.
const int kOpusMinBitrateBps = 6000;
const int kOpusMaxBitrateBps = 510000;
// iSAC bitrate should be <= 56000.
const int kIsacMaxBitrateBps = 56000;
const int kOpusBitrateFbBps = 32000;
// Default audio dscp value.
// See http://tools.ietf.org/html/rfc2474 for details.
@ -125,8 +107,15 @@ bool ValidateStreamParams(const StreamParams& sp) {
// Dumps an AudioCodec in RFC 2327-ish format.
std::string ToString(const AudioCodec& codec) {
std::stringstream ss;
ss << codec.name << "/" << codec.clockrate << "/" << codec.channels
<< " (" << codec.id << ")";
ss << codec.name << "/" << codec.clockrate << "/" << codec.channels;
if (!codec.params.empty()) {
ss << " {";
for (const auto& param : codec.params) {
ss << " " << param.first << "=" << param.second;
}
ss << " }";
}
ss << " (" << codec.id << ")";
return ss.str();
}
@ -134,10 +123,6 @@ bool IsCodec(const AudioCodec& codec, const char* ref_name) {
return (_stricmp(codec.name.c_str(), ref_name) == 0);
}
bool IsCodec(const webrtc::CodecInst& codec, const char* ref_name) {
return (_stricmp(codec.plname, ref_name) == 0);
}
bool FindCodec(const std::vector<AudioCodec>& codecs,
const AudioCodec& codec,
AudioCodec* found_codec) {
@ -165,12 +150,6 @@ bool VerifyUniquePayloadTypes(const std::vector<AudioCodec>& codecs) {
return it == payload_types.end();
}
// Return true if codec.params[feature] == "1", false otherwise.
bool IsCodecFeatureEnabled(const AudioCodec& codec, const char* feature) {
int value;
return codec.GetParam(feature, &value) && value == 1;
}
rtc::Optional<std::string> GetAudioNetworkAdaptorConfig(
const AudioOptions& options) {
if (options.audio_network_adaptor && *options.audio_network_adaptor &&
@ -182,85 +161,6 @@ rtc::Optional<std::string> GetAudioNetworkAdaptorConfig(
return rtc::Optional<std::string>();
}
// Returns integer parameter params[feature] if it is defined. Returns
// |default_value| otherwise.
int GetCodecFeatureInt(const AudioCodec& codec,
const char* feature,
int default_value) {
int value = 0;
if (codec.GetParam(feature, &value)) {
return value;
}
return default_value;
}
// Use params[kCodecParamMaxAverageBitrate] if it is defined, use codec.bitrate
// otherwise. If the value (either from params or codec.bitrate) <=0, use the
// default configuration. If the value is beyond feasible bit rate of Opus,
// clamp it. Returns the Opus bit rate for operation.
int GetOpusBitrate(const AudioCodec& codec, int max_playback_rate) {
int bitrate = 0;
bool use_param = true;
if (!codec.GetParam(kCodecParamMaxAverageBitrate, &bitrate)) {
bitrate = codec.bitrate;
use_param = false;
}
if (bitrate <= 0) {
if (max_playback_rate <= 8000) {
bitrate = kOpusBitrateNbBps;
} else if (max_playback_rate <= 16000) {
bitrate = kOpusBitrateWbBps;
} else {
bitrate = kOpusBitrateFbBps;
}
if (IsCodecFeatureEnabled(codec, kCodecParamStereo)) {
bitrate *= 2;
}
} else if (bitrate < kOpusMinBitrateBps || bitrate > kOpusMaxBitrateBps) {
bitrate = (bitrate < kOpusMinBitrateBps) ? kOpusMinBitrateBps
: kOpusMaxBitrateBps;
std::string rate_source =
use_param ? "Codec parameter \"maxaveragebitrate\"" :
"Supplied Opus bitrate";
LOG(LS_WARNING) << rate_source
<< " is invalid and is replaced by: "
<< bitrate;
}
return bitrate;
}
void GetOpusConfig(const AudioCodec& codec,
webrtc::CodecInst* voe_codec,
bool* enable_codec_fec,
int* max_playback_rate,
bool* enable_codec_dtx,
int* min_ptime_ms,
int* max_ptime_ms) {
*enable_codec_fec = IsCodecFeatureEnabled(codec, kCodecParamUseInbandFec);
*enable_codec_dtx = IsCodecFeatureEnabled(codec, kCodecParamUseDtx);
*max_playback_rate = GetCodecFeatureInt(codec, kCodecParamMaxPlaybackRate,
kOpusDefaultMaxPlaybackRate);
*max_ptime_ms =
GetCodecFeatureInt(codec, kCodecParamMaxPTime, kOpusDefaultMaxPTime);
*min_ptime_ms =
GetCodecFeatureInt(codec, kCodecParamMinPTime, kOpusDefaultMinPTime);
if (*max_ptime_ms < *min_ptime_ms) {
// If min ptime or max ptime defined by codec parameter is wrong, we use
// the default values.
*max_ptime_ms = kOpusDefaultMaxPTime;
*min_ptime_ms = kOpusDefaultMinPTime;
}
// If OPUS, change what we send according to the "stereo" codec
// parameter, and not the "channels" parameter. We set
// voe_codec.channels to 2 if "stereo=1" and 1 otherwise. If
// the bitrate is not specified, i.e. is <= zero, we set it to the
// appropriate default value for mono or stereo Opus.
voe_codec->channels = IsCodecFeatureEnabled(codec, kCodecParamStereo) ? 2 : 1;
voe_codec->rate = GetOpusBitrate(codec, *max_playback_rate);
}
webrtc::AudioState::Config MakeAudioStateConfig(
VoEWrapper* voe_wrapper,
rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer) {
@ -274,283 +174,41 @@ webrtc::AudioState::Config MakeAudioStateConfig(
return config;
}
class WebRtcVoiceCodecs final {
public:
// TODO(solenberg): Do this filtering once off-line, add a simple AudioCodec
// list and add a test which verifies VoE supports the listed codecs.
static std::vector<AudioCodec> SupportedSendCodecs() {
std::vector<AudioCodec> result;
// Iterate first over our preferred codecs list, so that the results are
// added in order of preference.
for (size_t i = 0; i < arraysize(kCodecPrefs); ++i) {
const CodecPref* pref = &kCodecPrefs[i];
for (webrtc::CodecInst voe_codec : webrtc::acm2::RentACodec::Database()) {
// Change the sample rate of G722 to 8000 to match SDP.
MaybeFixupG722(&voe_codec, 8000);
// Skip uncompressed formats.
if (IsCodec(voe_codec, kL16CodecName)) {
continue;
}
if (!IsCodec(voe_codec, pref->name) ||
pref->clockrate != voe_codec.plfreq ||
pref->channels != voe_codec.channels) {
// Not a match.
continue;
}
AudioCodec codec(pref->payload_type, voe_codec.plname, voe_codec.plfreq,
voe_codec.rate, voe_codec.channels);
LOG(LS_INFO) << "Adding supported codec: " << ToString(codec);
if (IsCodec(codec, kIsacCodecName)) {
// Indicate auto-bitrate in signaling.
codec.bitrate = 0;
}
if (IsCodec(codec, kOpusCodecName)) {
// Only add fmtp parameters that differ from the spec.
if (kPreferredMinPTime != kOpusDefaultMinPTime) {
codec.params[kCodecParamMinPTime] =
rtc::ToString(kPreferredMinPTime);
}
if (kPreferredMaxPTime != kOpusDefaultMaxPTime) {
codec.params[kCodecParamMaxPTime] =
rtc::ToString(kPreferredMaxPTime);
}
codec.SetParam(kCodecParamUseInbandFec, 1);
codec.AddFeedbackParam(
FeedbackParam(kRtcpFbParamTransportCc, kParamValueEmpty));
// TODO(hellner): Add ptime, sprop-stereo, and stereo
// when they can be set to values other than the default.
}
result.push_back(codec);
}
}
return result;
}
static bool ToCodecInst(const AudioCodec& in,
webrtc::CodecInst* out) {
for (webrtc::CodecInst voe_codec : webrtc::acm2::RentACodec::Database()) {
// Change the sample rate of G722 to 8000 to match SDP.
MaybeFixupG722(&voe_codec, 8000);
AudioCodec codec(voe_codec.pltype, voe_codec.plname, voe_codec.plfreq,
voe_codec.rate, voe_codec.channels);
bool multi_rate = IsCodecMultiRate(voe_codec);
// Allow arbitrary rates for ISAC to be specified.
if (multi_rate) {
// Set codec.bitrate to 0 so the check for codec.Matches() passes.
codec.bitrate = 0;
}
if (codec.Matches(in)) {
if (out) {
// Fixup the payload type.
voe_codec.pltype = in.id;
// Set bitrate if specified.
if (multi_rate && in.bitrate != 0) {
voe_codec.rate = in.bitrate;
}
// Reset G722 sample rate to 16000 to match WebRTC.
MaybeFixupG722(&voe_codec, 16000);
*out = voe_codec;
}
return true;
}
}
return false;
}
static bool IsCodecMultiRate(const webrtc::CodecInst& codec) {
for (size_t i = 0; i < arraysize(kCodecPrefs); ++i) {
if (IsCodec(codec, kCodecPrefs[i].name) &&
kCodecPrefs[i].clockrate == codec.plfreq) {
return kCodecPrefs[i].is_multi_rate;
}
}
return false;
}
static int MaxBitrateBps(const webrtc::CodecInst& codec) {
for (size_t i = 0; i < arraysize(kCodecPrefs); ++i) {
if (IsCodec(codec, kCodecPrefs[i].name) &&
kCodecPrefs[i].clockrate == codec.plfreq) {
return kCodecPrefs[i].max_bitrate_bps;
}
}
return 0;
}
static rtc::ArrayView<const int> GetPacketSizesMs(
const webrtc::CodecInst& codec) {
for (size_t i = 0; i < arraysize(kCodecPrefs); ++i) {
if (IsCodec(codec, kCodecPrefs[i].name)) {
size_t num_packet_sizes = kMaxNumPacketSize;
for (int index = 0; index < kMaxNumPacketSize; index++) {
if (kCodecPrefs[i].packet_sizes_ms[index] == 0) {
num_packet_sizes = index;
break;
}
}
return rtc::ArrayView<const int>(kCodecPrefs[i].packet_sizes_ms,
num_packet_sizes);
}
}
return rtc::ArrayView<const int>();
}
// If the AudioCodec param kCodecParamPTime is set, then we will set it to
// codec pacsize if it's valid, or we will pick the next smallest value we
// support.
// TODO(Brave): Query supported packet sizes from ACM when the API is ready.
static bool SetPTimeAsPacketSize(webrtc::CodecInst* codec, int ptime_ms) {
for (const CodecPref& codec_pref : kCodecPrefs) {
if ((IsCodec(*codec, codec_pref.name) &&
codec_pref.clockrate == codec->plfreq) ||
IsCodec(*codec, kG722CodecName)) {
int packet_size_ms = SelectPacketSize(codec_pref, ptime_ms);
if (packet_size_ms) {
// Convert unit from milli-seconds to samples.
codec->pacsize = (codec->plfreq / 1000) * packet_size_ms;
return true;
}
}
}
return false;
}
static const AudioCodec* GetPreferredCodec(
const std::vector<AudioCodec>& codecs,
webrtc::CodecInst* out) {
RTC_DCHECK(out);
// Select the preferred send codec (the first non-telephone-event/CN codec).
for (const AudioCodec& codec : codecs) {
if (IsCodec(codec, kDtmfCodecName) || IsCodec(codec, kCnCodecName)) {
// Skip telephone-event/CN codecs - they will be handled later.
continue;
}
// We'll use the first codec in the list to actually send audio data.
// Be sure to use the payload type requested by the remote side.
// Ignore codecs we don't know about. The negotiation step should prevent
// this, but double-check to be sure.
if (!ToCodecInst(codec, out)) {
LOG(LS_WARNING) << "Unknown codec " << ToString(codec);
continue;
}
return &codec;
}
return nullptr;
}
private:
static const int kMaxNumPacketSize = 6;
struct CodecPref {
const char* name;
int clockrate;
size_t channels;
int payload_type;
bool is_multi_rate;
int packet_sizes_ms[kMaxNumPacketSize];
int max_bitrate_bps;
};
// Note: keep the supported packet sizes in ascending order.
static const CodecPref kCodecPrefs[14];
static int SelectPacketSize(const CodecPref& codec_pref, int ptime_ms) {
int selected_packet_size_ms = codec_pref.packet_sizes_ms[0];
for (int packet_size_ms : codec_pref.packet_sizes_ms) {
if (packet_size_ms && packet_size_ms <= ptime_ms) {
selected_packet_size_ms = packet_size_ms;
}
}
return selected_packet_size_ms;
}
// Changes RTP timestamp rate of G722. This is due to the "bug" in the RFC
// which says that G722 should be advertised as 8 kHz although it is a 16 kHz
// codec.
static void MaybeFixupG722(webrtc::CodecInst* voe_codec, int new_plfreq) {
if (IsCodec(*voe_codec, kG722CodecName)) {
// If the DCHECK triggers, the codec definition in WebRTC VoiceEngine
// has changed, and this special case is no longer needed.
RTC_DCHECK(voe_codec->plfreq != new_plfreq);
voe_codec->plfreq = new_plfreq;
}
}
};
const WebRtcVoiceCodecs::CodecPref WebRtcVoiceCodecs::kCodecPrefs[14] = {
#if WEBRTC_OPUS_SUPPORT_120MS_PTIME
{kOpusCodecName, 48000, 2, 111, true, {10, 20, 40, 60, 120},
kOpusMaxBitrateBps},
#else
{kOpusCodecName, 48000, 2, 111, true, {10, 20, 40, 60}, kOpusMaxBitrateBps},
#endif
{kIsacCodecName, 16000, 1, 103, true, {30, 60}, kIsacMaxBitrateBps},
{kIsacCodecName, 32000, 1, 104, true, {30}, kIsacMaxBitrateBps},
// G722 should be advertised as 8000 Hz because of the RFC "bug".
{kG722CodecName, 8000, 1, 9, false, {10, 20, 30, 40, 50, 60}},
{kIlbcCodecName, 8000, 1, 102, false, {20, 30, 40, 60}},
{kPcmuCodecName, 8000, 1, 0, false, {10, 20, 30, 40, 50, 60}},
{kPcmaCodecName, 8000, 1, 8, false, {10, 20, 30, 40, 50, 60}},
{kCnCodecName, 32000, 1, 106, false, {}},
{kCnCodecName, 16000, 1, 105, false, {}},
{kCnCodecName, 8000, 1, 13, false, {}},
{kDtmfCodecName, 48000, 1, 110, false, {}},
{kDtmfCodecName, 32000, 1, 112, false, {}},
{kDtmfCodecName, 16000, 1, 113, false, {}},
{kDtmfCodecName, 8000, 1, 126, false, {}}
};
// |max_send_bitrate_bps| is the bitrate from "b=" in SDP.
// |rtp_max_bitrate_bps| is the bitrate from RtpSender::SetParameters.
rtc::Optional<int> ComputeSendBitrate(int max_send_bitrate_bps,
rtc::Optional<int> rtp_max_bitrate_bps,
const webrtc::CodecInst& codec_inst) {
const webrtc::AudioCodecSpec& spec) {
// If application-configured bitrate is set, take minimum of that and SDP
// bitrate.
const int bps = rtp_max_bitrate_bps
? MinPositive(max_send_bitrate_bps, *rtp_max_bitrate_bps)
: max_send_bitrate_bps;
const int codec_rate = codec_inst.rate;
if (bps <= 0) {
return rtc::Optional<int>(codec_rate);
return rtc::Optional<int>(spec.info.default_bitrate_bps);
}
if (codec_inst.pltype == -1) {
return rtc::Optional<int>(codec_rate);
;
}
if (WebRtcVoiceCodecs::IsCodecMultiRate(codec_inst)) {
// If codec is multi-rate then just set the bitrate.
return rtc::Optional<int>(
std::min(bps, WebRtcVoiceCodecs::MaxBitrateBps(codec_inst)));
}
if (bps < codec_inst.rate) {
if (bps < spec.info.min_bitrate_bps) {
// If codec is not multi-rate and |bps| is less than the fixed bitrate then
// fail. If codec is not multi-rate and |bps| exceeds or equal the fixed
// bitrate then ignore.
LOG(LS_ERROR) << "Failed to set codec " << codec_inst.plname
LOG(LS_ERROR) << "Failed to set codec " << spec.format.name
<< " to bitrate " << bps << " bps"
<< ", requires at least " << codec_inst.rate << " bps.";
<< ", requires at least " << spec.info.min_bitrate_bps
<< " bps.";
return rtc::Optional<int>();
}
return rtc::Optional<int>(codec_rate);
if (spec.info.HasFixedBitrate()) {
return rtc::Optional<int>(spec.info.default_bitrate_bps);
} else {
// If codec is multi-rate then just set the bitrate.
return rtc::Optional<int>(std::min(bps, spec.info.max_bitrate_bps));
}
}
} // namespace
bool WebRtcVoiceEngine::ToCodecInst(const AudioCodec& in,
webrtc::CodecInst* out) {
return WebRtcVoiceCodecs::ToCodecInst(in, out);
}
WebRtcVoiceEngine::WebRtcVoiceEngine(
webrtc::AudioDeviceModule* adm,
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory,
@ -565,7 +223,10 @@ WebRtcVoiceEngine::WebRtcVoiceEngine(
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory,
rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer,
VoEWrapper* voe_wrapper)
: adm_(adm), decoder_factory_(decoder_factory), voe_wrapper_(voe_wrapper) {
: adm_(adm),
encoder_factory_(webrtc::CreateBuiltinAudioEncoderFactory()),
decoder_factory_(decoder_factory),
voe_wrapper_(voe_wrapper) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
LOG(LS_INFO) << "WebRtcVoiceEngine::WebRtcVoiceEngine";
RTC_DCHECK(voe_wrapper);
@ -575,13 +236,13 @@ WebRtcVoiceEngine::WebRtcVoiceEngine(
// Load our audio codec list.
LOG(LS_INFO) << "Supported send codecs in order of preference:";
send_codecs_ = WebRtcVoiceCodecs::SupportedSendCodecs();
send_codecs_ = CollectCodecs(encoder_factory_->GetSupportedEncoders());
for (const AudioCodec& codec : send_codecs_) {
LOG(LS_INFO) << ToString(codec);
}
LOG(LS_INFO) << "Supported recv codecs in order of preference:";
recv_codecs_ = CollectRecvCodecs();
recv_codecs_ = CollectCodecs(decoder_factory_->GetSupportedDecoders());
for (const AudioCodec& codec : recv_codecs_) {
LOG(LS_INFO) << ToString(codec);
}
@ -1056,11 +717,10 @@ webrtc::voe::TransmitMixer* WebRtcVoiceEngine::transmit_mixer() {
return transmit_mixer_;
}
AudioCodecs WebRtcVoiceEngine::CollectRecvCodecs() const {
AudioCodecs WebRtcVoiceEngine::CollectCodecs(
const std::vector<webrtc::AudioCodecSpec>& specs) const {
PayloadTypeMapper mapper;
AudioCodecs out;
const std::vector<webrtc::AudioCodecSpec>& specs =
decoder_factory_->GetSupportedDecoders();
// Only generate CN payload types for these clockrates:
std::map<int, bool, std::greater<int>> generate_cn = {{ 8000, false },
@ -1140,12 +800,14 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream
webrtc::AudioTransport* voe_audio_transport,
uint32_t ssrc,
const std::string& c_name,
const webrtc::AudioSendStream::Config::SendCodecSpec& send_codec_spec,
const rtc::Optional<webrtc::AudioSendStream::Config::SendCodecSpec>&
send_codec_spec,
const std::vector<webrtc::RtpExtension>& extensions,
int max_send_bitrate_bps,
const rtc::Optional<std::string>& audio_network_adaptor_config,
webrtc::Call* call,
webrtc::Transport* send_transport)
webrtc::Transport* send_transport,
const rtc::scoped_refptr<webrtc::AudioEncoderFactory>& encoder_factory)
: voe_audio_transport_(voe_audio_transport),
call_(call),
config_(send_transport),
@ -1157,13 +819,20 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream
// TODO(solenberg): Once we're not using FakeWebRtcVoiceEngine anymore:
// RTC_DCHECK(voe_audio_transport);
RTC_DCHECK(call);
RTC_DCHECK(encoder_factory);
config_.rtp.ssrc = ssrc;
config_.rtp.c_name = c_name;
config_.voe_channel_id = ch;
config_.rtp.extensions = extensions;
config_.audio_network_adaptor_config = audio_network_adaptor_config;
config_.encoder_factory = encoder_factory;
rtp_parameters_.encodings[0].ssrc = rtc::Optional<uint32_t>(ssrc);
RecreateAudioSendStream(send_codec_spec);
if (send_codec_spec) {
UpdateSendCodecSpec(*send_codec_spec);
}
stream_ = call_->CreateAudioSendStream(config_);
}
~WebRtcAudioSendStream() override {
@ -1172,56 +841,45 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream
call_->DestroyAudioSendStream(stream_);
}
void RecreateAudioSendStream(
void SetSendCodecSpec(
const webrtc::AudioSendStream::Config::SendCodecSpec& send_codec_spec) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
send_codec_spec_ = send_codec_spec;
config_.rtp.nack.rtp_history_ms =
send_codec_spec_.nack_enabled ? kNackRtpHistoryMs : 0;
config_.send_codec_spec = send_codec_spec_;
auto send_rate = ComputeSendBitrate(
max_send_bitrate_bps_, rtp_parameters_.encodings[0].max_bitrate_bps,
send_codec_spec.codec_inst);
if (send_rate) {
// Apply a send rate that abides by |max_send_bitrate_bps_| and
// |rtp_parameters_| when possible. Otherwise use the codec rate.
config_.send_codec_spec.codec_inst.rate = *send_rate;
}
RecreateAudioSendStream();
UpdateSendCodecSpec(send_codec_spec);
ReconfigureAudioSendStream();
}
void RecreateAudioSendStream(
const std::vector<webrtc::RtpExtension>& extensions) {
void SetRtpExtensions(const std::vector<webrtc::RtpExtension>& extensions) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
config_.rtp.extensions = extensions;
RecreateAudioSendStream();
ReconfigureAudioSendStream();
}
void RecreateAudioSendStream(
void SetAudioNetworkAdaptorConfig(
const rtc::Optional<std::string>& audio_network_adaptor_config) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
if (config_.audio_network_adaptor_config == audio_network_adaptor_config) {
return;
}
config_.audio_network_adaptor_config = audio_network_adaptor_config;
RecreateAudioSendStream();
UpdateAllowedBitrateRange();
ReconfigureAudioSendStream();
}
bool SetMaxSendBitrate(int bps) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
auto send_rate =
ComputeSendBitrate(bps, rtp_parameters_.encodings[0].max_bitrate_bps,
send_codec_spec_.codec_inst);
RTC_DCHECK(config_.send_codec_spec);
RTC_DCHECK(audio_codec_spec_);
auto send_rate = ComputeSendBitrate(
bps, rtp_parameters_.encodings[0].max_bitrate_bps, *audio_codec_spec_);
if (!send_rate) {
return false;
}
max_send_bitrate_bps_ = bps;
if (config_.send_codec_spec.codec_inst.rate != *send_rate) {
// Recreate AudioSendStream with new bit rate.
config_.send_codec_spec.codec_inst.rate = *send_rate;
RecreateAudioSendStream();
if (send_rate != config_.send_codec_spec->target_bitrate_bps) {
config_.send_codec_spec->target_bitrate_bps = send_rate;
ReconfigureAudioSendStream();
}
return true;
}
@ -1337,11 +995,15 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream
if (!ValidateRtpParameters(parameters)) {
return false;
}
auto send_rate = ComputeSendBitrate(max_send_bitrate_bps_,
parameters.encodings[0].max_bitrate_bps,
send_codec_spec_.codec_inst);
if (!send_rate) {
return false;
rtc::Optional<int> send_rate;
if (audio_codec_spec_) {
send_rate = ComputeSendBitrate(max_send_bitrate_bps_,
parameters.encodings[0].max_bitrate_bps,
*audio_codec_spec_);
if (!send_rate) {
return false;
}
}
const rtc::Optional<int> old_rtp_max_bitrate =
@ -1350,9 +1012,12 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream
rtp_parameters_ = parameters;
if (rtp_parameters_.encodings[0].max_bitrate_bps != old_rtp_max_bitrate) {
// Recreate AudioSendStream with new bit rate.
config_.send_codec_spec.codec_inst.rate = *send_rate;
RecreateAudioSendStream();
// Reconfigure AudioSendStream with new bit rate.
if (send_rate) {
config_.send_codec_spec->target_bitrate_bps = send_rate;
}
UpdateAllowedBitrateRange();
ReconfigureAudioSendStream();
} else {
// parameters.encodings[0].active could have changed.
UpdateSendState();
@ -1372,18 +1037,17 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream
}
}
void RecreateAudioSendStream() {
void UpdateAllowedBitrateRange() {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
if (stream_) {
call_->DestroyAudioSendStream(stream_);
stream_ = nullptr;
}
RTC_DCHECK(!stream_);
if (webrtc::field_trial::IsEnabled("WebRTC-Audio-SendSideBwe")) {
const bool is_opus =
config_.send_codec_spec &&
!STR_CASE_CMP(config_.send_codec_spec->format.name.c_str(),
kOpusCodecName);
if (is_opus && webrtc::field_trial::IsEnabled("WebRTC-Audio-SendSideBwe")) {
config_.min_bitrate_bps = kOpusMinBitrateBps;
// This means that when RtpParameters is reset, we may change the
// encoder's bit rate immediately (through call_->CreateAudioSendStream),
// encoder's bit rate immediately (through ReconfigureAudioSendStream()),
// meanwhile change the cap to the output of BWE.
config_.max_bitrate_bps =
rtp_parameters_.encodings[0].max_bitrate_bps
@ -1393,48 +1057,65 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream
// TODO(mflodman): Keep testing this and set proper values.
// Note: This is an early experiment currently only supported by Opus.
if (send_side_bwe_with_overhead_) {
auto packet_sizes_ms = WebRtcVoiceCodecs::GetPacketSizesMs(
config_.send_codec_spec.codec_inst);
if (!packet_sizes_ms.empty()) {
int max_packet_size_ms =
*std::max_element(packet_sizes_ms.begin(), packet_sizes_ms.end());
const int max_packet_size_ms =
WEBRTC_OPUS_SUPPORT_120MS_PTIME ? 120 : 60;
// Audio network adaptor will just use 20ms and 60ms frame lengths.
// The adaptor will only be active for the Opus encoder.
if (config_.audio_network_adaptor_config &&
IsCodec(config_.send_codec_spec.codec_inst, kOpusCodecName)) {
#if WEBRTC_OPUS_SUPPORT_120MS_PTIME
max_packet_size_ms = 120;
#else
max_packet_size_ms = 60;
#endif
}
// OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12)
constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12;
// OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12)
constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12;
int min_overhead_bps =
kOverheadPerPacket * 8 * 1000 / max_packet_size_ms;
int min_overhead_bps =
kOverheadPerPacket * 8 * 1000 / max_packet_size_ms;
// We assume that |config_.max_bitrate_bps| before the next line is
// a hard limit on the payload bitrate, so we add min_overhead_bps to
// it to ensure that, when overhead is deducted, the payload rate
// never goes beyond the limit.
// Note: this also means that if a higher overhead is forced, we
// cannot reach the limit.
// TODO(minyue): Reconsider this when the signaling to BWE is done
// through a dedicated API.
config_.max_bitrate_bps += min_overhead_bps;
// We assume that |config_.max_bitrate_bps| before the next line is
// a hard limit on the payload bitrate, so we add min_overhead_bps to
// it to ensure that, when overhead is deducted, the payload rate
// never goes beyond the limit.
// Note: this also means that if a higher overhead is forced, we
// cannot reach the limit.
// TODO(minyue): Reconsider this when the signaling to BWE is done
// through a dedicated API.
config_.max_bitrate_bps += min_overhead_bps;
// In contrast to max_bitrate_bps, we let min_bitrate_bps always be
// reachable.
config_.min_bitrate_bps += min_overhead_bps;
}
// In contrast to max_bitrate_bps, we let min_bitrate_bps always be
// reachable.
config_.min_bitrate_bps += min_overhead_bps;
}
}
stream_ = call_->CreateAudioSendStream(config_);
RTC_CHECK(stream_);
UpdateSendState();
}
void UpdateSendCodecSpec(
const webrtc::AudioSendStream::Config::SendCodecSpec& send_codec_spec) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
config_.rtp.nack.rtp_history_ms =
send_codec_spec.nack_enabled ? kNackRtpHistoryMs : 0;
config_.send_codec_spec =
rtc::Optional<webrtc::AudioSendStream::Config::SendCodecSpec>(
send_codec_spec);
auto info =
config_.encoder_factory->QueryAudioEncoder(send_codec_spec.format);
RTC_DCHECK(info);
// If a specific target bitrate has been set for the stream, use that as
// the new default bitrate when computing send bitrate.
if (send_codec_spec.target_bitrate_bps) {
info->default_bitrate_bps = std::max(
info->min_bitrate_bps,
std::min(info->max_bitrate_bps, *send_codec_spec.target_bitrate_bps));
}
audio_codec_spec_.emplace(
webrtc::AudioCodecSpec{send_codec_spec.format, *info});
config_.send_codec_spec->target_bitrate_bps = ComputeSendBitrate(
max_send_bitrate_bps_, rtp_parameters_.encodings[0].max_bitrate_bps,
*audio_codec_spec_);
UpdateAllowedBitrateRange();
}
void ReconfigureAudioSendStream() {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
RTC_DCHECK(stream_);
stream_->Reconfigure(config_);
}
rtc::ThreadChecker worker_thread_checker_;
@ -1455,7 +1136,7 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream
bool muted_ = false;
int max_send_bitrate_bps_;
webrtc::RtpParameters rtp_parameters_;
webrtc::AudioSendStream::Config::SendCodecSpec send_codec_spec_;
rtc::Optional<webrtc::AudioCodecSpec> audio_codec_spec_;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcAudioSendStream);
};
@ -1651,7 +1332,7 @@ bool WebRtcVoiceMediaChannel::SetSendParameters(
if (send_rtp_extensions_ != filtered_extensions) {
send_rtp_extensions_.swap(filtered_extensions);
for (auto& it : send_streams_) {
it.second->RecreateAudioSendStream(send_rtp_extensions_);
it.second->SetRtpExtensions(send_rtp_extensions_);
}
}
@ -1818,10 +1499,10 @@ bool WebRtcVoiceMediaChannel::SetOptions(const AudioOptions& options) {
return false;
}
rtc::Optional<std::string> audio_network_adatptor_config =
rtc::Optional<std::string> audio_network_adaptor_config =
GetAudioNetworkAdaptorConfig(options_);
for (auto& it : send_streams_) {
it.second->RecreateAudioSendStream(audio_network_adatptor_config);
it.second->SetAudioNetworkAdaptorConfig(audio_network_adaptor_config);
}
LOG(LS_INFO) << "Set voice channel options. Current options: "
@ -1938,86 +1619,66 @@ bool WebRtcVoiceMediaChannel::SetSendCodecs(
}
}
// Scan through the list to figure out the codec to use for sending, along
// with the proper configuration for VAD, CNG, NACK and Opus-specific
// parameters.
// TODO(solenberg): Refactor this logic once we create AudioEncoders here.
webrtc::AudioSendStream::Config::SendCodecSpec send_codec_spec;
// Scan through the list to figure out the codec to use for sending.
rtc::Optional<webrtc::AudioSendStream::Config::SendCodecSpec> send_codec_spec;
webrtc::Call::Config::BitrateConfig bitrate_config;
{
send_codec_spec.nack_enabled = send_codec_spec_.nack_enabled;
rtc::Optional<webrtc::AudioCodecInfo> voice_codec_info;
for (const AudioCodec& voice_codec : codecs) {
if (!(IsCodec(voice_codec, kCnCodecName) ||
IsCodec(voice_codec, kDtmfCodecName) ||
IsCodec(voice_codec, kRedCodecName))) {
webrtc::SdpAudioFormat format(voice_codec.name, voice_codec.clockrate,
voice_codec.channels, voice_codec.params);
// Find send codec (the first non-telephone-event/CN codec).
const AudioCodec* codec = WebRtcVoiceCodecs::GetPreferredCodec(
codecs, &send_codec_spec.codec_inst);
if (!codec) {
LOG(LS_WARNING) << "Received empty list of codecs.";
return false;
}
send_codec_spec.transport_cc_enabled = HasTransportCc(*codec);
send_codec_spec.nack_enabled = HasNack(*codec);
bitrate_config = GetBitrateConfigForCodec(*codec);
// For Opus as the send codec, we are to determine inband FEC, maximum
// playback rate, and opus internal dtx.
if (IsCodec(*codec, kOpusCodecName)) {
GetOpusConfig(*codec, &send_codec_spec.codec_inst,
&send_codec_spec.enable_codec_fec,
&send_codec_spec.opus_max_playback_rate,
&send_codec_spec.enable_opus_dtx,
&send_codec_spec.min_ptime_ms,
&send_codec_spec.max_ptime_ms);
}
// Set packet size if the AudioCodec param kCodecParamPTime is set.
int ptime_ms = 0;
if (codec->GetParam(kCodecParamPTime, &ptime_ms)) {
if (!WebRtcVoiceCodecs::SetPTimeAsPacketSize(
&send_codec_spec.codec_inst, ptime_ms)) {
LOG(LS_WARNING) << "Failed to set packet size for codec "
<< send_codec_spec.codec_inst.plname;
return false;
}
}
// Loop through the codecs list again to find the CN codec.
// TODO(solenberg): Break out into a separate function?
for (const AudioCodec& cn_codec : codecs) {
// Ignore codecs we don't know about. The negotiation step should prevent
// this, but double-check to be sure.
webrtc::CodecInst voe_codec = {0};
if (!WebRtcVoiceEngine::ToCodecInst(cn_codec, &voe_codec)) {
LOG(LS_WARNING) << "Unknown codec " << ToString(cn_codec);
voice_codec_info = engine()->encoder_factory_->QueryAudioEncoder(format);
if (!voice_codec_info) {
LOG(LS_WARNING) << "Unknown codec " << ToString(voice_codec);
continue;
}
send_codec_spec =
rtc::Optional<webrtc::AudioSendStream::Config::SendCodecSpec>(
{voice_codec.id, format});
if (voice_codec.bitrate > 0) {
send_codec_spec->target_bitrate_bps =
rtc::Optional<int>(voice_codec.bitrate);
}
send_codec_spec->transport_cc_enabled = HasTransportCc(voice_codec);
send_codec_spec->nack_enabled = HasNack(voice_codec);
bitrate_config = GetBitrateConfigForCodec(voice_codec);
break;
}
}
if (!send_codec_spec) {
return false;
}
RTC_DCHECK(voice_codec_info);
if (voice_codec_info->allow_comfort_noise) {
// Loop through the codecs list again to find the CN codec.
// TODO(solenberg): Break out into a separate function?
for (const AudioCodec& cn_codec : codecs) {
if (IsCodec(cn_codec, kCnCodecName) &&
cn_codec.clockrate == codec->clockrate) {
// Turn voice activity detection/comfort noise on if supported.
// Set the wideband CN payload type appropriately.
// (narrowband always uses the static payload type 13).
int cng_plfreq = -1;
cn_codec.clockrate == send_codec_spec->format.clockrate_hz) {
switch (cn_codec.clockrate) {
case 8000:
case 16000:
case 32000:
cng_plfreq = cn_codec.clockrate;
send_codec_spec->cng_payload_type = rtc::Optional<int>(cn_codec.id);
break;
default:
LOG(LS_WARNING) << "CN frequency " << cn_codec.clockrate
<< " not supported.";
continue;
break;
}
send_codec_spec.cng_payload_type = cn_codec.id;
send_codec_spec.cng_plfreq = cng_plfreq;
break;
}
}
// Find the telephone-event PT exactly matching the preferred send codec.
for (const AudioCodec& dtmf_codec : dtmf_codecs) {
if (dtmf_codec.clockrate == codec->clockrate) {
if (dtmf_codec.clockrate == send_codec_spec->format.clockrate_hz) {
dtmf_payload_type_ = rtc::Optional<int>(dtmf_codec.id);
dtmf_payload_freq_ = dtmf_codec.clockrate;
break;
@ -2029,7 +1690,7 @@ bool WebRtcVoiceMediaChannel::SetSendCodecs(
send_codec_spec_ = std::move(send_codec_spec);
// Apply new settings to all streams.
for (const auto& kv : send_streams_) {
kv.second->RecreateAudioSendStream(send_codec_spec_);
kv.second->SetSendCodecSpec(*send_codec_spec_);
}
} else {
// If the codec isn't changing, set the start bitrate to -1 which means
@ -2040,12 +1701,12 @@ bool WebRtcVoiceMediaChannel::SetSendCodecs(
// Check if the transport cc feedback or NACK status has changed on the
// preferred send codec, and in that case reconfigure all receive streams.
if (recv_transport_cc_enabled_ != send_codec_spec_.transport_cc_enabled ||
recv_nack_enabled_ != send_codec_spec_.nack_enabled) {
if (recv_transport_cc_enabled_ != send_codec_spec_->transport_cc_enabled ||
recv_nack_enabled_ != send_codec_spec_->nack_enabled) {
LOG(LS_INFO) << "Recreate all the receive streams because the send "
"codec has changed.";
recv_transport_cc_enabled_ = send_codec_spec_.transport_cc_enabled;
recv_nack_enabled_ = send_codec_spec_.nack_enabled;
recv_transport_cc_enabled_ = send_codec_spec_->transport_cc_enabled;
recv_nack_enabled_ = send_codec_spec_->nack_enabled;
for (auto& kv : recv_streams_) {
kv.second->RecreateAudioReceiveStream(recv_transport_cc_enabled_,
recv_nack_enabled_);
@ -2168,7 +1829,7 @@ bool WebRtcVoiceMediaChannel::AddSendStream(const StreamParams& sp) {
WebRtcAudioSendStream* stream = new WebRtcAudioSendStream(
channel, audio_transport, ssrc, sp.cname, send_codec_spec_,
send_rtp_extensions_, max_send_bitrate_bps_, audio_network_adaptor_config,
call_, this);
call_, this, engine()->encoder_factory_);
send_streams_.insert(std::make_pair(ssrc, stream));
// At this point the stream's local SSRC has been updated. If it is the first

11
webrtc/media/engine/webrtcvoiceengine.h
View File

@ -29,6 +29,7 @@
#include "webrtc/media/engine/apm_helpers.h"
#include "webrtc/media/engine/webrtccommon.h"
#include "webrtc/media/engine/webrtcvoe.h"
#include "webrtc/modules/audio_coding/codecs/audio_encoder_factory.h"
#include "webrtc/modules/audio_processing/include/audio_processing.h"
#include "webrtc/pc/channel.h"
@ -51,9 +52,6 @@ class WebRtcVoiceMediaChannel;
class WebRtcVoiceEngine final : public webrtc::TraceCallback {
friend class WebRtcVoiceMediaChannel;
public:
// Exposed for the WVoE/MC unit test.
static bool ToCodecInst(const AudioCodec& in, webrtc::CodecInst* out);
WebRtcVoiceEngine(
webrtc::AudioDeviceModule* adm,
const rtc::scoped_refptr<webrtc::AudioDecoderFactory>& decoder_factory,
@ -114,13 +112,15 @@ class WebRtcVoiceEngine final : public webrtc::TraceCallback {
webrtc::AudioProcessing* apm();
webrtc::voe::TransmitMixer* transmit_mixer();
AudioCodecs CollectRecvCodecs() const;
AudioCodecs CollectCodecs(
const std::vector<webrtc::AudioCodecSpec>& specs) const;
rtc::ThreadChecker signal_thread_checker_;
rtc::ThreadChecker worker_thread_checker_;
// The audio device manager.
rtc::scoped_refptr<webrtc::AudioDeviceModule> adm_;
rtc::scoped_refptr<webrtc::AudioEncoderFactory> encoder_factory_;
rtc::scoped_refptr<webrtc::AudioDecoderFactory> decoder_factory_;
// Reference to the APM, owned by VoE.
webrtc::AudioProcessing* apm_ = nullptr;
@ -291,7 +291,8 @@ class WebRtcVoiceMediaChannel final : public VoiceMediaChannel,
std::map<uint32_t, WebRtcAudioReceiveStream*> recv_streams_;
std::vector<webrtc::RtpExtension> recv_rtp_extensions_;
webrtc::AudioSendStream::Config::SendCodecSpec send_codec_spec_;
rtc::Optional<webrtc::AudioSendStream::Config::SendCodecSpec>
send_codec_spec_;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcVoiceMediaChannel);
};

749
webrtc/media/engine/webrtcvoiceengine_unittest.cc
View File

File diff suppressed because it is too large Load Diff

4
webrtc/modules/audio_coding/codecs/mock/mock_audio_encoder.h
View File

@ -47,6 +47,10 @@ class MockAudioEncoder : public AudioEncoder {
MOCK_METHOD1(OnReceivedUplinkPacketLossFraction,
void(float uplink_packet_loss_fraction));
MOCK_METHOD2(EnableAudioNetworkAdaptor,
bool(const std::string& config_string,
RtcEventLog* event_log));
// Note, we explicitly chose not to create a mock for the Encode method.
MOCK_METHOD3(EncodeImpl,
EncodedInfo(uint32_t timestamp,

91
webrtc/modules/audio_coding/codecs/mock/mock_audio_encoder_factory.h Normal file
View File

@ -0,0 +1,91 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_AUDIO_CODING_CODECS_MOCK_MOCK_AUDIO_ENCODER_FACTORY_H_
#define WEBRTC_MODULES_AUDIO_CODING_CODECS_MOCK_MOCK_AUDIO_ENCODER_FACTORY_H_
#include <memory>
#include <vector>
#include "webrtc/base/scoped_ref_ptr.h"
#include "webrtc/modules/audio_coding/codecs/audio_encoder_factory.h"
#include "webrtc/test/gmock.h"
namespace webrtc {
class MockAudioEncoderFactory : public AudioEncoderFactory {
public:
MOCK_METHOD0(GetSupportedEncoders, std::vector<AudioCodecSpec>());
MOCK_METHOD1(QueryAudioEncoder,
rtc::Optional<AudioCodecInfo>(const SdpAudioFormat& format));
std::unique_ptr<AudioEncoder> MakeAudioEncoder(int payload_type,
const SdpAudioFormat& format) {
std::unique_ptr<AudioEncoder> return_value;
MakeAudioEncoderMock(payload_type, format, &return_value);
return return_value;
}
MOCK_METHOD3(MakeAudioEncoderMock,
void(int payload_type,
const SdpAudioFormat& format,
std::unique_ptr<AudioEncoder>* return_value));
// Creates a MockAudioEncoderFactory with no formats and that may not be
// invoked to create a codec - useful for initializing a voice engine, for
// example.
static rtc::scoped_refptr<webrtc::MockAudioEncoderFactory>
CreateUnusedFactory() {
using testing::_;
using testing::AnyNumber;
using testing::Return;
rtc::scoped_refptr<webrtc::MockAudioEncoderFactory> factory =
new rtc::RefCountedObject<webrtc::MockAudioEncoderFactory>;
ON_CALL(*factory.get(), GetSupportedEncoders())
.WillByDefault(Return(std::vector<webrtc::AudioCodecSpec>()));
ON_CALL(*factory.get(), QueryAudioEncoder(_))
.WillByDefault(Return(rtc::Optional<AudioCodecInfo>()));
EXPECT_CALL(*factory.get(), GetSupportedEncoders()).Times(AnyNumber());
EXPECT_CALL(*factory.get(), QueryAudioEncoder(_)).Times(AnyNumber());
EXPECT_CALL(*factory.get(), MakeAudioEncoderMock(_, _, _)).Times(0);
return factory;
}
// Creates a MockAudioEncoderFactory with no formats that may be invoked to
// create a codec any number of times. It will, though, return nullptr on each
// call, since it supports no codecs.
static rtc::scoped_refptr<webrtc::MockAudioEncoderFactory>
CreateEmptyFactory() {
using testing::_;
using testing::AnyNumber;
using testing::Return;
using testing::SetArgPointee;
rtc::scoped_refptr<webrtc::MockAudioEncoderFactory> factory =
new rtc::RefCountedObject<webrtc::MockAudioEncoderFactory>;
ON_CALL(*factory.get(), GetSupportedEncoders())
.WillByDefault(Return(std::vector<webrtc::AudioCodecSpec>()));
ON_CALL(*factory.get(), QueryAudioEncoder(_))
.WillByDefault(Return(rtc::Optional<AudioCodecInfo>()));
ON_CALL(*factory.get(), MakeAudioEncoderMock(_, _, _))
.WillByDefault(SetArgPointee<2>(nullptr));
EXPECT_CALL(*factory.get(), GetSupportedEncoders()).Times(AnyNumber());
EXPECT_CALL(*factory.get(), QueryAudioEncoder(_)).Times(AnyNumber());
EXPECT_CALL(*factory.get(), MakeAudioEncoderMock(_, _, _))
.Times(AnyNumber());
return factory;
}
};
} // namespace webrtc
#endif // WEBRTC_MODULES_AUDIO_CODING_CODECS_MOCK_MOCK_AUDIO_ENCODER_FACTORY_H_

1
webrtc/test/BUILD.gn
View File

@ -407,6 +407,7 @@ rtc_source_set("test_common") {
"../audio",
"../base:rtc_base_approved",
"../call",
"../modules/audio_coding:builtin_audio_encoder_factory",
"../modules/audio_mixer:audio_mixer_impl",
"../modules/audio_processing",
"../video",

8
webrtc/test/call_test.cc
View File

@ -15,6 +15,7 @@
#include "webrtc/api/audio_codecs/builtin_audio_decoder_factory.h"
#include "webrtc/base/checks.h"
#include "webrtc/config.h"
#include "webrtc/modules/audio_coding/codecs/builtin_audio_encoder_factory.h"
#include "webrtc/modules/audio_mixer/audio_mixer_impl.h"
#include "webrtc/test/testsupport/fileutils.h"
#include "webrtc/voice_engine/include/voe_base.h"
@ -38,6 +39,7 @@ CallTest::CallTest()
num_audio_streams_(0),
num_flexfec_streams_(0),
decoder_factory_(CreateBuiltinAudioDecoderFactory()),
encoder_factory_(CreateBuiltinAudioEncoderFactory()),
fake_send_audio_device_(nullptr),
fake_recv_audio_device_(nullptr) {}
@ -222,8 +224,10 @@ void CallTest::CreateSendConfig(size_t num_video_streams,
audio_send_config_ = AudioSendStream::Config(send_transport);
audio_send_config_.voe_channel_id = voe_send_.channel_id;
audio_send_config_.rtp.ssrc = kAudioSendSsrc;
audio_send_config_.send_codec_spec.codec_inst =
CodecInst{kAudioSendPayloadType, "OPUS", 48000, 960, 2, 64000};
audio_send_config_.send_codec_spec =
rtc::Optional<AudioSendStream::Config::SendCodecSpec>(
{kAudioSendPayloadType, {"OPUS", 48000, 2, {{"stereo", "1"}}}});
audio_send_config_.encoder_factory = encoder_factory_;
}
// TODO(brandtr): Update this when we support multistream protection.

1
webrtc/test/call_test.h
View File

@ -123,6 +123,7 @@ class CallTest : public ::testing::Test {
size_t num_audio_streams_;
size_t num_flexfec_streams_;
rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_;
rtc::scoped_refptr<AudioEncoderFactory> encoder_factory_;
test::FakeVideoRenderer fake_renderer_;
private:

16
webrtc/test/mock_voe_channel_proxy.h
View File

@ -30,6 +30,9 @@ class MockVoEChannelProxy : public voe::ChannelProxy {
MOCK_METHOD2(SetEncoderForMock,
bool(int payload_type,
std::unique_ptr<AudioEncoder>* encoder));
MOCK_METHOD1(
ModifyEncoder,
void(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier));
MOCK_METHOD1(SetRTCPStatus, void(bool enable));
MOCK_METHOD1(SetLocalSSRC, void(uint32_t ssrc));
MOCK_METHOD1(SetRTCP_CNAME, void(const std::string& c_name));
@ -68,11 +71,6 @@ class MockVoEChannelProxy : public voe::ChannelProxy {
MOCK_METHOD1(SetChannelOutputVolumeScaling, void(float scaling));
MOCK_METHOD1(SetRtcEventLog, void(RtcEventLog* event_log));
MOCK_METHOD1(SetRtcpRttStats, void(RtcpRttStats* rtcp_rtt_stats));
MOCK_METHOD1(EnableAudioNetworkAdaptor,
void(const std::string& config_string));
MOCK_METHOD0(DisableAudioNetworkAdaptor, void());
MOCK_METHOD2(SetReceiverFrameLengthRange,
void(int min_frame_length_ms, int max_frame_length_ms));
MOCK_METHOD2(GetAudioFrameWithInfo,
AudioMixer::Source::AudioFrameInfo(int sample_rate_hz,
AudioFrame* audio_frame));
@ -86,14 +84,6 @@ class MockVoEChannelProxy : public voe::ChannelProxy {
MOCK_CONST_METHOD0(GetPlayoutTimestamp, uint32_t());
MOCK_METHOD1(SetMinimumPlayoutDelay, void(int delay_ms));
MOCK_CONST_METHOD1(GetRecCodec, bool(CodecInst* codec_inst));
MOCK_CONST_METHOD1(GetSendCodec, bool(CodecInst* codec_inst));
MOCK_METHOD1(SetVADStatus, bool(bool enable));
MOCK_METHOD1(SetCodecFECStatus, bool(bool enable));
MOCK_METHOD1(SetOpusDtx, bool(bool enable));
MOCK_METHOD1(SetOpusMaxPlaybackRate, bool(int frequency_hz));
MOCK_METHOD1(SetSendCodec, bool(const CodecInst& codec_inst));
MOCK_METHOD2(SetSendCNPayloadType,
bool(int type, PayloadFrequencies frequency));
MOCK_METHOD1(SetReceiveCodecs,
void(const std::map<int, SdpAudioFormat>& codecs));
MOCK_METHOD1(OnTwccBasedUplinkPacketLossRate, void(float packet_loss_rate));

10
webrtc/video/video_quality_test.cc
View File

@ -1728,9 +1728,13 @@ void VideoQualityTest::SetupAudio(int send_channel_id,
audio_send_config_.min_bitrate_bps = kOpusMinBitrateBps;
audio_send_config_.max_bitrate_bps = kOpusBitrateFbBps;
}
audio_send_config_.send_codec_spec.codec_inst =
CodecInst{kAudioSendPayloadType, "OPUS", 48000, 960, 2, 64000};
audio_send_config_.send_codec_spec.enable_opus_dtx = params_.audio.dtx;
audio_send_config_.send_codec_spec =
rtc::Optional<AudioSendStream::Config::SendCodecSpec>(
{kAudioSendPayloadType,
{"OPUS", 48000, 2,
{{"usedtx", (params_.audio.dtx ? "1" : "0")},
{"stereo", "1"}}}});
audio_send_stream_ = call->CreateAudioSendStream(audio_send_config_);
AudioReceiveStream::Config audio_config;

19
webrtc/voice_engine/channel.cc
View File

@ -1303,9 +1303,15 @@ bool Channel::SetEncoder(int payload_type,
}
audio_coding_->SetEncoder(std::move(encoder));
codec_manager_.UnsetCodecInst();
return true;
}
void Channel::ModifyEncoder(
rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) {
audio_coding_->ModifyEncoder(modifier);
}
int32_t Channel::RegisterVoiceEngineObserver(VoiceEngineObserver& observer) {
WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId),
"Channel::RegisterVoiceEngineObserver()");
@ -1337,9 +1343,16 @@ int32_t Channel::DeRegisterVoiceEngineObserver() {
}
int32_t Channel::GetSendCodec(CodecInst& codec) {
auto send_codec = codec_manager_.GetCodecInst();
if (send_codec) {
codec = *send_codec;
{
const CodecInst* send_codec = codec_manager_.GetCodecInst();
if (send_codec) {
codec = *send_codec;
return 0;
}
}
rtc::Optional<CodecInst> acm_send_codec = audio_coding_->SendCodec();
if (acm_send_codec) {
codec = *acm_send_codec;
return 0;
}
return -1;

2
webrtc/voice_engine/channel.h
View File

@ -175,6 +175,8 @@ class Channel
// Send using this encoder, with this payload type.
bool SetEncoder(int payload_type, std::unique_ptr<AudioEncoder> encoder);
void ModifyEncoder(
rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier);
// API methods

87
webrtc/voice_engine/channel_proxy.cc
View File

@ -36,6 +36,12 @@ bool ChannelProxy::SetEncoder(int payload_type,
return channel()->SetEncoder(payload_type, std::move(encoder));
}
void ChannelProxy::ModifyEncoder(
rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
channel()->ModifyEncoder(modifier);
}
void ChannelProxy::SetRTCPStatus(bool enable) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
channel()->SetRTCPStatus(enable);
@ -235,24 +241,6 @@ void ChannelProxy::SetRtcEventLog(RtcEventLog* event_log) {
channel()->SetRtcEventLog(event_log);
}
void ChannelProxy::EnableAudioNetworkAdaptor(const std::string& config_string) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
bool ret = channel()->EnableAudioNetworkAdaptor(config_string);
RTC_DCHECK(ret);
;}
void ChannelProxy::DisableAudioNetworkAdaptor() {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
channel()->DisableAudioNetworkAdaptor();
}
void ChannelProxy::SetReceiverFrameLengthRange(int min_frame_length_ms,
int max_frame_length_ms) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
channel()->SetReceiverFrameLengthRange(min_frame_length_ms,
max_frame_length_ms);
}
AudioMixer::Source::AudioFrameInfo ChannelProxy::GetAudioFrameWithInfo(
int sample_rate_hz,
AudioFrame* audio_frame) {
@ -319,69 +307,6 @@ bool ChannelProxy::GetRecCodec(CodecInst* codec_inst) const {
return channel()->GetRecCodec(*codec_inst) == 0;
}
bool ChannelProxy::GetSendCodec(CodecInst* codec_inst) const {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
return channel()->GetSendCodec(*codec_inst) == 0;
}
bool ChannelProxy::SetVADStatus(bool enable) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
return channel()->SetVADStatus(enable, VADNormal, false) == 0;
}
bool ChannelProxy::SetCodecFECStatus(bool enable) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
return channel()->SetCodecFECStatus(enable) == 0;
}
bool ChannelProxy::SetOpusDtx(bool enable) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
return channel()->SetOpusDtx(enable) == 0;
}
bool ChannelProxy::SetOpusMaxPlaybackRate(int frequency_hz) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
return channel()->SetOpusMaxPlaybackRate(frequency_hz) == 0;
}
bool ChannelProxy::SetSendCodec(const CodecInst& codec_inst) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
// Validation code copied from VoECodecImpl::SetSendCodec().
if ((STR_CASE_CMP(codec_inst.plname, "L16") == 0) &&
(codec_inst.pacsize >= 960)) {
return false;
}
if (!STR_CASE_CMP(codec_inst.plname, "CN") ||
!STR_CASE_CMP(codec_inst.plname, "TELEPHONE-EVENT") ||
!STR_CASE_CMP(codec_inst.plname, "RED")) {
return false;
}
if ((codec_inst.channels != 1) && (codec_inst.channels != 2)) {
return false;
}
if (!AudioCodingModule::IsCodecValid(codec_inst)) {
return false;
}
return channel()->SetSendCodec(codec_inst) == 0;
}
bool ChannelProxy::SetSendCNPayloadType(int type,
PayloadFrequencies frequency) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
// Validation code copied from VoECodecImpl::SetSendCNPayloadType().
if (type < 96 || type > 127) {
// Only allow dynamic range: 96 to 127
return false;
}
if ((frequency != kFreq16000Hz) && (frequency != kFreq32000Hz)) {
// It is not possible to modify the payload type for CN/8000.
// We only allow modification of the CN payload type for CN/16000
// and CN/32000.
return false;
}
return channel()->SetSendCNPayloadType(type, frequency) == 0;
}
void ChannelProxy::OnTwccBasedUplinkPacketLossRate(float packet_loss_rate) {
// TODO(elad.alon): This fails in UT; fix and uncomment.
// See: https://bugs.chromium.org/p/webrtc/issues/detail?id=7405

13
webrtc/voice_engine/channel_proxy.h
View File

@ -58,6 +58,8 @@ class ChannelProxy {
virtual bool SetEncoder(int payload_type,
std::unique_ptr<AudioEncoder> encoder);
virtual void ModifyEncoder(
rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier);
virtual void SetRTCPStatus(bool enable);
virtual void SetLocalSSRC(uint32_t ssrc);
@ -98,10 +100,6 @@ class ChannelProxy {
GetAudioDecoderFactory() const;
virtual void SetChannelOutputVolumeScaling(float scaling);
virtual void SetRtcEventLog(RtcEventLog* event_log);
virtual void EnableAudioNetworkAdaptor(const std::string& config_string);
virtual void DisableAudioNetworkAdaptor();
virtual void SetReceiverFrameLengthRange(int min_frame_length_ms,
int max_frame_length_ms);
virtual AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo(
int sample_rate_hz,
AudioFrame* audio_frame);
@ -115,13 +113,6 @@ class ChannelProxy {
virtual void SetMinimumPlayoutDelay(int delay_ms);
virtual void SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats);
virtual bool GetRecCodec(CodecInst* codec_inst) const;
virtual bool GetSendCodec(CodecInst* codec_inst) const;
virtual bool SetVADStatus(bool enable);
virtual bool SetCodecFECStatus(bool enable);
virtual bool SetOpusDtx(bool enable);
virtual bool SetOpusMaxPlaybackRate(int frequency_hz);
virtual bool SetSendCodec(const CodecInst& codec_inst);
virtual bool SetSendCNPayloadType(int type, PayloadFrequencies frequency);
virtual void OnTwccBasedUplinkPacketLossRate(float packet_loss_rate);
virtual void OnRecoverableUplinkPacketLossRate(
float recoverable_packet_loss_rate);