admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Changed ramping functionality of the AudioMixer.

Browse Source 
BUG=webrtc:6346

Review-Url: https://codereview.webrtc.org/2398083005
Cr-Commit-Position: refs/heads/master@{#14607}
This commit is contained in:
aleloi 2016-10-12 02:14:59 -07:00 committed by Commit bot
parent 3ec3da6fc2
commit 4b8bfb8ed3
6 changed files with 161 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
webrtc/modules/BUILD.gn
View File

@ -336,6 +336,7 @@ if (rtc_include_tests) {
"audio_coding/neteq/tools/packet_unittest.cc",
"audio_conference_mixer/test/audio_conference_mixer_unittest.cc",
"audio_device/fine_audio_buffer_unittest.cc",
"audio_mixer/audio_frame_manipulator_unittest.cc",
"audio_mixer/test/audio_mixer_unittest.cc",
"audio_processing/aec/echo_cancellation_unittest.cc",
"audio_processing/aec/system_delay_unittest.cc",

52
webrtc/modules/audio_mixer/audio_frame_manipulator.cc
View File

@ -15,22 +15,6 @@
#include "webrtc/typedefs.h"
namespace webrtc {
namespace {
// Linear ramping over 80 samples.
// TODO(hellner): ramp using fix point?
const float kRampArray[] = {
0.0000f, 0.0127f, 0.0253f, 0.0380f, 0.0506f, 0.0633f, 0.0759f, 0.0886f,
0.1013f, 0.1139f, 0.1266f, 0.1392f, 0.1519f, 0.1646f, 0.1772f, 0.1899f,
0.2025f, 0.2152f, 0.2278f, 0.2405f, 0.2532f, 0.2658f, 0.2785f, 0.2911f,
0.3038f, 0.3165f, 0.3291f, 0.3418f, 0.3544f, 0.3671f, 0.3797f, 0.3924f,
0.4051f, 0.4177f, 0.4304f, 0.4430f, 0.4557f, 0.4684f, 0.4810f, 0.4937f,
0.5063f, 0.5190f, 0.5316f, 0.5443f, 0.5570f, 0.5696f, 0.5823f, 0.5949f,
0.6076f, 0.6203f, 0.6329f, 0.6456f, 0.6582f, 0.6709f, 0.6835f, 0.6962f,
0.7089f, 0.7215f, 0.7342f, 0.7468f, 0.7595f, 0.7722f, 0.7848f, 0.7975f,
0.8101f, 0.8228f, 0.8354f, 0.8481f, 0.8608f, 0.8734f, 0.8861f, 0.8987f,
0.9114f, 0.9241f, 0.9367f, 0.9494f, 0.9620f, 0.9747f, 0.9873f, 1.0000f};
const size_t kRampSize = sizeof(kRampArray) / sizeof(kRampArray[0]);
} // namespace
uint32_t AudioMixerCalculateEnergy(const AudioFrame& audio_frame) {
uint32_t energy = 0;
@ -42,34 +26,32 @@ uint32_t AudioMixerCalculateEnergy(const AudioFrame& audio_frame) {
return energy;
}
void NewMixerRampIn(AudioFrame* audio_frame) {
assert(kRampSize <= audio_frame->samples_per_channel_);
for (size_t i = 0; i < kRampSize; i++) {
audio_frame->data_[i] =
static_cast<int16_t>(kRampArray[i] * audio_frame->data_[i]);
}
}
void Ramp(float start_gain, float target_gain, AudioFrame* audio_frame) {
RTC_DCHECK(audio_frame);
RTC_DCHECK_GE(start_gain, 0.0f);
RTC_DCHECK_GE(target_gain, 0.0f);
void NewMixerRampOut(AudioFrame* audio_frame) {
assert(kRampSize <= audio_frame->samples_per_channel_);
for (size_t i = 0; i < kRampSize; i++) {
const size_t kRampPos = kRampSize - 1 - i;
audio_frame->data_[i] =
static_cast<int16_t>(kRampArray[kRampPos] * audio_frame->data_[i]);
size_t samples = audio_frame->samples_per_channel_;
RTC_DCHECK_LT(0u, samples);
float increment = (target_gain - start_gain) / samples;
float gain = start_gain;
for (size_t i = 0; i < samples; ++i) {
// If the audio is interleaved of several channels, we want to
// apply the same gain change to the ith sample of every channel.
for (size_t ch = 0; ch < audio_frame->num_channels_; ++ch) {
audio_frame->data_[audio_frame->num_channels_ * i + ch] *= gain;
}
gain += increment;
}
memset(&audio_frame->data_[kRampSize], 0,
(audio_frame->samples_per_channel_ - kRampSize) *
sizeof(audio_frame->data_[0]));
}
void RemixFrame(size_t target_number_of_channels, AudioFrame* frame) {
RTC_DCHECK_GE(target_number_of_channels, static_cast<size_t>(1));
RTC_DCHECK_LE(target_number_of_channels, static_cast<size_t>(2));
RTC_DCHECK_GE(target_number_of_channels, 1u);
RTC_DCHECK_LE(target_number_of_channels, 2u);
if (frame->num_channels_ == 1 && target_number_of_channels == 2) {
AudioFrameOperations::MonoToStereo(frame);
} else if (frame->num_channels_ == 2 && target_number_of_channels == 1) {
AudioFrameOperations::StereoToMono(frame);
}
}
} // namespace webrtc

6
webrtc/modules/audio_mixer/audio_frame_manipulator.h
View File

@ -19,9 +19,9 @@ namespace webrtc {
// Updates the audioFrame's energy (based on its samples).
uint32_t AudioMixerCalculateEnergy(const AudioFrame& audio_frame);
// Apply linear step function that ramps in/out the audio samples in audio_frame
void NewMixerRampIn(AudioFrame* audio_frame);
void NewMixerRampOut(AudioFrame* audio_frame);
// Ramps up or down the provided audio frame. Ramp(0, 1, frame) will
// linearly increase the samples in the frame from 0 to full volume.
void Ramp(float start_gain, float target_gain, AudioFrame* audio_frame);
// Downmixes or upmixes a frame between stereo and mono.
void RemixFrame(size_t target_number_of_channels, AudioFrame* frame);

63
webrtc/modules/audio_mixer/audio_frame_manipulator_unittest.cc Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <algorithm>
#include "webrtc/modules/audio_mixer/audio_frame_manipulator.h"
#include "webrtc/modules/include/module_common_types.h"
#include "webrtc/test/gtest.h"
namespace webrtc {
namespace {
void FillFrameWithConstants(size_t samples_per_channel,
size_t number_of_channels,
int16_t value,
AudioFrame* frame) {
frame->num_channels_ = number_of_channels;
frame->samples_per_channel_ = samples_per_channel;
std::fill(frame->data_,
frame->data_ + samples_per_channel * number_of_channels, value);
}
} // namespace
TEST(AudioFrameManipulator, CompareForwardRampWithExpectedResultStereo) {
constexpr int kSamplesPerChannel = 5;
constexpr int kNumberOfChannels = 2;
// Create a frame with values 5, 5, 5, ... and channels & samples as above.
AudioFrame frame;
FillFrameWithConstants(kSamplesPerChannel, kNumberOfChannels, 5, &frame);
Ramp(0.0f, 1.0f, &frame);
const int total_samples = kSamplesPerChannel * kNumberOfChannels;
const int16_t expected_result[total_samples] = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4};
EXPECT_TRUE(
std::equal(frame.data_, frame.data_ + total_samples, expected_result));
}
TEST(AudioFrameManipulator, CompareBackwardRampWithExpectedResultMono) {
constexpr int kSamplesPerChannel = 5;
constexpr int kNumberOfChannels = 1;
// Create a frame with values 5, 5, 5, ... and channels & samples as above.
AudioFrame frame;
FillFrameWithConstants(kSamplesPerChannel, kNumberOfChannels, 5, &frame);
Ramp(1.0f, 0.0f, &frame);
const int total_samples = kSamplesPerChannel * kNumberOfChannels;
const int16_t expected_result[total_samples] = {5, 4, 3, 2, 1};
EXPECT_TRUE(
std::equal(frame.data_, frame.data_ + total_samples, expected_result));
}
} // namespace webrtc

129
webrtc/modules/audio_mixer/audio_mixer_impl.cc
View File

@ -22,61 +22,59 @@
namespace webrtc {
namespace {
class SourceFrame {
public:
SourceFrame(AudioSourceWithMixStatus* audio_source,
struct SourceFrame {
SourceFrame(AudioMixerImpl::SourceStatus* source_status,
AudioFrame* audio_frame,
bool muted)
: audio_source_(audio_source), audio_frame_(audio_frame), muted_(muted) {
if (!muted_) {
energy_ = AudioMixerCalculateEnergy(*audio_frame);
: source_status(source_status), audio_frame(audio_frame), muted(muted) {
RTC_DCHECK(source_status);
RTC_DCHECK(audio_frame);
if (!muted) {
energy = AudioMixerCalculateEnergy(*audio_frame);
}
}
SourceFrame(AudioSourceWithMixStatus* audio_source,
SourceFrame(AudioMixerImpl::SourceStatus* source_status,
AudioFrame* audio_frame,
bool muted,
uint32_t energy)
: audio_source_(audio_source),
audio_frame_(audio_frame),
muted_(muted),
energy_(energy) {}
// a.ShouldMixBefore(b) is used to select mixer sources.
bool ShouldMixBefore(const SourceFrame& other) const {
if (muted_ != other.muted_) {
return other.muted_;
}
const auto our_activity = audio_frame_->vad_activity_;
const auto other_activity = other.audio_frame_->vad_activity_;
if (our_activity != other_activity) {
return our_activity == AudioFrame::kVadActive;
}
return energy_ > other.energy_;
: source_status(source_status),
audio_frame(audio_frame),
muted(muted),
energy(energy) {
RTC_DCHECK(source_status);
RTC_DCHECK(audio_frame);
}
AudioSourceWithMixStatus* audio_source_ = nullptr;
AudioFrame* audio_frame_ = nullptr;
bool muted_ = true;
uint32_t energy_ = 0;
AudioMixerImpl::SourceStatus* source_status = nullptr;
AudioFrame* audio_frame = nullptr;
bool muted = true;
uint32_t energy = 0;
};
// ShouldMixBefore(a, b) is used to select mixer sources.
bool ShouldMixBefore(const SourceFrame& a, const SourceFrame& b) {
if (a.muted != b.muted) {
return b.muted;
}
void Ramp(const std::vector<SourceFrame>& mixed_sources_and_frames) {
const auto a_activity = a.audio_frame->vad_activity_;
const auto b_activity = b.audio_frame->vad_activity_;
if (a_activity != b_activity) {
return a_activity == AudioFrame::kVadActive;
}
return a.energy > b.energy;
}
void RampAndUpdateGain(
const std::vector<SourceFrame>& mixed_sources_and_frames) {
for (const auto& source_frame : mixed_sources_and_frames) {
// Ramp in previously unmixed.
if (!source_frame.audio_source_->WasMixed()) {
NewMixerRampIn(source_frame.audio_frame_);
}
const bool is_mixed = source_frame.audio_source_->IsMixed();
// Ramp out currently unmixed.
if (source_frame.audio_source_->WasMixed() && !is_mixed) {
NewMixerRampOut(source_frame.audio_frame_);
}
float target_gain = source_frame.source_status->is_mixed ? 1.0f : 0.0f;
Ramp(source_frame.source_status->gain, target_gain,
source_frame.audio_frame);
source_frame.source_status->gain = target_gain;
}
}
@ -121,21 +119,21 @@ int32_t MixFromList(AudioFrame* mixed_audio,
return 0;
}
MixerAudioSourceList::const_iterator FindSourceInList(
AudioMixerImpl::SourceStatusList::const_iterator FindSourceInList(
AudioMixerImpl::Source const* audio_source,
MixerAudioSourceList const* audio_source_list) {
AudioMixerImpl::SourceStatusList const* audio_source_list) {
return std::find_if(audio_source_list->begin(), audio_source_list->end(),
[audio_source](const AudioSourceWithMixStatus& p) {
return p.audio_source() == audio_source;
[audio_source](const AudioMixerImpl::SourceStatus& p) {
return p.audio_source == audio_source;
});
}
MixerAudioSourceList::iterator FindSourceInList(
AudioMixerImpl::SourceStatusList::iterator FindSourceInList(
AudioMixerImpl::Source const* audio_source,
MixerAudioSourceList* audio_source_list) {
AudioMixerImpl::SourceStatusList* audio_source_list) {
return std::find_if(audio_source_list->begin(), audio_source_list->end(),
[audio_source](const AudioSourceWithMixStatus& p) {
return p.audio_source() == audio_source;
[audio_source](const AudioMixerImpl::SourceStatus& p) {
return p.audio_source == audio_source;
});
}
@ -362,7 +360,7 @@ AudioFrameList AudioMixerImpl::GetNonAnonymousAudio() {
// Get audio source audio and put it in the struct vector.
for (auto& source_and_status : audio_source_list_) {
auto audio_frame_with_info =
source_and_status.audio_source()->GetAudioFrameWithInfo(
source_and_status.audio_source->GetAudioFrameWithInfo(
id_, static_cast<int>(OutputFrequency()));
const auto audio_frame_info = audio_frame_with_info.audio_frame_info;
@ -380,16 +378,15 @@ AudioFrameList AudioMixerImpl::GetNonAnonymousAudio() {
// Sort frames by sorting function.
std::sort(audio_source_mixing_data_list.begin(),
audio_source_mixing_data_list.end(),
std::mem_fn(&SourceFrame::ShouldMixBefore));
audio_source_mixing_data_list.end(), ShouldMixBefore);
int max_audio_frame_counter = kMaximumAmountOfMixedAudioSources;
// Go through list in order and put unmuted frames in result list.
for (const auto& p : audio_source_mixing_data_list) {
// Filter muted.
if (p.muted_) {
p.audio_source_->SetIsMixed(false);
if (p.muted) {
p.source_status->is_mixed = false;
continue;
}
@ -397,13 +394,13 @@ AudioFrameList AudioMixerImpl::GetNonAnonymousAudio() {
bool is_mixed = false;
if (max_audio_frame_counter > 0) {
--max_audio_frame_counter;
result.push_back(p.audio_frame_);
ramp_list.emplace_back(p.audio_source_, p.audio_frame_, false, -1);
result.push_back(p.audio_frame);
ramp_list.emplace_back(p.source_status, p.audio_frame, false, -1);
is_mixed = true;
}
p.audio_source_->SetIsMixed(is_mixed);
p.source_status->is_mixed = is_mixed;
}
Ramp(ramp_list);
RampAndUpdateGain(ramp_list);
return result;
}
@ -415,7 +412,7 @@ AudioFrameList AudioMixerImpl::GetAnonymousAudio() {
AudioFrameList result;
for (auto& source_and_status : additional_audio_source_list_) {
const auto audio_frame_with_info =
source_and_status.audio_source()->GetAudioFrameWithInfo(
source_and_status.audio_source->GetAudioFrameWithInfo(
id_, OutputFrequency());
const auto ret = audio_frame_with_info.audio_frame_info;
AudioFrame* audio_frame = audio_frame_with_info.audio_frame;
@ -427,25 +424,25 @@ AudioFrameList AudioMixerImpl::GetAnonymousAudio() {
if (ret != Source::AudioFrameInfo::kMuted) {
result.push_back(audio_frame);
ramp_list.emplace_back(&source_and_status, audio_frame, false, 0);
source_and_status.SetIsMixed(true);
source_and_status.is_mixed = true;
}
}
Ramp(ramp_list);
RampAndUpdateGain(ramp_list);
return result;
}
bool AudioMixerImpl::AddAudioSourceToList(
Source* audio_source,
MixerAudioSourceList* audio_source_list) const {
SourceStatusList* audio_source_list) const {
WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, id_,
"AddAudioSourceToList(audio_source, audio_source_list)");
audio_source_list->emplace_back(audio_source);
audio_source_list->emplace_back(audio_source, false, 0);
return true;
}
bool AudioMixerImpl::RemoveAudioSourceFromList(
Source* audio_source,
MixerAudioSourceList* audio_source_list) const {
SourceStatusList* audio_source_list) const {
WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, id_,
"RemoveAudioSourceFromList(audio_source, audio_source_list)");
const auto iter = FindSourceInList(audio_source, audio_source_list);
@ -511,13 +508,13 @@ bool AudioMixerImpl::GetAudioSourceMixabilityStatusForTest(
const auto non_anonymous_iter =
FindSourceInList(audio_source, &audio_source_list_);
if (non_anonymous_iter != audio_source_list_.end()) {
return non_anonymous_iter->IsMixed();
return non_anonymous_iter->is_mixed;
}
const auto anonymous_iter =
FindSourceInList(audio_source, &additional_audio_source_list_);
if (anonymous_iter != audio_source_list_.end()) {
return anonymous_iter->IsMixed();
return anonymous_iter->is_mixed;
}
LOG(LS_ERROR) << "Audio source unknown";

20
webrtc/modules/audio_mixer/audio_mixer_impl.h
View File

@ -18,7 +18,6 @@
#include "webrtc/base/thread_annotations.h"
#include "webrtc/base/thread_checker.h"
#include "webrtc/modules/audio_mixer/audio_mixer.h"
#include "webrtc/modules/audio_mixer/audio_source_with_mix_status.h"
#include "webrtc/modules/audio_processing/include/audio_processing.h"
#include "webrtc/modules/include/module_common_types.h"
#include "webrtc/system_wrappers/include/critical_section_wrapper.h"
@ -28,10 +27,19 @@
namespace webrtc {
typedef std::vector<AudioFrame*> AudioFrameList;
typedef std::vector<AudioSourceWithMixStatus> MixerAudioSourceList;
class AudioMixerImpl : public AudioMixer {
public:
struct SourceStatus {
SourceStatus(Source* audio_source, bool is_mixed, float gain)
: audio_source(audio_source), is_mixed(is_mixed), gain(gain) {}
Source* audio_source = nullptr;
bool is_mixed = false;
float gain = 0.0f;
};
typedef std::vector<SourceStatus> SourceStatusList;
// AudioProcessing only accepts 10 ms frames.
static const int kFrameDurationInMs = 10;
@ -73,9 +81,9 @@ class AudioMixerImpl : public AudioMixer {
// Add/remove the MixerAudioSource to the specified
// MixerAudioSource list.
bool AddAudioSourceToList(Source* audio_source,
MixerAudioSourceList* audio_source_list) const;
SourceStatusList* audio_source_list) const;
bool RemoveAudioSourceFromList(Source* remove_audio_source,
MixerAudioSourceList* audio_source_list) const;
SourceStatusList* audio_source_list) const;
bool LimitMixedAudio(AudioFrame* mixed_audio) const;
@ -93,10 +101,10 @@ class AudioMixerImpl : public AudioMixer {
size_t sample_size_ ACCESS_ON(&thread_checker_);
// List of all audio sources. Note all lists are disjunct
MixerAudioSourceList audio_source_list_ GUARDED_BY(crit_); // May be mixed.
SourceStatusList audio_source_list_ GUARDED_BY(crit_); // May be mixed.
// Always mixed, anonymously.
MixerAudioSourceList additional_audio_source_list_ GUARDED_BY(crit_);
SourceStatusList additional_audio_source_list_ GUARDED_BY(crit_);
size_t num_mixed_audio_sources_ GUARDED_BY(crit_);
// Determines if we will use a limiter for clipping protection during