admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update RemixAndResample to use audio views

Browse Source 
Also adding more checks around parameter valuesm since the
`sample_rate_hz` argument is technically not needed, but will be
removed in a follow-up CL.

Bug: chromium:335805780
Change-Id: Ia7e50658f8a686ab71980f9c59cce5f097b0af40
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/353340
Reviewed-by: Per Åhgren <peah@webrtc.org>
Commit-Queue: Tomas Gunnarsson <tommi@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42581}
This commit is contained in:
Tommi 2024-07-03 09:34:18 +02:00 committed by WebRTC LUCI CQ
parent 5d56b7cdf0
commit 32c3398e1a
3 changed files with 38 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
audio/audio_transport_impl.cc
View File

@ -38,8 +38,8 @@ void InitializeCaptureFrame(int input_sample_rate,
RTC_DCHECK(audio_frame); RTC_DCHECK(audio_frame);
int min_processing_rate_hz = std::min(input_sample_rate, send_sample_rate_hz); int min_processing_rate_hz = std::min(input_sample_rate, send_sample_rate_hz);
for (int native_rate_hz : AudioProcessing::kNativeSampleRatesHz) { for (int native_rate_hz : AudioProcessing::kNativeSampleRatesHz) {
audio_frame->sample_rate_hz_ = native_rate_hz; audio_frame->SetSampleRateAndChannelSize(native_rate_hz);
if (audio_frame->sample_rate_hz_ >= min_processing_rate_hz) { if (native_rate_hz >= min_processing_rate_hz) {
break; break;
} }
} }
@ -149,6 +149,9 @@ int32_t AudioTransportImpl::RecordedDataIsAvailable(
RTC_DCHECK_LE(bytes_per_sample * number_of_frames * number_of_channels, RTC_DCHECK_LE(bytes_per_sample * number_of_frames * number_of_channels,
AudioFrame::kMaxDataSizeBytes); AudioFrame::kMaxDataSizeBytes);
InterleavedView<const int16_t> source(static_cast<const int16_t*>(audio_data),
number_of_frames, number_of_channels);
int send_sample_rate_hz = 0; int send_sample_rate_hz = 0;
size_t send_num_channels = 0; size_t send_num_channels = 0;
bool swap_stereo_channels = false; bool swap_stereo_channels = false;
@ -162,9 +165,8 @@ int32_t AudioTransportImpl::RecordedDataIsAvailable(
std::unique_ptr<AudioFrame> audio_frame(new AudioFrame()); std::unique_ptr<AudioFrame> audio_frame(new AudioFrame());
InitializeCaptureFrame(sample_rate, send_sample_rate_hz, number_of_channels, InitializeCaptureFrame(sample_rate, send_sample_rate_hz, number_of_channels,
send_num_channels, audio_frame.get()); send_num_channels, audio_frame.get());
voe::RemixAndResample(static_cast<const int16_t*>(audio_data), voe::RemixAndResample(source, sample_rate, &capture_resampler_,
number_of_frames, number_of_channels, sample_rate, audio_frame.get());
&capture_resampler_, audio_frame.get());
ProcessCaptureFrame(audio_delay_milliseconds, key_pressed, ProcessCaptureFrame(audio_delay_milliseconds, key_pressed,
swap_stereo_channels, audio_processing_, swap_stereo_channels, audio_processing_,
audio_frame.get()); audio_frame.get());

62
audio/remix_resample.cc
View File

@ -24,48 +24,49 @@ namespace voe {
void RemixAndResample(const AudioFrame& src_frame, void RemixAndResample(const AudioFrame& src_frame,
PushResampler<int16_t>* resampler, PushResampler<int16_t>* resampler,
AudioFrame* dst_frame) { AudioFrame* dst_frame) {
RemixAndResample(src_frame.data(), src_frame.samples_per_channel_, RemixAndResample(src_frame.data_view(), src_frame.sample_rate_hz_, resampler,
src_frame.num_channels_, src_frame.sample_rate_hz_, dst_frame);
resampler, dst_frame);
dst_frame->timestamp_ = src_frame.timestamp_; dst_frame->timestamp_ = src_frame.timestamp_;
dst_frame->elapsed_time_ms_ = src_frame.elapsed_time_ms_; dst_frame->elapsed_time_ms_ = src_frame.elapsed_time_ms_;
dst_frame->ntp_time_ms_ = src_frame.ntp_time_ms_; dst_frame->ntp_time_ms_ = src_frame.ntp_time_ms_;
dst_frame->packet_infos_ = src_frame.packet_infos_; dst_frame->packet_infos_ = src_frame.packet_infos_;
} }
// TODO: b/335805780 - Accept ArrayView. void RemixAndResample(InterleavedView<const int16_t> src_data,
void RemixAndResample(const int16_t* src_data,
size_t samples_per_channel,
size_t num_channels,
int sample_rate_hz, int sample_rate_hz,
PushResampler<int16_t>* resampler, PushResampler<int16_t>* resampler,
AudioFrame* dst_frame) { AudioFrame* dst_frame) {
const int16_t* audio_ptr = src_data; // The `samples_per_channel_` members must have been set correctly based on
size_t audio_ptr_num_channels = num_channels; // the associated sample rate and the assumed 10ms buffer size.
// TODO(tommi): Remove the `sample_rate_hz` param.
RTC_DCHECK_EQ(SampleRateToDefaultChannelSize(sample_rate_hz),
src_data.samples_per_channel());
RTC_DCHECK_EQ(SampleRateToDefaultChannelSize(dst_frame->sample_rate_hz_),
dst_frame->samples_per_channel());
// Temporary buffer in case downmixing is required.
std::array<int16_t, AudioFrame::kMaxDataSizeSamples> downmixed_audio; std::array<int16_t, AudioFrame::kMaxDataSizeSamples> downmixed_audio;
// Downmix before resampling. // Downmix before resampling.
if (num_channels > dst_frame->num_channels_) { if (src_data.num_channels() > dst_frame->num_channels_) {
RTC_DCHECK(num_channels == 2 || num_channels == 4) RTC_DCHECK(src_data.num_channels() == 2 || src_data.num_channels() == 4)
<< "num_channels: " << num_channels; << "num_channels: " << src_data.num_channels();
RTC_DCHECK(dst_frame->num_channels_ == 1 || dst_frame->num_channels_ == 2) RTC_DCHECK(dst_frame->num_channels_ == 1 || dst_frame->num_channels_ == 2)
<< "dst_frame->num_channels_: " << dst_frame->num_channels_; << "dst_frame->num_channels_: " << dst_frame->num_channels_;
AudioFrameOperations::DownmixChannels( InterleavedView<int16_t> downmixed(downmixed_audio.data(),
InterleavedView<const int16_t>(src_data, samples_per_channel, src_data.samples_per_channel(),
num_channels), dst_frame->num_channels_);
InterleavedView<int16_t>(&downmixed_audio[0], samples_per_channel, AudioFrameOperations::DownmixChannels(src_data, downmixed);
dst_frame->num_channels_)); src_data = downmixed;
audio_ptr = downmixed_audio.data();
audio_ptr_num_channels = dst_frame->num_channels_;
} }
if (resampler->InitializeIfNeeded(sample_rate_hz, dst_frame->sample_rate_hz_, if (resampler->InitializeIfNeeded(sample_rate_hz, dst_frame->sample_rate_hz_,
audio_ptr_num_channels) == -1) { src_data.num_channels()) == -1) {
RTC_FATAL() << "InitializeIfNeeded failed: sample_rate_hz = " RTC_FATAL() << "InitializeIfNeeded failed: sample_rate_hz = "
<< sample_rate_hz << ", dst_frame->sample_rate_hz_ = " << sample_rate_hz << ", dst_frame->sample_rate_hz_ = "
<< dst_frame->sample_rate_hz_ << dst_frame->sample_rate_hz_
<< ", audio_ptr_num_channels = " << audio_ptr_num_channels; << ", num_channels = " << src_data.num_channels();
} }
// TODO(yujo): for muted input frames, don't resample. Either 1) allow // TODO(yujo): for muted input frames, don't resample. Either 1) allow
@ -73,28 +74,19 @@ void RemixAndResample(const int16_t* src_data,
// how much to zero here; or 2) make resampler accept a hint that the input is // how much to zero here; or 2) make resampler accept a hint that the input is
// zeroed. // zeroed.
// Ensure the `samples_per_channel_` member is set correctly based on the
// destination sample rate, number of channels and assumed 10ms buffer size.
// TODO(tommi): Could we rather assume that this has been done by the caller?
dst_frame->SetSampleRateAndChannelSize(dst_frame->sample_rate_hz_);
InterleavedView<const int16_t> src_view(audio_ptr, samples_per_channel,
audio_ptr_num_channels);
// Stash away the originally requested number of channels. Then provide // Stash away the originally requested number of channels. Then provide
// `dst_frame` as a target buffer with the same number of channels as the // `dst_frame` as a target buffer with the same number of channels as the
// source. // source.
auto original_dst_number_of_channels = dst_frame->num_channels_; auto original_dst_number_of_channels = dst_frame->num_channels_;
int out_length = resampler->Resample( int out_length = resampler->Resample(
src_view, dst_frame->mutable_data(dst_frame->samples_per_channel_, src_data, dst_frame->mutable_data(dst_frame->samples_per_channel_,
src_view.num_channels())); src_data.num_channels()));
RTC_CHECK_NE(out_length, -1) << "Resample failed: audio_ptr = " << audio_ptr RTC_CHECK_NE(out_length, -1) << "src_data.size=" << src_data.size();
<< ", src_length = " << src_view.data().size();
RTC_DCHECK_EQ(dst_frame->samples_per_channel(), RTC_DCHECK_EQ(dst_frame->samples_per_channel(),
out_length / audio_ptr_num_channels); out_length / src_data.num_channels());
// Upmix after resampling. // Upmix after resampling.
if (num_channels == 1 && original_dst_number_of_channels == 2) { if (src_data.num_channels() == 1 && original_dst_number_of_channels == 2) {
// The audio in dst_frame really is mono at this point; MonoToStereo will // The audio in dst_frame really is mono at this point; MonoToStereo will
// set this back to stereo. // set this back to stereo.
RTC_DCHECK_EQ(dst_frame->num_channels_, 1); RTC_DCHECK_EQ(dst_frame->num_channels_, 1);

10
audio/remix_resample.h
View File

@ -12,6 +12,7 @@
#define AUDIO_REMIX_RESAMPLE_H_ #define AUDIO_REMIX_RESAMPLE_H_
#include "api/audio/audio_frame.h" #include "api/audio/audio_frame.h"
#include "api/audio/audio_view.h"
#include "common_audio/resampler/include/push_resampler.h" #include "common_audio/resampler/include/push_resampler.h"
namespace webrtc { namespace webrtc {
@ -30,12 +31,9 @@ void RemixAndResample(const AudioFrame& src_frame,
PushResampler<int16_t>* resampler, PushResampler<int16_t>* resampler,
AudioFrame* dst_frame); AudioFrame* dst_frame);
// This version has a pointer to the samples `src_data` as input and receives // TODO(tommi): The `sample_rate_hz` argument can probably be removed since it's
// `samples_per_channel`, `num_channels` and `sample_rate_hz` of the data as // always related to `src_data.samples_per_frame()'.
// parameters. void RemixAndResample(InterleavedView<const int16_t> src_data,
void RemixAndResample(const int16_t* src_data,
size_t samples_per_channel,
size_t num_channels,
int sample_rate_hz, int sample_rate_hz,
PushResampler<int16_t>* resampler, PushResampler<int16_t>* resampler,
AudioFrame* dst_frame); AudioFrame* dst_frame);