Implements RTCAudioSourceStats members: - audioLevel - totalAudioEnergy - totalSamplesDuration In this CL description these are collectively referred to as the audio levels. The audio levels are removed from sending "track" stats (in Chrome, these are now reported as undefined instead of 0). Background: For sending tracks, audio levels were always reported as 0 in Chrome (https://crbug.com/736403), while audio levels were correctly reported for receiving tracks. This problem affected the standard getStats() but not the legacy getStats(), blocking some people from migrating. This was likely not a problem in native third_party/webrtc code because the delivery of audio frames from device to send-stream uses a different code path outside of chromium. A recent PR (https://github.com/w3c/webrtc-stats/pull/451) moved the send-side audio levels to the RTCAudioSourceStats, while keeping the receive-side audio levels on the "track" stats. This allows an implementation to report the audio levels even if samples are not sent onto the network (such as if an ICE connection has not been established yet), reflecting some of the current implementation. Changes: 1. Audio levels are added to RTCAudioSourceStats. Send-side audio "track" stats are left undefined. Receive-side audio "track" stats are not changed in this CL and continue to work. 2. Audio level computation is moved from the AudioState and AudioTransportImpl to the AudioSendStream. This is because a) the AudioTransportImpl::RecordedDataIsAvailable() code path is not exercised in chromium, and b) audio levels should, per-spec, not be calculated on a per-call basis, for which the AudioState is defined. 3. The audio level computation is now performed in AudioSendStream::SendAudioData(), a code path used by both native and chromium code. 4. Comments are added to document behavior of existing code, such as AudioLevel and AudioSendStream::SendAudioData(). Note: In this CL, just like before this CL, audio level is only calculated after an AudioSendStream has been created. This means that before an O/A negotiation, audio levels are unavailable. According to spec, if we have an audio source, we should have audio levels. An immediate solution to this would have been to calculate the audio level at pc/rtp_sender.cc. The problem is that the LocalAudioSinkAdapter::OnData() code path, while exercised in chromium, is not exercised in native code. The issue of calculating audio levels on a per-source bases rather than on a per-send stream basis is left to https://crbug.com/webrtc/10771, an existing "media-source" bug. This CL can be verified manually in Chrome at: https://codepen.io/anon/pen/vqRGyq Bug: chromium:736403, webrtc:10771 Change-Id: I8036cd9984f3b187c3177470a8c0d6670a201a5a Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/143789 Reviewed-by: Oskar Sundbom <ossu@webrtc.org> Reviewed-by: Stefan Holmer <stefan@webrtc.org> Commit-Queue: Henrik Boström <hbos@webrtc.org> Cr-Commit-Position: refs/heads/master@{#28480}
179 lines
5.5 KiB
C++
179 lines
5.5 KiB
C++
/*
|
|
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "audio/audio_state.h"
|
|
|
|
#include <algorithm>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
#include "absl/memory/memory.h"
|
|
#include "audio/audio_receive_stream.h"
|
|
#include "modules/audio_device/include/audio_device.h"
|
|
#include "rtc_base/checks.h"
|
|
#include "rtc_base/logging.h"
|
|
#include "rtc_base/ref_counted_object.h"
|
|
#include "rtc_base/thread.h"
|
|
|
|
namespace webrtc {
|
|
namespace internal {
|
|
|
|
AudioState::AudioState(const AudioState::Config& config)
|
|
: config_(config),
|
|
audio_transport_(config_.audio_mixer, config_.audio_processing.get()) {
|
|
process_thread_checker_.Detach();
|
|
RTC_DCHECK(config_.audio_mixer);
|
|
RTC_DCHECK(config_.audio_device_module);
|
|
}
|
|
|
|
AudioState::~AudioState() {
|
|
RTC_DCHECK(thread_checker_.IsCurrent());
|
|
RTC_DCHECK(receiving_streams_.empty());
|
|
RTC_DCHECK(sending_streams_.empty());
|
|
}
|
|
|
|
AudioProcessing* AudioState::audio_processing() {
|
|
RTC_DCHECK(config_.audio_processing);
|
|
return config_.audio_processing.get();
|
|
}
|
|
|
|
AudioTransport* AudioState::audio_transport() {
|
|
return &audio_transport_;
|
|
}
|
|
|
|
bool AudioState::typing_noise_detected() const {
|
|
RTC_DCHECK(thread_checker_.IsCurrent());
|
|
return audio_transport_.typing_noise_detected();
|
|
}
|
|
|
|
void AudioState::AddReceivingStream(webrtc::AudioReceiveStream* stream) {
|
|
RTC_DCHECK(thread_checker_.IsCurrent());
|
|
RTC_DCHECK_EQ(0, receiving_streams_.count(stream));
|
|
receiving_streams_.insert(stream);
|
|
if (!config_.audio_mixer->AddSource(
|
|
static_cast<internal::AudioReceiveStream*>(stream))) {
|
|
RTC_DLOG(LS_ERROR) << "Failed to add source to mixer.";
|
|
}
|
|
|
|
// Make sure playback is initialized; start playing if enabled.
|
|
auto* adm = config_.audio_device_module.get();
|
|
if (!adm->Playing()) {
|
|
if (adm->InitPlayout() == 0) {
|
|
if (playout_enabled_) {
|
|
adm->StartPlayout();
|
|
}
|
|
} else {
|
|
RTC_DLOG_F(LS_ERROR) << "Failed to initialize playout.";
|
|
}
|
|
}
|
|
}
|
|
|
|
void AudioState::RemoveReceivingStream(webrtc::AudioReceiveStream* stream) {
|
|
RTC_DCHECK(thread_checker_.IsCurrent());
|
|
auto count = receiving_streams_.erase(stream);
|
|
RTC_DCHECK_EQ(1, count);
|
|
config_.audio_mixer->RemoveSource(
|
|
static_cast<internal::AudioReceiveStream*>(stream));
|
|
if (receiving_streams_.empty()) {
|
|
config_.audio_device_module->StopPlayout();
|
|
}
|
|
}
|
|
|
|
void AudioState::AddSendingStream(webrtc::AudioSendStream* stream,
|
|
int sample_rate_hz,
|
|
size_t num_channels) {
|
|
RTC_DCHECK(thread_checker_.IsCurrent());
|
|
auto& properties = sending_streams_[stream];
|
|
properties.sample_rate_hz = sample_rate_hz;
|
|
properties.num_channels = num_channels;
|
|
UpdateAudioTransportWithSendingStreams();
|
|
|
|
// Make sure recording is initialized; start recording if enabled.
|
|
auto* adm = config_.audio_device_module.get();
|
|
if (!adm->Recording()) {
|
|
if (adm->InitRecording() == 0) {
|
|
if (recording_enabled_) {
|
|
adm->StartRecording();
|
|
}
|
|
} else {
|
|
RTC_DLOG_F(LS_ERROR) << "Failed to initialize recording.";
|
|
}
|
|
}
|
|
}
|
|
|
|
void AudioState::RemoveSendingStream(webrtc::AudioSendStream* stream) {
|
|
RTC_DCHECK(thread_checker_.IsCurrent());
|
|
auto count = sending_streams_.erase(stream);
|
|
RTC_DCHECK_EQ(1, count);
|
|
UpdateAudioTransportWithSendingStreams();
|
|
if (sending_streams_.empty()) {
|
|
config_.audio_device_module->StopRecording();
|
|
}
|
|
}
|
|
|
|
void AudioState::SetPlayout(bool enabled) {
|
|
RTC_LOG(INFO) << "SetPlayout(" << enabled << ")";
|
|
RTC_DCHECK(thread_checker_.IsCurrent());
|
|
if (playout_enabled_ != enabled) {
|
|
playout_enabled_ = enabled;
|
|
if (enabled) {
|
|
null_audio_poller_.reset();
|
|
if (!receiving_streams_.empty()) {
|
|
config_.audio_device_module->StartPlayout();
|
|
}
|
|
} else {
|
|
config_.audio_device_module->StopPlayout();
|
|
null_audio_poller_ =
|
|
absl::make_unique<NullAudioPoller>(&audio_transport_);
|
|
}
|
|
}
|
|
}
|
|
|
|
void AudioState::SetRecording(bool enabled) {
|
|
RTC_LOG(INFO) << "SetRecording(" << enabled << ")";
|
|
RTC_DCHECK(thread_checker_.IsCurrent());
|
|
if (recording_enabled_ != enabled) {
|
|
recording_enabled_ = enabled;
|
|
if (enabled) {
|
|
if (!sending_streams_.empty()) {
|
|
config_.audio_device_module->StartRecording();
|
|
}
|
|
} else {
|
|
config_.audio_device_module->StopRecording();
|
|
}
|
|
}
|
|
}
|
|
|
|
void AudioState::SetStereoChannelSwapping(bool enable) {
|
|
RTC_DCHECK(thread_checker_.IsCurrent());
|
|
audio_transport_.SetStereoChannelSwapping(enable);
|
|
}
|
|
|
|
void AudioState::UpdateAudioTransportWithSendingStreams() {
|
|
RTC_DCHECK(thread_checker_.IsCurrent());
|
|
std::vector<webrtc::AudioSendStream*> sending_streams;
|
|
int max_sample_rate_hz = 8000;
|
|
size_t max_num_channels = 1;
|
|
for (const auto& kv : sending_streams_) {
|
|
sending_streams.push_back(kv.first);
|
|
max_sample_rate_hz = std::max(max_sample_rate_hz, kv.second.sample_rate_hz);
|
|
max_num_channels = std::max(max_num_channels, kv.second.num_channels);
|
|
}
|
|
audio_transport_.UpdateSendingStreams(std::move(sending_streams),
|
|
max_sample_rate_hz, max_num_channels);
|
|
}
|
|
} // namespace internal
|
|
|
|
rtc::scoped_refptr<AudioState> AudioState::Create(
|
|
const AudioState::Config& config) {
|
|
return new rtc::RefCountedObject<internal::AudioState>(config);
|
|
}
|
|
} // namespace webrtc
|