diff --git a/api/test/peerconnection_quality_test_fixture.h b/api/test/peerconnection_quality_test_fixture.h index 12907bc9ae..44bb1f02de 100644 --- a/api/test/peerconnection_quality_test_fixture.h +++ b/api/test/peerconnection_quality_test_fixture.h @@ -279,6 +279,13 @@ class PeerConnectionE2EQualityTestFixture { PeerConnectionInterface::BitrateParameters bitrate_params) = 0; }; + // Contains configuration for echo emulator. + struct EchoEmulationConfig { + // Delay which represents the echo path delay, i.e. how soon rendered signal + // should reach capturer. + TimeDelta echo_delay = TimeDelta::ms(50); + }; + // Contains parameters, that describe how long framework should run quality // test. struct RunParams { @@ -314,6 +321,10 @@ class PeerConnectionE2EQualityTestFixture { // If true will set conference mode in SDP media section for all video // tracks for all peers. bool use_conference_mode = false; + // If specified echo emulation will be done, by mixing the render audio into + // the capture signal. In such case input signal will be reduced by half to + // avoid saturation or compression in the echo path simulation. + absl::optional echo_emulation_config; }; // Represent an entity that will report quality metrics after test. diff --git a/rtc_base/swap_queue.h b/rtc_base/swap_queue.h index 891454829c..eb0b1fff0c 100644 --- a/rtc_base/swap_queue.h +++ b/rtc_base/swap_queue.h @@ -200,6 +200,16 @@ class SwapQueue { return true; } + // Returns the current number of elements in the queue. Since elements may be + // concurrently added to the queue, the caller must treat this as a lower + // bound, not an exact count. + // May only be called by the consumer. + size_t SizeAtLeast() const { + // Acquire memory ordering ensures that we wait for the producer to finish + // inserting any element in progress. + return std::atomic_load_explicit(&num_elements_, std::memory_order_acquire); + } + private: // Verify that the queue slots complies with the ItemVerifier test. This // function is not thread-safe and can only be used in the constructors. diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 6d24bbb476..440064b298 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -203,6 +203,20 @@ if (rtc_include_tests) { ] } + rtc_source_set("echo_emulation") { + visibility = [ "*" ] + testonly = true + sources = [ + "echo/echo_emulation.cc", + "echo/echo_emulation.h", + ] + deps = [ + "../../../api:peer_connection_quality_test_fixture_api", + "../../../modules/audio_device:audio_device_impl", + "../../../rtc_base:rtc_base_approved", + ] + } + rtc_source_set("test_peer") { visibility = [ "*" ] testonly = true @@ -211,6 +225,7 @@ if (rtc_include_tests) { "test_peer.h", ] deps = [ + ":echo_emulation", ":peer_connection_quality_test_params", ":video_quality_analyzer_injection_helper", "../../../api:peer_connection_quality_test_fixture_api", diff --git a/test/pc/e2e/echo/echo_emulation.cc b/test/pc/e2e/echo/echo_emulation.cc new file mode 100644 index 0000000000..1405570130 --- /dev/null +++ b/test/pc/e2e/echo/echo_emulation.cc @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/echo/echo_emulation.h" + +#include +#include + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +constexpr int kSingleBufferDurationMs = 10; + +} // namespace + +EchoEmulatingCapturer::EchoEmulatingCapturer( + std::unique_ptr capturer, + PeerConnectionE2EQualityTestFixture::EchoEmulationConfig config) + : delegate_(std::move(capturer)), + config_(config), + renderer_queue_(2 * config_.echo_delay.ms() / kSingleBufferDurationMs), + queue_input_(TestAudioDeviceModule::SamplesPerFrame( + delegate_->SamplingFrequency()) * + delegate_->NumChannels()), + queue_output_(TestAudioDeviceModule::SamplesPerFrame( + delegate_->SamplingFrequency()) * + delegate_->NumChannels()) { + renderer_thread_.Detach(); + capturer_thread_.Detach(); +} + +void EchoEmulatingCapturer::OnAudioRendered( + rtc::ArrayView data) { + RTC_DCHECK_RUN_ON(&renderer_thread_); + if (!recording_started_) { + // Because rendering can start before capturing in the beginning we can have + // a set of empty audio data frames. So we will skip them and will start + // fill the queue only after 1st non-empty audio data frame will arrive. + bool is_empty = true; + for (auto d : data) { + if (d != 0) { + is_empty = false; + break; + } + } + if (is_empty) { + return; + } + recording_started_ = true; + } + queue_input_.assign(data.begin(), data.end()); + if (!renderer_queue_.Insert(&queue_input_)) { + // Test audio device works too slow with sanitizers and on some platforms + // and can't properly process audio, so when capturer will be stopped + // renderer will quickly overfill the queue. + // TODO(crbug.com/webrtc/10850) remove it when test ADM will be fast enough. +#if !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER) && \ + !defined(ADDRESS_SANITIZER) && !defined(WEBRTC_ANDROID) && \ + !(defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)) + RTC_CHECK(false) << "Echo queue is full"; +#endif + } +} + +bool EchoEmulatingCapturer::Capture(rtc::BufferT* buffer) { + RTC_DCHECK_RUN_ON(&capturer_thread_); + bool result = delegate_->Capture(buffer); + // Now we have to reduce input signal to avoid saturation when mixing in the + // fake echo. + for (size_t i = 0; i < buffer->size(); ++i) { + (*buffer)[i] /= 2; + } + + // When we accumulated enough delay in the echo buffer we will pop from + // that buffer on each ::Capture(...) call. If the buffer become empty it + // will mean some bug, so we will crash during removing item from the queue. + if (!delay_accumulated_) { + delay_accumulated_ = + renderer_queue_.SizeAtLeast() >= + static_cast(config_.echo_delay.ms() / kSingleBufferDurationMs); + } + + if (delay_accumulated_) { + RTC_CHECK(renderer_queue_.Remove(&queue_output_)); + for (size_t i = 0; i < buffer->size() && i < queue_output_.size(); ++i) { + int32_t res = (*buffer)[i] + queue_output_[i]; + if (res < std::numeric_limits::min()) { + res = std::numeric_limits::min(); + } + if (res > std::numeric_limits::max()) { + res = std::numeric_limits::max(); + } + (*buffer)[i] = static_cast(res); + } + } + + return result; +} + +EchoEmulatingRenderer::EchoEmulatingRenderer( + std::unique_ptr renderer, + EchoEmulatingCapturer* echo_emulating_capturer) + : delegate_(std::move(renderer)), + echo_emulating_capturer_(echo_emulating_capturer) { + RTC_DCHECK(echo_emulating_capturer_); +} + +bool EchoEmulatingRenderer::Render(rtc::ArrayView data) { + if (data.size() > 0) { + echo_emulating_capturer_->OnAudioRendered(data); + } + return delegate_->Render(data); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/test/pc/e2e/echo/echo_emulation.h b/test/pc/e2e/echo/echo_emulation.h new file mode 100644 index 0000000000..d1d41f63a8 --- /dev/null +++ b/test/pc/e2e/echo/echo_emulation.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ECHO_ECHO_EMULATION_H_ +#define TEST_PC_E2E_ECHO_ECHO_EMULATION_H_ + +#include +#include +#include +#include + +#include "api/test/peerconnection_quality_test_fixture.h" +#include "modules/audio_device/include/test_audio_device.h" +#include "rtc_base/swap_queue.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Reduces audio input strength from provided capturer twice and adds input +// provided into EchoEmulatingCapturer::OnAudioRendered(...). +class EchoEmulatingCapturer : public TestAudioDeviceModule::Capturer { + public: + EchoEmulatingCapturer( + std::unique_ptr capturer, + PeerConnectionE2EQualityTestFixture::EchoEmulationConfig config); + + void OnAudioRendered(rtc::ArrayView data); + + int SamplingFrequency() const override { + return delegate_->SamplingFrequency(); + } + int NumChannels() const override { return delegate_->NumChannels(); } + bool Capture(rtc::BufferT* buffer) override; + + private: + std::unique_ptr delegate_; + const PeerConnectionE2EQualityTestFixture::EchoEmulationConfig config_; + + SwapQueue> renderer_queue_; + + SequenceChecker renderer_thread_; + std::vector queue_input_ RTC_GUARDED_BY(renderer_thread_); + bool recording_started_ RTC_GUARDED_BY(renderer_thread_) = false; + + SequenceChecker capturer_thread_; + std::vector queue_output_ RTC_GUARDED_BY(capturer_thread_); + bool delay_accumulated_ RTC_GUARDED_BY(capturer_thread_) = false; +}; + +// Renders output into provided renderer and also copy output into provided +// EchoEmulationCapturer. +class EchoEmulatingRenderer : public TestAudioDeviceModule::Renderer { + public: + EchoEmulatingRenderer( + std::unique_ptr renderer, + EchoEmulatingCapturer* echo_emulating_capturer); + + int SamplingFrequency() const override { + return delegate_->SamplingFrequency(); + } + int NumChannels() const override { return delegate_->NumChannels(); } + bool Render(rtc::ArrayView data) override; + + private: + std::unique_ptr delegate_; + EchoEmulatingCapturer* echo_emulating_capturer_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ECHO_ECHO_EMULATION_H_ diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc index a6f4b5e52c..dc1d819dc7 100644 --- a/test/pc/e2e/peer_connection_e2e_smoke_test.cc +++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc @@ -38,6 +38,8 @@ class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test { using ScrollingParams = PeerConnectionE2EQualityTestFixture::ScrollingParams; using VideoSimulcastConfig = PeerConnectionE2EQualityTestFixture::VideoSimulcastConfig; + using EchoEmulationConfig = + PeerConnectionE2EQualityTestFixture::EchoEmulationConfig; void RunTest(const std::string& test_case_name, const RunParams& run_params, @@ -136,6 +138,7 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) { run_params.use_flex_fec = true; run_params.use_ulp_fec = true; run_params.video_encoder_bitrate_multiplier = 1.1; + run_params.echo_emulation_config = EchoEmulationConfig(); RunTest( "smoke", run_params, [](PeerConfigurer* alice) { diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index e90b170099..c623cc9ce3 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -276,7 +276,7 @@ void PeerConnectionE2EQualityTest::Run(RunParams run_params) { [this]() { StartVideo(alice_video_sources_); }), video_quality_analyzer_injection_helper_.get(), signaling_thread.get(), alice_remote_audio_config, run_params.video_encoder_bitrate_multiplier, - task_queue_.get()); + run_params.echo_emulation_config, task_queue_.get()); bob_ = TestPeer::CreateTestPeer( std::move(bob_components), std::move(bob_params), absl::make_unique( @@ -287,7 +287,7 @@ void PeerConnectionE2EQualityTest::Run(RunParams run_params) { [this]() { StartVideo(bob_video_sources_); }), video_quality_analyzer_injection_helper_.get(), signaling_thread.get(), bob_remote_audio_config, run_params.video_encoder_bitrate_multiplier, - task_queue_.get()); + run_params.echo_emulation_config, task_queue_.get()); int num_cores = CpuInfo::DetectNumberOfCores(); RTC_DCHECK_GE(num_cores, 1); diff --git a/test/pc/e2e/test_peer.cc b/test/pc/e2e/test_peer.cc index 0e044b4ebc..6cc1168c5f 100644 --- a/test/pc/e2e/test_peer.cc +++ b/test/pc/e2e/test_peer.cc @@ -26,6 +26,7 @@ #include "modules/audio_processing/include/audio_processing.h" #include "p2p/client/basic_port_allocator.h" #include "rtc_base/location.h" +#include "test/pc/e2e/echo/echo_emulation.h" #include "test/testsupport/copy_to_file_audio_capturer.h" namespace webrtc { @@ -36,6 +37,8 @@ using RemotePeerAudioConfig = ::webrtc::webrtc_pc_e2e::TestPeer::RemotePeerAudioConfig; using AudioConfig = ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::AudioConfig; +using EchoEmulationConfig = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::EchoEmulationConfig; constexpr int16_t kGeneratedAudioMaxAmplitude = 32000; constexpr int kDefaultSamplingFrequencyInHz = 48000; @@ -72,13 +75,15 @@ class TestPeerComponents { rtc::Thread* signaling_thread, absl::optional remote_audio_config, double bitrate_multiplier, + absl::optional echo_emulation_config, rtc::TaskQueue* task_queue) : audio_config_opt_(params.audio_config), observer_(observer), video_analyzer_helper_(video_analyzer_helper), signaling_thread_(signaling_thread), remote_audio_config_(std::move(remote_audio_config)), - bitrate_multiplier_(bitrate_multiplier) { + bitrate_multiplier_(bitrate_multiplier), + echo_emulation_config_(std::move(echo_emulation_config)) { for (auto& video_config : params.video_configs) { // Stream label should be set by fixture implementation here. RTC_DCHECK(video_config.stream_label); @@ -177,31 +182,26 @@ class TestPeerComponents { rtc::scoped_refptr CreateAudioDeviceModule( TaskQueueFactory* task_queue_factory) { - std::unique_ptr capturer; - if (audio_config_opt_) { - capturer = CreateAudioCapturer(*audio_config_opt_); - if (audio_config_opt_->input_dump_file_name) { - capturer = absl::make_unique( - std::move(capturer), - audio_config_opt_->input_dump_file_name.value()); - } - } else { - // If we have no audio config we still need to provide some audio device. - // In such case use generated capturer. Despite of we provided audio here, - // in test media setup audio stream won't be added into peer connection. - capturer = TestAudioDeviceModule::CreatePulsedNoiseCapturer( - kGeneratedAudioMaxAmplitude, kDefaultSamplingFrequencyInHz); - } + std::unique_ptr renderer = + CreateAudioRenderer(remote_audio_config_); + std::unique_ptr capturer = + CreateAudioCapturer(audio_config_opt_); + RTC_DCHECK(renderer); RTC_DCHECK(capturer); - std::unique_ptr renderer; - if (remote_audio_config_ && remote_audio_config_->output_file_name) { - renderer = TestAudioDeviceModule::CreateBoundedWavFileWriter( - remote_audio_config_->output_file_name.value(), - remote_audio_config_->sampling_frequency_in_hz); - } else { - renderer = TestAudioDeviceModule::CreateDiscardRenderer( - kDefaultSamplingFrequencyInHz); + // Setup echo emulation if required. + if (echo_emulation_config_) { + capturer = absl::make_unique( + std::move(capturer), *echo_emulation_config_); + renderer = absl::make_unique( + std::move(renderer), + static_cast(capturer.get())); + } + + // Setup input stream dumping if required. + if (audio_config_opt_ && audio_config_opt_->input_dump_file_name) { + capturer = absl::make_unique( + std::move(capturer), audio_config_opt_->input_dump_file_name.value()); } return TestAudioDeviceModule::Create(task_queue_factory, @@ -209,19 +209,41 @@ class TestPeerComponents { std::move(renderer), /*speed=*/1.f); } + std::unique_ptr CreateAudioRenderer( + const absl::optional& config) { + if (!config) { + // Return default renderer because we always require some renderer. + return TestAudioDeviceModule::CreateDiscardRenderer( + kDefaultSamplingFrequencyInHz); + } + if (config->output_file_name) { + return TestAudioDeviceModule::CreateBoundedWavFileWriter( + config->output_file_name.value(), config->sampling_frequency_in_hz); + } + return TestAudioDeviceModule::CreateDiscardRenderer( + config->sampling_frequency_in_hz); + } + std::unique_ptr CreateAudioCapturer( - const AudioConfig& audio_config) { - if (audio_config.mode == AudioConfig::Mode::kGenerated) { + const absl::optional& audio_config) { + if (!audio_config) { + // If we have no audio config we still need to provide some audio device. + // In such case use generated capturer. Despite of we provided audio here, + // in test media setup audio stream won't be added into peer connection. return TestAudioDeviceModule::CreatePulsedNoiseCapturer( - kGeneratedAudioMaxAmplitude, audio_config.sampling_frequency_in_hz); + kGeneratedAudioMaxAmplitude, kDefaultSamplingFrequencyInHz); } - if (audio_config.mode == AudioConfig::Mode::kFile) { - RTC_DCHECK(audio_config.input_file_name); - return TestAudioDeviceModule::CreateWavFileReader( - audio_config.input_file_name.value(), /*repeat=*/true); + + switch (audio_config->mode) { + case AudioConfig::Mode::kGenerated: + return TestAudioDeviceModule::CreatePulsedNoiseCapturer( + kGeneratedAudioMaxAmplitude, + audio_config->sampling_frequency_in_hz); + case AudioConfig::Mode::kFile: + RTC_DCHECK(audio_config->input_file_name); + return TestAudioDeviceModule::CreateWavFileReader( + audio_config->input_file_name.value(), /*repeat=*/true); } - RTC_NOTREACHED() << "Unknown audio_config->mode"; - return nullptr; } std::unique_ptr CreateVideoEncoderFactory( @@ -290,6 +312,7 @@ class TestPeerComponents { rtc::Thread* signaling_thread_; absl::optional remote_audio_config_; double bitrate_multiplier_; + absl::optional echo_emulation_config_; }; } // namespace @@ -310,6 +333,7 @@ std::unique_ptr TestPeer::CreateTestPeer( rtc::Thread* signaling_thread, absl::optional remote_audio_config, double bitrate_multiplier, + absl::optional echo_emulation_config, rtc::TaskQueue* task_queue) { RTC_DCHECK(components); RTC_DCHECK(params); @@ -319,7 +343,7 @@ std::unique_ptr TestPeer::CreateTestPeer( TestPeerComponents tpc(std::move(components), *params, observer.get(), video_analyzer_helper, signaling_thread, std::move(remote_audio_config), bitrate_multiplier, - task_queue); + echo_emulation_config, task_queue); return absl::WrapUnique(new TestPeer( tpc.peer_connection_factory(), tpc.peer_connection(), std::move(observer), diff --git a/test/pc/e2e/test_peer.h b/test/pc/e2e/test_peer.h index 8cb8415f33..efacde5d17 100644 --- a/test/pc/e2e/test_peer.h +++ b/test/pc/e2e/test_peer.h @@ -36,6 +36,8 @@ class TestPeer final : public PeerConnectionWrapper { using PeerConnectionWrapper::PeerConnectionWrapper; using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; using AudioConfig = PeerConnectionE2EQualityTestFixture::AudioConfig; + using EchoEmulationConfig = + PeerConnectionE2EQualityTestFixture::EchoEmulationConfig; struct RemotePeerAudioConfig { RemotePeerAudioConfig(AudioConfig config) @@ -55,11 +57,8 @@ class TestPeer final : public PeerConnectionWrapper { // injection. // // |signaling_thread| will be provided by test fixture implementation. - // |params| - describes current peer paramters, like current peer video + // |params| - describes current peer parameters, like current peer video // streams and audio streams - // |audio_outpu_file_name| - the name of output file, where incoming audio - // stream should be written. It should be provided from remote peer - // |params.audio_config.output_file_name| static std::unique_ptr CreateTestPeer( std::unique_ptr components, std::unique_ptr params, @@ -68,6 +67,7 @@ class TestPeer final : public PeerConnectionWrapper { rtc::Thread* signaling_thread, absl::optional remote_audio_config, double bitrate_multiplier, + absl::optional echo_emulation_config, rtc::TaskQueue* task_queue); Params* params() const { return params_.get(); }