diff --git a/video/video_analyzer.cc b/video/video_analyzer.cc index 480e3ead5e..66c4db6de3 100644 --- a/video/video_analyzer.cc +++ b/video/video_analyzer.cc @@ -60,6 +60,7 @@ VideoAnalyzer::VideoAnalyzer(test::LayerFilteringTransport* transport, call_(nullptr), send_stream_(nullptr), receive_stream_(nullptr), + audio_receive_stream_(nullptr), captured_frame_forwarder_(this, clock), test_label_(test_label), graph_data_output_file_(graph_data_output_file), @@ -164,6 +165,12 @@ void VideoAnalyzer::SetReceiveStream(VideoReceiveStream* stream) { receive_stream_ = stream; } +void VideoAnalyzer::SetAudioReceiveStream(AudioReceiveStream* recv_stream) { + rtc::CritScope lock(&crit_); + RTC_CHECK(!audio_receive_stream_); + audio_receive_stream_ = recv_stream; +} + rtc::VideoSinkInterface* VideoAnalyzer::InputInterface() { return &captured_frame_forwarder_; } @@ -460,6 +467,14 @@ void VideoAnalyzer::PollStats() { decode_time_max_ms_.AddSample(receive_stats.max_decode_ms); } + if (audio_receive_stream_ != nullptr) { + AudioReceiveStream::Stats receive_stats = + audio_receive_stream_->GetStats(); + audio_expand_rate_.AddSample(receive_stats.expand_rate); + audio_accelerate_rate_.AddSample(receive_stats.accelerate_rate); + audio_jitter_buffer_ms_.AddSample(receive_stats.jitter_buffer_ms); + } + memory_usage_.AddSample(rtc::GetProcessResidentSizeBytes()); } } @@ -594,6 +609,12 @@ void VideoAnalyzer::PrintResults() { frame_writer.WriteFrame(worst_frame_->frame, 100 /*best quality*/)); } + if (audio_receive_stream_ != nullptr) { + PrintResult("audio_expand_rate", audio_expand_rate_, ""); + PrintResult("audio_accelerate_rate", audio_accelerate_rate_, ""); + PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, " ms"); + } + // Disable quality check for quick test, as quality checks may fail // because too few samples were collected. if (!is_quick_test_enabled_) { diff --git a/video/video_analyzer.h b/video/video_analyzer.h index 1b5a87d7d9..b4d31dbaea 100644 --- a/video/video_analyzer.h +++ b/video/video_analyzer.h @@ -50,6 +50,8 @@ class VideoAnalyzer : public PacketReceiver, void SetCall(Call* call); void SetSendStream(VideoSendStream* stream); void SetReceiveStream(VideoReceiveStream* stream); + void SetAudioReceiveStream(AudioReceiveStream* recv_stream); + rtc::VideoSinkInterface* InputInterface(); rtc::VideoSourceInterface* OutputInterface(); @@ -208,6 +210,7 @@ class VideoAnalyzer : public PacketReceiver, Call* call_; VideoSendStream* send_stream_; VideoReceiveStream* receive_stream_; + AudioReceiveStream* audio_receive_stream_; CapturedFrameForwarder captured_frame_forwarder_; const std::string test_label_; FILE* const graph_data_output_file_; @@ -239,6 +242,9 @@ class VideoAnalyzer : public PacketReceiver, test::Statistics send_bandwidth_bps_ RTC_GUARDED_BY(comparison_lock_); test::Statistics memory_usage_ RTC_GUARDED_BY(comparison_lock_); test::Statistics time_between_freezes_ RTC_GUARDED_BY(comparison_lock_); + test::Statistics audio_expand_rate_ RTC_GUARDED_BY(comparison_lock_); + test::Statistics audio_accelerate_rate_ RTC_GUARDED_BY(comparison_lock_); + test::Statistics audio_jitter_buffer_ms_ RTC_GUARDED_BY(comparison_lock_); // Rendered frame with worst PSNR is saved for further analysis. absl::optional worst_frame_ RTC_GUARDED_BY(comparison_lock_); diff --git a/video/video_quality_test.cc b/video/video_quality_test.cc index cad86aed0e..071fe5f635 100644 --- a/video/video_quality_test.cc +++ b/video/video_quality_test.cc @@ -757,6 +757,24 @@ void VideoQualityTest::CreateCapturers() { } } +void VideoQualityTest::StartThumbnails() { + for (VideoSendStream* send_stream : thumbnail_send_streams_) + send_stream->Start(); + for (VideoReceiveStream* receive_stream : thumbnail_receive_streams_) + receive_stream->Start(); + for (std::unique_ptr& capturer : thumbnail_capturers_) + capturer->Start(); +} + +void VideoQualityTest::StopThumbnails() { + for (std::unique_ptr& capturer : thumbnail_capturers_) + capturer->Stop(); + for (VideoReceiveStream* receive_stream : thumbnail_receive_streams_) + receive_stream->Stop(); + for (VideoSendStream* send_stream : thumbnail_send_streams_) + send_stream->Stop(); +} + std::unique_ptr VideoQualityTest::CreateSendTransport() { return absl::make_unique( @@ -781,8 +799,6 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { std::unique_ptr analyzer; params_ = params; - - RTC_CHECK(!params_.audio.enabled); // TODO(ivica): Merge with RunWithRenderer and use a flag / argument to // differentiate between the analyzer and the renderer case. CheckParams(); @@ -824,6 +840,10 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { task_queue_.SendTask([this, &send_call_config, &recv_call_config, &send_transport, &recv_transport]() { + if (params_.audio.enabled) { + InitializeAudioDevice(&send_call_config, &recv_call_config); + } + CreateCalls(send_call_config, recv_call_config); send_transport = CreateSendTransport(); recv_transport = CreateReceiveTransport(); @@ -886,39 +906,24 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { StartEncodedFrameLogs(GetVideoSendStream()); StartEncodedFrameLogs( video_receive_streams_[params_.ss[0].selected_stream]); - StartVideoStreams(); - for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_) - thumbnail_send_stream->Start(); - for (VideoReceiveStream* thumbnail_receive_stream : - thumbnail_receive_streams_) - thumbnail_receive_stream->Start(); - analyzer->StartMeasuringCpuProcessTime(); - StartVideoCapture(); - for (std::unique_ptr& video_caputurer : - thumbnail_capturers_) { - video_caputurer->Start(); + if (params_.audio.enabled) { + SetupAudio(send_transport.get()); } + StartThumbnails(); + Start(); + + if (params_.audio.enabled) { + analyzer->SetAudioReceiveStream(audio_receive_streams_[0]); + } + analyzer->StartMeasuringCpuProcessTime(); }); analyzer->Wait(); task_queue_.SendTask([&]() { - for (std::unique_ptr& video_caputurer : - thumbnail_capturers_) - video_caputurer->Stop(); - for (size_t video_idx = 0; video_idx < num_video_streams_; ++video_idx) { - video_capturers_[video_idx]->Stop(); - } - for (VideoReceiveStream* thumbnail_receive_stream : - thumbnail_receive_streams_) - thumbnail_receive_stream->Stop(); - for (VideoReceiveStream* receive_stream : video_receive_streams_) - receive_stream->Stop(); - for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_) - thumbnail_send_stream->Stop(); - for (VideoSendStream* video_send_stream : video_send_streams_) - video_send_stream->Stop(); + StopThumbnails(); + Stop(); DestroyStreams(); DestroyThumbnailStreams(); @@ -934,6 +939,24 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { }); } +void VideoQualityTest::InitializeAudioDevice(Call::Config* send_call_config, + Call::Config* recv_call_config) { + rtc::scoped_refptr fake_audio_device = + TestAudioDeviceModule::CreateTestAudioDeviceModule( + TestAudioDeviceModule::CreatePulsedNoiseCapturer(32000, 48000), + TestAudioDeviceModule::CreateDiscardRenderer(48000), 1.f); + + AudioState::Config audio_state_config; + audio_state_config.audio_mixer = AudioMixerImpl::Create(); + audio_state_config.audio_processing = AudioProcessingBuilder().Create(); + audio_state_config.audio_device_module = fake_audio_device; + send_call_config->audio_state = AudioState::Create(audio_state_config); + RTC_CHECK(fake_audio_device->RegisterAudioCallback( + send_call_config->audio_state->audio_transport()) == 0); + recv_call_config->audio_state = AudioState::Create(audio_state_config); + fake_audio_device->Init(); +} + void VideoQualityTest::SetupAudio(Transport* transport) { AudioSendStream::Config audio_send_config(transport); audio_send_config.rtp.ssrc = kAudioSendSsrc; @@ -959,7 +982,7 @@ void VideoQualityTest::SetupAudio(Transport* transport) { audio_send_config.encoder_factory = audio_encoder_factory_; SetAudioConfig(audio_send_config); - const char* sync_group = nullptr; + std::string sync_group; if (params_.video[0].enabled && params_.audio.sync_video) sync_group = kSyncGroup; @@ -984,22 +1007,13 @@ void VideoQualityTest::RunWithRenderers(const Params& params) { Call::Config call_config(&null_event_log); call_config.bitrate_config = params_.call.call_bitrate_config; - rtc::scoped_refptr fake_audio_device = - TestAudioDeviceModule::CreateTestAudioDeviceModule( - TestAudioDeviceModule::CreatePulsedNoiseCapturer(32000, 48000), - TestAudioDeviceModule::CreateDiscardRenderer(48000), 1.f); + Call::Config recv_call_config(&null_event_log); if (params_.audio.enabled) { - AudioState::Config audio_state_config; - audio_state_config.audio_mixer = AudioMixerImpl::Create(); - audio_state_config.audio_processing = AudioProcessingBuilder().Create(); - audio_state_config.audio_device_module = fake_audio_device; - call_config.audio_state = AudioState::Create(audio_state_config); - fake_audio_device->RegisterAudioCallback( - call_config.audio_state->audio_transport()); + InitializeAudioDevice(&call_config, &recv_call_config); } - CreateCalls(call_config, call_config); + CreateCalls(call_config, recv_call_config); // TODO(minyue): consider if this is a good transport even for audio only // calls. diff --git a/video/video_quality_test.h b/video/video_quality_test.h index 67facfda30..f00c748696 100644 --- a/video/video_quality_test.h +++ b/video/video_quality_test.h @@ -73,7 +73,11 @@ class VideoQualityTest : const SdpVideoFormat& format); void SetupVideo(Transport* send_transport, Transport* recv_transport); void SetupThumbnails(Transport* send_transport, Transport* recv_transport); + void StartThumbnails(); + void StopThumbnails(); void DestroyThumbnailStreams(); + void InitializeAudioDevice(Call::Config* send_call_config, + Call::Config* recv_call_config); void SetupAudio(Transport* transport); void StartEncodedFrameLogs(VideoSendStream* stream);