Enable audio in video_quality_test.

Allows enabling audio for RunWithAnalyzer method, and prints out audio jitterbuffer performance stats. Also fixes for RunWithRenderer when enabling audio (seg-faulted).

Bug: b/112299470
Change-Id: Ic7c0de1c455891f38cca317001c6c216e82f6ec3
Reviewed-on: https://webrtc-review.googlesource.com/92800
Commit-Queue: Christoffer Rodbro <crodbro@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24208}

video/video_analyzer.cc

@@ -60,6 +60,7 @@ VideoAnalyzer::VideoAnalyzer(test::LayerFilteringTransport* transport,
       call_(nullptr),
       send_stream_(nullptr),
       receive_stream_(nullptr),
+      audio_receive_stream_(nullptr),
       captured_frame_forwarder_(this, clock),
       test_label_(test_label),
       graph_data_output_file_(graph_data_output_file),
@@ -164,6 +165,12 @@ void VideoAnalyzer::SetReceiveStream(VideoReceiveStream* stream) {
   receive_stream_ = stream;
 }
 
+void VideoAnalyzer::SetAudioReceiveStream(AudioReceiveStream* recv_stream) {
+  rtc::CritScope lock(&crit_);
+  RTC_CHECK(!audio_receive_stream_);
+  audio_receive_stream_ = recv_stream;
+}
+
 rtc::VideoSinkInterface<VideoFrame>* VideoAnalyzer::InputInterface() {
   return &captured_frame_forwarder_;
 }
@@ -460,6 +467,14 @@ void VideoAnalyzer::PollStats() {
         decode_time_max_ms_.AddSample(receive_stats.max_decode_ms);
     }
 
+    if (audio_receive_stream_ != nullptr) {
+      AudioReceiveStream::Stats receive_stats =
+          audio_receive_stream_->GetStats();
+      audio_expand_rate_.AddSample(receive_stats.expand_rate);
+      audio_accelerate_rate_.AddSample(receive_stats.accelerate_rate);
+      audio_jitter_buffer_ms_.AddSample(receive_stats.jitter_buffer_ms);
+    }
+
     memory_usage_.AddSample(rtc::GetProcessResidentSizeBytes());
   }
 }
@@ -594,6 +609,12 @@ void VideoAnalyzer::PrintResults() {
         frame_writer.WriteFrame(worst_frame_->frame, 100 /*best quality*/));
   }
 
+  if (audio_receive_stream_ != nullptr) {
+    PrintResult("audio_expand_rate", audio_expand_rate_, "");
+    PrintResult("audio_accelerate_rate", audio_accelerate_rate_, "");
+    PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, " ms");
+  }
+
   //  Disable quality check for quick test, as quality checks may fail
   //  because too few samples were collected.
   if (!is_quick_test_enabled_) {

video/video_analyzer.h

@@ -50,6 +50,8 @@ class VideoAnalyzer : public PacketReceiver,
   void SetCall(Call* call);
   void SetSendStream(VideoSendStream* stream);
   void SetReceiveStream(VideoReceiveStream* stream);
+  void SetAudioReceiveStream(AudioReceiveStream* recv_stream);
+
   rtc::VideoSinkInterface<VideoFrame>* InputInterface();
   rtc::VideoSourceInterface<VideoFrame>* OutputInterface();
 
@@ -208,6 +210,7 @@ class VideoAnalyzer : public PacketReceiver,
   Call* call_;
   VideoSendStream* send_stream_;
   VideoReceiveStream* receive_stream_;
+  AudioReceiveStream* audio_receive_stream_;
   CapturedFrameForwarder captured_frame_forwarder_;
   const std::string test_label_;
   FILE* const graph_data_output_file_;
@@ -239,6 +242,9 @@ class VideoAnalyzer : public PacketReceiver,
   test::Statistics send_bandwidth_bps_ RTC_GUARDED_BY(comparison_lock_);
   test::Statistics memory_usage_ RTC_GUARDED_BY(comparison_lock_);
   test::Statistics time_between_freezes_ RTC_GUARDED_BY(comparison_lock_);
+  test::Statistics audio_expand_rate_ RTC_GUARDED_BY(comparison_lock_);
+  test::Statistics audio_accelerate_rate_ RTC_GUARDED_BY(comparison_lock_);
+  test::Statistics audio_jitter_buffer_ms_ RTC_GUARDED_BY(comparison_lock_);
   // Rendered frame with worst PSNR is saved for further analysis.
   absl::optional<FrameWithPsnr> worst_frame_ RTC_GUARDED_BY(comparison_lock_);
 

video/video_quality_test.cc

@@ -757,6 +757,24 @@ void VideoQualityTest::CreateCapturers() {
   }
 }
 
+void VideoQualityTest::StartThumbnails() {
+  for (VideoSendStream* send_stream : thumbnail_send_streams_)
+    send_stream->Start();
+  for (VideoReceiveStream* receive_stream : thumbnail_receive_streams_)
+    receive_stream->Start();
+  for (std::unique_ptr<test::VideoCapturer>& capturer : thumbnail_capturers_)
+    capturer->Start();
+}
+
+void VideoQualityTest::StopThumbnails() {
+  for (std::unique_ptr<test::VideoCapturer>& capturer : thumbnail_capturers_)
+    capturer->Stop();
+  for (VideoReceiveStream* receive_stream : thumbnail_receive_streams_)
+    receive_stream->Stop();
+  for (VideoSendStream* send_stream : thumbnail_send_streams_)
+    send_stream->Stop();
+}
+
 std::unique_ptr<test::LayerFilteringTransport>
 VideoQualityTest::CreateSendTransport() {
   return absl::make_unique<test::LayerFilteringTransport>(
@@ -781,8 +799,6 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) {
   std::unique_ptr<VideoAnalyzer> analyzer;
 
   params_ = params;
-
-  RTC_CHECK(!params_.audio.enabled);
   // TODO(ivica): Merge with RunWithRenderer and use a flag / argument to
   // differentiate between the analyzer and the renderer case.
   CheckParams();
@@ -824,6 +840,10 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) {
 
   task_queue_.SendTask([this, &send_call_config, &recv_call_config,
                         &send_transport, &recv_transport]() {
+    if (params_.audio.enabled) {
+      InitializeAudioDevice(&send_call_config, &recv_call_config);
+    }
+
     CreateCalls(send_call_config, recv_call_config);
     send_transport = CreateSendTransport();
     recv_transport = CreateReceiveTransport();
@@ -886,39 +906,24 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) {
     StartEncodedFrameLogs(GetVideoSendStream());
     StartEncodedFrameLogs(
         video_receive_streams_[params_.ss[0].selected_stream]);
-    StartVideoStreams();
-    for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_)
-      thumbnail_send_stream->Start();
-    for (VideoReceiveStream* thumbnail_receive_stream :
-         thumbnail_receive_streams_)
-      thumbnail_receive_stream->Start();
 
-    analyzer->StartMeasuringCpuProcessTime();
+    if (params_.audio.enabled) {
-    StartVideoCapture();
+      SetupAudio(send_transport.get());
-    for (std::unique_ptr<test::VideoCapturer>& video_caputurer :
-         thumbnail_capturers_) {
-      video_caputurer->Start();
     }
+    StartThumbnails();
+    Start();
+
+    if (params_.audio.enabled) {
+      analyzer->SetAudioReceiveStream(audio_receive_streams_[0]);
+    }
+    analyzer->StartMeasuringCpuProcessTime();
   });
 
   analyzer->Wait();
 
   task_queue_.SendTask([&]() {
-    for (std::unique_ptr<test::VideoCapturer>& video_caputurer :
+    StopThumbnails();
-         thumbnail_capturers_)
+    Stop();
-      video_caputurer->Stop();
-    for (size_t video_idx = 0; video_idx < num_video_streams_; ++video_idx) {
-      video_capturers_[video_idx]->Stop();
-    }
-    for (VideoReceiveStream* thumbnail_receive_stream :
-         thumbnail_receive_streams_)
-      thumbnail_receive_stream->Stop();
-    for (VideoReceiveStream* receive_stream : video_receive_streams_)
-      receive_stream->Stop();
-    for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_)
-      thumbnail_send_stream->Stop();
-    for (VideoSendStream* video_send_stream : video_send_streams_)
-      video_send_stream->Stop();
 
     DestroyStreams();
     DestroyThumbnailStreams();
@@ -934,6 +939,24 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) {
   });
 }
 
+void VideoQualityTest::InitializeAudioDevice(Call::Config* send_call_config,
+                                             Call::Config* recv_call_config) {
+  rtc::scoped_refptr<TestAudioDeviceModule> fake_audio_device =
+      TestAudioDeviceModule::CreateTestAudioDeviceModule(
+          TestAudioDeviceModule::CreatePulsedNoiseCapturer(32000, 48000),
+          TestAudioDeviceModule::CreateDiscardRenderer(48000), 1.f);
+
+  AudioState::Config audio_state_config;
+  audio_state_config.audio_mixer = AudioMixerImpl::Create();
+  audio_state_config.audio_processing = AudioProcessingBuilder().Create();
+  audio_state_config.audio_device_module = fake_audio_device;
+  send_call_config->audio_state = AudioState::Create(audio_state_config);
+  RTC_CHECK(fake_audio_device->RegisterAudioCallback(
+                send_call_config->audio_state->audio_transport()) == 0);
+  recv_call_config->audio_state = AudioState::Create(audio_state_config);
+  fake_audio_device->Init();
+}
+
 void VideoQualityTest::SetupAudio(Transport* transport) {
   AudioSendStream::Config audio_send_config(transport);
   audio_send_config.rtp.ssrc = kAudioSendSsrc;
@@ -959,7 +982,7 @@ void VideoQualityTest::SetupAudio(Transport* transport) {
   audio_send_config.encoder_factory = audio_encoder_factory_;
   SetAudioConfig(audio_send_config);
 
-  const char* sync_group = nullptr;
+  std::string sync_group;
   if (params_.video[0].enabled && params_.audio.sync_video)
     sync_group = kSyncGroup;
 
@@ -984,22 +1007,13 @@ void VideoQualityTest::RunWithRenderers(const Params& params) {
     Call::Config call_config(&null_event_log);
     call_config.bitrate_config = params_.call.call_bitrate_config;
 
-    rtc::scoped_refptr<TestAudioDeviceModule> fake_audio_device =
+    Call::Config recv_call_config(&null_event_log);
-        TestAudioDeviceModule::CreateTestAudioDeviceModule(
-            TestAudioDeviceModule::CreatePulsedNoiseCapturer(32000, 48000),
-            TestAudioDeviceModule::CreateDiscardRenderer(48000), 1.f);
 
     if (params_.audio.enabled) {
-      AudioState::Config audio_state_config;
+      InitializeAudioDevice(&call_config, &recv_call_config);
-      audio_state_config.audio_mixer = AudioMixerImpl::Create();
-      audio_state_config.audio_processing = AudioProcessingBuilder().Create();
-      audio_state_config.audio_device_module = fake_audio_device;
-      call_config.audio_state = AudioState::Create(audio_state_config);
-      fake_audio_device->RegisterAudioCallback(
-          call_config.audio_state->audio_transport());
     }
 
-    CreateCalls(call_config, call_config);
+    CreateCalls(call_config, recv_call_config);
 
     // TODO(minyue): consider if this is a good transport even for audio only
     // calls.

video/video_quality_test.h

@@ -73,7 +73,11 @@ class VideoQualityTest :
       const SdpVideoFormat& format);
   void SetupVideo(Transport* send_transport, Transport* recv_transport);
   void SetupThumbnails(Transport* send_transport, Transport* recv_transport);
+  void StartThumbnails();
+  void StopThumbnails();
   void DestroyThumbnailStreams();
+  void InitializeAudioDevice(Call::Config* send_call_config,
+                             Call::Config* recv_call_config);
   void SetupAudio(Transport* transport);
 
   void StartEncodedFrameLogs(VideoSendStream* stream);