From 732086609177d0e657cc7bc96737db7117e72cec Mon Sep 17 00:00:00 2001 From: minyue Date: Thu, 18 Aug 2016 06:28:55 -0700 Subject: [PATCH] Reland of Adding audio to video_quality_test. The original commit was https://codereview.webrtc.org/2136573002/. BUG= Review-Url: https://codereview.webrtc.org/2259783002 Cr-Commit-Position: refs/heads/master@{#13818} --- webrtc/video/screenshare_loopback.cc | 30 +++--- webrtc/video/video_loopback.cc | 34 +++--- webrtc/video/video_quality_test.cc | 154 +++++++++++++++++++++++++-- webrtc/video/video_quality_test.h | 6 +- 4 files changed, 183 insertions(+), 41 deletions(-) diff --git a/webrtc/video/screenshare_loopback.cc b/webrtc/video/screenshare_loopback.cc index 05ebc2e02f..82e51dd939 100644 --- a/webrtc/video/screenshare_loopback.cc +++ b/webrtc/video/screenshare_loopback.cc @@ -223,19 +223,21 @@ void Loopback() { call_bitrate_config.start_bitrate_bps = flags::StartBitrateKbps() * 1000; call_bitrate_config.max_bitrate_bps = flags::MaxBitrateKbps() * 1000; - VideoQualityTest::Params params{ - {flags::Width(), flags::Height(), flags::Fps(), - flags::MinBitrateKbps() * 1000, flags::TargetBitrateKbps() * 1000, - flags::MaxBitrateKbps() * 1000, false, flags::Codec(), - flags::NumTemporalLayers(), flags::SelectedTL(), - flags::MinTransmitBitrateKbps() * 1000, flags::FLAGS_send_side_bwe, - false, call_bitrate_config}, - {}, // Video specific. - {true, flags::SlideChangeInterval(), flags::ScrollDuration()}, - {"screenshare", 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename(), - flags::GraphTitle()}, - pipe_config, - flags::FLAGS_logs}; + VideoQualityTest::Params params; + params.common = {flags::Width(), flags::Height(), flags::Fps(), + flags::MinBitrateKbps() * 1000, flags::TargetBitrateKbps() * 1000, + flags::MaxBitrateKbps() * 1000, false, flags::Codec(), + flags::NumTemporalLayers(), flags::SelectedTL(), + flags::MinTransmitBitrateKbps() * 1000, flags::FLAGS_send_side_bwe, + false, call_bitrate_config}; + params.screenshare = {true, flags::SlideChangeInterval(), + flags::ScrollDuration()}; + params.analyzer = {"screenshare", 0.0, 0.0, flags::DurationSecs(), + flags::OutputFilename(), flags::GraphTitle()}; + params.pipe = pipe_config; + params.logs = flags::FLAGS_logs; + params.audio = false; + params.audio_video_sync = false; std::vector stream_descriptors; stream_descriptors.push_back(flags::Stream0()); @@ -251,7 +253,7 @@ void Loopback() { if (flags::DurationSecs()) { test.RunWithAnalyzer(params); } else { - test.RunWithVideoRenderer(params); + test.RunWithRenderers(params); } } } // namespace webrtc diff --git a/webrtc/video/video_loopback.cc b/webrtc/video/video_loopback.cc index 518129479f..10998c8867 100644 --- a/webrtc/video/video_loopback.cc +++ b/webrtc/video/video_loopback.cc @@ -191,6 +191,11 @@ DEFINE_bool(allow_reordering, false, "Allow packet reordering to occur"); DEFINE_bool(use_fec, false, "Use forward error correction."); +DEFINE_bool(audio, false, "Add audio stream"); + +DEFINE_bool(audio_video_sync, false, "Sync audio and video stream (no effect if" + " audio is false)"); + DEFINE_string( force_fieldtrials, "", @@ -224,19 +229,20 @@ void Loopback() { call_bitrate_config.start_bitrate_bps = flags::StartBitrateKbps() * 1000; call_bitrate_config.max_bitrate_bps = flags::MaxBitrateKbps() * 1000; - VideoQualityTest::Params params{ - {flags::Width(), flags::Height(), flags::Fps(), - flags::MinBitrateKbps() * 1000, flags::TargetBitrateKbps() * 1000, - flags::MaxBitrateKbps() * 1000, flags::FLAGS_suspend_below_min_bitrate, - flags::Codec(), flags::NumTemporalLayers(), flags::SelectedTL(), - 0, // No min transmit bitrate. - flags::FLAGS_send_side_bwe, flags::FLAGS_use_fec, call_bitrate_config}, - {flags::Clip()}, - {}, // Screenshare specific. - {"video", 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename(), - flags::GraphTitle()}, - pipe_config, - flags::FLAGS_logs}; + VideoQualityTest::Params params; + params.common = {flags::Width(), flags::Height(), flags::Fps(), + flags::MinBitrateKbps() * 1000, flags::TargetBitrateKbps() * 1000, + flags::MaxBitrateKbps() * 1000, flags::FLAGS_suspend_below_min_bitrate, + flags::Codec(), flags::NumTemporalLayers(), flags::SelectedTL(), + 0, // No min transmit bitrate. + flags::FLAGS_send_side_bwe, flags::FLAGS_use_fec, call_bitrate_config}; + params.video = {flags::Clip()}; + params.analyzer = {"video", 0.0, 0.0, flags::DurationSecs(), + flags::OutputFilename(), flags::GraphTitle()}; + params.pipe = pipe_config; + params.logs = flags::FLAGS_logs; + params.audio = flags::FLAGS_audio, + params.audio_video_sync = flags::FLAGS_audio_video_sync; std::vector stream_descriptors; stream_descriptors.push_back(flags::Stream0()); @@ -252,7 +258,7 @@ void Loopback() { if (flags::DurationSecs()) { test.RunWithAnalyzer(params); } else { - test.RunWithVideoRenderer(params); + test.RunWithRenderers(params); } } } // namespace webrtc diff --git a/webrtc/video/video_quality_test.cc b/webrtc/video/video_quality_test.cc index a401d6dd42..51160c91c6 100644 --- a/webrtc/video/video_quality_test.cc +++ b/webrtc/video/video_quality_test.cc @@ -34,15 +34,68 @@ #include "webrtc/test/testsupport/fileutils.h" #include "webrtc/test/video_renderer.h" #include "webrtc/video/video_quality_test.h" +#include "webrtc/voice_engine/include/voe_base.h" +#include "webrtc/voice_engine/include/voe_codec.h" + +namespace { + +constexpr int kSendStatsPollingIntervalMs = 1000; +constexpr int kPayloadTypeH264 = 122; +constexpr int kPayloadTypeVP8 = 123; +constexpr int kPayloadTypeVP9 = 124; +constexpr size_t kMaxComparisons = 10; +constexpr char kSyncGroup[] = "av_sync"; +constexpr int kOpusMinBitrate = 6000; +constexpr int kOpusBitrateFb = 32000; + +struct VoiceEngineState { + VoiceEngineState() + : voice_engine(nullptr), + base(nullptr), + codec(nullptr), + send_channel_id(-1), + receive_channel_id(-1) {} + + webrtc::VoiceEngine* voice_engine; + webrtc::VoEBase* base; + webrtc::VoECodec* codec; + int send_channel_id; + int receive_channel_id; +}; + +void CreateVoiceEngine(VoiceEngineState* voe, + rtc::scoped_refptr + decoder_factory) { + voe->voice_engine = webrtc::VoiceEngine::Create(); + voe->base = webrtc::VoEBase::GetInterface(voe->voice_engine); + voe->codec = webrtc::VoECodec::GetInterface(voe->voice_engine); + EXPECT_EQ(0, voe->base->Init(nullptr, nullptr, decoder_factory)); + webrtc::Config voe_config; + voe_config.Set(new webrtc::VoicePacing(true)); + voe->send_channel_id = voe->base->CreateChannel(voe_config); + EXPECT_GE(voe->send_channel_id, 0); + voe->receive_channel_id = voe->base->CreateChannel(); + EXPECT_GE(voe->receive_channel_id, 0); +} + +void DestroyVoiceEngine(VoiceEngineState* voe) { + voe->base->DeleteChannel(voe->send_channel_id); + voe->send_channel_id = -1; + voe->base->DeleteChannel(voe->receive_channel_id); + voe->receive_channel_id = -1; + voe->base->Release(); + voe->base = nullptr; + voe->codec->Release(); + voe->codec = nullptr; + + webrtc::VoiceEngine::Delete(voe->voice_engine); + voe->voice_engine = nullptr; +} + +} // namespace namespace webrtc { -static const int kSendStatsPollingIntervalMs = 1000; -static const int kPayloadTypeH264 = 122; -static const int kPayloadTypeVP8 = 123; -static const int kPayloadTypeVP9 = 124; -static const size_t kMaxComparisons = 10; - class VideoAnalyzer : public PacketReceiver, public Transport, public rtc::VideoSinkInterface, @@ -1002,6 +1055,7 @@ void VideoQualityTest::CreateCapturer(VideoCaptureInput* input) { void VideoQualityTest::RunWithAnalyzer(const Params& params) { params_ = params; + RTC_CHECK(!params_.audio); // TODO(ivica): Merge with RunWithRenderer and use a flag / argument to // differentiate between the analyzer and the renderer case. CheckParams(); @@ -1099,7 +1153,7 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { fclose(graph_data_output_file); } -void VideoQualityTest::RunWithVideoRenderer(const Params& params) { +void VideoQualityTest::RunWithRenderers(const Params& params) { params_ = params; CheckParams(); @@ -1123,6 +1177,15 @@ void VideoQualityTest::RunWithVideoRenderer(const Params& params) { // match the full stack tests. Call::Config call_config; call_config.bitrate_config = params_.common.call_bitrate_config; + + ::VoiceEngineState voe; + if (params_.audio) { + CreateVoiceEngine(&voe, decoder_factory_); + AudioState::Config audio_state_config; + audio_state_config.voice_engine = voe.voice_engine; + call_config.audio_state = AudioState::Create(audio_state_config); + } + std::unique_ptr call(Call::Create(call_config)); test::LayerFilteringTransport transport( @@ -1137,6 +1200,8 @@ void VideoQualityTest::RunWithVideoRenderer(const Params& params) { video_send_config_.local_renderer = local_preview.get(); video_receive_configs_[stream_id].renderer = loopback_video.get(); + if (params_.audio && params_.audio_video_sync) + video_receive_configs_[stream_id].sync_group = kSyncGroup; video_send_config_.suspend_below_min_bitrate = params_.common.suspend_below_min_bitrate; @@ -1155,24 +1220,91 @@ void VideoQualityTest::RunWithVideoRenderer(const Params& params) { video_send_stream_ = call->CreateVideoSendStream(video_send_config_, video_encoder_config_); - VideoReceiveStream* receive_stream = + VideoReceiveStream* video_receive_stream = call->CreateVideoReceiveStream(video_receive_configs_[stream_id].Copy()); CreateCapturer(video_send_stream_->Input()); - receive_stream->Start(); + AudioReceiveStream* audio_receive_stream = nullptr; + if (params_.audio) { + audio_send_config_ = AudioSendStream::Config(&transport); + audio_send_config_.voe_channel_id = voe.send_channel_id; + audio_send_config_.rtp.ssrc = kAudioSendSsrc; + + // Add extension to enable audio send side BWE, and allow audio bit rate + // adaptation. + audio_send_config_.rtp.extensions.clear(); + if (params_.common.send_side_bwe) { + audio_send_config_.rtp.extensions.push_back(webrtc::RtpExtension( + webrtc::RtpExtension::kTransportSequenceNumberUri, + test::kTransportSequenceNumberExtensionId)); + audio_send_config_.min_bitrate_kbps = kOpusMinBitrate / 1000; + audio_send_config_.max_bitrate_kbps = kOpusBitrateFb / 1000; + } + + audio_send_stream_ = call->CreateAudioSendStream(audio_send_config_); + + AudioReceiveStream::Config audio_config; + audio_config.rtp.local_ssrc = kReceiverLocalAudioSsrc; + audio_config.rtcp_send_transport = &transport; + audio_config.voe_channel_id = voe.receive_channel_id; + audio_config.rtp.remote_ssrc = audio_send_config_.rtp.ssrc; + audio_config.rtp.transport_cc = params_.common.send_side_bwe; + audio_config.rtp.extensions = audio_send_config_.rtp.extensions; + audio_config.decoder_factory = decoder_factory_; + if (params_.audio_video_sync) + audio_config.sync_group = kSyncGroup; + + audio_receive_stream =call->CreateAudioReceiveStream(audio_config); + + const CodecInst kOpusInst = {120, "OPUS", 48000, 960, 2, 64000}; + EXPECT_EQ(0, voe.codec->SetSendCodec(voe.send_channel_id, kOpusInst)); + } + + // Start sending and receiving video. + video_receive_stream->Start(); video_send_stream_->Start(); capturer_->Start(); + if (params_.audio) { + // Start receiving audio. + audio_receive_stream->Start(); + EXPECT_EQ(0, voe.base->StartPlayout(voe.receive_channel_id)); + EXPECT_EQ(0, voe.base->StartReceive(voe.receive_channel_id)); + + // Start sending audio. + audio_send_stream_->Start(); + EXPECT_EQ(0, voe.base->StartSend(voe.send_channel_id)); + } + test::PressEnterToContinue(); + if (params_.audio) { + // Stop sending audio. + EXPECT_EQ(0, voe.base->StopSend(voe.send_channel_id)); + audio_send_stream_->Stop(); + + // Stop receiving audio. + EXPECT_EQ(0, voe.base->StopReceive(voe.receive_channel_id)); + EXPECT_EQ(0, voe.base->StopPlayout(voe.receive_channel_id)); + audio_receive_stream->Stop(); + } + + // Stop receiving and sending video. capturer_->Stop(); video_send_stream_->Stop(); - receive_stream->Stop(); + video_receive_stream->Stop(); - call->DestroyVideoReceiveStream(receive_stream); + call->DestroyVideoReceiveStream(video_receive_stream); call->DestroyVideoSendStream(video_send_stream_); + if (params_.audio) { + call->DestroyAudioSendStream(audio_send_stream_); + call->DestroyAudioReceiveStream(audio_receive_stream); + } + transport.StopSending(); + if (params_.audio) + DestroyVoiceEngine(&voe); } } // namespace webrtc diff --git a/webrtc/video/video_quality_test.h b/webrtc/video/video_quality_test.h index 9c7f0e7e1d..e5c85aa231 100644 --- a/webrtc/video/video_quality_test.h +++ b/webrtc/video/video_quality_test.h @@ -62,7 +62,7 @@ class VideoQualityTest : public test::CallTest { } analyzer; FakeNetworkPipe::Config pipe; bool logs; - struct { // Spatial scalability. + struct { // Spatial scalability. std::vector streams; // If empty, one stream is assumed. size_t selected_stream; int num_spatial_layers; @@ -70,6 +70,8 @@ class VideoQualityTest : public test::CallTest { // If empty, bitrates are generated in VP9Impl automatically. std::vector spatial_layers; } ss; + bool audio; + bool audio_video_sync; }; // (*) Set to -1.1 if generating graph data for simulcast or SVC and the // selected stream/layer doesn't have the same resolution as the largest @@ -77,7 +79,7 @@ class VideoQualityTest : public test::CallTest { VideoQualityTest(); void RunWithAnalyzer(const Params& params); - void RunWithVideoRenderer(const Params& params); + void RunWithRenderers(const Params& params); static void FillScalabilitySettings( Params* params,