From 6957abeff181b3f34c40b097078df80bf1a5960c Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Tue, 29 Jan 2019 16:33:04 +0100 Subject: [PATCH] Reland "Always use real VideoStreamsFactory in full stack tests" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reland with fixes. Previous iteration affected media bitrate in bunch of tests. Always use real VideoStreamsFactory in full stack tests Because quality scaling is enabled now in full stack test, correct factory should be used to compute actual resolution. Also, since analyzed stream may be disabled completely now, change how analyzer considers the test finished --- count captured frames and stop if required amount of frames is captured and no new comparison were made. Original Reviewed-on: https://webrtc-review.googlesource.com/c/118687 Bug: webrtc:10204 Change-Id: Id1d9066add185d56fe3cb6856b700d350576c6b2 Reviewed-on: https://webrtc-review.googlesource.com/c/119950 Commit-Queue: Ilya Nikolaevskiy Reviewed-by: Åsa Persson Reviewed-by: Erik Språng Reviewed-by: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#26460} --- media/engine/webrtc_video_engine.cc | 15 +++++++- video/video_analyzer.cc | 60 ++++++++++++++++++++++------- video/video_analyzer.h | 7 +++- video/video_quality_test.cc | 53 +++++++++++++++---------- 4 files changed, 97 insertions(+), 38 deletions(-) diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index fc74a38b88..724fe8f523 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -2741,11 +2741,16 @@ std::vector EncoderStreamFactory::CreateEncoderStreams( layers[i].max_bitrate_bps = encoder_config.simulcast_layers[i].max_bitrate_bps; } + if (encoder_config.simulcast_layers[i].target_bitrate_bps > 0) { + layers[i].target_bitrate_bps = + encoder_config.simulcast_layers[i].target_bitrate_bps; + } if (encoder_config.simulcast_layers[i].min_bitrate_bps > 0 && encoder_config.simulcast_layers[i].max_bitrate_bps > 0) { // Min and max bitrate are configured. // Set target to 3/4 of the max bitrate (or to max if below min). - layers[i].target_bitrate_bps = layers[i].max_bitrate_bps * 3 / 4; + if (encoder_config.simulcast_layers[i].target_bitrate_bps <= 0) + layers[i].target_bitrate_bps = layers[i].max_bitrate_bps * 3 / 4; if (layers[i].target_bitrate_bps < layers[i].min_bitrate_bps) layers[i].target_bitrate_bps = layers[i].max_bitrate_bps; } else if (encoder_config.simulcast_layers[i].min_bitrate_bps > 0) { @@ -2811,7 +2816,13 @@ std::vector EncoderStreamFactory::CreateEncoderStreams( // In the case that the application sets a max bitrate that's lower than the // min bitrate, we adjust it down (see bugs.webrtc.org/9141). layer.min_bitrate_bps = std::min(min_bitrate_bps, max_bitrate_bps); - layer.target_bitrate_bps = layer.max_bitrate_bps = max_bitrate_bps; + if (encoder_config.simulcast_layers[0].target_bitrate_bps <= 0) { + layer.target_bitrate_bps = max_bitrate_bps; + } else { + layer.target_bitrate_bps = + encoder_config.simulcast_layers[0].target_bitrate_bps; + } + layer.max_bitrate_bps = max_bitrate_bps; layer.max_qp = max_qp_; layer.bitrate_priority = encoder_config.bitrate_priority; diff --git a/video/video_analyzer.cc b/video/video_analyzer.cc index c81c9b7c43..6535133a39 100644 --- a/video/video_analyzer.cc +++ b/video/video_analyzer.cc @@ -36,6 +36,12 @@ namespace webrtc { namespace { constexpr int kSendStatsPollingIntervalMs = 1000; constexpr size_t kMaxComparisons = 10; +// How often is keep alive message printed. +constexpr int kKeepAliveIntervalSeconds = 30; +// Interval between checking that the test is over. +constexpr int kProbingIntervalMs = 500; +constexpr int kKeepAliveIntervalIterations = + kKeepAliveIntervalSeconds * 1000 / kProbingIntervalMs; bool IsFlexfec(int payload_type) { return payload_type == test::CallTest::kFlexfecPayloadType; @@ -63,7 +69,7 @@ VideoAnalyzer::VideoAnalyzer(test::LayerFilteringTransport* transport, send_stream_(nullptr), receive_stream_(nullptr), audio_receive_stream_(nullptr), - captured_frame_forwarder_(this, clock), + captured_frame_forwarder_(this, clock, duration_frames), test_label_(test_label), graph_data_output_file_(graph_data_output_file), graph_title_(graph_title), @@ -77,6 +83,7 @@ VideoAnalyzer::VideoAnalyzer(test::LayerFilteringTransport* transport, frames_recorded_(0), frames_processed_(0), dropped_frames_(0), + captured_frames_(0), dropped_frames_before_first_encode_(0), dropped_frames_before_rendering_(0), last_render_time_(0), @@ -333,12 +340,16 @@ void VideoAnalyzer::Wait() { stats_polling_thread_.Start(); int last_frames_processed = -1; + int last_frames_captured = -1; int iteration = 0; - while (!done_.Wait(test::CallTest::kDefaultTimeoutMs)) { + + while (!done_.Wait(kProbingIntervalMs)) { int frames_processed; + int frames_captured; { rtc::CritScope crit(&comparison_lock_); frames_processed = frames_processed_; + frames_captured = captured_frames_; } // Print some output so test infrastructure won't think we've crashed. @@ -346,24 +357,35 @@ void VideoAnalyzer::Wait() { "Uh, I'm-I'm not quite dead, sir.", "Uh, I-I think uh, I could pull through, sir.", "Actually, I think I'm all right to come with you--"}; - printf("- %s\n", kKeepAliveMessages[iteration++ % 3]); + if (++iteration % kKeepAliveIntervalIterations == 0) { + printf("- %s\n", kKeepAliveMessages[iteration % 3]); + } if (last_frames_processed == -1) { last_frames_processed = frames_processed; + last_frames_captured = frames_captured; continue; } - if (frames_processed == last_frames_processed) { - EXPECT_GT(frames_processed, last_frames_processed) - << "Analyzer stalled while waiting for test to finish."; + if (frames_processed == last_frames_processed && + last_frames_captured == frames_captured) { + if (frames_captured < frames_to_process_) { + EXPECT_GT(frames_processed, last_frames_processed) + << "Analyzer stalled while waiting for test to finish."; + } done_.Set(); break; } last_frames_processed = frames_processed; + last_frames_captured = frames_captured; } if (iteration > 0) printf("- Farewell, sweet Concorde!\n"); + PrintResults(); + if (graph_data_output_file_) + PrintSamplesToFile(); + stats_polling_thread_.Stop(); } @@ -516,9 +538,6 @@ bool VideoAnalyzer::CompareFrames() { StopExcludingCpuThreadTime(); if (FrameProcessed()) { - PrintResults(); - if (graph_data_output_file_) - PrintSamplesToFile(); done_.Set(); comparison_available_event_.Set(); return false; @@ -564,6 +583,11 @@ bool VideoAnalyzer::FrameProcessed() { void VideoAnalyzer::PrintResults() { StopMeasuringCpuProcessTime(); + int frames_left; + { + rtc::CritScope crit(&crit_); + frames_left = frames_.size(); + } rtc::CritScope crit(&comparison_lock_); // Record the time from the last freeze until the last rendered frame to // ensure we cover the full timespan of the session. Otherwise the metric @@ -592,7 +616,8 @@ void VideoAnalyzer::PrintResults() { if (receive_stream_ != nullptr) { PrintResult("decode_time", decode_time_ms_, " ms"); } - + dropped_frames_ += dropped_frames_before_first_encode_ + + dropped_frames_before_rendering_ + frames_left; test::PrintResult("dropped_frames", "", test_label_.c_str(), dropped_frames_, "frames", false); test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(), @@ -753,7 +778,10 @@ double VideoAnalyzer::GetAverageMediaBitrateBps() { void VideoAnalyzer::AddCapturedFrameForComparison( const VideoFrame& video_frame) { rtc::CritScope lock(&crit_); - frames_.push_back(video_frame); + if (captured_frames_ < frames_to_process_) { + ++captured_frames_; + frames_.push_back(video_frame); + } } void VideoAnalyzer::AddFrameComparison(const VideoFrame& reference, @@ -844,11 +872,14 @@ VideoAnalyzer::Sample::Sample(int dropped, VideoAnalyzer::CapturedFrameForwarder::CapturedFrameForwarder( VideoAnalyzer* analyzer, - Clock* clock) + Clock* clock, + int frames_to_process) : analyzer_(analyzer), send_stream_input_(nullptr), video_source_(nullptr), - clock_(clock) {} + clock_(clock), + captured_frames_(0), + frames_to_process_(frames_to_process) {} void VideoAnalyzer::CapturedFrameForwarder::SetSource( VideoSourceInterface* video_source) { @@ -866,7 +897,8 @@ void VideoAnalyzer::CapturedFrameForwarder::OnFrame( copy.set_timestamp(copy.ntp_time_ms() * 90); analyzer_->AddCapturedFrameForComparison(copy); rtc::CritScope lock(&crit_); - if (send_stream_input_) + ++captured_frames_; + if (send_stream_input_ && captured_frames_ <= frames_to_process_) send_stream_input_->OnFrame(copy); } diff --git a/video/video_analyzer.h b/video/video_analyzer.h index 4e2abcd4e1..8caaf1420a 100644 --- a/video/video_analyzer.h +++ b/video/video_analyzer.h @@ -136,7 +136,9 @@ class VideoAnalyzer : public PacketReceiver, class CapturedFrameForwarder : public rtc::VideoSinkInterface, public rtc::VideoSourceInterface { public: - explicit CapturedFrameForwarder(VideoAnalyzer* analyzer, Clock* clock); + CapturedFrameForwarder(VideoAnalyzer* analyzer, + Clock* clock, + int frames_to_process); void SetSource(rtc::VideoSourceInterface* video_source); private: @@ -155,6 +157,8 @@ class VideoAnalyzer : public PacketReceiver, RTC_GUARDED_BY(crit_); VideoSourceInterface* video_source_; Clock* clock_; + int captured_frames_ RTC_GUARDED_BY(crit_); + int frames_to_process_ RTC_GUARDED_BY(crit_); }; struct FrameWithPsnr { @@ -240,6 +244,7 @@ class VideoAnalyzer : public PacketReceiver, int frames_recorded_; int frames_processed_; int dropped_frames_; + int captured_frames_; int dropped_frames_before_first_encode_; int dropped_frames_before_rendering_; int64_t last_render_time_; diff --git a/video/video_quality_test.cc b/video/video_quality_test.cc index 08fea89f99..809ea380c3 100644 --- a/video/video_quality_test.cc +++ b/video/video_quality_test.cc @@ -630,19 +630,17 @@ void VideoQualityTest::SetupVideo(Transport* send_transport, video_encoder_configs_[video_idx].max_bitrate_bps += params_.ss[video_idx].streams[i].max_bitrate_bps; } - if (params_.ss[video_idx].infer_streams) { + video_encoder_configs_[video_idx].simulcast_layers = + std::vector(params_.ss[video_idx].streams.size()); + if (!params_.ss[video_idx].infer_streams) { video_encoder_configs_[video_idx].simulcast_layers = - std::vector(params_.ss[video_idx].streams.size()); - video_encoder_configs_[video_idx].video_stream_factory = - new rtc::RefCountedObject( - params_.video[video_idx].codec, - params_.ss[video_idx].streams[0].max_qp, - params_.screenshare[video_idx].enabled, true); - } else { - video_encoder_configs_[video_idx].video_stream_factory = - new rtc::RefCountedObject( - params_.ss[video_idx].streams); + params_.ss[video_idx].streams; } + video_encoder_configs_[video_idx].video_stream_factory = + new rtc::RefCountedObject( + params_.video[video_idx].codec, + params_.ss[video_idx].streams[0].max_qp, + params_.screenshare[video_idx].enabled, true); video_encoder_configs_[video_idx].spatial_layers = params_.ss[video_idx].spatial_layers; @@ -719,6 +717,26 @@ void VideoQualityTest::SetupVideo(Transport* send_transport, << params_.video[video_idx].codec << ", stream " << video_idx; } + } else { + // Default mode. Single SL, no automatic_scaling, + if (params_.video[video_idx].codec == "VP8") { + VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings(); + vp8_settings.automaticResizeOn = false; + video_encoder_configs_[video_idx].encoder_specific_settings = + new rtc::RefCountedObject< + VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings); + } else if (params_.video[video_idx].codec == "VP9") { + VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings(); + vp9_settings.automaticResizeOn = false; + video_encoder_configs_[video_idx].encoder_specific_settings = + new rtc::RefCountedObject< + VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings); + } else if (params_.video[video_idx].codec == "H264") { + VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings(); + video_encoder_configs_[video_idx].encoder_specific_settings = + new rtc::RefCountedObject< + VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings); + } } total_streams_used += num_video_substreams; } @@ -791,16 +809,9 @@ void VideoQualityTest::SetupThumbnails(Transport* send_transport, params_.video[0].suspend_below_min_bitrate; thumbnail_encoder_config.number_of_streams = 1; thumbnail_encoder_config.max_bitrate_bps = 50000; - if (params_.ss[0].infer_streams) { - thumbnail_encoder_config.video_stream_factory = - new rtc::RefCountedObject(params_.ss[0].streams); - } else { - thumbnail_encoder_config.simulcast_layers = std::vector(1); - thumbnail_encoder_config.video_stream_factory = - new rtc::RefCountedObject( - params_.video[0].codec, params_.ss[0].streams[0].max_qp, - params_.screenshare[0].enabled, true); - } + std::vector streams{params_.ss[0].streams[0]}; + thumbnail_encoder_config.video_stream_factory = + new rtc::RefCountedObject(streams); thumbnail_encoder_config.spatial_layers = params_.ss[0].spatial_layers; thumbnail_encoder_configs_.push_back(thumbnail_encoder_config.Copy());