From b17976763d58e9fdf5b25747ec45f86344225627 Mon Sep 17 00:00:00 2001 From: stefan Date: Thu, 11 Aug 2016 07:00:57 -0700 Subject: [PATCH] Add an HD resolution perf test. Also update existing perf tests to use send side bwe. BUG=webrtc:4604, chromium:522001 Review-Url: https://codereview.webrtc.org/2227733004 Cr-Commit-Position: refs/heads/master@{#13726} --- .../ConferenceMotion_1280_720_50.yuv.sha1 | 1 + webrtc/video/full_stack.cc | 65 +++++++++++++------ webrtc/video/screenshare_loopback.cc | 4 +- webrtc/video/video_loopback.cc | 10 +-- webrtc/video/video_quality_test.cc | 25 +++++-- webrtc/video/video_quality_test.h | 4 +- webrtc/webrtc_perf_tests.isolate | 1 + 7 files changed, 74 insertions(+), 36 deletions(-) create mode 100644 resources/ConferenceMotion_1280_720_50.yuv.sha1 diff --git a/resources/ConferenceMotion_1280_720_50.yuv.sha1 b/resources/ConferenceMotion_1280_720_50.yuv.sha1 new file mode 100644 index 0000000000..c929e3fc2a --- /dev/null +++ b/resources/ConferenceMotion_1280_720_50.yuv.sha1 @@ -0,0 +1 @@ +60a92ea32e238bc2801ac2ca26827b8b10155978 \ No newline at end of file diff --git a/webrtc/video/full_stack.cc b/webrtc/video/full_stack.cc index 368341370a..26aae3c1ff 100644 --- a/webrtc/video/full_stack.cc +++ b/webrtc/video/full_stack.cc @@ -68,7 +68,7 @@ TEST_F(FullStackTest, ForemanCifPlr5Vp9) { TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) { VideoQualityTest::Params paris_qcif = { - {176, 144, 30, 300000, 300000, 300000, false, "VP8", 1}, + {176, 144, 30, 300000, 300000, 300000, false, "VP8", 1, 0, 0, true}, {"paris_qcif"}, {}, {"net_delay_0_0_plr_0", 36.0, 0.96, kFullStackTestDurationSecs}}; @@ -78,17 +78,17 @@ TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) { TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) { // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif. VideoQualityTest::Params foreman_cif = { - {352, 288, 30, 700000, 700000, 700000, false, "VP8", 1}, + {352, 288, 30, 700000, 700000, 700000, false, "VP8", 1, 0, 0, true}, {"foreman_cif"}, {}, - {"foreman_cif_net_delay_0_0_plr_0", 0.0, 0.0, kFullStackTestDurationSecs} - }; + {"foreman_cif_net_delay_0_0_plr_0", 0.0, 0.0, + kFullStackTestDurationSecs}}; RunTest(foreman_cif); } TEST_F(FullStackTest, ForemanCifPlr5) { VideoQualityTest::Params foreman_cif = { - {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1}, + {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, 0, 0, true}, {"foreman_cif"}, {}, {"foreman_cif_delay_50_0_plr_5", 0.0, 0.0, kFullStackTestDurationSecs}}; @@ -99,7 +99,7 @@ TEST_F(FullStackTest, ForemanCifPlr5) { TEST_F(FullStackTest, ForemanCif500kbps) { VideoQualityTest::Params foreman_cif = { - {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1}, + {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, 0, 0, true}, {"foreman_cif"}, {}, {"foreman_cif_500kbps", 0.0, 0.0, kFullStackTestDurationSecs}}; @@ -111,11 +111,11 @@ TEST_F(FullStackTest, ForemanCif500kbps) { TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) { VideoQualityTest::Params foreman_cif = { - {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1}, + {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, 0, 0, true}, {"foreman_cif"}, {}, - {"foreman_cif_500kbps_32pkts_queue", 0.0, 0.0, kFullStackTestDurationSecs} - }; + {"foreman_cif_500kbps_32pkts_queue", 0.0, 0.0, + kFullStackTestDurationSecs}}; foreman_cif.pipe.queue_length_packets = 32; foreman_cif.pipe.queue_delay_ms = 0; foreman_cif.pipe.link_capacity_kbps = 500; @@ -124,7 +124,7 @@ TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) { TEST_F(FullStackTest, ForemanCif500kbps100ms) { VideoQualityTest::Params foreman_cif = { - {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1}, + {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, 0, 0, true}, {"foreman_cif"}, {}, {"foreman_cif_500kbps_100ms", 0.0, 0.0, kFullStackTestDurationSecs}}; @@ -136,11 +136,24 @@ TEST_F(FullStackTest, ForemanCif500kbps100ms) { TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) { VideoQualityTest::Params foreman_cif = { - {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1}, + {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, 0, 0, true}, {"foreman_cif"}, {}, {"foreman_cif_500kbps_100ms_32pkts_queue", 0.0, 0.0, - kFullStackTestDurationSecs}}; + kFullStackTestDurationSecs}}; + foreman_cif.pipe.queue_length_packets = 32; + foreman_cif.pipe.queue_delay_ms = 100; + foreman_cif.pipe.link_capacity_kbps = 500; + RunTest(foreman_cif); +} + +TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueueRecvBwe) { + VideoQualityTest::Params foreman_cif = { + {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, 0, 0, false}, + {"foreman_cif"}, + {}, + {"foreman_cif_500kbps_100ms_32pkts_queue", 0.0, 0.0, + kFullStackTestDurationSecs}}; foreman_cif.pipe.queue_length_packets = 32; foreman_cif.pipe.queue_delay_ms = 100; foreman_cif.pipe.link_capacity_kbps = 500; @@ -149,20 +162,33 @@ TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) { TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) { VideoQualityTest::Params foreman_cif = { - {352, 288, 30, 30000, 2000000, 2000000, false, "VP8", 1}, + {352, 288, 30, 30000, 2000000, 2000000, false, "VP8", 1, 0, 0, true}, {"foreman_cif"}, {}, {"foreman_cif_1000kbps_100ms_32pkts_queue", 0.0, 0.0, - kFullStackTestDurationSecs}}; + kFullStackTestDurationSecs}}; foreman_cif.pipe.queue_length_packets = 32; foreman_cif.pipe.queue_delay_ms = 100; foreman_cif.pipe.link_capacity_kbps = 1000; RunTest(foreman_cif); } +TEST_F(FullStackTest, ConferenceMotionHd2000kbps100msLimitedQueue) { + VideoQualityTest::Params conf_motion_hd = { + {1280, 720, 50, 30000, 3000000, 3000000, false, "VP8", 1, 0, 0, true}, + {"ConferenceMotion_1280_720_50"}, + {}, + {"conference_motion_hd_2000kbps_100ms_32pkts_queue", 0.0, 0.0, + kFullStackTestDurationSecs}}; + conf_motion_hd.pipe.queue_length_packets = 32; + conf_motion_hd.pipe.queue_delay_ms = 100; + conf_motion_hd.pipe.link_capacity_kbps = 2000; + RunTest(conf_motion_hd); +} + TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL) { VideoQualityTest::Params screenshare = { - {1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000}, + {1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000, true}, {}, {true, 10}, {"screenshare_slides", 0.0, 0.0, kFullStackTestDurationSecs}}; @@ -171,7 +197,8 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL) { TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_Scroll) { VideoQualityTest::Params config = { - {1850, 1110 / 2, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000}, + {1850, 1110 / 2, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000, + true}, {}, {true, 10, 2}, {"screenshare_slides_scrolling", 0.0, 0.0, kFullStackTestDurationSecs}}; @@ -180,7 +207,7 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_Scroll) { TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_LossyNet) { VideoQualityTest::Params screenshare = { - {1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000}, + {1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000, true}, {}, // Video-specific. {true, 10}, // Screenshare-specific. {"screenshare_slides_lossy_net", 0.0, 0.0, kFullStackTestDurationSecs}}; @@ -192,7 +219,7 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_LossyNet) { TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_VeryLossyNet) { VideoQualityTest::Params screenshare = { - {1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000}, + {1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000, true}, {}, // Video-specific. {true, 10}, // Screenshare-specific. {"screenshare_slides_very_lossy", 0.0, 0.0, kFullStackTestDurationSecs}}; @@ -205,7 +232,7 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_VeryLossyNet) { #if !defined(RTC_DISABLE_VP9) TEST_F(FullStackTest, ScreenshareSlidesVP9_2SL) { VideoQualityTest::Params screenshare = { - {1850, 1110, 5, 50000, 200000, 2000000, false, "VP9", 1, 0, 400000}, + {1850, 1110, 5, 50000, 200000, 2000000, false, "VP9", 1, 0, 400000, true}, {}, {true, 10}, {"screenshare_slides_vp9_2sl", 0.0, 0.0, kFullStackTestDurationSecs}, diff --git a/webrtc/video/screenshare_loopback.cc b/webrtc/video/screenshare_loopback.cc index 8508cd3a1f..05ebc2e02f 100644 --- a/webrtc/video/screenshare_loopback.cc +++ b/webrtc/video/screenshare_loopback.cc @@ -228,8 +228,8 @@ void Loopback() { flags::MinBitrateKbps() * 1000, flags::TargetBitrateKbps() * 1000, flags::MaxBitrateKbps() * 1000, false, flags::Codec(), flags::NumTemporalLayers(), flags::SelectedTL(), - flags::MinTransmitBitrateKbps() * 1000, call_bitrate_config, - flags::FLAGS_send_side_bwe}, + flags::MinTransmitBitrateKbps() * 1000, flags::FLAGS_send_side_bwe, + false, call_bitrate_config}, {}, // Video specific. {true, flags::SlideChangeInterval(), flags::ScrollDuration()}, {"screenshare", 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename(), diff --git a/webrtc/video/video_loopback.cc b/webrtc/video/video_loopback.cc index f3e13de346..518129479f 100644 --- a/webrtc/video/video_loopback.cc +++ b/webrtc/video/video_loopback.cc @@ -227,14 +227,10 @@ void Loopback() { VideoQualityTest::Params params{ {flags::Width(), flags::Height(), flags::Fps(), flags::MinBitrateKbps() * 1000, flags::TargetBitrateKbps() * 1000, - flags::MaxBitrateKbps() * 1000, - flags::FLAGS_suspend_below_min_bitrate, - flags::Codec(), - flags::NumTemporalLayers(), flags::SelectedTL(), + flags::MaxBitrateKbps() * 1000, flags::FLAGS_suspend_below_min_bitrate, + flags::Codec(), flags::NumTemporalLayers(), flags::SelectedTL(), 0, // No min transmit bitrate. - call_bitrate_config, - flags::FLAGS_send_side_bwe, - flags::FLAGS_use_fec}, + flags::FLAGS_send_side_bwe, flags::FLAGS_use_fec, call_bitrate_config}, {flags::Clip()}, {}, // Screenshare specific. {"video", 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename(), diff --git a/webrtc/video/video_quality_test.cc b/webrtc/video/video_quality_test.cc index 66210d0864..a401d6dd42 100644 --- a/webrtc/video/video_quality_test.cc +++ b/webrtc/video/video_quality_test.cc @@ -41,6 +41,7 @@ static const int kSendStatsPollingIntervalMs = 1000; static const int kPayloadTypeH264 = 122; static const int kPayloadTypeVP8 = 123; static const int kPayloadTypeVP9 = 124; +static const size_t kMaxComparisons = 10; class VideoAnalyzer : public PacketReceiver, public Transport, @@ -408,10 +409,16 @@ class VideoAnalyzer : public PacketReceiver, VideoFrame reference_copy; VideoFrame render_copy; - reference_copy.CopyFrame(reference); - render_copy.CopyFrame(render); rtc::CritScope crit(&comparison_lock_); + if (comparisons_.size() < kMaxComparisons) { + reference_copy.CopyFrame(reference); + render_copy.CopyFrame(render); + } else { + // Copy the time to ensure that delay calculations can still be made. + reference_copy.set_ntp_time_ms(reference.ntp_time_ms()); + render_copy.set_ntp_time_ms(render.ntp_time_ms()); + } comparisons_.push_back(FrameComparison(reference_copy, render_copy, dropped, send_time_ms, recv_time_ms, render_time_ms, encoded_size)); @@ -546,8 +553,12 @@ class VideoAnalyzer : public PacketReceiver, void PerformFrameComparison(const FrameComparison& comparison) { // Perform expensive psnr and ssim calculations while not holding lock. - double psnr = I420PSNR(&comparison.reference, &comparison.render); - double ssim = I420SSIM(&comparison.reference, &comparison.render); + double psnr = -1.0; + double ssim = -1.0; + if (!comparison.reference.IsZeroSize()) { + psnr = I420PSNR(&comparison.reference, &comparison.render); + ssim = I420SSIM(&comparison.reference, &comparison.render); + } int64_t input_time_ms = comparison.reference.ntp_time_ms(); @@ -558,8 +569,10 @@ class VideoAnalyzer : public PacketReceiver, comparison.recv_time_ms, comparison.render_time_ms, comparison.encoded_frame_size, psnr, ssim)); } - psnr_.AddSample(psnr); - ssim_.AddSample(ssim); + if (psnr >= 0.0) + psnr_.AddSample(psnr); + if (ssim >= 0.0) + ssim_.AddSample(ssim); if (comparison.dropped) { ++dropped_frames_; diff --git a/webrtc/video/video_quality_test.h b/webrtc/video/video_quality_test.h index b1f59e540d..9c7f0e7e1d 100644 --- a/webrtc/video/video_quality_test.h +++ b/webrtc/video/video_quality_test.h @@ -39,10 +39,10 @@ class VideoQualityTest : public test::CallTest { int num_temporal_layers; int selected_tl; int min_transmit_bps; - - Call::Config::BitrateConfig call_bitrate_config; bool send_side_bwe; bool fec; + + Call::Config::BitrateConfig call_bitrate_config; } common; struct { // Video-specific settings. std::string clip_name; diff --git a/webrtc/webrtc_perf_tests.isolate b/webrtc/webrtc_perf_tests.isolate index b66054c320..7d58fd6a5d 100644 --- a/webrtc/webrtc_perf_tests.isolate +++ b/webrtc/webrtc_perf_tests.isolate @@ -12,6 +12,7 @@ 'files': [ '<(DEPTH)/resources/audio_coding/speech_mono_16kHz.pcm', '<(DEPTH)/resources/audio_coding/testfile32kHz.pcm', + '<(DEPTH)/resources/ConferenceMotion_1280_720_50.yuv', '<(DEPTH)/resources/foreman_cif.yuv', '<(DEPTH)/resources/paris_qcif.yuv', '<(DEPTH)/resources/voice_engine/audio_long16.pcm',