From df232992597bd2865fdbede3e9d7e45445e4f122 Mon Sep 17 00:00:00 2001 From: brandtr Date: Thu, 7 Sep 2017 07:50:06 -0700 Subject: [PATCH] Add VideoProcessorIntegrationTest for MediaCodec implementations. * Add VideoProcessorIntegrationTestMediaCodec tests, including a standard foreman test and a forced SW fallback test. * Morph PlotVideoProcessorIntegrationTest into VideoProcessorIntegrationTestParameterized. This test is intended to be patched locally, depending on what metric is of interest. It is run on the bots to ensure that it doesn't break. * Remove the plot_videoprocessor_integrationtest binary. The test above is instead moved to the modules_tests binary. BUG=webrtc:6634 Review-Url: https://codereview.webrtc.org/3011043002 Cr-Commit-Position: refs/heads/master@{#19727} --- webrtc/BUILD.gn | 1 - webrtc/modules/BUILD.gn | 8 +- webrtc/modules/video_coding/BUILD.gn | 151 +++++------------- .../codecs/test/plot_webrtc_test_logs.py | 5 +- .../test/videoprocessor_integrationtest.cc | 7 +- ...deoprocessor_integrationtest_mediacodec.cc | 98 ++++++++++++ ...rocessor_integrationtest_parameterized.cc} | 31 ++-- 7 files changed, 161 insertions(+), 140 deletions(-) create mode 100644 webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc rename webrtc/modules/video_coding/codecs/test/{plot_videoprocessor_integrationtest.cc => videoprocessor_integrationtest_parameterized.cc} (75%) diff --git a/webrtc/BUILD.gn b/webrtc/BUILD.gn index 210d2086df..b11016cc70 100644 --- a/webrtc/BUILD.gn +++ b/webrtc/BUILD.gn @@ -307,7 +307,6 @@ if (!build_with_chromium) { "modules/remote_bitrate_estimator:bwe_simulations_tests", "modules/rtp_rtcp:test_packet_masks_metrics", "modules/video_capture:video_capture_internal_impl", - "modules/video_coding:plot_videoprocessor_integrationtest", "ortc:ortc_unittests", "pc:peerconnection_unittests", "pc:rtc_pc_unittests", diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index dd8e54b68b..ead7855755 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -77,7 +77,13 @@ if (rtc_include_tests) { data = modules_tests_resources if (is_android) { - deps += [ "//testing/android/native_test:native_test_native_code" ] + deps += [ + # NOTE(brandtr): Including Java classes seems only to be possible from + # rtc_test targets. Therefore we include this target here, instead of + # in video_coding_modules_tests, where it is actually used. + "../sdk/android:libjingle_peerconnection_java", + "//testing/android/native_test:native_test_native_code", + ] shard_timeout = 900 } diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index 3bf221be17..3d3a3af342 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -366,55 +366,20 @@ if (rtc_include_tests) { ] } - rtc_source_set("video_coding_videoprocessor_integration_test") { - testonly = true + video_coding_modules_tests_resources = [ + "../../../resources/foreman_128x96.yuv", + "../../../resources/foreman_160x120.yuv", + "../../../resources/foreman_176x144.yuv", + "../../../resources/foreman_320x240.yuv", + ] - sources = [ - "codecs/test/videoprocessor_integrationtest.cc", - "codecs/test/videoprocessor_integrationtest.h", - ] - - deps = [ - ":video_codecs_test_framework", - ":video_coding", - ":video_coding_utility", - ":webrtc_h264", - ":webrtc_vp8", - ":webrtc_vp9", - "../..:webrtc_common", - "../../media:rtc_media", - "../../rtc_base:rtc_base_approved", - "../../rtc_base:rtc_task_queue", - "../../system_wrappers:system_wrappers", - "../../test:test_support", - "../../test:video_test_common", - "../../test:video_test_support", - ] - - if (is_android) { - sources += [ - "codecs/test/android_test_initializer.cc", - "codecs/test/android_test_initializer.h", + if (is_ios || is_mac) { + bundle_data("video_coding_modules_tests_resources_bundle_data") { + testonly = true + sources = video_coding_modules_tests_resources + outputs = [ + "{{bundle_resources_dir}}/{{source_file_part}}", ] - - deps += [ - "../../rtc_base:rtc_base_approved", - "../../sdk/android:libjingle_peerconnection_jni", - "//base", - ] - } - - if (is_ios || is_mac) { - deps += [ - ":codec_test_objc", - "../../media:rtc_media_base", - "../../sdk:videotoolbox_objc", - ] - } - - if (!build_with_chromium && is_clang) { - # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). - suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] } } @@ -429,105 +394,63 @@ if (rtc_include_tests) { } sources = [ "codecs/h264/test/h264_impl_unittest.cc", + "codecs/test/videoprocessor_integrationtest.cc", + "codecs/test/videoprocessor_integrationtest.h", "codecs/test/videoprocessor_integrationtest_libvpx.cc", "codecs/test/videoprocessor_integrationtest_openh264.cc", + "codecs/test/videoprocessor_integrationtest_parameterized.cc", "codecs/vp8/test/vp8_impl_unittest.cc", "codecs/vp9/test/vp9_impl_unittest.cc", ] deps = [ ":video_codecs_test_framework", - ":video_coding_videoprocessor_integration_test", + ":video_coding", + ":video_coding_utility", ":webrtc_h264", ":webrtc_vp8", ":webrtc_vp9", + "../..:webrtc_common", "../../api:optional", "../../api:video_frame_api", - "../../common_video:common_video", + "../../common_video", + "../../media:rtc_audio_video", "../../rtc_base:rtc_base_approved", + "../../system_wrappers", "../../test:field_trial", "../../test:test_support", "../../test:video_test_common", + "../../test:video_test_support", "../video_capture", ] - if (is_ios || is_mac) { - deps += [ ":codec_test_objc" ] - } - - if (rtc_use_h264) { - defines = [ "WEBRTC_USE_H264" ] - } - - if (!build_with_chromium && is_clang) { - # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). - suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] - } - } - - plot_videoprocessor_integrationtest_resources = [ - "../../../resources/foreman_128x96.yuv", - "../../../resources/foreman_160x120.yuv", - "../../../resources/foreman_176x144.yuv", - "../../../resources/foreman_320x240.yuv", - "../../../resources/foreman_cif.yuv", - ] - - if (is_ios || is_mac) { - bundle_data("plot_videoprocessor_integrationtest_bundle_data") { - testonly = true - sources = plot_videoprocessor_integrationtest_resources - outputs = [ - "{{bundle_resources_dir}}/{{source_file_part}}", - ] - } - } - - # This executable is meant for local codec perf testing and should not be run - # on the trybots/buildbots, hence the existence of this special build target. - rtc_test("plot_videoprocessor_integrationtest") { - testonly = true - - sources = [ - "codecs/test/plot_videoprocessor_integrationtest.cc", - ] - - deps = [ - ":video_coding_videoprocessor_integration_test", - "../../test:test_main", - "../../test:video_test_common", - "../video_capture", - ] - - data = plot_videoprocessor_integrationtest_resources + data = video_coding_modules_tests_resources if (is_android) { - deps += [ - "../../rtc_base:rtc_base_approved", - - # TODO(brandtr): Figure out if the java dep below could be moved into - # :video_coding_videoprocessor_integration_test, where it belongs. - # When that is done, support for Android HW codecs can be added to the - # modules_tests target as well. - "../../sdk/android:libjingle_peerconnection_java", - "../../sdk/android:libjingle_peerconnection_jni", - "//base", - "//testing/android/native_test:native_test_support", + sources += [ + "codecs/test/android_test_initializer.cc", + "codecs/test/android_test_initializer.h", + "codecs/test/videoprocessor_integrationtest_mediacodec.cc", ] - shard_timeout = 900 + deps += [ + "../../sdk/android:libjingle_peerconnection_jni", + "//base", + ] } if (is_ios || is_mac) { deps += [ ":codec_test_objc", - ":plot_videoprocessor_integrationtest_bundle_data", + ":video_coding_modules_tests_resources_bundle_data", + "../../media:rtc_media_base", + "../../sdk:videotoolbox_objc", ] } - # TODO(brandtr): Remove this define when the modules_tests target properly - # loads the Java classes mentioned above. - defines = [ "WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED" ] + if (rtc_use_h264) { + defines = [ "WEBRTC_USE_H264" ] + } if (!build_with_chromium && is_clang) { # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). diff --git a/webrtc/modules/video_coding/codecs/test/plot_webrtc_test_logs.py b/webrtc/modules/video_coding/codecs/test/plot_webrtc_test_logs.py index d598ffb66d..0a8b094075 100755 --- a/webrtc/modules/video_coding/codecs/test/plot_webrtc_test_logs.py +++ b/webrtc/modules/video_coding/codecs/test/plot_webrtc_test_logs.py @@ -18,8 +18,9 @@ import re import matplotlib.pyplot as plt # Log events. -EVENT_START = 'RUN ] CodecSettings/PlotVideoProcessorIntegrationTest.' -EVENT_END = 'OK ] CodecSettings/PlotVideoProcessorIntegrationTest.' +EVENT_START = \ + 'RUN ] CodecSettings/VideoProcessorIntegrationTestParameterized.' +EVENT_END = 'OK ] CodecSettings/VideoProcessorIntegrationTestParameterized.' # Metrics to plot, tuple: (name to parse in file, label to use when plotting). BITRATE = ('Target bitrate', 'target bitrate (kbps)') diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index cbc251896d..5d54caa753 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -71,8 +71,7 @@ int NumberOfTemporalLayers(const VideoCodec& codec_settings) { } // namespace VideoProcessorIntegrationTest::VideoProcessorIntegrationTest() { -#if defined(WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED) && \ - defined(WEBRTC_ANDROID) +#if defined(WEBRTC_ANDROID) InitializeAndroidObjects(); #endif } @@ -276,7 +275,6 @@ void VideoProcessorIntegrationTest::ProcessFramesAndMaybeVerify( void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { std::unique_ptr encoder_factory; if (config_.hw_encoder) { -#if defined(WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED) #if defined(WEBRTC_ANDROID) encoder_factory.reset(new jni::MediaCodecVideoEncoderFactory()); #elif defined(WEBRTC_IOS) @@ -286,13 +284,11 @@ void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { #else RTC_NOTREACHED() << "Only support HW encoder on Android and iOS."; #endif -#endif // WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED } else { encoder_factory.reset(new cricket::InternalEncoderFactory()); } if (config_.hw_decoder) { -#if defined(WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED) #if defined(WEBRTC_ANDROID) decoder_factory_.reset(new jni::MediaCodecVideoDecoderFactory()); #elif defined(WEBRTC_IOS) @@ -302,7 +298,6 @@ void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { #else RTC_NOTREACHED() << "Only support HW decoder on Android and iOS."; #endif -#endif // WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED } else { decoder_factory_.reset(new cricket::InternalDecoderFactory()); } diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc new file mode 100644 index 0000000000..47734d79a5 --- /dev/null +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h" + +#include + +#include "webrtc/test/field_trial.h" +#include "webrtc/test/testsupport/fileutils.h" + +namespace webrtc { +namespace test { + +#if defined(WEBRTC_ANDROID) + +namespace { +const int kForemanNumFrames = 300; +const std::nullptr_t kNoVisualizationParams = nullptr; +} // namespace + +class VideoProcessorIntegrationTestMediaCodec + : public VideoProcessorIntegrationTest { + protected: + VideoProcessorIntegrationTestMediaCodec() { + config_.filename = "foreman_cif"; + config_.input_filename = ResourcePath(config_.filename, "yuv"); + config_.output_filename = + TempFilename(OutputPath(), "videoprocessor_integrationtest_mediacodec"); + config_.verbose = false; + config_.hw_encoder = true; + config_.hw_decoder = true; + } +}; + +TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsVp8) { + SetCodecSettings(&config_, kVideoCodecVP8, 1, false, false, false, false, + false, 352, 288); + + RateProfile rate_profile; + SetRateProfile(&rate_profile, 0, 500, 30, 0); // Start below |low_kbps|. + rate_profile.frame_index_rate_update[1] = kForemanNumFrames + 1; + rate_profile.num_frames = kForemanNumFrames; + + // The thresholds below may have to be tweaked to let even poor MediaCodec + // implementations pass. If this test fails on the bots, disable it and + // ping brandtr@. + std::vector rc_thresholds; + AddRateControlThresholds(5, 95, 20, 10, 10, 0, 1, &rc_thresholds); + + QualityThresholds quality_thresholds(30.0, 15.0, 0.90, 0.40); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); +} + +TEST_F(VideoProcessorIntegrationTestMediaCodec, + Foreman240p100kbpsVp8WithForcedSwFallback) { + ScopedFieldTrials override_field_trials( + "WebRTC-VP8-Forced-Fallback-Encoder/Enabled-150,175,10000,1/"); + + config_.filename = "foreman_320x240"; + config_.input_filename = ResourcePath(config_.filename, "yuv"); + config_.sw_fallback_encoder = true; + SetCodecSettings(&config_, kVideoCodecVP8, 1, false, false, false, false, + false, 320, 240); + + RateProfile rate_profile; + SetRateProfile(&rate_profile, 0, 100, 10, 0); // Start below |low_kbps|. + SetRateProfile(&rate_profile, 1, 100, 10, 80); // Fallback in this bucket. + SetRateProfile(&rate_profile, 2, 200, 10, 200); // Switch back here. + rate_profile.frame_index_rate_update[3] = kForemanNumFrames + 1; + rate_profile.num_frames = kForemanNumFrames; + + // The thresholds below may have to be tweaked to let even poor MediaCodec + // implementations pass. If this test fails on the bots, disable it and + // ping brandtr@. + std::vector rc_thresholds; + AddRateControlThresholds(0, 50, 75, 70, 10, 0, 1, &rc_thresholds); + AddRateControlThresholds(0, 50, 25, 12, 60, 0, 1, &rc_thresholds); + AddRateControlThresholds(0, 65, 15, 5, 5, 0, 1, &rc_thresholds); + + QualityThresholds quality_thresholds(33.0, 30.0, 0.90, 0.85); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); +} + +#endif // defined(WEBRTC_ANDROID) + +} // namespace test +} // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc similarity index 75% rename from webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc rename to webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc index ef6dbde545..1a1555856b 100644 --- a/webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc @@ -18,7 +18,7 @@ namespace test { namespace { // Loop variables. -const int kBitrates[] = {30, 50, 100, 200, 300, 500, 1000}; +const int kBitrates[] = {500}; const VideoCodecType kVideoCodecType[] = {kVideoCodecVP8}; const bool kHwCodec[] = {false}; @@ -37,21 +37,21 @@ const VisualizationParams kVisualizationParams = { false, // save_decoded_y4m }; -const int kNumFrames = 300; +const int kNumFrames = 30; } // namespace // Tests for plotting statistics from logs. -class PlotVideoProcessorIntegrationTest +class VideoProcessorIntegrationTestParameterized : public VideoProcessorIntegrationTest, public ::testing::WithParamInterface< ::testing::tuple> { protected: - PlotVideoProcessorIntegrationTest() + VideoProcessorIntegrationTestParameterized() : bitrate_(::testing::get<0>(GetParam())), codec_type_(::testing::get<1>(GetParam())), hw_codec_(::testing::get<2>(GetParam())) {} - ~PlotVideoProcessorIntegrationTest() override = default; + ~VideoProcessorIntegrationTestParameterized() override = default; void RunTest(int width, int height, @@ -86,30 +86,29 @@ class PlotVideoProcessorIntegrationTest const bool hw_codec_; }; -INSTANTIATE_TEST_CASE_P( - CodecSettings, - PlotVideoProcessorIntegrationTest, - ::testing::Combine(::testing::ValuesIn(kBitrates), - ::testing::ValuesIn(kVideoCodecType), - ::testing::ValuesIn(kHwCodec))); +INSTANTIATE_TEST_CASE_P(CodecSettings, + VideoProcessorIntegrationTestParameterized, + ::testing::Combine(::testing::ValuesIn(kBitrates), + ::testing::ValuesIn(kVideoCodecType), + ::testing::ValuesIn(kHwCodec))); -TEST_P(PlotVideoProcessorIntegrationTest, Process_128x96_30fps) { +TEST_P(VideoProcessorIntegrationTestParameterized, Process_128x96_30fps) { RunTest(128, 96, 30, "foreman_128x96"); } -TEST_P(PlotVideoProcessorIntegrationTest, Process_160x120_30fps) { +TEST_P(VideoProcessorIntegrationTestParameterized, Process_160x120_30fps) { RunTest(160, 120, 30, "foreman_160x120"); } -TEST_P(PlotVideoProcessorIntegrationTest, Process_176x144_30fps) { +TEST_P(VideoProcessorIntegrationTestParameterized, Process_176x144_30fps) { RunTest(176, 144, 30, "foreman_176x144"); } -TEST_P(PlotVideoProcessorIntegrationTest, Process_320x240_30fps) { +TEST_P(VideoProcessorIntegrationTestParameterized, Process_320x240_30fps) { RunTest(320, 240, 30, "foreman_320x240"); } -TEST_P(PlotVideoProcessorIntegrationTest, Process_352x288_30fps) { +TEST_P(VideoProcessorIntegrationTestParameterized, Process_352x288_30fps) { RunTest(352, 288, 30, "foreman_cif"); }