From 9d96e923169dc237b056f3e2b432a6759155bf2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A1ri=20Tristan=20Helgason?= Date: Fri, 4 May 2018 11:56:55 +0200 Subject: [PATCH] Rewrite videoprocessor integrationtest to use public fixture. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL creates a test fixture for the videoprocessor integration tests and exposes it as part of the public API. It also rewrites the current versions of the tests to build on this new paradigm. The motivation for this is to easily allow projects that build on top of webrtc to add integration-level tests for their own custom codec implementations in a way that does not link them too tightly to the internal implementations of said tests. Bug: None Change-Id: I7cf9f29322a6934b3cfc32da02ea7dfa5858c2b2 Reviewed-on: https://webrtc-review.googlesource.com/72481 Commit-Queue: Kári Helgason Reviewed-by: Rasmus Brandt Reviewed-by: Fredrik Solenberg Reviewed-by: Patrik Höglund Cr-Commit-Position: refs/heads/master@{#23118} --- api/BUILD.gn | 32 ++ api/test/DEPS | 6 + api/test/create_videocodec_test_fixture.cc | 38 ++ api/test/create_videocodec_test_fixture.h | 36 ++ api/test/videocodec_test_fixture.h | 74 +++ modules/video_coding/BUILD.gn | 80 ++- ...est.cc => videocodec_test_fixture_impl.cc} | 69 ++- ...ntest.h => videocodec_test_fixture_impl.h} | 87 +-- .../codecs/test/videocodec_test_libvpx.cc | 494 ++++++++++++++++++ ...codec.cc => videocodec_test_mediacodec.cc} | 98 ++-- .../codecs/test/videocodec_test_openh264.cc | 93 ++++ ...ed.cc => videocodec_test_parameterized.cc} | 38 +- .../test/videocodec_test_videotoolbox.cc | 93 ++++ .../videoprocessor_integrationtest_libvpx.cc | 444 ---------------- ...videoprocessor_integrationtest_openh264.cc | 85 --- ...oprocessor_integrationtest_videotoolbox.cc | 97 ---- 16 files changed, 1062 insertions(+), 802 deletions(-) create mode 100644 api/test/DEPS create mode 100644 api/test/create_videocodec_test_fixture.cc create mode 100644 api/test/create_videocodec_test_fixture.h create mode 100644 api/test/videocodec_test_fixture.h rename modules/video_coding/codecs/test/{videoprocessor_integrationtest.cc => videocodec_test_fixture_impl.cc} (89%) rename modules/video_coding/codecs/test/{videoprocessor_integrationtest.h => videocodec_test_fixture_impl.h} (62%) create mode 100644 modules/video_coding/codecs/test/videocodec_test_libvpx.cc rename modules/video_coding/codecs/test/{videoprocessor_integrationtest_mediacodec.cc => videocodec_test_mediacodec.cc} (52%) create mode 100644 modules/video_coding/codecs/test/videocodec_test_openh264.cc rename modules/video_coding/codecs/test/{videoprocessor_integrationtest_parameterized.cc => videocodec_test_parameterized.cc} (76%) create mode 100644 modules/video_coding/codecs/test/videocodec_test_videotoolbox.cc delete mode 100644 modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc delete mode 100644 modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc delete mode 100644 modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc diff --git a/api/BUILD.gn b/api/BUILD.gn index bdff109179..d5da1821bb 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -375,6 +375,38 @@ if (rtc_include_tests) { } } + rtc_source_set("videocodec_test_fixture_api") { + visibility = [ "*" ] + testonly = true + sources = [ + "test/videocodec_test_fixture.h", + ] + deps = [ + "../modules/video_coding:video_codecs_test_framework", + "video_codecs:video_codecs_api", + ] + } + + rtc_source_set("create_videocodec_test_fixture_api") { + visibility = [ "*" ] + testonly = true + sources = [ + "test/create_videocodec_test_fixture.cc", + "test/create_videocodec_test_fixture.h", + ] + deps = [ + ":videocodec_test_fixture_api", + "../modules/video_coding:video_codecs_test_framework", + "../modules/video_coding:videocodec_test_impl", + "../rtc_base:rtc_base_approved", + "video_codecs:video_codecs_api", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + rtc_source_set("mock_audio_mixer") { testonly = true sources = [ diff --git a/api/test/DEPS b/api/test/DEPS new file mode 100644 index 0000000000..cd91a9ff38 --- /dev/null +++ b/api/test/DEPS @@ -0,0 +1,6 @@ +# TODO(kthelgason): Move the relevant included files to api/test. +specific_include_rules = { + ".*": [ + "+modules/video_coding/codecs/test", + ], +} diff --git a/api/test/create_videocodec_test_fixture.cc b/api/test/create_videocodec_test_fixture.cc new file mode 100644 index 0000000000..9a0aacc7c2 --- /dev/null +++ b/api/test/create_videocodec_test_fixture.cc @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_videocodec_test_fixture.h" + +#include +#include + +#include "api/test/videocodec_test_fixture.h" +#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h" +#include "rtc_base/ptr_util.h" + +namespace webrtc { +namespace test { + +std::unique_ptr +CreateVideoCodecTestFixture(const TestConfig& config) { + return rtc::MakeUnique(config); +} + +std::unique_ptr +CreateVideoCodecTestFixture( + const TestConfig& config, + std::unique_ptr decoder_factory, + std::unique_ptr encoder_factory) { + return rtc::MakeUnique( + config, std::move(decoder_factory), std::move(encoder_factory)); +} + +} // namespace test +} // namespace webrtc diff --git a/api/test/create_videocodec_test_fixture.h b/api/test/create_videocodec_test_fixture.h new file mode 100644 index 0000000000..24e17fd8d2 --- /dev/null +++ b/api/test/create_videocodec_test_fixture.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_CREATE_VIDEOCODEC_TEST_FIXTURE_H_ +#define API_TEST_CREATE_VIDEOCODEC_TEST_FIXTURE_H_ + +#include + +#include "api/test/videocodec_test_fixture.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/video_coding/codecs/test/test_config.h" + +namespace webrtc { +namespace test { + +std::unique_ptr +CreateVideoCodecTestFixture(const TestConfig& config); + +std::unique_ptr +CreateVideoCodecTestFixture( + const TestConfig& config, + std::unique_ptr decoder_factory, + std::unique_ptr encoder_factory); + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_CREATE_VIDEOCODEC_TEST_FIXTURE_H_ diff --git a/api/test/videocodec_test_fixture.h b/api/test/videocodec_test_fixture.h new file mode 100644 index 0000000000..88da7df317 --- /dev/null +++ b/api/test/videocodec_test_fixture.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEOCODEC_TEST_FIXTURE_H_ +#define API_TEST_VIDEOCODEC_TEST_FIXTURE_H_ + +#include + +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/video_coding/codecs/test/stats.h" + +namespace webrtc { +namespace test { + +// Rates for the encoder and the frame number when to change profile. +struct RateProfile { + size_t target_kbps; + size_t input_fps; + size_t frame_index_rate_update; +}; + +struct RateControlThresholds { + double max_avg_bitrate_mismatch_percent; + double max_time_to_reach_target_bitrate_sec; + // TODO(ssilkin): Use absolute threshold for framerate. + double max_avg_framerate_mismatch_percent; + double max_avg_buffer_level_sec; + double max_max_key_frame_delay_sec; + double max_max_delta_frame_delay_sec; + size_t max_num_spatial_resizes; + size_t max_num_key_frames; +}; + +struct QualityThresholds { + double min_avg_psnr; + double min_min_psnr; + double min_avg_ssim; + double min_min_ssim; +}; + +struct BitstreamThresholds { + size_t max_max_nalu_size_bytes; +}; + +// Should video files be saved persistently to disk for post-run visualization? +struct VisualizationParams { + bool save_encoded_ivf; + bool save_decoded_y4m; +}; + +class VideoCodecTestFixture { + public: + virtual ~VideoCodecTestFixture() = default; + + virtual void RunTest(const std::vector& rate_profiles, + const std::vector* rc_thresholds, + const std::vector* quality_thresholds, + const BitstreamThresholds* bs_thresholds, + const VisualizationParams* visualization_params) = 0; + virtual Stats GetStats() = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEOCODEC_TEST_FIXTURE_H_ diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 77b4fb96aa..bd06f3fda3 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -670,43 +670,25 @@ if (rtc_include_tests) { } } - rtc_source_set("video_coding_modules_tests") { + rtc_source_set("videocodec_test_impl") { testonly = true - sources = [ - "codecs/h264/test/h264_impl_unittest.cc", - "codecs/multiplex/test/multiplex_adapter_unittest.cc", - "codecs/test/video_encoder_decoder_instantiation_tests.cc", - "codecs/test/videoprocessor_integrationtest.cc", - "codecs/test/videoprocessor_integrationtest.h", - "codecs/test/videoprocessor_integrationtest_libvpx.cc", - "codecs/test/videoprocessor_integrationtest_openh264.cc", - "codecs/test/videoprocessor_integrationtest_parameterized.cc", - "codecs/vp8/test/vp8_impl_unittest.cc", - "codecs/vp9/test/vp9_impl_unittest.cc", + "codecs/test/videocodec_test_fixture_impl.cc", + "codecs/test/videocodec_test_fixture_impl.h", ] - deps = [ ":video_codec_interface", ":video_codecs_test_framework", ":video_coding", ":video_coding_utility", - ":webrtc_h264", ":webrtc_multiplex", ":webrtc_vp8_helpers", - ":webrtc_vp9", ":webrtc_vp9_helpers", "../..:webrtc_common", - "../../api:mock_video_codec_factory", - "../../api:optional", - "../../api:video_frame_api", - "../../api:video_frame_api_i420", + "../../api:videocodec_test_fixture_api", "../../api/video_codecs:video_codecs_api", "../../common_video", - "../../media:rtc_audio_video", - "../../media:rtc_h264_profile_id", "../../media:rtc_internal_video_codecs", - "../../media:rtc_media_base", "../../media:rtc_software_fallback_wrappers", "../../rtc_base:checks", "../../rtc_base:rtc_base", @@ -714,22 +696,66 @@ if (rtc_include_tests) { "../../system_wrappers", "../../test:fileutils", "../../test:test_support", - "../../test:video_test_common", "../../test:video_test_support", - "../video_capture", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + if (is_android) { + deps += [ ":android_codec_factory_helper" ] + } + } + + rtc_source_set("video_coding_modules_tests") { + testonly = true + + sources = [ + "codecs/h264/test/h264_impl_unittest.cc", + "codecs/multiplex/test/multiplex_adapter_unittest.cc", + "codecs/test/video_encoder_decoder_instantiation_tests.cc", + "codecs/test/videocodec_test_libvpx.cc", + "codecs/test/videocodec_test_openh264.cc", + "codecs/test/videocodec_test_parameterized.cc", + "codecs/vp8/test/vp8_impl_unittest.cc", + "codecs/vp9/test/vp9_impl_unittest.cc", + ] + + deps = [ + ":video_codecs_test_framework", + ":video_coding_utility", + ":videocodec_test_impl", + ":webrtc_h264", + ":webrtc_multiplex", + ":webrtc_vp8_helpers", + ":webrtc_vp9", + ":webrtc_vp9_helpers", + "../..:webrtc_common", + "../../api:create_videocodec_test_fixture_api", + "../../api:mock_video_codec_factory", + "../../api:optional", + "../../api:video_frame_api_i420", + "../../api:videocodec_test_fixture_api", + "../../api/video_codecs:video_codecs_api", + "../../common_video", + "../../media:rtc_h264_profile_id", + "../../media:rtc_media_base", + "../../rtc_base:rtc_base", + "../../test:fileutils", + "../../test:test_support", + "../../test:video_test_common", ] data = video_coding_modules_tests_resources if (is_android) { - sources += [ "codecs/test/videoprocessor_integrationtest_mediacodec.cc" ] + sources += [ "codecs/test/videocodec_test_mediacodec.cc" ] deps += [ ":android_codec_factory_helper" ] } if (is_ios || is_mac) { - sources += - [ "codecs/test/videoprocessor_integrationtest_videotoolbox.cc" ] + sources += [ "codecs/test/videocodec_test_videotoolbox.cc" ] deps += [ ":objc_codec_factory_helper" ] diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc similarity index 89% rename from modules/video_coding/codecs/test/videoprocessor_integrationtest.cc rename to modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc index 3079ed30f1..f669d902a3 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h" +#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h" #include #include @@ -34,6 +34,7 @@ #include "rtc_base/file.h" #include "rtc_base/ptr_util.h" #include "system_wrappers/include/sleep.h" +#include "test/gtest.h" #include "test/testsupport/fileutils.h" namespace webrtc { @@ -55,9 +56,11 @@ bool RunEncodeInRealTime(const TestConfig& config) { } // namespace -void VideoProcessorIntegrationTest::H264KeyframeChecker::CheckEncodedFrame( - webrtc::VideoCodecType codec, - const EncodedImage& encoded_frame) const { +// TODO(kthelgason): Move this out of the test fixture impl and +// make available as a shared utility class. +void VideoCodecTestFixtureImpl::H264KeyframeChecker:: + CheckEncodedFrame(webrtc::VideoCodecType codec, + const EncodedImage& encoded_frame) const { EXPECT_EQ(kVideoCodecH264, codec); bool contains_sps = false; bool contains_pps = false; @@ -89,7 +92,7 @@ void VideoProcessorIntegrationTest::H264KeyframeChecker::CheckEncodedFrame( } } -class VideoProcessorIntegrationTest::CpuProcessTime final { +class VideoCodecTestFixtureImpl::CpuProcessTime final { public: explicit CpuProcessTime(const TestConfig& config) : config_(config) {} ~CpuProcessTime() {} @@ -124,16 +127,32 @@ class VideoProcessorIntegrationTest::CpuProcessTime final { int64_t wallclock_time_ = 0; }; -VideoProcessorIntegrationTest::VideoProcessorIntegrationTest() { +VideoCodecTestFixtureImpl:: + VideoCodecTestFixtureImpl(TestConfig config) + : config_(config) { #if defined(WEBRTC_ANDROID) InitializeAndroidObjects(); #endif } -VideoProcessorIntegrationTest::~VideoProcessorIntegrationTest() = default; +VideoCodecTestFixtureImpl:: + VideoCodecTestFixtureImpl( + TestConfig config, + std::unique_ptr decoder_factory, + std::unique_ptr encoder_factory) + : decoder_factory_(std::move(decoder_factory)), + encoder_factory_(std::move(encoder_factory)), + config_(config) { +#if defined(WEBRTC_ANDROID) + InitializeAndroidObjects(); +#endif +} + +VideoCodecTestFixtureImpl:: + ~VideoCodecTestFixtureImpl() = default; // Processes all frames in the clip and verifies the result. -void VideoProcessorIntegrationTest::ProcessFramesAndMaybeVerify( +void VideoCodecTestFixtureImpl::RunTest( const std::vector& rate_profiles, const std::vector* rc_thresholds, const std::vector* quality_thresholds, @@ -156,7 +175,7 @@ void VideoProcessorIntegrationTest::ProcessFramesAndMaybeVerify( bs_thresholds); } -void VideoProcessorIntegrationTest::ProcessAllFrames( +void VideoCodecTestFixtureImpl::ProcessAllFrames( rtc::TaskQueue* task_queue, const std::vector& rate_profiles) { // Process all frames. @@ -206,7 +225,7 @@ void VideoProcessorIntegrationTest::ProcessAllFrames( cpu_process_time_->Stop(); } -void VideoProcessorIntegrationTest::AnalyzeAllFrames( +void VideoCodecTestFixtureImpl::AnalyzeAllFrames( const std::vector& rate_profiles, const std::vector* rc_thresholds, const std::vector* quality_thresholds, @@ -252,7 +271,7 @@ void VideoProcessorIntegrationTest::AnalyzeAllFrames( printf("\n"); } -void VideoProcessorIntegrationTest::VerifyVideoStatistic( +void VideoCodecTestFixtureImpl::VerifyVideoStatistic( const VideoStatistics& video_stat, const RateControlThresholds* rc_thresholds, const QualityThresholds* quality_thresholds, @@ -297,7 +316,7 @@ void VideoProcessorIntegrationTest::VerifyVideoStatistic( } std::unique_ptr -VideoProcessorIntegrationTest::CreateDecoderFactory() { +VideoCodecTestFixtureImpl::CreateDecoderFactory() { if (config_.hw_decoder) { #if defined(WEBRTC_ANDROID) return CreateAndroidDecoderFactory(); @@ -311,7 +330,7 @@ VideoProcessorIntegrationTest::CreateDecoderFactory() { } std::unique_ptr -VideoProcessorIntegrationTest::CreateEncoderFactory() { +VideoCodecTestFixtureImpl::CreateEncoderFactory() { if (config_.hw_encoder) { #if defined(WEBRTC_ANDROID) return CreateAndroidEncoderFactory(); @@ -324,11 +343,12 @@ VideoProcessorIntegrationTest::CreateEncoderFactory() { } } -void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { - encoder_factory_ = CreateEncoderFactory(); - std::unique_ptr decoder_factory = CreateDecoderFactory(); - +void VideoCodecTestFixtureImpl::CreateEncoderAndDecoder() { const SdpVideoFormat format = config_.ToSdpVideoFormat(); + if (!decoder_factory_) + decoder_factory_ = CreateDecoderFactory(); + if (!encoder_factory_) + encoder_factory_ = CreateEncoderFactory(); if (config_.simulcast_adapted_encoder) { EXPECT_EQ("VP8", format.name); encoder_.reset(new SimulcastEncoderAdapter(encoder_factory_.get())); @@ -341,7 +361,7 @@ void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { for (size_t i = 0; i < num_simulcast_or_spatial_layers; ++i) { decoders_.push_back(std::unique_ptr( - decoder_factory->CreateVideoDecoder(format))); + decoder_factory_->CreateVideoDecoder(format))); } if (config_.sw_fallback_encoder) { @@ -367,13 +387,16 @@ void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { } } -void VideoProcessorIntegrationTest::DestroyEncoderAndDecoder() { +void VideoCodecTestFixtureImpl::DestroyEncoderAndDecoder() { decoders_.clear(); encoder_.reset(); - encoder_factory_.reset(); } -void VideoProcessorIntegrationTest::SetUpAndInitObjects( +Stats VideoCodecTestFixtureImpl::GetStats() { + return stats_; +} + +void VideoCodecTestFixtureImpl::SetUpAndInitObjects( rtc::TaskQueue* task_queue, int initial_bitrate_kbps, int initial_framerate_fps, @@ -437,7 +460,7 @@ void VideoProcessorIntegrationTest::SetUpAndInitObjects( sync_event.Wait(rtc::Event::kForever); } -void VideoProcessorIntegrationTest::ReleaseAndCloseObjects( +void VideoCodecTestFixtureImpl::ReleaseAndCloseObjects( rtc::TaskQueue* task_queue) { rtc::Event sync_event(false, false); task_queue->PostTask([this, &sync_event]() { @@ -462,7 +485,7 @@ void VideoProcessorIntegrationTest::ReleaseAndCloseObjects( decoded_frame_writers_.clear(); } -void VideoProcessorIntegrationTest::PrintSettings( +void VideoCodecTestFixtureImpl::PrintSettings( rtc::TaskQueue* task_queue) const { printf("==> TestConfig\n"); printf("%s\n", config_.ToString().c_str()); diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.h b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.h similarity index 62% rename from modules/video_coding/codecs/test/videoprocessor_integrationtest.h rename to modules/video_coding/codecs/test/videocodec_test_fixture_impl.h index 32a9160773..7ad7171013 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.h +++ b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.h @@ -8,15 +8,14 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_ -#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_ +#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEOCODEC_TEST_FIXTURE_IMPL_H_ +#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEOCODEC_TEST_FIXTURE_IMPL_H_ -#include -#include #include #include #include +#include "api/test/videocodec_test_fixture.h" #include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_encoder_factory.h" #include "common_types.h" // NOLINT(build/include) @@ -25,84 +24,39 @@ #include "modules/video_coding/codecs/test/test_config.h" #include "modules/video_coding/codecs/test/videoprocessor.h" #include "modules/video_coding/utility/ivf_file_writer.h" -#include "test/gtest.h" #include "test/testsupport/frame_reader.h" #include "test/testsupport/frame_writer.h" namespace webrtc { namespace test { -// Rates for the encoder and the frame number when to change profile. -struct RateProfile { - size_t target_kbps; - size_t input_fps; - size_t frame_index_rate_update; -}; - -struct RateControlThresholds { - double max_avg_bitrate_mismatch_percent; - double max_time_to_reach_target_bitrate_sec; - // TODO(ssilkin): Use absolute threshold for framerate. - double max_avg_framerate_mismatch_percent; - double max_avg_buffer_level_sec; - double max_max_key_frame_delay_sec; - double max_max_delta_frame_delay_sec; - size_t max_num_spatial_resizes; - size_t max_num_key_frames; -}; - -struct QualityThresholds { - double min_avg_psnr; - double min_min_psnr; - double min_avg_ssim; - double min_min_ssim; -}; - -struct BitstreamThresholds { - size_t max_max_nalu_size_bytes; -}; - -// Should video files be saved persistently to disk for post-run visualization? -struct VisualizationParams { - bool save_encoded_ivf; - bool save_decoded_y4m; -}; - // Integration test for video processor. It does rate control and frame quality // analysis using frame statistics collected by video processor and logs the // results. If thresholds are specified it checks that corresponding metrics // are in desirable range. -class VideoProcessorIntegrationTest : public testing::Test { - protected: +class VideoCodecTestFixtureImpl : public VideoCodecTestFixture { // Verifies that all H.264 keyframes contain SPS/PPS/IDR NALUs. + public: class H264KeyframeChecker : public TestConfig::EncodedFrameChecker { public: void CheckEncodedFrame(webrtc::VideoCodecType codec, const EncodedImage& encoded_frame) const override; }; - VideoProcessorIntegrationTest(); - ~VideoProcessorIntegrationTest() override; + explicit VideoCodecTestFixtureImpl(TestConfig config); + VideoCodecTestFixtureImpl( + TestConfig config, + std::unique_ptr decoder_factory, + std::unique_ptr encoder_factory); + ~VideoCodecTestFixtureImpl() override; - void ProcessFramesAndMaybeVerify( - const std::vector& rate_profiles, - const std::vector* rc_thresholds, - const std::vector* quality_thresholds, - const BitstreamThresholds* bs_thresholds, - const VisualizationParams* visualization_params); + void RunTest(const std::vector& rate_profiles, + const std::vector* rc_thresholds, + const std::vector* quality_thresholds, + const BitstreamThresholds* bs_thresholds, + const VisualizationParams* visualization_params) override; - // Config. - TestConfig config_; - - Stats stats_; - - // Can be used by all H.264 tests. - const H264KeyframeChecker h264_keyframe_checker_; - - protected: - // Overwrite in subclasses for custom codec factories. - virtual std::unique_ptr CreateDecoderFactory(); - virtual std::unique_ptr CreateEncoderFactory(); + Stats GetStats() override; private: class CpuProcessTime; @@ -131,13 +85,18 @@ class VideoProcessorIntegrationTest : public testing::Test { float input_framerate_fps); void PrintSettings(rtc::TaskQueue* task_queue) const; + std::unique_ptr CreateDecoderFactory(); + std::unique_ptr CreateEncoderFactory(); // Codecs. + std::unique_ptr decoder_factory_; std::unique_ptr encoder_factory_; std::unique_ptr encoder_; VideoProcessor::VideoDecoderList decoders_; // Helper objects. + TestConfig config_; + Stats stats_; std::unique_ptr source_frame_reader_; VideoProcessor::IvfFileWriterList encoded_frame_writers_; VideoProcessor::FrameWriterList decoded_frame_writers_; @@ -148,4 +107,4 @@ class VideoProcessorIntegrationTest : public testing::Test { } // namespace test } // namespace webrtc -#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_ +#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEOCODEC_TEST_FIXTURE_IMPL_H_ diff --git a/modules/video_coding/codecs/test/videocodec_test_libvpx.cc b/modules/video_coding/codecs/test/videocodec_test_libvpx.cc new file mode 100644 index 0000000000..3e7332ae01 --- /dev/null +++ b/modules/video_coding/codecs/test/videocodec_test_libvpx.cc @@ -0,0 +1,494 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/test/create_videocodec_test_fixture.h" +#include "media/base/mediaconstants.h" +#include "modules/video_coding/codecs/test/test_config.h" +#include "modules/video_coding/utility/vp8_header_parser.h" +#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" +#include "rtc_base/ptr_util.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace test { + +namespace { +// Codec settings. +const int kCifWidth = 352; +const int kCifHeight = 288; +#if !defined(WEBRTC_IOS) +const int kNumFramesShort = 100; +#endif +const int kNumFramesLong = 300; +const size_t kBitrateRdPerfKbps[] = {100, 200, 300, 400, 500, 600, + 700, 800, 1000, 1250, 1400, 1600, + 1800, 2000, 2200, 2500}; +const size_t kNumFirstFramesToSkipAtRdPerfAnalysis = 60; + +class QpFrameChecker : public TestConfig::EncodedFrameChecker { + public: + void CheckEncodedFrame(webrtc::VideoCodecType codec, + const EncodedImage& encoded_frame) const override { + int qp; + if (codec == kVideoCodecVP8) { + EXPECT_TRUE( + vp8::GetQp(encoded_frame._buffer, encoded_frame._length, &qp)); + } else if (codec == kVideoCodecVP9) { + EXPECT_TRUE( + vp9::GetQp(encoded_frame._buffer, encoded_frame._length, &qp)); + } else { + RTC_NOTREACHED(); + } + EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP."; + } +}; + +TestConfig CreateTestConfig() { + TestConfig config; + config.filename = "foreman_cif"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = kNumFramesLong; + config.use_single_core = true; + config.hw_encoder = false; + config.hw_decoder = false; + return config; +} + +void PrintRdPerf(std::map> rd_stats) { + printf("--> Summary\n"); + printf("%11s %5s %6s %11s %12s %11s %13s %13s %5s %7s %7s %7s %13s %13s\n", + "uplink_kbps", "width", "height", "spatial_idx", "temporal_idx", + "target_kbps", "downlink_kbps", "framerate_fps", "psnr", "psnr_y", + "psnr_u", "psnr_v", "enc_speed_fps", "dec_speed_fps"); + for (const auto& rd_stat : rd_stats) { + const size_t bitrate_kbps = rd_stat.first; + for (const auto& layer_stat : rd_stat.second) { + printf( + "%11zu %5zu %6zu %11zu %12zu %11zu %13zu %13.2f %5.2f %7.2f %7.2f " + "%7.2f" + "%13.2f %13.2f\n", + bitrate_kbps, layer_stat.width, layer_stat.height, + layer_stat.spatial_idx, layer_stat.temporal_idx, + layer_stat.target_bitrate_kbps, layer_stat.bitrate_kbps, + layer_stat.framerate_fps, layer_stat.avg_psnr, layer_stat.avg_psnr_y, + layer_stat.avg_psnr_u, layer_stat.avg_psnr_v, + layer_stat.enc_speed_fps, layer_stat.dec_speed_fps); + } + } +} +} // namespace + +// Fails on iOS. See webrtc:4755. +#if !defined(WEBRTC_IOS) + +#if !defined(RTC_DISABLE_VP9) +TEST(VideoProcessorIntegrationTestLibvpx, HighBitrateVP9) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false, + kCifWidth, kCifHeight); + config.num_frames = kNumFramesShort; + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{500, 30, kNumFramesShort}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.11, 0.3, 0.1, 0, 1}}; + + std::vector quality_thresholds = {{37, 36, 0.94, 0.92}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +TEST(VideoProcessorIntegrationTestLibvpx, ChangeBitrateVP9) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false, + kCifWidth, kCifHeight); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = { + {200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update + {700, 30, 200}, + {500, 30, kNumFramesLong}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.15, 0.5, 0.1, 0, 1}, + {15, 2, 0, 0.2, 0.5, 0.1, 0, 0}, + {10, 1, 0, 0.3, 0.5, 0.1, 0, 0}}; + + std::vector quality_thresholds = { + {34, 33, 0.90, 0.88}, {38, 35, 0.95, 0.91}, {35, 34, 0.93, 0.90}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +TEST(VideoProcessorIntegrationTestLibvpx, ChangeFramerateVP9) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false, + kCifWidth, kCifHeight); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = { + {100, 24, 100}, // target_kbps, input_fps, frame_index_rate_update + {100, 15, 200}, + {100, 10, kNumFramesLong}}; + + // Framerate mismatch should be lower for lower framerate. + std::vector rc_thresholds = { + {10, 2, 40, 0.4, 0.5, 0.2, 0, 1}, + {8, 2, 5, 0.2, 0.5, 0.2, 0, 0}, + {5, 2, 0, 0.21, 0.5, 0.3, 0, 0}}; + + // Quality should be higher for lower framerates for the same content. + std::vector quality_thresholds = { + {33, 32, 0.89, 0.87}, {33.5, 32, 0.90, 0.86}, {33.5, 31.5, 0.90, 0.85}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +TEST(VideoProcessorIntegrationTestLibvpx, DenoiserOnVP9) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, true, true, false, + kCifWidth, kCifHeight); + config.num_frames = kNumFramesShort; + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{500, 30, kNumFramesShort}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.11, 0.3, 0.1, 0, 1}}; + + std::vector quality_thresholds = {{37.5, 36, 0.94, 0.93}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +TEST(VideoProcessorIntegrationTestLibvpx, VeryLowBitrateVP9) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, true, + kCifWidth, kCifHeight); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{50, 30, kNumFramesLong}}; + + std::vector rc_thresholds = { + {15, 3, 75, 1.0, 0.5, 0.4, 1, 1}}; + + std::vector quality_thresholds = {{28, 25, 0.80, 0.65}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// TODO(marpan): Add temporal layer test for VP9, once changes are in +// vp9 wrapper for this. + +#endif // !defined(RTC_DISABLE_VP9) + +TEST(VideoProcessorIntegrationTestLibvpx, HighBitrateVP8) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false, + kCifWidth, kCifHeight); + config.num_frames = kNumFramesShort; + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{500, 30, kNumFramesShort}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; + + // std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; + // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse + // than quality of x86 version. Use lower thresholds for now. + std::vector quality_thresholds = {{35, 33, 0.91, 0.89}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +#endif // !defined(WEBRTC_IOS) + +// The tests below are currently disabled for Android. For ARM, the encoder +// uses |cpu_speed| = 12, as opposed to default |cpu_speed| <= 6 for x86, +// which leads to significantly different quality. The quality and rate control +// settings in the tests below are defined for encoder speed setting +// |cpu_speed| <= ~6. A number of settings would need to be significantly +// modified for the |cpu_speed| = 12 case. For now, keep the tests below +// disabled on Android. Some quality parameter in the above test has been +// adjusted to also pass for |cpu_speed| <= 12. + +// Too slow to finish before timeout on iOS. See webrtc:4755. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_ChangeBitrateVP8 DISABLED_ChangeBitrateVP8 +#else +#define MAYBE_ChangeBitrateVP8 ChangeBitrateVP8 +#endif +TEST(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeBitrateVP8) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false, + kCifWidth, kCifHeight); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = { + {200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update + {800, 30, 200}, + {500, 30, kNumFramesLong}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, + {15.5, 1, 0, 0.1, 0.2, 0.1, 0, 0}, + {15, 1, 0, 0.3, 0.2, 0.1, 0, 0}}; + + // std::vector quality_thresholds = { + // {33, 32, 0.89, 0.88}, {38, 36, 0.94, 0.93}, {35, 34, 0.92, 0.91}}; + // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse + // than quality of x86 version. Use lower thresholds for now. + std::vector quality_thresholds = { + {31.8, 31, 0.86, 0.85}, {36, 34.8, 0.92, 0.90}, {33.5, 32, 0.90, 0.88}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// Too slow to finish before timeout on iOS. See webrtc:4755. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_ChangeFramerateVP8 DISABLED_ChangeFramerateVP8 +#else +#define MAYBE_ChangeFramerateVP8 ChangeFramerateVP8 +#endif +TEST(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeFramerateVP8) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false, + kCifWidth, kCifHeight); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = { + {80, 24, 100}, // target_kbps, input_fps, frame_index_rate_update + {80, 15, 200}, + {80, 10, kNumFramesLong}}; + + // std::vector rc_thresholds = { + // {10, 2, 20, 0.4, 0.3, 0.1, 0, 1}, + // {5, 2, 5, 0.3, 0.3, 0.1, 0, 0}, + // {4, 2, 1, 0.2, 0.3, 0.2, 0, 0}}; + // TODO(webrtc:8757): ARM VP8 drops more frames than x86 version. Use lower + // thresholds for now. + std::vector rc_thresholds = { + {10, 2, 60, 0.5, 0.3, 0.3, 0, 1}, + {10, 2, 30, 0.3, 0.3, 0.3, 0, 0}, + {10, 2, 10, 0.2, 0.3, 0.2, 0, 0}}; + + // std::vector quality_thresholds = { + // {31, 30, 0.87, 0.86}, {32, 31, 0.89, 0.86}, {32, 30, 0.87, 0.82}}; + // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse + // than quality of x86 version. Use lower thresholds for now. + std::vector quality_thresholds = { + {31, 30, 0.85, 0.84}, {31.5, 30.5, 0.86, 0.84}, {30.5, 29, 0.83, 0.78}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// Too slow to finish before timeout on iOS. See webrtc:4755. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_TemporalLayersVP8 DISABLED_TemporalLayersVP8 +#else +#define MAYBE_TemporalLayersVP8 TemporalLayersVP8 +#endif +TEST(VideoProcessorIntegrationTestLibvpx, MAYBE_TemporalLayersVP8) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 3, true, true, false, + kCifWidth, kCifHeight); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{200, 30, 150}, + {400, 30, kNumFramesLong}}; + + // std::vector rc_thresholds = { + // {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, {10, 2, 0, 0.1, 0.2, 0.1, 0, 1}}; + // TODO(webrtc:8757): ARM VP8 drops more frames than x86 version. Use lower + // thresholds for now. + std::vector rc_thresholds = { + {10, 1, 2, 0.3, 0.2, 0.1, 0, 1}, {12, 2, 3, 0.1, 0.2, 0.1, 0, 1}}; + + // Min SSIM drops because of high motion scene with complex backgound (trees). + // std::vector quality_thresholds = {{32, 30, 0.88, 0.85}, + // {33, 30, 0.89, 0.83}}; + // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse + // than quality of x86 version. Use lower thresholds for now. + std::vector quality_thresholds = {{31, 30, 0.85, 0.84}, + {31, 28, 0.85, 0.75}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// Might be too slow on mobile platforms. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_MultiresVP8 DISABLED_MultiresVP8 +#else +#define MAYBE_MultiresVP8 MultiresVP8 +#endif +TEST(VideoProcessorIntegrationTestLibvpx, MAYBE_MultiresVP8) { + auto config = CreateTestConfig(); + config.filename = "ConferenceMotion_1280_720_50"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = 100; + config.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false, + 1280, 720); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{1500, 30, config.num_frames}}; + + std::vector rc_thresholds = { + {5, 1, 5, 0.2, 0.3, 0.1, 0, 1}}; + std::vector quality_thresholds = {{34, 32, 0.90, 0.88}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// Might be too slow on mobile platforms. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_SimulcastVP8 DISABLED_SimulcastVP8 +#else +#define MAYBE_SimulcastVP8 SimulcastVP8 +#endif +TEST(VideoProcessorIntegrationTestLibvpx, MAYBE_SimulcastVP8) { + auto config = CreateTestConfig(); + config.filename = "ConferenceMotion_1280_720_50"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = 100; + config.simulcast_adapted_encoder = true; + config.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false, + 1280, 720); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{1500, 30, config.num_frames}}; + + std::vector rc_thresholds = { + {20, 5, 90, 0.8, 0.5, 0.3, 0, 1}}; + std::vector quality_thresholds = {{34, 32, 0.90, 0.88}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// Might be too slow on mobile platforms. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_SvcVP9 DISABLED_SvcVP9 +#else +#define MAYBE_SvcVP9 SvcVP9 +#endif +TEST(VideoProcessorIntegrationTestLibvpx, MAYBE_SvcVP9) { + auto config = CreateTestConfig(); + config.filename = "ConferenceMotion_1280_720_50"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = 100; + config.SetCodecSettings(cricket::kVp9CodecName, 1, 3, 3, true, true, false, + 1280, 720); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{1500, 30, config.num_frames}}; + + std::vector rc_thresholds = { + {5, 1, 5, 0.2, 0.3, 0.1, 0, 1}}; + std::vector quality_thresholds = {{36, 34, 0.93, 0.91}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +TEST(VideoProcessorIntegrationTestLibvpx, DISABLED_MultiresVP8RdPerf) { + auto config = CreateTestConfig(); + config.filename = "FourPeople_1280x720_30"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = 300; + config.print_frame_level_stats = true; + config.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false, + 1280, 720); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::map> rd_stats; + for (size_t bitrate_kbps : kBitrateRdPerfKbps) { + std::vector rate_profiles = { + {bitrate_kbps, 30, config.num_frames}}; + + fixture->RunTest(rate_profiles, nullptr, nullptr, nullptr, nullptr); + + rd_stats[bitrate_kbps] = + fixture->GetStats().SliceAndCalcLayerVideoStatistic( + kNumFirstFramesToSkipAtRdPerfAnalysis, config.num_frames - 1); + } + + PrintRdPerf(rd_stats); +} + +TEST(VideoProcessorIntegrationTestLibvpx, DISABLED_SvcVP9RdPerf) { + auto config = CreateTestConfig(); + config.filename = "FourPeople_1280x720_30"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = 300; + config.print_frame_level_stats = true; + config.SetCodecSettings(cricket::kVp9CodecName, 1, 3, 3, true, true, false, + 1280, 720); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::map> rd_stats; + for (size_t bitrate_kbps : kBitrateRdPerfKbps) { + std::vector rate_profiles = { + {bitrate_kbps, 30, config.num_frames}}; + + fixture->RunTest(rate_profiles, nullptr, nullptr, nullptr, nullptr); + + rd_stats[bitrate_kbps] = + fixture->GetStats().SliceAndCalcLayerVideoStatistic( + kNumFirstFramesToSkipAtRdPerfAnalysis, config.num_frames - 1); + } + + PrintRdPerf(rd_stats); +} + +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc b/modules/video_coding/codecs/test/videocodec_test_mediacodec.cc similarity index 52% rename from modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc rename to modules/video_coding/codecs/test/videocodec_test_mediacodec.cc index dd64eb7d8e..f4ca157507 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc +++ b/modules/video_coding/codecs/test/videocodec_test_mediacodec.cc @@ -8,14 +8,15 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h" - #include #include #include +#include "api/test/create_videocodec_test_fixture.h" #include "common_types.h" // NOLINT(build/include) #include "media/base/mediaconstants.h" +#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h" +#include "test/gtest.h" #include "test/testsupport/fileutils.h" namespace webrtc { @@ -24,23 +25,23 @@ namespace test { namespace { const int kForemanNumFrames = 300; const int kForemanFramerateFps = 30; + +TestConfig CreateTestConfig() { + TestConfig config; + config.filename = "foreman_cif"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = kForemanNumFrames; + config.hw_encoder = true; + config.hw_decoder = true; + return config; +} } // namespace -class VideoProcessorIntegrationTestMediaCodec - : public VideoProcessorIntegrationTest { - protected: - VideoProcessorIntegrationTestMediaCodec() { - config_.filename = "foreman_cif"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = kForemanNumFrames; - config_.hw_encoder = true; - config_.hw_decoder = true; - } -}; - -TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsVp8) { - config_.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, false, false, false, - 352, 288); +TEST(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsVp8) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, false, false, false, + 352, 288); + auto fixture = CreateVideoCodecTestFixture(config); std::vector rate_profiles = { {500, kForemanFramerateFps, kForemanNumFrames}}; @@ -53,14 +54,18 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsVp8) { std::vector quality_thresholds = {{36, 31, 0.92, 0.86}}; - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); } -TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsH264CBP) { - config_.encoded_frame_checker = &h264_keyframe_checker_; - config_.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, - false, 352, 288); +TEST(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsH264CBP) { + auto config = CreateTestConfig(); + const auto frame_checker = rtc::MakeUnique< + VideoCodecTestFixtureImpl::H264KeyframeChecker>(); + config.encoded_frame_checker = frame_checker.get(); + config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false, + 352, 288); + auto fixture = CreateVideoCodecTestFixture(config); std::vector rate_profiles = { {500, kForemanFramerateFps, kForemanNumFrames}}; @@ -73,18 +78,23 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsH264CBP) { std::vector quality_thresholds = {{36, 31, 0.92, 0.86}}; - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); } // TODO(brandtr): Enable this test when we have trybots/buildbots with // HW encoders that support CHP. -TEST_F(VideoProcessorIntegrationTestMediaCodec, - DISABLED_ForemanCif500kbpsH264CHP) { - config_.h264_codec_settings.profile = H264::kProfileConstrainedHigh; - config_.encoded_frame_checker = &h264_keyframe_checker_; - config_.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, - false, 352, 288); +TEST(VideoProcessorIntegrationTestMediaCodec, + DISABLED_ForemanCif500kbpsH264CHP) { + auto config = CreateTestConfig(); + const auto frame_checker = rtc::MakeUnique< + VideoCodecTestFixtureImpl::H264KeyframeChecker>(); + + config.h264_codec_settings.profile = H264::kProfileConstrainedHigh; + config.encoded_frame_checker = frame_checker.get(); + config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false, + 352, 288); + auto fixture = CreateVideoCodecTestFixture(config); std::vector rate_profiles = { {500, kForemanFramerateFps, kForemanNumFrames}}; @@ -97,11 +107,12 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); } -TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanMixedRes100kbpsVp8H264) { +TEST(VideoProcessorIntegrationTestMediaCodec, ForemanMixedRes100kbpsVp8H264) { + auto config = CreateTestConfig(); const int kNumFrames = 30; // TODO(brandtr): Add H.264 when we have fixed the encoder. const std::vector codecs = {cricket::kVp8CodecName}; @@ -116,16 +127,17 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanMixedRes100kbpsVp8H264) { for (const auto& resolution : resolutions) { const int width = std::get<0>(resolution); const int height = std::get<1>(resolution); - config_.filename = std::string("foreman_") + std::to_string(width) + "x" + - std::to_string(height); - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = kNumFrames; - config_.SetCodecSettings(codec, 1, 1, 1, false, false, false, - width, height); + config.filename = std::string("foreman_") + std::to_string(width) + "x" + + std::to_string(height); + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = kNumFrames; + config.SetCodecSettings(codec, 1, 1, 1, false, false, false, width, + height); - ProcessFramesAndMaybeVerify( - rate_profiles, nullptr /* rc_thresholds */, &quality_thresholds, - nullptr /* bs_thresholds */, nullptr /* visualization_params */); + auto fixture = CreateVideoCodecTestFixture(config); + fixture->RunTest(rate_profiles, nullptr /* rc_thresholds */, + &quality_thresholds, nullptr /* bs_thresholds */, + nullptr /* visualization_params */); } } } diff --git a/modules/video_coding/codecs/test/videocodec_test_openh264.cc b/modules/video_coding/codecs/test/videocodec_test_openh264.cc new file mode 100644 index 0000000000..2bfd12fdff --- /dev/null +++ b/modules/video_coding/codecs/test/videocodec_test_openh264.cc @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/test/create_videocodec_test_fixture.h" +#include "media/base/mediaconstants.h" +#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace test { + +#if defined(WEBRTC_USE_H264) + +namespace { +// Codec settings. +const int kCifWidth = 352; +const int kCifHeight = 288; +const int kNumFrames = 100; + +TestConfig CreateTestConfig() { + TestConfig config; + config.filename = "foreman_cif"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = kNumFrames; + // Only allow encoder/decoder to use single core, for predictability. + config.use_single_core = true; + config.hw_encoder = false; + config.hw_decoder = false; + return config; +} +} // namespace + +TEST(VideoProcessorIntegrationTestOpenH264, ConstantHighBitrate) { + auto frame_checker = rtc::MakeUnique< + VideoCodecTestFixtureImpl::H264KeyframeChecker>(); + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, true, false, + kCifWidth, kCifHeight); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{500, 30, kNumFrames}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; + + std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// H264: Enable SingleNalUnit packetization mode. Encoder should split +// large frames into multiple slices and limit length of NAL units. +TEST(VideoProcessorIntegrationTestOpenH264, SingleNalUnit) { + auto frame_checker = rtc::MakeUnique< + VideoCodecTestFixtureImpl::H264KeyframeChecker>(); + auto config = CreateTestConfig(); + config.h264_codec_settings.packetization_mode = + H264PacketizationMode::SingleNalUnit; + config.max_payload_size_bytes = 500; + config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, true, false, + kCifWidth, kCifHeight); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{500, 30, kNumFrames}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; + + std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; + + BitstreamThresholds bs_thresholds = {config.max_payload_size_bytes}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, + &bs_thresholds, nullptr); +} + +#endif // defined(WEBRTC_USE_H264) + +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc b/modules/video_coding/codecs/test/videocodec_test_parameterized.cc similarity index 76% rename from modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc rename to modules/video_coding/codecs/test/videocodec_test_parameterized.cc index 2499ede07e..200af91536 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc +++ b/modules/video_coding/codecs/test/videocodec_test_parameterized.cc @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h" - +#include "api/test/create_videocodec_test_fixture.h" +#include "test/gtest.h" #include "test/testsupport/fileutils.h" namespace webrtc { @@ -36,14 +36,12 @@ const VisualizationParams kVisualizationParams = { false, // save_encoded_ivf false, // save_decoded_y4m }; - const int kNumFrames = 30; - } // namespace // Tests for plotting statistics from logs. class VideoProcessorIntegrationTestParameterized - : public VideoProcessorIntegrationTest, + : public ::testing::Test, public ::testing::WithParamInterface< ::testing::tuple> { protected: @@ -57,13 +55,14 @@ class VideoProcessorIntegrationTestParameterized size_t height, size_t framerate, const std::string& filename) { - config_.filename = filename; - config_.filepath = ResourcePath(filename, "yuv"); - config_.use_single_core = kUseSingleCore; - config_.measure_cpu = kMeasureCpu; - config_.hw_encoder = hw_codec_; - config_.hw_decoder = hw_codec_; - config_.num_frames = kNumFrames; + TestConfig config; + config.filename = filename; + config.filepath = ResourcePath(filename, "yuv"); + config.use_single_core = kUseSingleCore; + config.measure_cpu = kMeasureCpu; + config.hw_encoder = hw_codec_; + config.hw_decoder = hw_codec_; + config.num_frames = kNumFrames; const size_t num_simulcast_streams = codec_type_ == kVideoCodecVP8 ? kNumSpatialLayers : 1; @@ -71,18 +70,19 @@ class VideoProcessorIntegrationTestParameterized codec_type_ == kVideoCodecVP9 ? kNumSpatialLayers : 1; const std::string codec_name = CodecTypeToPayloadString(codec_type_); - config_.SetCodecSettings(codec_name, num_simulcast_streams, - num_spatial_layers, kNumTemporalLayers, - kDenoisingOn, kFrameDropperOn, kSpatialResizeOn, - width, height); + config.SetCodecSettings(codec_name, num_simulcast_streams, + num_spatial_layers, kNumTemporalLayers, + kDenoisingOn, kFrameDropperOn, kSpatialResizeOn, + width, height); std::vector rate_profiles = { {bitrate_, framerate, kNumFrames}}; - ProcessFramesAndMaybeVerify(rate_profiles, nullptr, nullptr, nullptr, - &kVisualizationParams); + fixture_ = CreateVideoCodecTestFixture(config); + fixture_->RunTest(rate_profiles, nullptr, nullptr, nullptr, + &kVisualizationParams); } - + std::unique_ptr fixture_; const size_t bitrate_; const VideoCodecType codec_type_; const bool hw_codec_; diff --git a/modules/video_coding/codecs/test/videocodec_test_videotoolbox.cc b/modules/video_coding/codecs/test/videocodec_test_videotoolbox.cc new file mode 100644 index 0000000000..67232b4f96 --- /dev/null +++ b/modules/video_coding/codecs/test/videocodec_test_videotoolbox.cc @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/test/create_videocodec_test_fixture.h" +#include "media/base/mediaconstants.h" +#include "modules/video_coding/codecs/test/objc_codec_factory_helper.h" +#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace test { + +namespace { +const int kForemanNumFrames = 300; + +TestConfig CreateTestConfig() { + TestConfig config; + config.filename = "foreman_cif"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = kForemanNumFrames; + config.hw_encoder = true; + config.hw_decoder = true; + return config; +} + +std::unique_ptr CreateTestFixtureWithConfig( + TestConfig config) { + auto decoder_factory = CreateObjCDecoderFactory(); + auto encoder_factory = CreateObjCEncoderFactory(); + return CreateVideoCodecTestFixture( + config, std::move(decoder_factory), std::move(encoder_factory)); +} +} // namespace + +// TODO(webrtc:9099): Disabled until the issue is fixed. +// HW codecs don't work on simulators. Only run these tests on device. +// #if TARGET_OS_IPHONE && !TARGET_IPHONE_SIMULATOR +// #define MAYBE_TEST TEST +// #else +#define MAYBE_TEST(s, name) TEST(s, DISABLED_##name) +// #endif + +// TODO(kthelgason): Use RC Thresholds when the internal bitrateAdjuster is no +// longer in use. +MAYBE_TEST(VideoProcessorIntegrationTestVideoToolbox, + ForemanCif500kbpsH264CBP) { + const auto frame_checker = rtc::MakeUnique< + VideoCodecTestFixtureImpl::H264KeyframeChecker>(); + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false, + 352, 288); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateTestFixtureWithConfig(config); + + std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; + + std::vector quality_thresholds = {{33, 29, 0.9, 0.82}}; + + fixture->RunTest(rate_profiles, nullptr, &quality_thresholds, nullptr, + nullptr); +} + +MAYBE_TEST(VideoProcessorIntegrationTestVideoToolbox, + ForemanCif500kbpsH264CHP) { + const auto frame_checker = rtc::MakeUnique< + VideoCodecTestFixtureImpl::H264KeyframeChecker>(); + auto config = CreateTestConfig(); + config.h264_codec_settings.profile = H264::kProfileConstrainedHigh; + config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false, + 352, 288); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateTestFixtureWithConfig(config); + + std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; + + std::vector quality_thresholds = {{33, 30, 0.91, 0.83}}; + + fixture->RunTest(rate_profiles, nullptr, &quality_thresholds, nullptr, + nullptr); +} + +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc deleted file mode 100644 index 05abfdfc2d..0000000000 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc +++ /dev/null @@ -1,444 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h" - -#include - -#include "media/base/mediaconstants.h" -#include "modules/video_coding/codecs/test/test_config.h" -#include "modules/video_coding/utility/vp8_header_parser.h" -#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" -#include "rtc_base/ptr_util.h" -#include "test/testsupport/fileutils.h" - -namespace webrtc { -namespace test { - -namespace { -// Codec settings. -const int kCifWidth = 352; -const int kCifHeight = 288; -#if !defined(WEBRTC_IOS) -const int kNumFramesShort = 100; -#endif -const int kNumFramesLong = 300; -const size_t kBitrateRdPerfKbps[] = {100, 200, 300, 400, 500, 600, - 700, 800, 1000, 1250, 1400, 1600, - 1800, 2000, 2200, 2500}; -const size_t kNumFirstFramesToSkipAtRdPerfAnalysis = 60; -} // namespace - -class VideoProcessorIntegrationTestLibvpx - : public VideoProcessorIntegrationTest { - protected: - VideoProcessorIntegrationTestLibvpx() { - config_.filename = "foreman_cif"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = kNumFramesLong; - // Only allow encoder/decoder to use single core, for predictability. - config_.use_single_core = true; - config_.hw_encoder = false; - config_.hw_decoder = false; - config_.encoded_frame_checker = &qp_frame_checker_; - } - - void PrintRdPerf(std::map> rd_stats) { - printf("--> Summary\n"); - printf("%11s %5s %6s %11s %12s %11s %13s %13s %5s %7s %7s %7s %13s %13s\n", - "uplink_kbps", "width", "height", "spatial_idx", "temporal_idx", - "target_kbps", "downlink_kbps", "framerate_fps", "psnr", "psnr_y", - "psnr_u", "psnr_v", "enc_speed_fps", "dec_speed_fps"); - for (const auto& rd_stat : rd_stats) { - const size_t bitrate_kbps = rd_stat.first; - for (const auto& layer_stat : rd_stat.second) { - printf( - "%11zu %5zu %6zu %11zu %12zu %11zu %13zu %13.2f %5.2f %7.2f %7.2f " - "%7.2f" - "%13.2f %13.2f\n", - bitrate_kbps, layer_stat.width, layer_stat.height, - layer_stat.spatial_idx, layer_stat.temporal_idx, - layer_stat.target_bitrate_kbps, layer_stat.bitrate_kbps, - layer_stat.framerate_fps, layer_stat.avg_psnr, - layer_stat.avg_psnr_y, layer_stat.avg_psnr_u, layer_stat.avg_psnr_v, - layer_stat.enc_speed_fps, layer_stat.dec_speed_fps); - } - } - } - - private: - // Verify that the QP parser returns the same QP as the encoder does. - const class QpFrameChecker : public TestConfig::EncodedFrameChecker { - public: - void CheckEncodedFrame(webrtc::VideoCodecType codec, - const EncodedImage& encoded_frame) const override { - int qp; - if (codec == kVideoCodecVP8) { - EXPECT_TRUE( - vp8::GetQp(encoded_frame._buffer, encoded_frame._length, &qp)); - } else if (codec == kVideoCodecVP9) { - EXPECT_TRUE( - vp9::GetQp(encoded_frame._buffer, encoded_frame._length, &qp)); - } else { - RTC_NOTREACHED(); - } - EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP."; - } - } qp_frame_checker_; -}; - -// Fails on iOS. See webrtc:4755. -#if !defined(WEBRTC_IOS) - -#if !defined(RTC_DISABLE_VP9) -TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP9) { - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false, - kCifWidth, kCifHeight); - config_.num_frames = kNumFramesShort; - - std::vector rate_profiles = {{500, 30, kNumFramesShort}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.11, 0.3, 0.1, 0, 1}}; - - std::vector quality_thresholds = {{37, 36, 0.94, 0.92}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -TEST_F(VideoProcessorIntegrationTestLibvpx, ChangeBitrateVP9) { - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = { - {200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update - {700, 30, 200}, - {500, 30, kNumFramesLong}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.15, 0.5, 0.1, 0, 1}, - {15, 2, 0, 0.2, 0.5, 0.1, 0, 0}, - {10, 1, 0, 0.3, 0.5, 0.1, 0, 0}}; - - std::vector quality_thresholds = { - {34, 33, 0.90, 0.88}, {38, 35, 0.95, 0.91}, {35, 34, 0.93, 0.90}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -TEST_F(VideoProcessorIntegrationTestLibvpx, ChangeFramerateVP9) { - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = { - {100, 24, 100}, // target_kbps, input_fps, frame_index_rate_update - {100, 15, 200}, - {100, 10, kNumFramesLong}}; - - // Framerate mismatch should be lower for lower framerate. - std::vector rc_thresholds = { - {10, 2, 40, 0.4, 0.5, 0.2, 0, 1}, - {8, 2, 5, 0.2, 0.5, 0.2, 0, 0}, - {5, 2, 0, 0.21, 0.5, 0.3, 0, 0}}; - - // Quality should be higher for lower framerates for the same content. - std::vector quality_thresholds = { - {33, 32, 0.89, 0.87}, {33.5, 32, 0.90, 0.86}, {33.5, 31.5, 0.90, 0.85}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -TEST_F(VideoProcessorIntegrationTestLibvpx, DenoiserOnVP9) { - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, true, true, false, - kCifWidth, kCifHeight); - config_.num_frames = kNumFramesShort; - - std::vector rate_profiles = {{500, 30, kNumFramesShort}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.11, 0.3, 0.1, 0, 1}}; - - std::vector quality_thresholds = {{37.5, 36, 0.94, 0.93}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -TEST_F(VideoProcessorIntegrationTestLibvpx, VeryLowBitrateVP9) { - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, true, - kCifWidth, kCifHeight); - - std::vector rate_profiles = {{50, 30, kNumFramesLong}}; - - std::vector rc_thresholds = { - {15, 3, 75, 1.0, 0.5, 0.4, 1, 1}}; - - std::vector quality_thresholds = {{28, 25, 0.80, 0.65}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// TODO(marpan): Add temporal layer test for VP9, once changes are in -// vp9 wrapper for this. - -#endif // !defined(RTC_DISABLE_VP9) - -TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP8) { - config_.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false, - kCifWidth, kCifHeight); - config_.num_frames = kNumFramesShort; - - std::vector rate_profiles = {{500, 30, kNumFramesShort}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; - - // std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; - // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse - // than quality of x86 version. Use lower thresholds for now. - std::vector quality_thresholds = {{35, 33, 0.91, 0.89}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -#endif // !defined(WEBRTC_IOS) - -// The tests below are currently disabled for Android. For ARM, the encoder -// uses |cpu_speed| = 12, as opposed to default |cpu_speed| <= 6 for x86, -// which leads to significantly different quality. The quality and rate control -// settings in the tests below are defined for encoder speed setting -// |cpu_speed| <= ~6. A number of settings would need to be significantly -// modified for the |cpu_speed| = 12 case. For now, keep the tests below -// disabled on Android. Some quality parameter in the above test has been -// adjusted to also pass for |cpu_speed| <= 12. - -// Too slow to finish before timeout on iOS. See webrtc:4755. -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_ChangeBitrateVP8 DISABLED_ChangeBitrateVP8 -#else -#define MAYBE_ChangeBitrateVP8 ChangeBitrateVP8 -#endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeBitrateVP8) { - config_.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = { - {200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update - {800, 30, 200}, - {500, 30, kNumFramesLong}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, - {15.5, 1, 0, 0.1, 0.2, 0.1, 0, 0}, - {15, 1, 0, 0.3, 0.2, 0.1, 0, 0}}; - - // std::vector quality_thresholds = { - // {33, 32, 0.89, 0.88}, {38, 36, 0.94, 0.93}, {35, 34, 0.92, 0.91}}; - // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse - // than quality of x86 version. Use lower thresholds for now. - std::vector quality_thresholds = { - {31.8, 31, 0.86, 0.85}, {36, 34.8, 0.92, 0.90}, {33.5, 32, 0.90, 0.88}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// Too slow to finish before timeout on iOS. See webrtc:4755. -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_ChangeFramerateVP8 DISABLED_ChangeFramerateVP8 -#else -#define MAYBE_ChangeFramerateVP8 ChangeFramerateVP8 -#endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeFramerateVP8) { - config_.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = { - {80, 24, 100}, // target_kbps, input_fps, frame_index_rate_update - {80, 15, 200}, - {80, 10, kNumFramesLong}}; - - // std::vector rc_thresholds = { - // {10, 2, 20, 0.4, 0.3, 0.1, 0, 1}, - // {5, 2, 5, 0.3, 0.3, 0.1, 0, 0}, - // {4, 2, 1, 0.2, 0.3, 0.2, 0, 0}}; - // TODO(webrtc:8757): ARM VP8 drops more frames than x86 version. Use lower - // thresholds for now. - std::vector rc_thresholds = { - {10, 2, 60, 0.5, 0.3, 0.3, 0, 1}, - {10, 2, 30, 0.3, 0.3, 0.3, 0, 0}, - {10, 2, 10, 0.2, 0.3, 0.2, 0, 0}}; - - // std::vector quality_thresholds = { - // {31, 30, 0.87, 0.86}, {32, 31, 0.89, 0.86}, {32, 30, 0.87, 0.82}}; - // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse - // than quality of x86 version. Use lower thresholds for now. - std::vector quality_thresholds = { - {31, 30, 0.85, 0.84}, {31.5, 30.5, 0.86, 0.84}, {30.5, 29, 0.83, 0.78}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// Too slow to finish before timeout on iOS. See webrtc:4755. -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_TemporalLayersVP8 DISABLED_TemporalLayersVP8 -#else -#define MAYBE_TemporalLayersVP8 TemporalLayersVP8 -#endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_TemporalLayersVP8) { - config_.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 3, true, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = {{200, 30, 150}, - {400, 30, kNumFramesLong}}; - - // std::vector rc_thresholds = { - // {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, {10, 2, 0, 0.1, 0.2, 0.1, 0, 1}}; - // TODO(webrtc:8757): ARM VP8 drops more frames than x86 version. Use lower - // thresholds for now. - std::vector rc_thresholds = { - {10, 1, 2, 0.3, 0.2, 0.1, 0, 1}, {12, 2, 3, 0.1, 0.2, 0.1, 0, 1}}; - - // Min SSIM drops because of high motion scene with complex backgound (trees). - // std::vector quality_thresholds = {{32, 30, 0.88, 0.85}, - // {33, 30, 0.89, 0.83}}; - // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse - // than quality of x86 version. Use lower thresholds for now. - std::vector quality_thresholds = {{31, 30, 0.85, 0.84}, - {31, 28, 0.85, 0.75}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// Might be too slow on mobile platforms. -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_MultiresVP8 DISABLED_MultiresVP8 -#else -#define MAYBE_MultiresVP8 MultiresVP8 -#endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_MultiresVP8) { - config_.filename = "ConferenceMotion_1280_720_50"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = 100; - config_.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false, - 1280, 720); - - std::vector rate_profiles = {{1500, 30, config_.num_frames}}; - - std::vector rc_thresholds = { - {5, 1, 5, 0.2, 0.3, 0.1, 0, 1}}; - std::vector quality_thresholds = {{34, 32, 0.90, 0.88}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// Might be too slow on mobile platforms. -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_SimulcastVP8 DISABLED_SimulcastVP8 -#else -#define MAYBE_SimulcastVP8 SimulcastVP8 -#endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_SimulcastVP8) { - config_.filename = "ConferenceMotion_1280_720_50"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = 100; - config_.simulcast_adapted_encoder = true; - config_.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false, - 1280, 720); - - std::vector rate_profiles = {{1500, 30, config_.num_frames}}; - - std::vector rc_thresholds = { - {20, 5, 90, 0.8, 0.5, 0.3, 0, 1}}; - std::vector quality_thresholds = {{34, 32, 0.90, 0.88}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// Might be too slow on mobile platforms. -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_SvcVP9 DISABLED_SvcVP9 -#else -#define MAYBE_SvcVP9 SvcVP9 -#endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_SvcVP9) { - config_.filename = "ConferenceMotion_1280_720_50"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = 100; - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 3, 3, true, true, false, - 1280, 720); - - std::vector rate_profiles = {{1500, 30, config_.num_frames}}; - - std::vector rc_thresholds = { - {5, 1, 5, 0.2, 0.3, 0.1, 0, 1}}; - std::vector quality_thresholds = {{36, 34, 0.93, 0.91}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -TEST_F(VideoProcessorIntegrationTestLibvpx, DISABLED_MultiresVP8RdPerf) { - config_.filename = "FourPeople_1280x720_30"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = 300; - config_.print_frame_level_stats = true; - config_.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false, - 1280, 720); - - std::map> rd_stats; - for (size_t bitrate_kbps : kBitrateRdPerfKbps) { - std::vector rate_profiles = { - {bitrate_kbps, 30, config_.num_frames}}; - - ProcessFramesAndMaybeVerify(rate_profiles, nullptr, nullptr, nullptr, - nullptr); - - rd_stats[bitrate_kbps] = stats_.SliceAndCalcLayerVideoStatistic( - kNumFirstFramesToSkipAtRdPerfAnalysis, config_.num_frames - 1); - } - - PrintRdPerf(rd_stats); -} - -TEST_F(VideoProcessorIntegrationTestLibvpx, DISABLED_SvcVP9RdPerf) { - config_.filename = "FourPeople_1280x720_30"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = 300; - config_.print_frame_level_stats = true; - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 3, 3, true, true, false, - 1280, 720); - - std::map> rd_stats; - for (size_t bitrate_kbps : kBitrateRdPerfKbps) { - std::vector rate_profiles = { - {bitrate_kbps, 30, config_.num_frames}}; - - ProcessFramesAndMaybeVerify(rate_profiles, nullptr, nullptr, nullptr, - nullptr); - - rd_stats[bitrate_kbps] = stats_.SliceAndCalcLayerVideoStatistic( - kNumFirstFramesToSkipAtRdPerfAnalysis, config_.num_frames - 1); - } - - PrintRdPerf(rd_stats); -} - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc deleted file mode 100644 index d218b0c660..0000000000 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h" - -#include - -#include "media/base/mediaconstants.h" -#include "test/testsupport/fileutils.h" - -namespace webrtc { -namespace test { - -#if defined(WEBRTC_USE_H264) - -namespace { -// Codec settings. -const int kCifWidth = 352; -const int kCifHeight = 288; -const int kNumFrames = 100; -} // namespace - -class VideoProcessorIntegrationTestOpenH264 - : public VideoProcessorIntegrationTest { - protected: - VideoProcessorIntegrationTestOpenH264() { - config_.filename = "foreman_cif"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = kNumFrames; - // Only allow encoder/decoder to use single core, for predictability. - config_.use_single_core = true; - config_.hw_encoder = false; - config_.hw_decoder = false; - config_.encoded_frame_checker = &h264_keyframe_checker_; - } -}; - -TEST_F(VideoProcessorIntegrationTestOpenH264, ConstantHighBitrate) { - config_.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = {{500, 30, kNumFrames}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; - - std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// H264: Enable SingleNalUnit packetization mode. Encoder should split -// large frames into multiple slices and limit length of NAL units. -TEST_F(VideoProcessorIntegrationTestOpenH264, SingleNalUnit) { - config_.h264_codec_settings.packetization_mode = - H264PacketizationMode::SingleNalUnit; - config_.max_payload_size_bytes = 500; - config_.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = {{500, 30, kNumFrames}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; - - std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; - - BitstreamThresholds bs_thresholds = {config_.max_payload_size_bytes}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, &bs_thresholds, nullptr); -} - -#endif // defined(WEBRTC_USE_H264) - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc deleted file mode 100644 index 6ff6a57afc..0000000000 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h" - -#include - -#include "media/base/mediaconstants.h" -#include "modules/video_coding/codecs/test/objc_codec_factory_helper.h" -#include "test/testsupport/fileutils.h" - -namespace webrtc { -namespace test { - -namespace { -const int kForemanNumFrames = 300; -} // namespace - -class VideoProcessorIntegrationTestVideoToolbox - : public VideoProcessorIntegrationTest { - protected: - VideoProcessorIntegrationTestVideoToolbox() { - config_.filename = "foreman_cif"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = kForemanNumFrames; - config_.hw_encoder = true; - config_.hw_decoder = true; - config_.encoded_frame_checker = &h264_keyframe_checker_; - } - - std::unique_ptr CreateDecoderFactory() override { - if (config_.hw_decoder) { - EXPECT_EQ(kVideoCodecH264, config_.codec_settings.codecType) - << "iOS HW codecs only support H264."; - return CreateObjCDecoderFactory(); - } - RTC_NOTREACHED() << "Only support HW decoder on iOS."; - return nullptr; - } - - std::unique_ptr CreateEncoderFactory() override { - if (config_.hw_encoder) { - EXPECT_EQ(kVideoCodecH264, config_.codec_settings.codecType) - << "iOS HW codecs only support H264."; - return CreateObjCEncoderFactory(); - } - RTC_NOTREACHED() << "Only support HW encoder on iOS."; - return nullptr; - } -}; - -// TODO(webrtc:9099): Disabled until the issue is fixed. -// HW codecs don't work on simulators. Only run these tests on device. -// #if TARGET_OS_IPHONE && !TARGET_IPHONE_SIMULATOR -// #define MAYBE_TEST_F TEST_F -// #else -#define MAYBE_TEST_F(s, name) TEST_F(s, DISABLED_##name) -// #endif - -// TODO(kthelgason): Use RC Thresholds when the internal bitrateAdjuster is no -// longer in use. -MAYBE_TEST_F(VideoProcessorIntegrationTestVideoToolbox, - ForemanCif500kbpsH264CBP) { - config_.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, - false, 352, 288); - - std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; - - std::vector quality_thresholds = {{33, 29, 0.9, 0.82}}; - - ProcessFramesAndMaybeVerify(rate_profiles, nullptr, - &quality_thresholds, nullptr, nullptr); -} - -MAYBE_TEST_F(VideoProcessorIntegrationTestVideoToolbox, - ForemanCif500kbpsH264CHP) { - config_.h264_codec_settings.profile = H264::kProfileConstrainedHigh; - config_.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, - false, 352, 288); - - std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; - - std::vector quality_thresholds = {{33, 30, 0.91, 0.83}}; - - ProcessFramesAndMaybeVerify(rate_profiles, nullptr, - &quality_thresholds, nullptr, nullptr); -} - -} // namespace test -} // namespace webrtc