diff --git a/api/BUILD.gn b/api/BUILD.gn index bdff109179..d5da1821bb 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -375,6 +375,38 @@ if (rtc_include_tests) { } } + rtc_source_set("videocodec_test_fixture_api") { + visibility = [ "*" ] + testonly = true + sources = [ + "test/videocodec_test_fixture.h", + ] + deps = [ + "../modules/video_coding:video_codecs_test_framework", + "video_codecs:video_codecs_api", + ] + } + + rtc_source_set("create_videocodec_test_fixture_api") { + visibility = [ "*" ] + testonly = true + sources = [ + "test/create_videocodec_test_fixture.cc", + "test/create_videocodec_test_fixture.h", + ] + deps = [ + ":videocodec_test_fixture_api", + "../modules/video_coding:video_codecs_test_framework", + "../modules/video_coding:videocodec_test_impl", + "../rtc_base:rtc_base_approved", + "video_codecs:video_codecs_api", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + rtc_source_set("mock_audio_mixer") { testonly = true sources = [ diff --git a/api/test/DEPS b/api/test/DEPS new file mode 100644 index 0000000000..cd91a9ff38 --- /dev/null +++ b/api/test/DEPS @@ -0,0 +1,6 @@ +# TODO(kthelgason): Move the relevant included files to api/test. +specific_include_rules = { + ".*": [ + "+modules/video_coding/codecs/test", + ], +} diff --git a/api/test/create_videocodec_test_fixture.cc b/api/test/create_videocodec_test_fixture.cc new file mode 100644 index 0000000000..9a0aacc7c2 --- /dev/null +++ b/api/test/create_videocodec_test_fixture.cc @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_videocodec_test_fixture.h" + +#include +#include + +#include "api/test/videocodec_test_fixture.h" +#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h" +#include "rtc_base/ptr_util.h" + +namespace webrtc { +namespace test { + +std::unique_ptr +CreateVideoCodecTestFixture(const TestConfig& config) { + return rtc::MakeUnique(config); +} + +std::unique_ptr +CreateVideoCodecTestFixture( + const TestConfig& config, + std::unique_ptr decoder_factory, + std::unique_ptr encoder_factory) { + return rtc::MakeUnique( + config, std::move(decoder_factory), std::move(encoder_factory)); +} + +} // namespace test +} // namespace webrtc diff --git a/api/test/create_videocodec_test_fixture.h b/api/test/create_videocodec_test_fixture.h new file mode 100644 index 0000000000..24e17fd8d2 --- /dev/null +++ b/api/test/create_videocodec_test_fixture.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_CREATE_VIDEOCODEC_TEST_FIXTURE_H_ +#define API_TEST_CREATE_VIDEOCODEC_TEST_FIXTURE_H_ + +#include + +#include "api/test/videocodec_test_fixture.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/video_coding/codecs/test/test_config.h" + +namespace webrtc { +namespace test { + +std::unique_ptr +CreateVideoCodecTestFixture(const TestConfig& config); + +std::unique_ptr +CreateVideoCodecTestFixture( + const TestConfig& config, + std::unique_ptr decoder_factory, + std::unique_ptr encoder_factory); + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_CREATE_VIDEOCODEC_TEST_FIXTURE_H_ diff --git a/api/test/videocodec_test_fixture.h b/api/test/videocodec_test_fixture.h new file mode 100644 index 0000000000..88da7df317 --- /dev/null +++ b/api/test/videocodec_test_fixture.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEOCODEC_TEST_FIXTURE_H_ +#define API_TEST_VIDEOCODEC_TEST_FIXTURE_H_ + +#include + +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/video_coding/codecs/test/stats.h" + +namespace webrtc { +namespace test { + +// Rates for the encoder and the frame number when to change profile. +struct RateProfile { + size_t target_kbps; + size_t input_fps; + size_t frame_index_rate_update; +}; + +struct RateControlThresholds { + double max_avg_bitrate_mismatch_percent; + double max_time_to_reach_target_bitrate_sec; + // TODO(ssilkin): Use absolute threshold for framerate. + double max_avg_framerate_mismatch_percent; + double max_avg_buffer_level_sec; + double max_max_key_frame_delay_sec; + double max_max_delta_frame_delay_sec; + size_t max_num_spatial_resizes; + size_t max_num_key_frames; +}; + +struct QualityThresholds { + double min_avg_psnr; + double min_min_psnr; + double min_avg_ssim; + double min_min_ssim; +}; + +struct BitstreamThresholds { + size_t max_max_nalu_size_bytes; +}; + +// Should video files be saved persistently to disk for post-run visualization? +struct VisualizationParams { + bool save_encoded_ivf; + bool save_decoded_y4m; +}; + +class VideoCodecTestFixture { + public: + virtual ~VideoCodecTestFixture() = default; + + virtual void RunTest(const std::vector& rate_profiles, + const std::vector* rc_thresholds, + const std::vector* quality_thresholds, + const BitstreamThresholds* bs_thresholds, + const VisualizationParams* visualization_params) = 0; + virtual Stats GetStats() = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEOCODEC_TEST_FIXTURE_H_ diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 77b4fb96aa..bd06f3fda3 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -670,43 +670,25 @@ if (rtc_include_tests) { } } - rtc_source_set("video_coding_modules_tests") { + rtc_source_set("videocodec_test_impl") { testonly = true - sources = [ - "codecs/h264/test/h264_impl_unittest.cc", - "codecs/multiplex/test/multiplex_adapter_unittest.cc", - "codecs/test/video_encoder_decoder_instantiation_tests.cc", - "codecs/test/videoprocessor_integrationtest.cc", - "codecs/test/videoprocessor_integrationtest.h", - "codecs/test/videoprocessor_integrationtest_libvpx.cc", - "codecs/test/videoprocessor_integrationtest_openh264.cc", - "codecs/test/videoprocessor_integrationtest_parameterized.cc", - "codecs/vp8/test/vp8_impl_unittest.cc", - "codecs/vp9/test/vp9_impl_unittest.cc", + "codecs/test/videocodec_test_fixture_impl.cc", + "codecs/test/videocodec_test_fixture_impl.h", ] - deps = [ ":video_codec_interface", ":video_codecs_test_framework", ":video_coding", ":video_coding_utility", - ":webrtc_h264", ":webrtc_multiplex", ":webrtc_vp8_helpers", - ":webrtc_vp9", ":webrtc_vp9_helpers", "../..:webrtc_common", - "../../api:mock_video_codec_factory", - "../../api:optional", - "../../api:video_frame_api", - "../../api:video_frame_api_i420", + "../../api:videocodec_test_fixture_api", "../../api/video_codecs:video_codecs_api", "../../common_video", - "../../media:rtc_audio_video", - "../../media:rtc_h264_profile_id", "../../media:rtc_internal_video_codecs", - "../../media:rtc_media_base", "../../media:rtc_software_fallback_wrappers", "../../rtc_base:checks", "../../rtc_base:rtc_base", @@ -714,22 +696,66 @@ if (rtc_include_tests) { "../../system_wrappers", "../../test:fileutils", "../../test:test_support", - "../../test:video_test_common", "../../test:video_test_support", - "../video_capture", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + if (is_android) { + deps += [ ":android_codec_factory_helper" ] + } + } + + rtc_source_set("video_coding_modules_tests") { + testonly = true + + sources = [ + "codecs/h264/test/h264_impl_unittest.cc", + "codecs/multiplex/test/multiplex_adapter_unittest.cc", + "codecs/test/video_encoder_decoder_instantiation_tests.cc", + "codecs/test/videocodec_test_libvpx.cc", + "codecs/test/videocodec_test_openh264.cc", + "codecs/test/videocodec_test_parameterized.cc", + "codecs/vp8/test/vp8_impl_unittest.cc", + "codecs/vp9/test/vp9_impl_unittest.cc", + ] + + deps = [ + ":video_codecs_test_framework", + ":video_coding_utility", + ":videocodec_test_impl", + ":webrtc_h264", + ":webrtc_multiplex", + ":webrtc_vp8_helpers", + ":webrtc_vp9", + ":webrtc_vp9_helpers", + "../..:webrtc_common", + "../../api:create_videocodec_test_fixture_api", + "../../api:mock_video_codec_factory", + "../../api:optional", + "../../api:video_frame_api_i420", + "../../api:videocodec_test_fixture_api", + "../../api/video_codecs:video_codecs_api", + "../../common_video", + "../../media:rtc_h264_profile_id", + "../../media:rtc_media_base", + "../../rtc_base:rtc_base", + "../../test:fileutils", + "../../test:test_support", + "../../test:video_test_common", ] data = video_coding_modules_tests_resources if (is_android) { - sources += [ "codecs/test/videoprocessor_integrationtest_mediacodec.cc" ] + sources += [ "codecs/test/videocodec_test_mediacodec.cc" ] deps += [ ":android_codec_factory_helper" ] } if (is_ios || is_mac) { - sources += - [ "codecs/test/videoprocessor_integrationtest_videotoolbox.cc" ] + sources += [ "codecs/test/videocodec_test_videotoolbox.cc" ] deps += [ ":objc_codec_factory_helper" ] diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc similarity index 89% rename from modules/video_coding/codecs/test/videoprocessor_integrationtest.cc rename to modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc index 3079ed30f1..f669d902a3 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h" +#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h" #include #include @@ -34,6 +34,7 @@ #include "rtc_base/file.h" #include "rtc_base/ptr_util.h" #include "system_wrappers/include/sleep.h" +#include "test/gtest.h" #include "test/testsupport/fileutils.h" namespace webrtc { @@ -55,9 +56,11 @@ bool RunEncodeInRealTime(const TestConfig& config) { } // namespace -void VideoProcessorIntegrationTest::H264KeyframeChecker::CheckEncodedFrame( - webrtc::VideoCodecType codec, - const EncodedImage& encoded_frame) const { +// TODO(kthelgason): Move this out of the test fixture impl and +// make available as a shared utility class. +void VideoCodecTestFixtureImpl::H264KeyframeChecker:: + CheckEncodedFrame(webrtc::VideoCodecType codec, + const EncodedImage& encoded_frame) const { EXPECT_EQ(kVideoCodecH264, codec); bool contains_sps = false; bool contains_pps = false; @@ -89,7 +92,7 @@ void VideoProcessorIntegrationTest::H264KeyframeChecker::CheckEncodedFrame( } } -class VideoProcessorIntegrationTest::CpuProcessTime final { +class VideoCodecTestFixtureImpl::CpuProcessTime final { public: explicit CpuProcessTime(const TestConfig& config) : config_(config) {} ~CpuProcessTime() {} @@ -124,16 +127,32 @@ class VideoProcessorIntegrationTest::CpuProcessTime final { int64_t wallclock_time_ = 0; }; -VideoProcessorIntegrationTest::VideoProcessorIntegrationTest() { +VideoCodecTestFixtureImpl:: + VideoCodecTestFixtureImpl(TestConfig config) + : config_(config) { #if defined(WEBRTC_ANDROID) InitializeAndroidObjects(); #endif } -VideoProcessorIntegrationTest::~VideoProcessorIntegrationTest() = default; +VideoCodecTestFixtureImpl:: + VideoCodecTestFixtureImpl( + TestConfig config, + std::unique_ptr decoder_factory, + std::unique_ptr encoder_factory) + : decoder_factory_(std::move(decoder_factory)), + encoder_factory_(std::move(encoder_factory)), + config_(config) { +#if defined(WEBRTC_ANDROID) + InitializeAndroidObjects(); +#endif +} + +VideoCodecTestFixtureImpl:: + ~VideoCodecTestFixtureImpl() = default; // Processes all frames in the clip and verifies the result. -void VideoProcessorIntegrationTest::ProcessFramesAndMaybeVerify( +void VideoCodecTestFixtureImpl::RunTest( const std::vector& rate_profiles, const std::vector* rc_thresholds, const std::vector* quality_thresholds, @@ -156,7 +175,7 @@ void VideoProcessorIntegrationTest::ProcessFramesAndMaybeVerify( bs_thresholds); } -void VideoProcessorIntegrationTest::ProcessAllFrames( +void VideoCodecTestFixtureImpl::ProcessAllFrames( rtc::TaskQueue* task_queue, const std::vector& rate_profiles) { // Process all frames. @@ -206,7 +225,7 @@ void VideoProcessorIntegrationTest::ProcessAllFrames( cpu_process_time_->Stop(); } -void VideoProcessorIntegrationTest::AnalyzeAllFrames( +void VideoCodecTestFixtureImpl::AnalyzeAllFrames( const std::vector& rate_profiles, const std::vector* rc_thresholds, const std::vector* quality_thresholds, @@ -252,7 +271,7 @@ void VideoProcessorIntegrationTest::AnalyzeAllFrames( printf("\n"); } -void VideoProcessorIntegrationTest::VerifyVideoStatistic( +void VideoCodecTestFixtureImpl::VerifyVideoStatistic( const VideoStatistics& video_stat, const RateControlThresholds* rc_thresholds, const QualityThresholds* quality_thresholds, @@ -297,7 +316,7 @@ void VideoProcessorIntegrationTest::VerifyVideoStatistic( } std::unique_ptr -VideoProcessorIntegrationTest::CreateDecoderFactory() { +VideoCodecTestFixtureImpl::CreateDecoderFactory() { if (config_.hw_decoder) { #if defined(WEBRTC_ANDROID) return CreateAndroidDecoderFactory(); @@ -311,7 +330,7 @@ VideoProcessorIntegrationTest::CreateDecoderFactory() { } std::unique_ptr -VideoProcessorIntegrationTest::CreateEncoderFactory() { +VideoCodecTestFixtureImpl::CreateEncoderFactory() { if (config_.hw_encoder) { #if defined(WEBRTC_ANDROID) return CreateAndroidEncoderFactory(); @@ -324,11 +343,12 @@ VideoProcessorIntegrationTest::CreateEncoderFactory() { } } -void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { - encoder_factory_ = CreateEncoderFactory(); - std::unique_ptr decoder_factory = CreateDecoderFactory(); - +void VideoCodecTestFixtureImpl::CreateEncoderAndDecoder() { const SdpVideoFormat format = config_.ToSdpVideoFormat(); + if (!decoder_factory_) + decoder_factory_ = CreateDecoderFactory(); + if (!encoder_factory_) + encoder_factory_ = CreateEncoderFactory(); if (config_.simulcast_adapted_encoder) { EXPECT_EQ("VP8", format.name); encoder_.reset(new SimulcastEncoderAdapter(encoder_factory_.get())); @@ -341,7 +361,7 @@ void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { for (size_t i = 0; i < num_simulcast_or_spatial_layers; ++i) { decoders_.push_back(std::unique_ptr( - decoder_factory->CreateVideoDecoder(format))); + decoder_factory_->CreateVideoDecoder(format))); } if (config_.sw_fallback_encoder) { @@ -367,13 +387,16 @@ void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { } } -void VideoProcessorIntegrationTest::DestroyEncoderAndDecoder() { +void VideoCodecTestFixtureImpl::DestroyEncoderAndDecoder() { decoders_.clear(); encoder_.reset(); - encoder_factory_.reset(); } -void VideoProcessorIntegrationTest::SetUpAndInitObjects( +Stats VideoCodecTestFixtureImpl::GetStats() { + return stats_; +} + +void VideoCodecTestFixtureImpl::SetUpAndInitObjects( rtc::TaskQueue* task_queue, int initial_bitrate_kbps, int initial_framerate_fps, @@ -437,7 +460,7 @@ void VideoProcessorIntegrationTest::SetUpAndInitObjects( sync_event.Wait(rtc::Event::kForever); } -void VideoProcessorIntegrationTest::ReleaseAndCloseObjects( +void VideoCodecTestFixtureImpl::ReleaseAndCloseObjects( rtc::TaskQueue* task_queue) { rtc::Event sync_event(false, false); task_queue->PostTask([this, &sync_event]() { @@ -462,7 +485,7 @@ void VideoProcessorIntegrationTest::ReleaseAndCloseObjects( decoded_frame_writers_.clear(); } -void VideoProcessorIntegrationTest::PrintSettings( +void VideoCodecTestFixtureImpl::PrintSettings( rtc::TaskQueue* task_queue) const { printf("==> TestConfig\n"); printf("%s\n", config_.ToString().c_str()); diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.h b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.h similarity index 62% rename from modules/video_coding/codecs/test/videoprocessor_integrationtest.h rename to modules/video_coding/codecs/test/videocodec_test_fixture_impl.h index 32a9160773..7ad7171013 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.h +++ b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.h @@ -8,15 +8,14 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_ -#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_ +#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEOCODEC_TEST_FIXTURE_IMPL_H_ +#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEOCODEC_TEST_FIXTURE_IMPL_H_ -#include -#include #include #include #include +#include "api/test/videocodec_test_fixture.h" #include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_encoder_factory.h" #include "common_types.h" // NOLINT(build/include) @@ -25,84 +24,39 @@ #include "modules/video_coding/codecs/test/test_config.h" #include "modules/video_coding/codecs/test/videoprocessor.h" #include "modules/video_coding/utility/ivf_file_writer.h" -#include "test/gtest.h" #include "test/testsupport/frame_reader.h" #include "test/testsupport/frame_writer.h" namespace webrtc { namespace test { -// Rates for the encoder and the frame number when to change profile. -struct RateProfile { - size_t target_kbps; - size_t input_fps; - size_t frame_index_rate_update; -}; - -struct RateControlThresholds { - double max_avg_bitrate_mismatch_percent; - double max_time_to_reach_target_bitrate_sec; - // TODO(ssilkin): Use absolute threshold for framerate. - double max_avg_framerate_mismatch_percent; - double max_avg_buffer_level_sec; - double max_max_key_frame_delay_sec; - double max_max_delta_frame_delay_sec; - size_t max_num_spatial_resizes; - size_t max_num_key_frames; -}; - -struct QualityThresholds { - double min_avg_psnr; - double min_min_psnr; - double min_avg_ssim; - double min_min_ssim; -}; - -struct BitstreamThresholds { - size_t max_max_nalu_size_bytes; -}; - -// Should video files be saved persistently to disk for post-run visualization? -struct VisualizationParams { - bool save_encoded_ivf; - bool save_decoded_y4m; -}; - // Integration test for video processor. It does rate control and frame quality // analysis using frame statistics collected by video processor and logs the // results. If thresholds are specified it checks that corresponding metrics // are in desirable range. -class VideoProcessorIntegrationTest : public testing::Test { - protected: +class VideoCodecTestFixtureImpl : public VideoCodecTestFixture { // Verifies that all H.264 keyframes contain SPS/PPS/IDR NALUs. + public: class H264KeyframeChecker : public TestConfig::EncodedFrameChecker { public: void CheckEncodedFrame(webrtc::VideoCodecType codec, const EncodedImage& encoded_frame) const override; }; - VideoProcessorIntegrationTest(); - ~VideoProcessorIntegrationTest() override; + explicit VideoCodecTestFixtureImpl(TestConfig config); + VideoCodecTestFixtureImpl( + TestConfig config, + std::unique_ptr decoder_factory, + std::unique_ptr encoder_factory); + ~VideoCodecTestFixtureImpl() override; - void ProcessFramesAndMaybeVerify( - const std::vector& rate_profiles, - const std::vector* rc_thresholds, - const std::vector* quality_thresholds, - const BitstreamThresholds* bs_thresholds, - const VisualizationParams* visualization_params); + void RunTest(const std::vector& rate_profiles, + const std::vector* rc_thresholds, + const std::vector* quality_thresholds, + const BitstreamThresholds* bs_thresholds, + const VisualizationParams* visualization_params) override; - // Config. - TestConfig config_; - - Stats stats_; - - // Can be used by all H.264 tests. - const H264KeyframeChecker h264_keyframe_checker_; - - protected: - // Overwrite in subclasses for custom codec factories. - virtual std::unique_ptr CreateDecoderFactory(); - virtual std::unique_ptr CreateEncoderFactory(); + Stats GetStats() override; private: class CpuProcessTime; @@ -131,13 +85,18 @@ class VideoProcessorIntegrationTest : public testing::Test { float input_framerate_fps); void PrintSettings(rtc::TaskQueue* task_queue) const; + std::unique_ptr CreateDecoderFactory(); + std::unique_ptr CreateEncoderFactory(); // Codecs. + std::unique_ptr decoder_factory_; std::unique_ptr encoder_factory_; std::unique_ptr encoder_; VideoProcessor::VideoDecoderList decoders_; // Helper objects. + TestConfig config_; + Stats stats_; std::unique_ptr source_frame_reader_; VideoProcessor::IvfFileWriterList encoded_frame_writers_; VideoProcessor::FrameWriterList decoded_frame_writers_; @@ -148,4 +107,4 @@ class VideoProcessorIntegrationTest : public testing::Test { } // namespace test } // namespace webrtc -#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_ +#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEOCODEC_TEST_FIXTURE_IMPL_H_ diff --git a/modules/video_coding/codecs/test/videocodec_test_libvpx.cc b/modules/video_coding/codecs/test/videocodec_test_libvpx.cc new file mode 100644 index 0000000000..3e7332ae01 --- /dev/null +++ b/modules/video_coding/codecs/test/videocodec_test_libvpx.cc @@ -0,0 +1,494 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/test/create_videocodec_test_fixture.h" +#include "media/base/mediaconstants.h" +#include "modules/video_coding/codecs/test/test_config.h" +#include "modules/video_coding/utility/vp8_header_parser.h" +#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" +#include "rtc_base/ptr_util.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace test { + +namespace { +// Codec settings. +const int kCifWidth = 352; +const int kCifHeight = 288; +#if !defined(WEBRTC_IOS) +const int kNumFramesShort = 100; +#endif +const int kNumFramesLong = 300; +const size_t kBitrateRdPerfKbps[] = {100, 200, 300, 400, 500, 600, + 700, 800, 1000, 1250, 1400, 1600, + 1800, 2000, 2200, 2500}; +const size_t kNumFirstFramesToSkipAtRdPerfAnalysis = 60; + +class QpFrameChecker : public TestConfig::EncodedFrameChecker { + public: + void CheckEncodedFrame(webrtc::VideoCodecType codec, + const EncodedImage& encoded_frame) const override { + int qp; + if (codec == kVideoCodecVP8) { + EXPECT_TRUE( + vp8::GetQp(encoded_frame._buffer, encoded_frame._length, &qp)); + } else if (codec == kVideoCodecVP9) { + EXPECT_TRUE( + vp9::GetQp(encoded_frame._buffer, encoded_frame._length, &qp)); + } else { + RTC_NOTREACHED(); + } + EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP."; + } +}; + +TestConfig CreateTestConfig() { + TestConfig config; + config.filename = "foreman_cif"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = kNumFramesLong; + config.use_single_core = true; + config.hw_encoder = false; + config.hw_decoder = false; + return config; +} + +void PrintRdPerf(std::map> rd_stats) { + printf("--> Summary\n"); + printf("%11s %5s %6s %11s %12s %11s %13s %13s %5s %7s %7s %7s %13s %13s\n", + "uplink_kbps", "width", "height", "spatial_idx", "temporal_idx", + "target_kbps", "downlink_kbps", "framerate_fps", "psnr", "psnr_y", + "psnr_u", "psnr_v", "enc_speed_fps", "dec_speed_fps"); + for (const auto& rd_stat : rd_stats) { + const size_t bitrate_kbps = rd_stat.first; + for (const auto& layer_stat : rd_stat.second) { + printf( + "%11zu %5zu %6zu %11zu %12zu %11zu %13zu %13.2f %5.2f %7.2f %7.2f " + "%7.2f" + "%13.2f %13.2f\n", + bitrate_kbps, layer_stat.width, layer_stat.height, + layer_stat.spatial_idx, layer_stat.temporal_idx, + layer_stat.target_bitrate_kbps, layer_stat.bitrate_kbps, + layer_stat.framerate_fps, layer_stat.avg_psnr, layer_stat.avg_psnr_y, + layer_stat.avg_psnr_u, layer_stat.avg_psnr_v, + layer_stat.enc_speed_fps, layer_stat.dec_speed_fps); + } + } +} +} // namespace + +// Fails on iOS. See webrtc:4755. +#if !defined(WEBRTC_IOS) + +#if !defined(RTC_DISABLE_VP9) +TEST(VideoProcessorIntegrationTestLibvpx, HighBitrateVP9) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false, + kCifWidth, kCifHeight); + config.num_frames = kNumFramesShort; + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{500, 30, kNumFramesShort}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.11, 0.3, 0.1, 0, 1}}; + + std::vector quality_thresholds = {{37, 36, 0.94, 0.92}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +TEST(VideoProcessorIntegrationTestLibvpx, ChangeBitrateVP9) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false, + kCifWidth, kCifHeight); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = { + {200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update + {700, 30, 200}, + {500, 30, kNumFramesLong}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.15, 0.5, 0.1, 0, 1}, + {15, 2, 0, 0.2, 0.5, 0.1, 0, 0}, + {10, 1, 0, 0.3, 0.5, 0.1, 0, 0}}; + + std::vector quality_thresholds = { + {34, 33, 0.90, 0.88}, {38, 35, 0.95, 0.91}, {35, 34, 0.93, 0.90}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +TEST(VideoProcessorIntegrationTestLibvpx, ChangeFramerateVP9) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false, + kCifWidth, kCifHeight); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = { + {100, 24, 100}, // target_kbps, input_fps, frame_index_rate_update + {100, 15, 200}, + {100, 10, kNumFramesLong}}; + + // Framerate mismatch should be lower for lower framerate. + std::vector rc_thresholds = { + {10, 2, 40, 0.4, 0.5, 0.2, 0, 1}, + {8, 2, 5, 0.2, 0.5, 0.2, 0, 0}, + {5, 2, 0, 0.21, 0.5, 0.3, 0, 0}}; + + // Quality should be higher for lower framerates for the same content. + std::vector quality_thresholds = { + {33, 32, 0.89, 0.87}, {33.5, 32, 0.90, 0.86}, {33.5, 31.5, 0.90, 0.85}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +TEST(VideoProcessorIntegrationTestLibvpx, DenoiserOnVP9) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, true, true, false, + kCifWidth, kCifHeight); + config.num_frames = kNumFramesShort; + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{500, 30, kNumFramesShort}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.11, 0.3, 0.1, 0, 1}}; + + std::vector quality_thresholds = {{37.5, 36, 0.94, 0.93}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +TEST(VideoProcessorIntegrationTestLibvpx, VeryLowBitrateVP9) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, true, + kCifWidth, kCifHeight); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{50, 30, kNumFramesLong}}; + + std::vector rc_thresholds = { + {15, 3, 75, 1.0, 0.5, 0.4, 1, 1}}; + + std::vector quality_thresholds = {{28, 25, 0.80, 0.65}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// TODO(marpan): Add temporal layer test for VP9, once changes are in +// vp9 wrapper for this. + +#endif // !defined(RTC_DISABLE_VP9) + +TEST(VideoProcessorIntegrationTestLibvpx, HighBitrateVP8) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false, + kCifWidth, kCifHeight); + config.num_frames = kNumFramesShort; + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{500, 30, kNumFramesShort}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; + + // std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; + // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse + // than quality of x86 version. Use lower thresholds for now. + std::vector quality_thresholds = {{35, 33, 0.91, 0.89}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +#endif // !defined(WEBRTC_IOS) + +// The tests below are currently disabled for Android. For ARM, the encoder +// uses |cpu_speed| = 12, as opposed to default |cpu_speed| <= 6 for x86, +// which leads to significantly different quality. The quality and rate control +// settings in the tests below are defined for encoder speed setting +// |cpu_speed| <= ~6. A number of settings would need to be significantly +// modified for the |cpu_speed| = 12 case. For now, keep the tests below +// disabled on Android. Some quality parameter in the above test has been +// adjusted to also pass for |cpu_speed| <= 12. + +// Too slow to finish before timeout on iOS. See webrtc:4755. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_ChangeBitrateVP8 DISABLED_ChangeBitrateVP8 +#else +#define MAYBE_ChangeBitrateVP8 ChangeBitrateVP8 +#endif +TEST(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeBitrateVP8) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false, + kCifWidth, kCifHeight); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = { + {200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update + {800, 30, 200}, + {500, 30, kNumFramesLong}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, + {15.5, 1, 0, 0.1, 0.2, 0.1, 0, 0}, + {15, 1, 0, 0.3, 0.2, 0.1, 0, 0}}; + + // std::vector quality_thresholds = { + // {33, 32, 0.89, 0.88}, {38, 36, 0.94, 0.93}, {35, 34, 0.92, 0.91}}; + // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse + // than quality of x86 version. Use lower thresholds for now. + std::vector quality_thresholds = { + {31.8, 31, 0.86, 0.85}, {36, 34.8, 0.92, 0.90}, {33.5, 32, 0.90, 0.88}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// Too slow to finish before timeout on iOS. See webrtc:4755. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_ChangeFramerateVP8 DISABLED_ChangeFramerateVP8 +#else +#define MAYBE_ChangeFramerateVP8 ChangeFramerateVP8 +#endif +TEST(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeFramerateVP8) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false, + kCifWidth, kCifHeight); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = { + {80, 24, 100}, // target_kbps, input_fps, frame_index_rate_update + {80, 15, 200}, + {80, 10, kNumFramesLong}}; + + // std::vector rc_thresholds = { + // {10, 2, 20, 0.4, 0.3, 0.1, 0, 1}, + // {5, 2, 5, 0.3, 0.3, 0.1, 0, 0}, + // {4, 2, 1, 0.2, 0.3, 0.2, 0, 0}}; + // TODO(webrtc:8757): ARM VP8 drops more frames than x86 version. Use lower + // thresholds for now. + std::vector rc_thresholds = { + {10, 2, 60, 0.5, 0.3, 0.3, 0, 1}, + {10, 2, 30, 0.3, 0.3, 0.3, 0, 0}, + {10, 2, 10, 0.2, 0.3, 0.2, 0, 0}}; + + // std::vector quality_thresholds = { + // {31, 30, 0.87, 0.86}, {32, 31, 0.89, 0.86}, {32, 30, 0.87, 0.82}}; + // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse + // than quality of x86 version. Use lower thresholds for now. + std::vector quality_thresholds = { + {31, 30, 0.85, 0.84}, {31.5, 30.5, 0.86, 0.84}, {30.5, 29, 0.83, 0.78}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// Too slow to finish before timeout on iOS. See webrtc:4755. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_TemporalLayersVP8 DISABLED_TemporalLayersVP8 +#else +#define MAYBE_TemporalLayersVP8 TemporalLayersVP8 +#endif +TEST(VideoProcessorIntegrationTestLibvpx, MAYBE_TemporalLayersVP8) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 3, true, true, false, + kCifWidth, kCifHeight); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{200, 30, 150}, + {400, 30, kNumFramesLong}}; + + // std::vector rc_thresholds = { + // {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, {10, 2, 0, 0.1, 0.2, 0.1, 0, 1}}; + // TODO(webrtc:8757): ARM VP8 drops more frames than x86 version. Use lower + // thresholds for now. + std::vector rc_thresholds = { + {10, 1, 2, 0.3, 0.2, 0.1, 0, 1}, {12, 2, 3, 0.1, 0.2, 0.1, 0, 1}}; + + // Min SSIM drops because of high motion scene with complex backgound (trees). + // std::vector quality_thresholds = {{32, 30, 0.88, 0.85}, + // {33, 30, 0.89, 0.83}}; + // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse + // than quality of x86 version. Use lower thresholds for now. + std::vector quality_thresholds = {{31, 30, 0.85, 0.84}, + {31, 28, 0.85, 0.75}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// Might be too slow on mobile platforms. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_MultiresVP8 DISABLED_MultiresVP8 +#else +#define MAYBE_MultiresVP8 MultiresVP8 +#endif +TEST(VideoProcessorIntegrationTestLibvpx, MAYBE_MultiresVP8) { + auto config = CreateTestConfig(); + config.filename = "ConferenceMotion_1280_720_50"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = 100; + config.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false, + 1280, 720); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{1500, 30, config.num_frames}}; + + std::vector rc_thresholds = { + {5, 1, 5, 0.2, 0.3, 0.1, 0, 1}}; + std::vector quality_thresholds = {{34, 32, 0.90, 0.88}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// Might be too slow on mobile platforms. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_SimulcastVP8 DISABLED_SimulcastVP8 +#else +#define MAYBE_SimulcastVP8 SimulcastVP8 +#endif +TEST(VideoProcessorIntegrationTestLibvpx, MAYBE_SimulcastVP8) { + auto config = CreateTestConfig(); + config.filename = "ConferenceMotion_1280_720_50"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = 100; + config.simulcast_adapted_encoder = true; + config.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false, + 1280, 720); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{1500, 30, config.num_frames}}; + + std::vector rc_thresholds = { + {20, 5, 90, 0.8, 0.5, 0.3, 0, 1}}; + std::vector quality_thresholds = {{34, 32, 0.90, 0.88}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// Might be too slow on mobile platforms. +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) +#define MAYBE_SvcVP9 DISABLED_SvcVP9 +#else +#define MAYBE_SvcVP9 SvcVP9 +#endif +TEST(VideoProcessorIntegrationTestLibvpx, MAYBE_SvcVP9) { + auto config = CreateTestConfig(); + config.filename = "ConferenceMotion_1280_720_50"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = 100; + config.SetCodecSettings(cricket::kVp9CodecName, 1, 3, 3, true, true, false, + 1280, 720); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{1500, 30, config.num_frames}}; + + std::vector rc_thresholds = { + {5, 1, 5, 0.2, 0.3, 0.1, 0, 1}}; + std::vector quality_thresholds = {{36, 34, 0.93, 0.91}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +TEST(VideoProcessorIntegrationTestLibvpx, DISABLED_MultiresVP8RdPerf) { + auto config = CreateTestConfig(); + config.filename = "FourPeople_1280x720_30"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = 300; + config.print_frame_level_stats = true; + config.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false, + 1280, 720); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::map> rd_stats; + for (size_t bitrate_kbps : kBitrateRdPerfKbps) { + std::vector rate_profiles = { + {bitrate_kbps, 30, config.num_frames}}; + + fixture->RunTest(rate_profiles, nullptr, nullptr, nullptr, nullptr); + + rd_stats[bitrate_kbps] = + fixture->GetStats().SliceAndCalcLayerVideoStatistic( + kNumFirstFramesToSkipAtRdPerfAnalysis, config.num_frames - 1); + } + + PrintRdPerf(rd_stats); +} + +TEST(VideoProcessorIntegrationTestLibvpx, DISABLED_SvcVP9RdPerf) { + auto config = CreateTestConfig(); + config.filename = "FourPeople_1280x720_30"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = 300; + config.print_frame_level_stats = true; + config.SetCodecSettings(cricket::kVp9CodecName, 1, 3, 3, true, true, false, + 1280, 720); + const auto frame_checker = rtc::MakeUnique(); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::map> rd_stats; + for (size_t bitrate_kbps : kBitrateRdPerfKbps) { + std::vector rate_profiles = { + {bitrate_kbps, 30, config.num_frames}}; + + fixture->RunTest(rate_profiles, nullptr, nullptr, nullptr, nullptr); + + rd_stats[bitrate_kbps] = + fixture->GetStats().SliceAndCalcLayerVideoStatistic( + kNumFirstFramesToSkipAtRdPerfAnalysis, config.num_frames - 1); + } + + PrintRdPerf(rd_stats); +} + +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc b/modules/video_coding/codecs/test/videocodec_test_mediacodec.cc similarity index 52% rename from modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc rename to modules/video_coding/codecs/test/videocodec_test_mediacodec.cc index dd64eb7d8e..f4ca157507 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc +++ b/modules/video_coding/codecs/test/videocodec_test_mediacodec.cc @@ -8,14 +8,15 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h" - #include #include #include +#include "api/test/create_videocodec_test_fixture.h" #include "common_types.h" // NOLINT(build/include) #include "media/base/mediaconstants.h" +#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h" +#include "test/gtest.h" #include "test/testsupport/fileutils.h" namespace webrtc { @@ -24,23 +25,23 @@ namespace test { namespace { const int kForemanNumFrames = 300; const int kForemanFramerateFps = 30; + +TestConfig CreateTestConfig() { + TestConfig config; + config.filename = "foreman_cif"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = kForemanNumFrames; + config.hw_encoder = true; + config.hw_decoder = true; + return config; +} } // namespace -class VideoProcessorIntegrationTestMediaCodec - : public VideoProcessorIntegrationTest { - protected: - VideoProcessorIntegrationTestMediaCodec() { - config_.filename = "foreman_cif"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = kForemanNumFrames; - config_.hw_encoder = true; - config_.hw_decoder = true; - } -}; - -TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsVp8) { - config_.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, false, false, false, - 352, 288); +TEST(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsVp8) { + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, false, false, false, + 352, 288); + auto fixture = CreateVideoCodecTestFixture(config); std::vector rate_profiles = { {500, kForemanFramerateFps, kForemanNumFrames}}; @@ -53,14 +54,18 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsVp8) { std::vector quality_thresholds = {{36, 31, 0.92, 0.86}}; - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); } -TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsH264CBP) { - config_.encoded_frame_checker = &h264_keyframe_checker_; - config_.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, - false, 352, 288); +TEST(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsH264CBP) { + auto config = CreateTestConfig(); + const auto frame_checker = rtc::MakeUnique< + VideoCodecTestFixtureImpl::H264KeyframeChecker>(); + config.encoded_frame_checker = frame_checker.get(); + config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false, + 352, 288); + auto fixture = CreateVideoCodecTestFixture(config); std::vector rate_profiles = { {500, kForemanFramerateFps, kForemanNumFrames}}; @@ -73,18 +78,23 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsH264CBP) { std::vector quality_thresholds = {{36, 31, 0.92, 0.86}}; - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); } // TODO(brandtr): Enable this test when we have trybots/buildbots with // HW encoders that support CHP. -TEST_F(VideoProcessorIntegrationTestMediaCodec, - DISABLED_ForemanCif500kbpsH264CHP) { - config_.h264_codec_settings.profile = H264::kProfileConstrainedHigh; - config_.encoded_frame_checker = &h264_keyframe_checker_; - config_.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, - false, 352, 288); +TEST(VideoProcessorIntegrationTestMediaCodec, + DISABLED_ForemanCif500kbpsH264CHP) { + auto config = CreateTestConfig(); + const auto frame_checker = rtc::MakeUnique< + VideoCodecTestFixtureImpl::H264KeyframeChecker>(); + + config.h264_codec_settings.profile = H264::kProfileConstrainedHigh; + config.encoded_frame_checker = frame_checker.get(); + config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false, + 352, 288); + auto fixture = CreateVideoCodecTestFixture(config); std::vector rate_profiles = { {500, kForemanFramerateFps, kForemanNumFrames}}; @@ -97,11 +107,12 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); } -TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanMixedRes100kbpsVp8H264) { +TEST(VideoProcessorIntegrationTestMediaCodec, ForemanMixedRes100kbpsVp8H264) { + auto config = CreateTestConfig(); const int kNumFrames = 30; // TODO(brandtr): Add H.264 when we have fixed the encoder. const std::vector codecs = {cricket::kVp8CodecName}; @@ -116,16 +127,17 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanMixedRes100kbpsVp8H264) { for (const auto& resolution : resolutions) { const int width = std::get<0>(resolution); const int height = std::get<1>(resolution); - config_.filename = std::string("foreman_") + std::to_string(width) + "x" + - std::to_string(height); - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = kNumFrames; - config_.SetCodecSettings(codec, 1, 1, 1, false, false, false, - width, height); + config.filename = std::string("foreman_") + std::to_string(width) + "x" + + std::to_string(height); + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = kNumFrames; + config.SetCodecSettings(codec, 1, 1, 1, false, false, false, width, + height); - ProcessFramesAndMaybeVerify( - rate_profiles, nullptr /* rc_thresholds */, &quality_thresholds, - nullptr /* bs_thresholds */, nullptr /* visualization_params */); + auto fixture = CreateVideoCodecTestFixture(config); + fixture->RunTest(rate_profiles, nullptr /* rc_thresholds */, + &quality_thresholds, nullptr /* bs_thresholds */, + nullptr /* visualization_params */); } } } diff --git a/modules/video_coding/codecs/test/videocodec_test_openh264.cc b/modules/video_coding/codecs/test/videocodec_test_openh264.cc new file mode 100644 index 0000000000..2bfd12fdff --- /dev/null +++ b/modules/video_coding/codecs/test/videocodec_test_openh264.cc @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/test/create_videocodec_test_fixture.h" +#include "media/base/mediaconstants.h" +#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace test { + +#if defined(WEBRTC_USE_H264) + +namespace { +// Codec settings. +const int kCifWidth = 352; +const int kCifHeight = 288; +const int kNumFrames = 100; + +TestConfig CreateTestConfig() { + TestConfig config; + config.filename = "foreman_cif"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = kNumFrames; + // Only allow encoder/decoder to use single core, for predictability. + config.use_single_core = true; + config.hw_encoder = false; + config.hw_decoder = false; + return config; +} +} // namespace + +TEST(VideoProcessorIntegrationTestOpenH264, ConstantHighBitrate) { + auto frame_checker = rtc::MakeUnique< + VideoCodecTestFixtureImpl::H264KeyframeChecker>(); + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, true, false, + kCifWidth, kCifHeight); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{500, 30, kNumFrames}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; + + std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, + nullptr); +} + +// H264: Enable SingleNalUnit packetization mode. Encoder should split +// large frames into multiple slices and limit length of NAL units. +TEST(VideoProcessorIntegrationTestOpenH264, SingleNalUnit) { + auto frame_checker = rtc::MakeUnique< + VideoCodecTestFixtureImpl::H264KeyframeChecker>(); + auto config = CreateTestConfig(); + config.h264_codec_settings.packetization_mode = + H264PacketizationMode::SingleNalUnit; + config.max_payload_size_bytes = 500; + config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, true, false, + kCifWidth, kCifHeight); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector rate_profiles = {{500, 30, kNumFrames}}; + + std::vector rc_thresholds = { + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; + + std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; + + BitstreamThresholds bs_thresholds = {config.max_payload_size_bytes}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, + &bs_thresholds, nullptr); +} + +#endif // defined(WEBRTC_USE_H264) + +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc b/modules/video_coding/codecs/test/videocodec_test_parameterized.cc similarity index 76% rename from modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc rename to modules/video_coding/codecs/test/videocodec_test_parameterized.cc index 2499ede07e..200af91536 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc +++ b/modules/video_coding/codecs/test/videocodec_test_parameterized.cc @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h" - +#include "api/test/create_videocodec_test_fixture.h" +#include "test/gtest.h" #include "test/testsupport/fileutils.h" namespace webrtc { @@ -36,14 +36,12 @@ const VisualizationParams kVisualizationParams = { false, // save_encoded_ivf false, // save_decoded_y4m }; - const int kNumFrames = 30; - } // namespace // Tests for plotting statistics from logs. class VideoProcessorIntegrationTestParameterized - : public VideoProcessorIntegrationTest, + : public ::testing::Test, public ::testing::WithParamInterface< ::testing::tuple> { protected: @@ -57,13 +55,14 @@ class VideoProcessorIntegrationTestParameterized size_t height, size_t framerate, const std::string& filename) { - config_.filename = filename; - config_.filepath = ResourcePath(filename, "yuv"); - config_.use_single_core = kUseSingleCore; - config_.measure_cpu = kMeasureCpu; - config_.hw_encoder = hw_codec_; - config_.hw_decoder = hw_codec_; - config_.num_frames = kNumFrames; + TestConfig config; + config.filename = filename; + config.filepath = ResourcePath(filename, "yuv"); + config.use_single_core = kUseSingleCore; + config.measure_cpu = kMeasureCpu; + config.hw_encoder = hw_codec_; + config.hw_decoder = hw_codec_; + config.num_frames = kNumFrames; const size_t num_simulcast_streams = codec_type_ == kVideoCodecVP8 ? kNumSpatialLayers : 1; @@ -71,18 +70,19 @@ class VideoProcessorIntegrationTestParameterized codec_type_ == kVideoCodecVP9 ? kNumSpatialLayers : 1; const std::string codec_name = CodecTypeToPayloadString(codec_type_); - config_.SetCodecSettings(codec_name, num_simulcast_streams, - num_spatial_layers, kNumTemporalLayers, - kDenoisingOn, kFrameDropperOn, kSpatialResizeOn, - width, height); + config.SetCodecSettings(codec_name, num_simulcast_streams, + num_spatial_layers, kNumTemporalLayers, + kDenoisingOn, kFrameDropperOn, kSpatialResizeOn, + width, height); std::vector rate_profiles = { {bitrate_, framerate, kNumFrames}}; - ProcessFramesAndMaybeVerify(rate_profiles, nullptr, nullptr, nullptr, - &kVisualizationParams); + fixture_ = CreateVideoCodecTestFixture(config); + fixture_->RunTest(rate_profiles, nullptr, nullptr, nullptr, + &kVisualizationParams); } - + std::unique_ptr fixture_; const size_t bitrate_; const VideoCodecType codec_type_; const bool hw_codec_; diff --git a/modules/video_coding/codecs/test/videocodec_test_videotoolbox.cc b/modules/video_coding/codecs/test/videocodec_test_videotoolbox.cc new file mode 100644 index 0000000000..67232b4f96 --- /dev/null +++ b/modules/video_coding/codecs/test/videocodec_test_videotoolbox.cc @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/test/create_videocodec_test_fixture.h" +#include "media/base/mediaconstants.h" +#include "modules/video_coding/codecs/test/objc_codec_factory_helper.h" +#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h" +#include "test/gtest.h" +#include "test/testsupport/fileutils.h" + +namespace webrtc { +namespace test { + +namespace { +const int kForemanNumFrames = 300; + +TestConfig CreateTestConfig() { + TestConfig config; + config.filename = "foreman_cif"; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = kForemanNumFrames; + config.hw_encoder = true; + config.hw_decoder = true; + return config; +} + +std::unique_ptr CreateTestFixtureWithConfig( + TestConfig config) { + auto decoder_factory = CreateObjCDecoderFactory(); + auto encoder_factory = CreateObjCEncoderFactory(); + return CreateVideoCodecTestFixture( + config, std::move(decoder_factory), std::move(encoder_factory)); +} +} // namespace + +// TODO(webrtc:9099): Disabled until the issue is fixed. +// HW codecs don't work on simulators. Only run these tests on device. +// #if TARGET_OS_IPHONE && !TARGET_IPHONE_SIMULATOR +// #define MAYBE_TEST TEST +// #else +#define MAYBE_TEST(s, name) TEST(s, DISABLED_##name) +// #endif + +// TODO(kthelgason): Use RC Thresholds when the internal bitrateAdjuster is no +// longer in use. +MAYBE_TEST(VideoProcessorIntegrationTestVideoToolbox, + ForemanCif500kbpsH264CBP) { + const auto frame_checker = rtc::MakeUnique< + VideoCodecTestFixtureImpl::H264KeyframeChecker>(); + auto config = CreateTestConfig(); + config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false, + 352, 288); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateTestFixtureWithConfig(config); + + std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; + + std::vector quality_thresholds = {{33, 29, 0.9, 0.82}}; + + fixture->RunTest(rate_profiles, nullptr, &quality_thresholds, nullptr, + nullptr); +} + +MAYBE_TEST(VideoProcessorIntegrationTestVideoToolbox, + ForemanCif500kbpsH264CHP) { + const auto frame_checker = rtc::MakeUnique< + VideoCodecTestFixtureImpl::H264KeyframeChecker>(); + auto config = CreateTestConfig(); + config.h264_codec_settings.profile = H264::kProfileConstrainedHigh; + config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false, + 352, 288); + config.encoded_frame_checker = frame_checker.get(); + auto fixture = CreateTestFixtureWithConfig(config); + + std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; + + std::vector quality_thresholds = {{33, 30, 0.91, 0.83}}; + + fixture->RunTest(rate_profiles, nullptr, &quality_thresholds, nullptr, + nullptr); +} + +} // namespace test +} // namespace webrtc diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc deleted file mode 100644 index 05abfdfc2d..0000000000 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc +++ /dev/null @@ -1,444 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h" - -#include - -#include "media/base/mediaconstants.h" -#include "modules/video_coding/codecs/test/test_config.h" -#include "modules/video_coding/utility/vp8_header_parser.h" -#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" -#include "rtc_base/ptr_util.h" -#include "test/testsupport/fileutils.h" - -namespace webrtc { -namespace test { - -namespace { -// Codec settings. -const int kCifWidth = 352; -const int kCifHeight = 288; -#if !defined(WEBRTC_IOS) -const int kNumFramesShort = 100; -#endif -const int kNumFramesLong = 300; -const size_t kBitrateRdPerfKbps[] = {100, 200, 300, 400, 500, 600, - 700, 800, 1000, 1250, 1400, 1600, - 1800, 2000, 2200, 2500}; -const size_t kNumFirstFramesToSkipAtRdPerfAnalysis = 60; -} // namespace - -class VideoProcessorIntegrationTestLibvpx - : public VideoProcessorIntegrationTest { - protected: - VideoProcessorIntegrationTestLibvpx() { - config_.filename = "foreman_cif"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = kNumFramesLong; - // Only allow encoder/decoder to use single core, for predictability. - config_.use_single_core = true; - config_.hw_encoder = false; - config_.hw_decoder = false; - config_.encoded_frame_checker = &qp_frame_checker_; - } - - void PrintRdPerf(std::map> rd_stats) { - printf("--> Summary\n"); - printf("%11s %5s %6s %11s %12s %11s %13s %13s %5s %7s %7s %7s %13s %13s\n", - "uplink_kbps", "width", "height", "spatial_idx", "temporal_idx", - "target_kbps", "downlink_kbps", "framerate_fps", "psnr", "psnr_y", - "psnr_u", "psnr_v", "enc_speed_fps", "dec_speed_fps"); - for (const auto& rd_stat : rd_stats) { - const size_t bitrate_kbps = rd_stat.first; - for (const auto& layer_stat : rd_stat.second) { - printf( - "%11zu %5zu %6zu %11zu %12zu %11zu %13zu %13.2f %5.2f %7.2f %7.2f " - "%7.2f" - "%13.2f %13.2f\n", - bitrate_kbps, layer_stat.width, layer_stat.height, - layer_stat.spatial_idx, layer_stat.temporal_idx, - layer_stat.target_bitrate_kbps, layer_stat.bitrate_kbps, - layer_stat.framerate_fps, layer_stat.avg_psnr, - layer_stat.avg_psnr_y, layer_stat.avg_psnr_u, layer_stat.avg_psnr_v, - layer_stat.enc_speed_fps, layer_stat.dec_speed_fps); - } - } - } - - private: - // Verify that the QP parser returns the same QP as the encoder does. - const class QpFrameChecker : public TestConfig::EncodedFrameChecker { - public: - void CheckEncodedFrame(webrtc::VideoCodecType codec, - const EncodedImage& encoded_frame) const override { - int qp; - if (codec == kVideoCodecVP8) { - EXPECT_TRUE( - vp8::GetQp(encoded_frame._buffer, encoded_frame._length, &qp)); - } else if (codec == kVideoCodecVP9) { - EXPECT_TRUE( - vp9::GetQp(encoded_frame._buffer, encoded_frame._length, &qp)); - } else { - RTC_NOTREACHED(); - } - EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP."; - } - } qp_frame_checker_; -}; - -// Fails on iOS. See webrtc:4755. -#if !defined(WEBRTC_IOS) - -#if !defined(RTC_DISABLE_VP9) -TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP9) { - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false, - kCifWidth, kCifHeight); - config_.num_frames = kNumFramesShort; - - std::vector rate_profiles = {{500, 30, kNumFramesShort}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.11, 0.3, 0.1, 0, 1}}; - - std::vector quality_thresholds = {{37, 36, 0.94, 0.92}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -TEST_F(VideoProcessorIntegrationTestLibvpx, ChangeBitrateVP9) { - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = { - {200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update - {700, 30, 200}, - {500, 30, kNumFramesLong}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.15, 0.5, 0.1, 0, 1}, - {15, 2, 0, 0.2, 0.5, 0.1, 0, 0}, - {10, 1, 0, 0.3, 0.5, 0.1, 0, 0}}; - - std::vector quality_thresholds = { - {34, 33, 0.90, 0.88}, {38, 35, 0.95, 0.91}, {35, 34, 0.93, 0.90}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -TEST_F(VideoProcessorIntegrationTestLibvpx, ChangeFramerateVP9) { - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = { - {100, 24, 100}, // target_kbps, input_fps, frame_index_rate_update - {100, 15, 200}, - {100, 10, kNumFramesLong}}; - - // Framerate mismatch should be lower for lower framerate. - std::vector rc_thresholds = { - {10, 2, 40, 0.4, 0.5, 0.2, 0, 1}, - {8, 2, 5, 0.2, 0.5, 0.2, 0, 0}, - {5, 2, 0, 0.21, 0.5, 0.3, 0, 0}}; - - // Quality should be higher for lower framerates for the same content. - std::vector quality_thresholds = { - {33, 32, 0.89, 0.87}, {33.5, 32, 0.90, 0.86}, {33.5, 31.5, 0.90, 0.85}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -TEST_F(VideoProcessorIntegrationTestLibvpx, DenoiserOnVP9) { - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, true, true, false, - kCifWidth, kCifHeight); - config_.num_frames = kNumFramesShort; - - std::vector rate_profiles = {{500, 30, kNumFramesShort}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.11, 0.3, 0.1, 0, 1}}; - - std::vector quality_thresholds = {{37.5, 36, 0.94, 0.93}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -TEST_F(VideoProcessorIntegrationTestLibvpx, VeryLowBitrateVP9) { - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 1, 1, false, true, true, - kCifWidth, kCifHeight); - - std::vector rate_profiles = {{50, 30, kNumFramesLong}}; - - std::vector rc_thresholds = { - {15, 3, 75, 1.0, 0.5, 0.4, 1, 1}}; - - std::vector quality_thresholds = {{28, 25, 0.80, 0.65}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// TODO(marpan): Add temporal layer test for VP9, once changes are in -// vp9 wrapper for this. - -#endif // !defined(RTC_DISABLE_VP9) - -TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP8) { - config_.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false, - kCifWidth, kCifHeight); - config_.num_frames = kNumFramesShort; - - std::vector rate_profiles = {{500, 30, kNumFramesShort}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; - - // std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; - // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse - // than quality of x86 version. Use lower thresholds for now. - std::vector quality_thresholds = {{35, 33, 0.91, 0.89}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -#endif // !defined(WEBRTC_IOS) - -// The tests below are currently disabled for Android. For ARM, the encoder -// uses |cpu_speed| = 12, as opposed to default |cpu_speed| <= 6 for x86, -// which leads to significantly different quality. The quality and rate control -// settings in the tests below are defined for encoder speed setting -// |cpu_speed| <= ~6. A number of settings would need to be significantly -// modified for the |cpu_speed| = 12 case. For now, keep the tests below -// disabled on Android. Some quality parameter in the above test has been -// adjusted to also pass for |cpu_speed| <= 12. - -// Too slow to finish before timeout on iOS. See webrtc:4755. -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_ChangeBitrateVP8 DISABLED_ChangeBitrateVP8 -#else -#define MAYBE_ChangeBitrateVP8 ChangeBitrateVP8 -#endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeBitrateVP8) { - config_.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = { - {200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update - {800, 30, 200}, - {500, 30, kNumFramesLong}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, - {15.5, 1, 0, 0.1, 0.2, 0.1, 0, 0}, - {15, 1, 0, 0.3, 0.2, 0.1, 0, 0}}; - - // std::vector quality_thresholds = { - // {33, 32, 0.89, 0.88}, {38, 36, 0.94, 0.93}, {35, 34, 0.92, 0.91}}; - // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse - // than quality of x86 version. Use lower thresholds for now. - std::vector quality_thresholds = { - {31.8, 31, 0.86, 0.85}, {36, 34.8, 0.92, 0.90}, {33.5, 32, 0.90, 0.88}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// Too slow to finish before timeout on iOS. See webrtc:4755. -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_ChangeFramerateVP8 DISABLED_ChangeFramerateVP8 -#else -#define MAYBE_ChangeFramerateVP8 ChangeFramerateVP8 -#endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeFramerateVP8) { - config_.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, true, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = { - {80, 24, 100}, // target_kbps, input_fps, frame_index_rate_update - {80, 15, 200}, - {80, 10, kNumFramesLong}}; - - // std::vector rc_thresholds = { - // {10, 2, 20, 0.4, 0.3, 0.1, 0, 1}, - // {5, 2, 5, 0.3, 0.3, 0.1, 0, 0}, - // {4, 2, 1, 0.2, 0.3, 0.2, 0, 0}}; - // TODO(webrtc:8757): ARM VP8 drops more frames than x86 version. Use lower - // thresholds for now. - std::vector rc_thresholds = { - {10, 2, 60, 0.5, 0.3, 0.3, 0, 1}, - {10, 2, 30, 0.3, 0.3, 0.3, 0, 0}, - {10, 2, 10, 0.2, 0.3, 0.2, 0, 0}}; - - // std::vector quality_thresholds = { - // {31, 30, 0.87, 0.86}, {32, 31, 0.89, 0.86}, {32, 30, 0.87, 0.82}}; - // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse - // than quality of x86 version. Use lower thresholds for now. - std::vector quality_thresholds = { - {31, 30, 0.85, 0.84}, {31.5, 30.5, 0.86, 0.84}, {30.5, 29, 0.83, 0.78}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// Too slow to finish before timeout on iOS. See webrtc:4755. -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_TemporalLayersVP8 DISABLED_TemporalLayersVP8 -#else -#define MAYBE_TemporalLayersVP8 TemporalLayersVP8 -#endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_TemporalLayersVP8) { - config_.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 3, true, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = {{200, 30, 150}, - {400, 30, kNumFramesLong}}; - - // std::vector rc_thresholds = { - // {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, {10, 2, 0, 0.1, 0.2, 0.1, 0, 1}}; - // TODO(webrtc:8757): ARM VP8 drops more frames than x86 version. Use lower - // thresholds for now. - std::vector rc_thresholds = { - {10, 1, 2, 0.3, 0.2, 0.1, 0, 1}, {12, 2, 3, 0.1, 0.2, 0.1, 0, 1}}; - - // Min SSIM drops because of high motion scene with complex backgound (trees). - // std::vector quality_thresholds = {{32, 30, 0.88, 0.85}, - // {33, 30, 0.89, 0.83}}; - // TODO(webrtc:8757): ARM VP8 encoder's quality is significantly worse - // than quality of x86 version. Use lower thresholds for now. - std::vector quality_thresholds = {{31, 30, 0.85, 0.84}, - {31, 28, 0.85, 0.75}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// Might be too slow on mobile platforms. -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_MultiresVP8 DISABLED_MultiresVP8 -#else -#define MAYBE_MultiresVP8 MultiresVP8 -#endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_MultiresVP8) { - config_.filename = "ConferenceMotion_1280_720_50"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = 100; - config_.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false, - 1280, 720); - - std::vector rate_profiles = {{1500, 30, config_.num_frames}}; - - std::vector rc_thresholds = { - {5, 1, 5, 0.2, 0.3, 0.1, 0, 1}}; - std::vector quality_thresholds = {{34, 32, 0.90, 0.88}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// Might be too slow on mobile platforms. -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_SimulcastVP8 DISABLED_SimulcastVP8 -#else -#define MAYBE_SimulcastVP8 SimulcastVP8 -#endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_SimulcastVP8) { - config_.filename = "ConferenceMotion_1280_720_50"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = 100; - config_.simulcast_adapted_encoder = true; - config_.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false, - 1280, 720); - - std::vector rate_profiles = {{1500, 30, config_.num_frames}}; - - std::vector rc_thresholds = { - {20, 5, 90, 0.8, 0.5, 0.3, 0, 1}}; - std::vector quality_thresholds = {{34, 32, 0.90, 0.88}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// Might be too slow on mobile platforms. -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_SvcVP9 DISABLED_SvcVP9 -#else -#define MAYBE_SvcVP9 SvcVP9 -#endif -TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_SvcVP9) { - config_.filename = "ConferenceMotion_1280_720_50"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = 100; - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 3, 3, true, true, false, - 1280, 720); - - std::vector rate_profiles = {{1500, 30, config_.num_frames}}; - - std::vector rc_thresholds = { - {5, 1, 5, 0.2, 0.3, 0.1, 0, 1}}; - std::vector quality_thresholds = {{36, 34, 0.93, 0.91}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -TEST_F(VideoProcessorIntegrationTestLibvpx, DISABLED_MultiresVP8RdPerf) { - config_.filename = "FourPeople_1280x720_30"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = 300; - config_.print_frame_level_stats = true; - config_.SetCodecSettings(cricket::kVp8CodecName, 3, 1, 3, true, true, false, - 1280, 720); - - std::map> rd_stats; - for (size_t bitrate_kbps : kBitrateRdPerfKbps) { - std::vector rate_profiles = { - {bitrate_kbps, 30, config_.num_frames}}; - - ProcessFramesAndMaybeVerify(rate_profiles, nullptr, nullptr, nullptr, - nullptr); - - rd_stats[bitrate_kbps] = stats_.SliceAndCalcLayerVideoStatistic( - kNumFirstFramesToSkipAtRdPerfAnalysis, config_.num_frames - 1); - } - - PrintRdPerf(rd_stats); -} - -TEST_F(VideoProcessorIntegrationTestLibvpx, DISABLED_SvcVP9RdPerf) { - config_.filename = "FourPeople_1280x720_30"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = 300; - config_.print_frame_level_stats = true; - config_.SetCodecSettings(cricket::kVp9CodecName, 1, 3, 3, true, true, false, - 1280, 720); - - std::map> rd_stats; - for (size_t bitrate_kbps : kBitrateRdPerfKbps) { - std::vector rate_profiles = { - {bitrate_kbps, 30, config_.num_frames}}; - - ProcessFramesAndMaybeVerify(rate_profiles, nullptr, nullptr, nullptr, - nullptr); - - rd_stats[bitrate_kbps] = stats_.SliceAndCalcLayerVideoStatistic( - kNumFirstFramesToSkipAtRdPerfAnalysis, config_.num_frames - 1); - } - - PrintRdPerf(rd_stats); -} - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc deleted file mode 100644 index d218b0c660..0000000000 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h" - -#include - -#include "media/base/mediaconstants.h" -#include "test/testsupport/fileutils.h" - -namespace webrtc { -namespace test { - -#if defined(WEBRTC_USE_H264) - -namespace { -// Codec settings. -const int kCifWidth = 352; -const int kCifHeight = 288; -const int kNumFrames = 100; -} // namespace - -class VideoProcessorIntegrationTestOpenH264 - : public VideoProcessorIntegrationTest { - protected: - VideoProcessorIntegrationTestOpenH264() { - config_.filename = "foreman_cif"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = kNumFrames; - // Only allow encoder/decoder to use single core, for predictability. - config_.use_single_core = true; - config_.hw_encoder = false; - config_.hw_decoder = false; - config_.encoded_frame_checker = &h264_keyframe_checker_; - } -}; - -TEST_F(VideoProcessorIntegrationTestOpenH264, ConstantHighBitrate) { - config_.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = {{500, 30, kNumFrames}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; - - std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, nullptr, nullptr); -} - -// H264: Enable SingleNalUnit packetization mode. Encoder should split -// large frames into multiple slices and limit length of NAL units. -TEST_F(VideoProcessorIntegrationTestOpenH264, SingleNalUnit) { - config_.h264_codec_settings.packetization_mode = - H264PacketizationMode::SingleNalUnit; - config_.max_payload_size_bytes = 500; - config_.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, true, false, - kCifWidth, kCifHeight); - - std::vector rate_profiles = {{500, 30, kNumFrames}}; - - std::vector rc_thresholds = { - {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; - - std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; - - BitstreamThresholds bs_thresholds = {config_.max_payload_size_bytes}; - - ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, - &quality_thresholds, &bs_thresholds, nullptr); -} - -#endif // defined(WEBRTC_USE_H264) - -} // namespace test -} // namespace webrtc diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc deleted file mode 100644 index 6ff6a57afc..0000000000 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h" - -#include - -#include "media/base/mediaconstants.h" -#include "modules/video_coding/codecs/test/objc_codec_factory_helper.h" -#include "test/testsupport/fileutils.h" - -namespace webrtc { -namespace test { - -namespace { -const int kForemanNumFrames = 300; -} // namespace - -class VideoProcessorIntegrationTestVideoToolbox - : public VideoProcessorIntegrationTest { - protected: - VideoProcessorIntegrationTestVideoToolbox() { - config_.filename = "foreman_cif"; - config_.filepath = ResourcePath(config_.filename, "yuv"); - config_.num_frames = kForemanNumFrames; - config_.hw_encoder = true; - config_.hw_decoder = true; - config_.encoded_frame_checker = &h264_keyframe_checker_; - } - - std::unique_ptr CreateDecoderFactory() override { - if (config_.hw_decoder) { - EXPECT_EQ(kVideoCodecH264, config_.codec_settings.codecType) - << "iOS HW codecs only support H264."; - return CreateObjCDecoderFactory(); - } - RTC_NOTREACHED() << "Only support HW decoder on iOS."; - return nullptr; - } - - std::unique_ptr CreateEncoderFactory() override { - if (config_.hw_encoder) { - EXPECT_EQ(kVideoCodecH264, config_.codec_settings.codecType) - << "iOS HW codecs only support H264."; - return CreateObjCEncoderFactory(); - } - RTC_NOTREACHED() << "Only support HW encoder on iOS."; - return nullptr; - } -}; - -// TODO(webrtc:9099): Disabled until the issue is fixed. -// HW codecs don't work on simulators. Only run these tests on device. -// #if TARGET_OS_IPHONE && !TARGET_IPHONE_SIMULATOR -// #define MAYBE_TEST_F TEST_F -// #else -#define MAYBE_TEST_F(s, name) TEST_F(s, DISABLED_##name) -// #endif - -// TODO(kthelgason): Use RC Thresholds when the internal bitrateAdjuster is no -// longer in use. -MAYBE_TEST_F(VideoProcessorIntegrationTestVideoToolbox, - ForemanCif500kbpsH264CBP) { - config_.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, - false, 352, 288); - - std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; - - std::vector quality_thresholds = {{33, 29, 0.9, 0.82}}; - - ProcessFramesAndMaybeVerify(rate_profiles, nullptr, - &quality_thresholds, nullptr, nullptr); -} - -MAYBE_TEST_F(VideoProcessorIntegrationTestVideoToolbox, - ForemanCif500kbpsH264CHP) { - config_.h264_codec_settings.profile = H264::kProfileConstrainedHigh; - config_.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, - false, 352, 288); - - std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; - - std::vector quality_thresholds = {{33, 30, 0.91, 0.83}}; - - ProcessFramesAndMaybeVerify(rate_profiles, nullptr, - &quality_thresholds, nullptr, nullptr); -} - -} // namespace test -} // namespace webrtc