From b6c6201b0fc5a769c7e6cf00e6f878e8eb181f62 Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Tue, 8 Jan 2019 14:58:23 +0100 Subject: [PATCH] Introduce peer connection end-2-end quality test fixture interface. Also introduce interface for video quality analyze and mock interface, that then will be extended for audio quality analyze. Bug: webrtc:10138 Change-Id: I0e3957fb2af1b12e796f154765580ddf562c7814 Reviewed-on: https://webrtc-review.googlesource.com/c/116500 Commit-Queue: Artem Titov Reviewed-by: Stefan Holmer Reviewed-by: Yves Gerey Reviewed-by: Ilya Nikolaevskiy Reviewed-by: Mirko Bonadei Cr-Commit-Position: refs/heads/master@{#26157} --- test/BUILD.gn | 1 + test/pc/e2e/api/BUILD.gn | 53 +++++++ .../api/audio_quality_analyzer_interface.h | 20 +++ .../api/peerconnection_quality_test_fixture.h | 138 ++++++++++++++++++ .../api/video_quality_analyzer_interface.h | 70 +++++++++ 5 files changed, 282 insertions(+) create mode 100644 test/pc/e2e/api/BUILD.gn create mode 100644 test/pc/e2e/api/audio_quality_analyzer_interface.h create mode 100644 test/pc/e2e/api/peerconnection_quality_test_fixture.h create mode 100644 test/pc/e2e/api/video_quality_analyzer_interface.h diff --git a/test/BUILD.gn b/test/BUILD.gn index b9f3d0b51a..ffa6d01ddb 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -21,6 +21,7 @@ group("test") { ":test_renderer", ":test_support", ":video_test_common", + "pc/e2e/api:peer_connection_quality_test_fixture_api", ] if (rtc_include_tests) { diff --git a/test/pc/e2e/api/BUILD.gn b/test/pc/e2e/api/BUILD.gn new file mode 100644 index 0000000000..f9fc5387a2 --- /dev/null +++ b/test/pc/e2e/api/BUILD.gn @@ -0,0 +1,53 @@ +# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../../webrtc.gni") + +rtc_source_set("video_quality_analyzer_api") { + visibility = [ "*" ] + sources = [ + "video_quality_analyzer_interface.h", + ] + + deps = [ + "../../../../api/video:encoded_image", + "../../../../api/video:video_frame", + "../../../../api/video_codecs:video_codecs_api", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_source_set("audio_quality_analyzer_api") { + visibility = [ "*" ] + sources = [ + "audio_quality_analyzer_interface.h", + ] + + deps = [] +} + +rtc_source_set("peer_connection_quality_test_fixture_api") { + visibility = [ "*" ] + sources = [ + "peerconnection_quality_test_fixture.h", + ] + + deps = [ + ":audio_quality_analyzer_api", + ":video_quality_analyzer_api", + "../../../../api:callfactory_api", + "../../../../api:fec_controller_api", + "../../../../api:libjingle_peerconnection_api", + "../../../../api:simulated_network_api", + "../../../../api/transport:network_control", + "../../../../api/video_codecs:video_codecs_api", + "../../../../logging:rtc_event_log_api", + "../../../../rtc_base:rtc_base", + "//third_party/abseil-cpp/absl/types:optional", + ] +} diff --git a/test/pc/e2e/api/audio_quality_analyzer_interface.h b/test/pc/e2e/api/audio_quality_analyzer_interface.h new file mode 100644 index 0000000000..25f7410c39 --- /dev/null +++ b/test/pc/e2e/api/audio_quality_analyzer_interface.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_API_AUDIO_QUALITY_ANALYZER_INTERFACE_H_ +#define TEST_PC_E2E_API_AUDIO_QUALITY_ANALYZER_INTERFACE_H_ + +namespace webrtc { + +class AudioQualityAnalyzerInterface {}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_API_AUDIO_QUALITY_ANALYZER_INTERFACE_H_ diff --git a/test/pc/e2e/api/peerconnection_quality_test_fixture.h b/test/pc/e2e/api/peerconnection_quality_test_fixture.h new file mode 100644 index 0000000000..1640340457 --- /dev/null +++ b/test/pc/e2e/api/peerconnection_quality_test_fixture.h @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PC_E2E_API_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ +#define TEST_PC_E2E_API_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ + +#include +#include +#include + +#include "api/asyncresolverfactory.h" +#include "api/call/callfactoryinterface.h" +#include "api/fec_controller.h" +#include "api/media_transport_interface.h" +#include "api/peerconnectioninterface.h" +#include "api/test/simulated_network.h" +#include "api/transport/network_control.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "logging/rtc_event_log/rtc_event_log_factory_interface.h" +#include "rtc_base/network.h" +#include "rtc_base/rtccertificategenerator.h" +#include "rtc_base/sslcertificate.h" +#include "rtc_base/thread.h" +#include "test/pc/e2e/api/audio_quality_analyzer_interface.h" +#include "test/pc/e2e/api/video_quality_analyzer_interface.h" + +namespace webrtc { + +// TODO(titovartem) move to API when it will be stabilized. +class PeerConnectionE2EQualityTestFixture { + public: + struct PeerConnectionFactoryComponents { + std::unique_ptr call_factory; + std::unique_ptr event_log_factory; + std::unique_ptr fec_controller_factory; + std::unique_ptr + network_controller_factory; + std::unique_ptr media_transport_factory; + + // Will be passed to MediaEngineInterface, that will be used in + // PeerConnectionFactory. + std::unique_ptr video_encoder_factory; + std::unique_ptr video_decoder_factory; + }; + + struct PeerConnectionComponents { + std::unique_ptr network_manager; + std::unique_ptr async_resolver_factory; + std::unique_ptr cert_generator; + std::unique_ptr tls_cert_verifier; + }; + + struct InjectableComponents { + explicit InjectableComponents(rtc::Thread* network_thread) + : network_thread(network_thread) {} + + rtc::Thread* network_thread; + + std::unique_ptr pcf_dependencies; + std::unique_ptr pc_dependencies; + }; + + struct ScreenShareConfig { + // If true, slides will be generated programmatically. + bool generate_slides; + int32_t slide_change_interval; + // If equal to 0, no scrolling will be applied. + int32_t scroll_duration; + // If empty, default set of slides will be used. + std::vector slides_yuv_file_names; + }; + + struct VideoConfig { + size_t width; + size_t height; + int32_t fps; + // Have to be unique among all specified configs for all peers in the call. + absl::optional stream_label; + // Only single from 3 next fields can be specified. + // If specified generator with this name will be used as input. + absl::optional generator_name; + // If specified this file will be used as input. + absl::optional input_file_name; + // If specified screen share video stream will be created as input. + absl::optional screen_share_config; + // If specified the input stream will be also copied to specified file. + absl::optional input_dump_file_name; + // If specified this file will be used as output on the receiver side for + // this stream. If multiple streams will be produced by input stream, + // output files will be appended with indexes. + absl::optional output_file_name; + }; + + struct AudioConfig { + enum Mode { + kGenerated, + kFile, + }; + Mode mode; + // Have to be specified only if mode = kFile + absl::optional input_file_name; + // If specified the input stream will be also copied to specified file. + absl::optional input_dump_file_name; + // If specified the output stream will be copied to specified file. + absl::optional output_file_name; + // Audio options to use. + cricket::AudioOptions audio_options; + }; + + struct Params { + // If |video_configs| is empty - no video should be added to the test call. + std::vector video_configs; + // If |audio_config| is presented audio stream will be configured + absl::optional audio_config; + + PeerConnectionInterface::RTCConfiguration rtc_configuration; + }; + + struct Analyzers { + std::unique_ptr audio_quality_analyzer; + std::unique_ptr video_quality_analyzer; + }; + + virtual void Run() = 0; + virtual ~PeerConnectionE2EQualityTestFixture() = default; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_API_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ diff --git a/test/pc/e2e/api/video_quality_analyzer_interface.h b/test/pc/e2e/api/video_quality_analyzer_interface.h new file mode 100644 index 0000000000..ae4a428b8c --- /dev/null +++ b/test/pc/e2e/api/video_quality_analyzer_interface.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_API_VIDEO_QUALITY_ANALYZER_INTERFACE_H_ +#define TEST_PC_E2E_API_VIDEO_QUALITY_ANALYZER_INTERFACE_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_encoder.h" + +namespace webrtc { + +class VideoQualityAnalyzerInterface { + public: + virtual ~VideoQualityAnalyzerInterface() = default; + + // Will be called by framework before test. |threads_count| is number of + // threads, that analyzer can use for heavy calculations. Analyzer can perform + // simple calculations on the calling thread in each method, but should + // remember, that is the same thread, that is used in video pipeline. + virtual void Start(uint16_t threads_count) {} + + // Will be called when frame was generated from the input stream. + // Returns frame id, that will be set by framework to the frame. + virtual uint16_t OnFrameCaptured(std::string stream_label, + const VideoFrame& frame) = 0; + // Will be called before calling the real encoder. + virtual void OnFramePreEncode(const VideoFrame& frame) {} + // Will be called for each EncodedImage received from encoder. Single + // VideoFrame can produce multiple EncodedImages. Each encoded image will + // have id from VideoFrame. + virtual void OnFrameEncoded(uint16_t frame_id, + const EncodedImage& encoded_image) {} + // Will be called for each frame dropped by encoder. + virtual void OnFrameDropped(EncodedImageCallback::DropReason reason) {} + // Will be called before calling the real decoder. + virtual void OnFrameReceived(uint16_t frame_id, + const EncodedImage& encoded_image) {} + // Will be called after decoding the frame. |decode_time_ms| is a decode + // time provided by decoder itself. If decoder doesn’t produce such + // information can be omitted. + virtual void OnFrameDecoded(const VideoFrame& frame, + absl::optional decode_time_ms, + absl::optional qp) {} + // Will be called when frame will be obtained from PeerConnection stack. + virtual void OnFrameRendered(const VideoFrame& frame) {} + // Will be called if real encoder return not WEBRTC_VIDEO_CODEC_OK. + virtual void OnEncoderError(const VideoFrame& frame, int32_t error_code) {} + // Will be called if real decoder return not WEBRTC_VIDEO_CODEC_OK. + virtual void OnDecoderError(uint16_t frame_id, int32_t error_code) {} + + // Tells analyzer, that analysis complete and it should calculate final + // statistics. + virtual void Stop() {} +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_API_VIDEO_QUALITY_ANALYZER_INTERFACE_H_