From 5983585ee868938ca06239a10de996fff31f2a9c Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Wed, 27 Feb 2019 17:44:13 +0100 Subject: [PATCH] Introduce test case name in peer connection e2e test framework. Introduce test case name for proper metrics reporting across different parts of framework. Bug: webrtc:10138 Change-Id: I7c501413ca2f2ee40314d988855dec0c28381c47 Reviewed-on: https://webrtc-review.googlesource.com/c/124740 Reviewed-by: Mirko Bonadei Commit-Queue: Artem Titov Cr-Commit-Position: refs/heads/master@{#26886} --- .../audio/default_audio_quality_analyzer.cc | 4 ++++ .../analyzer/audio/default_audio_quality_analyzer.h | 4 ++++ .../video/default_video_quality_analyzer.cc | 8 +++++--- .../analyzer/video/default_video_quality_analyzer.h | 8 ++++---- .../video/example_video_quality_analyzer.cc | 3 ++- .../analyzer/video/example_video_quality_analyzer.h | 2 +- .../video_quality_analyzer_injection_helper.cc | 5 +++-- .../video/video_quality_analyzer_injection_helper.h | 2 +- test/pc/e2e/api/audio_quality_analyzer_interface.h | 5 +++++ .../create_peerconnection_quality_test_fixture.cc | 4 +++- .../create_peerconnection_quality_test_fixture.h | 3 +++ test/pc/e2e/api/video_quality_analyzer_interface.h | 13 ++++++++----- test/pc/e2e/peer_connection_e2e_smoke_test.cc | 5 +++-- test/pc/e2e/peer_connection_quality_test.cc | 9 +++++++-- test/pc/e2e/peer_connection_quality_test.h | 2 ++ 15 files changed, 55 insertions(+), 22 deletions(-) diff --git a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc index 33315eb1b9..a5b3e1b786 100644 --- a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc @@ -15,6 +15,10 @@ namespace webrtc { namespace test { +void DefaultAudioQualityAnalyzer::Start(std::string test_case_name) { + test_case_name_ = std::move(test_case_name); +} + void DefaultAudioQualityAnalyzer::OnStatsReports( absl::string_view pc_label, const StatsReports& stats_reports) { diff --git a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h index aecd3e341f..83bcbfcb80 100644 --- a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h +++ b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h @@ -20,8 +20,12 @@ namespace test { class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface { public: + void Start(std::string test_case_name) override; void OnStatsReports(absl::string_view pc_label, const StatsReports& stats_reports) override; + + private: + std::string test_case_name_; }; } // namespace test diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index 2c91351bd7..601cd2bc78 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -46,13 +46,15 @@ double RateCounter::GetEventsPerSecond() const { (event_last_time_ - event_first_time_).us() * kMicrosPerSecond; } -DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(std::string test_label) - : test_label_(std::move(test_label)), clock_(Clock::GetRealTimeClock()) {} +DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer() + : clock_(Clock::GetRealTimeClock()) {} DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() { Stop(); } -void DefaultVideoQualityAnalyzer::Start(int max_threads_count) { +void DefaultVideoQualityAnalyzer::Start(std::string test_case_name, + int max_threads_count) { + test_label_ = std::move(test_case_name); for (int i = 0; i < max_threads_count; i++) { auto thread = absl::make_unique( &DefaultVideoQualityAnalyzer::ProcessComparisonsThread, this, diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index de6d09d9e8..1d30ccfe50 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -117,10 +117,10 @@ struct AnalyzerStats { class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { public: - explicit DefaultVideoQualityAnalyzer(std::string test_label); + DefaultVideoQualityAnalyzer(); ~DefaultVideoQualityAnalyzer() override; - void Start(int max_threads_count) override; + void Start(std::string test_case_name, int max_threads_count) override; uint16_t OnFrameCaptured(const std::string& stream_label, const VideoFrame& frame) override; void OnFramePreEncode(const VideoFrame& frame) override; @@ -247,11 +247,11 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { std::string GetTestCaseName(const std::string& stream_label) const; Timestamp Now(); - const std::string test_label_; - webrtc::Clock* const clock_; std::atomic next_frame_id_{0}; + std::string test_label_; + rtc::CriticalSection lock_; State state_ RTC_GUARDED_BY(lock_) = State::kNew; // Frames that were captured by all streams and still aren't rendered by any diff --git a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc index 2937bcb188..2ff5633391 100644 --- a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc @@ -18,7 +18,8 @@ namespace test { ExampleVideoQualityAnalyzer::ExampleVideoQualityAnalyzer() = default; ExampleVideoQualityAnalyzer::~ExampleVideoQualityAnalyzer() = default; -void ExampleVideoQualityAnalyzer::Start(int max_threads_count) {} +void ExampleVideoQualityAnalyzer::Start(std::string test_case_name, + int max_threads_count) {} uint16_t ExampleVideoQualityAnalyzer::OnFrameCaptured( const std::string& stream_label, diff --git a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h index 7e96d4c218..7bd93823f5 100644 --- a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h @@ -33,7 +33,7 @@ class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { ExampleVideoQualityAnalyzer(); ~ExampleVideoQualityAnalyzer() override; - void Start(int max_threads_count) override; + void Start(std::string test_case_name, int max_threads_count) override; uint16_t OnFrameCaptured(const std::string& stream_label, const VideoFrame& frame) override; void OnFramePreEncode(const VideoFrame& frame) override; diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc index d980110483..d146e71da9 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc @@ -133,8 +133,9 @@ VideoQualityAnalyzerInjectionHelper::CreateVideoSink( return absl::make_unique(analyzer_.get(), writer); } -void VideoQualityAnalyzerInjectionHelper::Start(int max_threads_count) { - analyzer_->Start(max_threads_count); +void VideoQualityAnalyzerInjectionHelper::Start(std::string test_case_name, + int max_threads_count) { + analyzer_->Start(std::move(test_case_name), max_threads_count); } void VideoQualityAnalyzerInjectionHelper::OnStatsReports( diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h index bf817ada17..fbe93cb430 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h @@ -64,7 +64,7 @@ class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface { std::unique_ptr> CreateVideoSink( VideoFrameWriter* writer) const; - void Start(int max_threads_count); + void Start(std::string test_case_name, int max_threads_count); // Forwards |stats_reports| for Peer Connection |pc_label| to // |analyzer_|. diff --git a/test/pc/e2e/api/audio_quality_analyzer_interface.h b/test/pc/e2e/api/audio_quality_analyzer_interface.h index 99d119c382..66da75a5e5 100644 --- a/test/pc/e2e/api/audio_quality_analyzer_interface.h +++ b/test/pc/e2e/api/audio_quality_analyzer_interface.h @@ -19,6 +19,11 @@ namespace test { class AudioQualityAnalyzerInterface : public StatsObserverInterface { public: ~AudioQualityAnalyzerInterface() override = default; + + // Will be called by framework before test. + // |test_case_name| is name of test case, that should be used to report all + // audio metrics. + virtual void Start(std::string test_case_name) = 0; }; } // namespace test diff --git a/test/pc/e2e/api/create_peerconnection_quality_test_fixture.cc b/test/pc/e2e/api/create_peerconnection_quality_test_fixture.cc index 8bb3712276..3b69d56919 100644 --- a/test/pc/e2e/api/create_peerconnection_quality_test_fixture.cc +++ b/test/pc/e2e/api/create_peerconnection_quality_test_fixture.cc @@ -20,10 +20,12 @@ namespace test { std::unique_ptr CreatePeerConnectionE2EQualityTestFixture( + std::string test_case_name, std::unique_ptr audio_quality_analyzer, std::unique_ptr video_quality_analyzer) { return absl::make_unique( - std::move(audio_quality_analyzer), std::move(video_quality_analyzer)); + std::move(test_case_name), std::move(audio_quality_analyzer), + std::move(video_quality_analyzer)); } } // namespace test diff --git a/test/pc/e2e/api/create_peerconnection_quality_test_fixture.h b/test/pc/e2e/api/create_peerconnection_quality_test_fixture.h index 52c8f9fe7f..b4dd2ab81d 100644 --- a/test/pc/e2e/api/create_peerconnection_quality_test_fixture.h +++ b/test/pc/e2e/api/create_peerconnection_quality_test_fixture.h @@ -22,8 +22,11 @@ namespace test { // API is in development. Can be changed/removed without notice. // Create test fixture to establish test call between Alice and Bob. // During the test Alice will be caller and Bob will answer the call. +// |test_case_name| is a name of test case, that will be used for all metrics +// reporting. std::unique_ptr CreatePeerConnectionE2EQualityTestFixture( + std::string test_case_name, std::unique_ptr audio_quality_analyzer, std::unique_ptr video_quality_analyzer); diff --git a/test/pc/e2e/api/video_quality_analyzer_interface.h b/test/pc/e2e/api/video_quality_analyzer_interface.h index 056a632647..5d90ab94c5 100644 --- a/test/pc/e2e/api/video_quality_analyzer_interface.h +++ b/test/pc/e2e/api/video_quality_analyzer_interface.h @@ -54,11 +54,14 @@ class VideoQualityAnalyzerInterface : public StatsObserverInterface { public: ~VideoQualityAnalyzerInterface() override = default; - // Will be called by framework before test. |threads_count| is number of - // threads that analyzer can use for heavy calculations. Analyzer can perform - // simple calculations on the calling thread in each method, but should - // remember, that it is the same thread, that is used in video pipeline. - virtual void Start(int max_threads_count) {} + // Will be called by framework before test. + // |test_case_name| is name of test case, that should be used to report all + // video metrics. + // |threads_count| is number of threads that analyzer can use for heavy + // calculations. Analyzer can perform simple calculations on the calling + // thread in each method, but should remember, that it is the same thread, + // that is used in video pipeline. + virtual void Start(std::string test_case_name, int max_threads_count) {} // Will be called when frame was generated from the input stream. // Returns frame id, that will be set by framework to the frame. diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc index 5e4a7c0b98..f96c9b3ae9 100644 --- a/test/pc/e2e/peer_connection_e2e_smoke_test.cc +++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc @@ -100,7 +100,7 @@ TEST(PeerConnectionE2EQualityTestSmokeTest, RunWithEmulatedNetwork) { // Create analyzers. std::unique_ptr video_quality_analyzer = - absl::make_unique("smoke_test"); + absl::make_unique(); // This is only done for the sake of smoke testing. In general there should // be no need to explicitly pull data from analyzers after the run. auto* video_analyzer_ptr = @@ -110,7 +110,8 @@ TEST(PeerConnectionE2EQualityTestSmokeTest, RunWithEmulatedNetwork) { absl::make_unique(); auto fixture = CreatePeerConnectionE2EQualityTestFixture( - std::move(audio_quality_analyzer), std::move(video_quality_analyzer)); + "smoke_test", std::move(audio_quality_analyzer), + std::move(video_quality_analyzer)); fixture->Run(std::move(alice_components), std::move(alice_params), std::move(bob_components), absl::make_unique(), RunParams{TimeDelta::seconds(5)}); diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index 22a42112f1..2f983d3d6b 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -97,9 +97,12 @@ class FixturePeerConnectionObserver : public MockPeerConnectionObserver { } // namespace PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest( + std::string test_case_name, std::unique_ptr audio_quality_analyzer, std::unique_ptr video_quality_analyzer) - : clock_(Clock::GetRealTimeClock()), task_queue_("pc_e2e_quality_test") { + : clock_(Clock::GetRealTimeClock()), + test_case_name_(std::move(test_case_name)), + task_queue_("pc_e2e_quality_test") { // Create default video quality analyzer. We will always create an analyzer, // even if there are no video streams, because it will be installed into video // encoder/decoder factories. @@ -194,7 +197,9 @@ void PeerConnectionE2EQualityTest::Run( std::min(video_analyzer_threads, kMaxVideoAnalyzerThreads); RTC_LOG(INFO) << "video_analyzer_threads=" << video_analyzer_threads; - video_quality_analyzer_injection_helper_->Start(video_analyzer_threads); + video_quality_analyzer_injection_helper_->Start(test_case_name_, + video_analyzer_threads); + audio_quality_analyzer_->Start(test_case_name_); // Start RTCEventLog recording if requested. if (alice_->params()->rtc_event_log_path) { diff --git a/test/pc/e2e/peer_connection_quality_test.h b/test/pc/e2e/peer_connection_quality_test.h index 968a637ca8..6388f4b1c9 100644 --- a/test/pc/e2e/peer_connection_quality_test.h +++ b/test/pc/e2e/peer_connection_quality_test.h @@ -42,6 +42,7 @@ class PeerConnectionE2EQualityTest using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; PeerConnectionE2EQualityTest( + std::string test_case_name, std::unique_ptr audio_quality_analyzer, std::unique_ptr video_quality_analyzer); @@ -82,6 +83,7 @@ class PeerConnectionE2EQualityTest const VideoConfig& config); Clock* const clock_; + std::string test_case_name_; std::unique_ptr video_quality_analyzer_injection_helper_; std::unique_ptr