admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce test case name in peer connection e2e test framework.

Browse Source 
Introduce test case name for proper metrics reporting across different
parts of framework.

Bug: webrtc:10138
Change-Id: I7c501413ca2f2ee40314d988855dec0c28381c47
Reviewed-on: https://webrtc-review.googlesource.com/c/124740
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26886}
This commit is contained in:
Artem Titov 2019-02-27 17:44:13 +01:00 committed by Commit Bot
parent f5e5f0d643
commit 5983585ee8
15 changed files with 55 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc
View File

@ -15,6 +15,10 @@
namespace webrtc {
namespace test {
void DefaultAudioQualityAnalyzer::Start(std::string test_case_name) {
test_case_name_ = std::move(test_case_name);
}
void DefaultAudioQualityAnalyzer::OnStatsReports(
absl::string_view pc_label,
const StatsReports& stats_reports) {

4
test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h
View File

@ -20,8 +20,12 @@ namespace test {
class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface {
public:
void Start(std::string test_case_name) override;
void OnStatsReports(absl::string_view pc_label,
const StatsReports& stats_reports) override;
private:
std::string test_case_name_;
};
} // namespace test

8
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View File

@ -46,13 +46,15 @@ double RateCounter::GetEventsPerSecond() const {
(event_last_time_ - event_first_time_).us() * kMicrosPerSecond;
}
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(std::string test_label)
: test_label_(std::move(test_label)), clock_(Clock::GetRealTimeClock()) {}
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer()
: clock_(Clock::GetRealTimeClock()) {}
DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() {
Stop();
}
void DefaultVideoQualityAnalyzer::Start(int max_threads_count) {
void DefaultVideoQualityAnalyzer::Start(std::string test_case_name,
int max_threads_count) {
test_label_ = std::move(test_case_name);
for (int i = 0; i < max_threads_count; i++) {
auto thread = absl::make_unique<rtc::PlatformThread>(
&DefaultVideoQualityAnalyzer::ProcessComparisonsThread, this,

8
test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
View File

@ -117,10 +117,10 @@ struct AnalyzerStats {
class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
public:
explicit DefaultVideoQualityAnalyzer(std::string test_label);
DefaultVideoQualityAnalyzer();
~DefaultVideoQualityAnalyzer() override;
void Start(int max_threads_count) override;
void Start(std::string test_case_name, int max_threads_count) override;
uint16_t OnFrameCaptured(const std::string& stream_label,
const VideoFrame& frame) override;
void OnFramePreEncode(const VideoFrame& frame) override;
@ -247,11 +247,11 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
std::string GetTestCaseName(const std::string& stream_label) const;
Timestamp Now();
const std::string test_label_;
webrtc::Clock* const clock_;
std::atomic<uint16_t> next_frame_id_{0};
std::string test_label_;
rtc::CriticalSection lock_;
State state_ RTC_GUARDED_BY(lock_) = State::kNew;
// Frames that were captured by all streams and still aren't rendered by any

3
test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc
View File

@ -18,7 +18,8 @@ namespace test {
ExampleVideoQualityAnalyzer::ExampleVideoQualityAnalyzer() = default;
ExampleVideoQualityAnalyzer::~ExampleVideoQualityAnalyzer() = default;
void ExampleVideoQualityAnalyzer::Start(int max_threads_count) {}
void ExampleVideoQualityAnalyzer::Start(std::string test_case_name,
int max_threads_count) {}
uint16_t ExampleVideoQualityAnalyzer::OnFrameCaptured(
const std::string& stream_label,

2
test/pc/e2e/analyzer/video/example_video_quality_analyzer.h
View File

@ -33,7 +33,7 @@ class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
ExampleVideoQualityAnalyzer();
~ExampleVideoQualityAnalyzer() override;
void Start(int max_threads_count) override;
void Start(std::string test_case_name, int max_threads_count) override;
uint16_t OnFrameCaptured(const std::string& stream_label,
const VideoFrame& frame) override;
void OnFramePreEncode(const VideoFrame& frame) override;

5
test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc
View File

@ -133,8 +133,9 @@ VideoQualityAnalyzerInjectionHelper::CreateVideoSink(
return absl::make_unique<AnalyzingVideoSink>(analyzer_.get(), writer);
}
void VideoQualityAnalyzerInjectionHelper::Start(int max_threads_count) {
analyzer_->Start(max_threads_count);
void VideoQualityAnalyzerInjectionHelper::Start(std::string test_case_name,
int max_threads_count) {
analyzer_->Start(std::move(test_case_name), max_threads_count);
}
void VideoQualityAnalyzerInjectionHelper::OnStatsReports(

2
test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h
View File

@ -64,7 +64,7 @@ class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface {
std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> CreateVideoSink(
VideoFrameWriter* writer) const;
void Start(int max_threads_count);
void Start(std::string test_case_name, int max_threads_count);
// Forwards |stats_reports| for Peer Connection |pc_label| to
// |analyzer_|.

5
test/pc/e2e/api/audio_quality_analyzer_interface.h
View File

@ -19,6 +19,11 @@ namespace test {
class AudioQualityAnalyzerInterface : public StatsObserverInterface {
public:
~AudioQualityAnalyzerInterface() override = default;
// Will be called by framework before test.
// |test_case_name| is name of test case, that should be used to report all
// audio metrics.
virtual void Start(std::string test_case_name) = 0;
};
} // namespace test

4
test/pc/e2e/api/create_peerconnection_quality_test_fixture.cc
View File

@ -20,10 +20,12 @@ namespace test {
std::unique_ptr<PeerConnectionE2EQualityTestFixture>
CreatePeerConnectionE2EQualityTestFixture(
std::string test_case_name,
std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer,
std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer) {
return absl::make_unique<webrtc::test::PeerConnectionE2EQualityTest>(
std::move(audio_quality_analyzer), std::move(video_quality_analyzer));
std::move(test_case_name), std::move(audio_quality_analyzer),
std::move(video_quality_analyzer));
}
} // namespace test

3
test/pc/e2e/api/create_peerconnection_quality_test_fixture.h
View File

@ -22,8 +22,11 @@ namespace test {
// API is in development. Can be changed/removed without notice.
// Create test fixture to establish test call between Alice and Bob.
// During the test Alice will be caller and Bob will answer the call.
// |test_case_name| is a name of test case, that will be used for all metrics
// reporting.
std::unique_ptr<PeerConnectionE2EQualityTestFixture>
CreatePeerConnectionE2EQualityTestFixture(
std::string test_case_name,
std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer,
std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer);

13
test/pc/e2e/api/video_quality_analyzer_interface.h
View File

@ -54,11 +54,14 @@ class VideoQualityAnalyzerInterface : public StatsObserverInterface {
public:
~VideoQualityAnalyzerInterface() override = default;
// Will be called by framework before test. |threads_count| is number of
// threads that analyzer can use for heavy calculations. Analyzer can perform
// simple calculations on the calling thread in each method, but should
// remember, that it is the same thread, that is used in video pipeline.
virtual void Start(int max_threads_count) {}
// Will be called by framework before test.
// |test_case_name| is name of test case, that should be used to report all
// video metrics.
// |threads_count| is number of threads that analyzer can use for heavy
// calculations. Analyzer can perform simple calculations on the calling
// thread in each method, but should remember, that it is the same thread,
// that is used in video pipeline.
virtual void Start(std::string test_case_name, int max_threads_count) {}
// Will be called when frame was generated from the input stream.
// Returns frame id, that will be set by framework to the frame.

5
test/pc/e2e/peer_connection_e2e_smoke_test.cc
View File

@ -100,7 +100,7 @@ TEST(PeerConnectionE2EQualityTestSmokeTest, RunWithEmulatedNetwork) {
// Create analyzers.
std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer =
absl::make_unique<DefaultVideoQualityAnalyzer>("smoke_test");
absl::make_unique<DefaultVideoQualityAnalyzer>();
// This is only done for the sake of smoke testing. In general there should
// be no need to explicitly pull data from analyzers after the run.
auto* video_analyzer_ptr =
@ -110,7 +110,8 @@ TEST(PeerConnectionE2EQualityTestSmokeTest, RunWithEmulatedNetwork) {
absl::make_unique<DefaultAudioQualityAnalyzer>();
auto fixture = CreatePeerConnectionE2EQualityTestFixture(
std::move(audio_quality_analyzer), std::move(video_quality_analyzer));
"smoke_test", std::move(audio_quality_analyzer),
std::move(video_quality_analyzer));
fixture->Run(std::move(alice_components), std::move(alice_params),
std::move(bob_components), absl::make_unique<Params>(),
RunParams{TimeDelta::seconds(5)});

9
test/pc/e2e/peer_connection_quality_test.cc
View File

@ -97,9 +97,12 @@ class FixturePeerConnectionObserver : public MockPeerConnectionObserver {
} // namespace
PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest(
std::string test_case_name,
std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer,
std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer)
: clock_(Clock::GetRealTimeClock()), task_queue_("pc_e2e_quality_test") {
: clock_(Clock::GetRealTimeClock()),
test_case_name_(std::move(test_case_name)),
task_queue_("pc_e2e_quality_test") {
// Create default video quality analyzer. We will always create an analyzer,
// even if there are no video streams, because it will be installed into video
// encoder/decoder factories.
@ -194,7 +197,9 @@ void PeerConnectionE2EQualityTest::Run(
std::min(video_analyzer_threads, kMaxVideoAnalyzerThreads);
RTC_LOG(INFO) << "video_analyzer_threads=" << video_analyzer_threads;
video_quality_analyzer_injection_helper_->Start(video_analyzer_threads);
video_quality_analyzer_injection_helper_->Start(test_case_name_,
video_analyzer_threads);
audio_quality_analyzer_->Start(test_case_name_);
// Start RTCEventLog recording if requested.
if (alice_->params()->rtc_event_log_path) {

2
test/pc/e2e/peer_connection_quality_test.h
View File

@ -42,6 +42,7 @@ class PeerConnectionE2EQualityTest
using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig;
PeerConnectionE2EQualityTest(
std::string test_case_name,
std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer,
std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer);
@ -82,6 +83,7 @@ class PeerConnectionE2EQualityTest
const VideoConfig& config);
Clock* const clock_;
std::string test_case_name_;
std::unique_ptr<VideoQualityAnalyzerInjectionHelper>
video_quality_analyzer_injection_helper_;
std::unique_ptr<SingleProcessEncodedImageDataInjector>