diff --git a/test/scenario/BUILD.gn b/test/scenario/BUILD.gn index b9d0a8a4c2..aafafa54c4 100644 --- a/test/scenario/BUILD.gn +++ b/test/scenario/BUILD.gn @@ -59,15 +59,17 @@ if (rtc_include_tests) { "hardware_codecs.h", "network_node.cc", "network_node.h", - "quality_info.h", - "quality_stats.cc", - "quality_stats.h", + "performance_stats.h", "scenario.cc", "scenario.h", "scenario_config.cc", "scenario_config.h", "simulated_time.cc", "simulated_time.h", + "stats_collection.cc", + "stats_collection.h", + "video_frame_matcher.cc", + "video_frame_matcher.h", "video_stream.cc", "video_stream.h", ] @@ -156,8 +158,8 @@ if (rtc_include_tests) { rtc_source_set("scenario_unittests") { testonly = true sources = [ - "quality_stats_unittest.cc", "scenario_unittest.cc", + "stats_collection_unittest.cc", "video_stream_unittest.cc", ] deps = [ diff --git a/test/scenario/quality_info.h b/test/scenario/performance_stats.h similarity index 64% rename from test/scenario/quality_info.h rename to test/scenario/performance_stats.h index 0cf6eafff6..e58dab37bf 100644 --- a/test/scenario/quality_info.h +++ b/test/scenario/performance_stats.h @@ -1,5 +1,5 @@ /* - * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * Copyright 2019 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -7,17 +7,21 @@ * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ -#ifndef TEST_SCENARIO_QUALITY_INFO_H_ -#define TEST_SCENARIO_QUALITY_INFO_H_ +#ifndef TEST_SCENARIO_PERFORMANCE_STATS_H_ +#define TEST_SCENARIO_PERFORMANCE_STATS_H_ +#include "api/units/data_rate.h" +#include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/video_frame_buffer.h" +#include "test/statistics.h" namespace webrtc { namespace test { + struct VideoFramePair { - rtc::scoped_refptr captured; - rtc::scoped_refptr decoded; + rtc::scoped_refptr captured; + rtc::scoped_refptr decoded; Timestamp capture_time = Timestamp::MinusInfinity(); Timestamp render_time = Timestamp::PlusInfinity(); // A unique identifier for the spatial/temporal layer the decoded frame @@ -30,6 +34,16 @@ struct VideoFramePair { // decoded frame has matched differend captured frames. int repeated = 0; }; + +struct VideoQualityStats { + int captures_count = 0; + int valid_count = 0; + int lost_count = 0; + Statistics end_to_end_seconds; + Statistics frame_size; + Statistics psnr; +}; + } // namespace test } // namespace webrtc -#endif // TEST_SCENARIO_QUALITY_INFO_H_ +#endif // TEST_SCENARIO_PERFORMANCE_STATS_H_ diff --git a/test/scenario/scenario_config.h b/test/scenario/scenario_config.h index bc0fd0150a..84caa9b2ae 100644 --- a/test/scenario/scenario_config.h +++ b/test/scenario/scenario_config.h @@ -22,7 +22,7 @@ #include "api/units/time_delta.h" #include "api/video/video_codec_type.h" #include "test/frame_generator.h" -#include "test/scenario/quality_info.h" +#include "test/scenario/performance_stats.h" namespace webrtc { namespace test { diff --git a/test/scenario/scenario_unittest.cc b/test/scenario/scenario_unittest.cc index 2b1cec6394..e3abd48f50 100644 --- a/test/scenario/scenario_unittest.cc +++ b/test/scenario/scenario_unittest.cc @@ -9,8 +9,10 @@ */ #include -#include "test/scenario/scenario.h" #include "test/gtest.h" +#include "test/scenario/scenario.h" +#include "test/scenario/stats_collection.h" + namespace webrtc { namespace test { TEST(ScenarioTest, StartsAndStopsWithoutErrors) { diff --git a/test/scenario/stats_collection.cc b/test/scenario/stats_collection.cc new file mode 100644 index 0000000000..77b2ae0438 --- /dev/null +++ b/test/scenario/stats_collection.cc @@ -0,0 +1,68 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/scenario/stats_collection.h" + +#include + +#include "common_video/libyuv/include/webrtc_libyuv.h" + +namespace webrtc { +namespace test { + +VideoQualityAnalyzer::VideoQualityAnalyzer( + VideoQualityAnalyzerConfig config, + std::unique_ptr writer) + : config_(config), writer_(std::move(writer)) { + if (writer_) { + PrintHeaders(); + } +} + +VideoQualityAnalyzer::~VideoQualityAnalyzer() = default; + +void VideoQualityAnalyzer::PrintHeaders() { + writer_->Write( + "capture_time render_time capture_width capture_height render_width " + "render_height psnr\n"); +} + +std::function VideoQualityAnalyzer::Handler() { + return [this](VideoFramePair pair) { HandleFramePair(pair); }; +} + +void VideoQualityAnalyzer::HandleFramePair(VideoFramePair sample) { + double psnr = NAN; + RTC_CHECK(sample.captured); + ++stats_.captures_count; + if (!sample.decoded) { + ++stats_.lost_count; + } else { + psnr = I420PSNR(*sample.captured->ToI420(), *sample.decoded->ToI420()); + ++stats_.valid_count; + stats_.end_to_end_seconds.AddSample( + (sample.render_time - sample.capture_time).seconds()); + stats_.psnr.AddSample(psnr); + } + if (writer_) { + LogWriteFormat(writer_.get(), "%.3f %.3f %.3f %i %i %i %i %.3f\n", + sample.capture_time.seconds(), + sample.render_time.seconds(), + sample.captured->width(), sample.captured->height(), + sample.decoded->width(), sample.decoded->height(), psnr); + } +} + +VideoQualityStats VideoQualityAnalyzer::stats() const { + return stats_; +} + +} // namespace test +} // namespace webrtc diff --git a/test/scenario/stats_collection.h b/test/scenario/stats_collection.h new file mode 100644 index 0000000000..d1b46e4c78 --- /dev/null +++ b/test/scenario/stats_collection.h @@ -0,0 +1,46 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_SCENARIO_STATS_COLLECTION_H_ +#define TEST_SCENARIO_STATS_COLLECTION_H_ + +#include + +#include "test/logging/log_writer.h" +#include "test/scenario/performance_stats.h" + +namespace webrtc { +namespace test { + +struct VideoQualityAnalyzerConfig { + double psnr_coverage = 1; +}; + +class VideoQualityAnalyzer { + public: + explicit VideoQualityAnalyzer( + VideoQualityAnalyzerConfig config = VideoQualityAnalyzerConfig(), + std::unique_ptr writer = nullptr); + ~VideoQualityAnalyzer(); + void HandleFramePair(VideoFramePair sample); + VideoQualityStats stats() const; + void PrintHeaders(); + void PrintFrameInfo(const VideoFramePair& sample); + std::function Handler(); + + private: + const VideoQualityAnalyzerConfig config_; + VideoQualityStats stats_; + const std::unique_ptr writer_; +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_SCENARIO_STATS_COLLECTION_H_ diff --git a/test/scenario/quality_stats_unittest.cc b/test/scenario/stats_collection_unittest.cc similarity index 98% rename from test/scenario/quality_stats_unittest.cc rename to test/scenario/stats_collection_unittest.cc index bef9f4b6cc..ee01ad53c2 100644 --- a/test/scenario/quality_stats_unittest.cc +++ b/test/scenario/stats_collection_unittest.cc @@ -7,6 +7,7 @@ * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ +#include "test/scenario/stats_collection.h" #include "test/gtest.h" #include "test/scenario/scenario.h" diff --git a/test/scenario/quality_stats.cc b/test/scenario/video_frame_matcher.cc similarity index 74% rename from test/scenario/quality_stats.cc rename to test/scenario/video_frame_matcher.cc index 61d6edd13c..c0d7925db4 100644 --- a/test/scenario/quality_stats.cc +++ b/test/scenario/video_frame_matcher.cc @@ -7,7 +7,7 @@ * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ -#include "test/scenario/quality_stats.h" +#include "test/scenario/video_frame_matcher.h" #include @@ -83,7 +83,7 @@ void VideoFrameMatcher::OnDecodedFrame(const VideoFrame& frame, } while (!layer.captured_frames.empty() && layer.captured_frames.front().matched) { - HandleMatch(layer.captured_frames.front(), layer_id); + HandleMatch(std::move(layer.captured_frames.front()), layer_id); layer.captured_frames.pop_front(); } }); @@ -93,10 +93,27 @@ bool VideoFrameMatcher::Active() const { return !frame_pair_handlers_.empty(); } +void VideoFrameMatcher::HandleMatch(VideoFrameMatcher::CapturedFrame captured, + int layer_id) { + VideoFramePair frame_pair; + frame_pair.layer_id = layer_id; + frame_pair.captured = captured.frame; + frame_pair.capture_id = captured.id; + if (captured.best_decode) { + frame_pair.decode_id = captured.best_decode->id; + frame_pair.capture_time = captured.capture_time; + frame_pair.decoded = captured.best_decode->frame; + frame_pair.render_time = captured.best_decode->render_time; + frame_pair.repeated = captured.best_decode->repeat_count++; + } + for (auto& handler : frame_pair_handlers_) + handler(frame_pair); +} + void VideoFrameMatcher::Finalize() { for (auto& layer : layers_) { while (!layer.second.captured_frames.empty()) { - HandleMatch(layer.second.captured_frames.front(), layer.first); + HandleMatch(std::move(layer.second.captured_frames.front()), layer.first); layer.second.captured_frames.pop_front(); } } @@ -143,52 +160,5 @@ void DecodedFrameTap::OnFrame(const VideoFrame& frame) { layer_id_); } -VideoQualityAnalyzer::VideoQualityAnalyzer( - VideoQualityAnalyzerConfig config, - std::unique_ptr writer) - : config_(config), writer_(std::move(writer)) { - if (writer_) { - PrintHeaders(); - } -} - -VideoQualityAnalyzer::~VideoQualityAnalyzer() = default; - -void VideoQualityAnalyzer::PrintHeaders() { - writer_->Write( - "capture_time render_time capture_width capture_height render_width " - "render_height psnr\n"); -} - -std::function VideoQualityAnalyzer::Handler() { - return [this](VideoFramePair pair) { HandleFramePair(pair); }; -} - -void VideoQualityAnalyzer::HandleFramePair(VideoFramePair sample) { - double psnr = NAN; - RTC_CHECK(sample.captured); - ++stats_.captures_count; - if (!sample.decoded) { - ++stats_.lost_count; - } else { - psnr = I420PSNR(*sample.captured->ToI420(), *sample.decoded->ToI420()); - ++stats_.valid_count; - stats_.end_to_end_seconds.AddSample( - (sample.render_time - sample.capture_time).seconds()); - stats_.psnr.AddSample(psnr); - } - if (writer_) { - LogWriteFormat(writer_.get(), "%.3f %.3f %.3f %i %i %i %i %.3f\n", - sample.capture_time.seconds(), - sample.render_time.seconds(), - sample.captured->width(), sample.captured->height(), - sample.decoded->width(), sample.decoded->height(), psnr); - } -} - -VideoQualityStats VideoQualityAnalyzer::stats() const { - return stats_; -} - } // namespace test } // namespace webrtc diff --git a/test/scenario/quality_stats.h b/test/scenario/video_frame_matcher.h similarity index 67% rename from test/scenario/quality_stats.h rename to test/scenario/video_frame_matcher.h index 26dd6a32ac..1a4a60a086 100644 --- a/test/scenario/quality_stats.h +++ b/test/scenario/video_frame_matcher.h @@ -7,8 +7,8 @@ * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ -#ifndef TEST_SCENARIO_QUALITY_STATS_H_ -#define TEST_SCENARIO_QUALITY_STATS_H_ +#ifndef TEST_SCENARIO_VIDEO_FRAME_MATCHER_H_ +#define TEST_SCENARIO_VIDEO_FRAME_MATCHER_H_ #include #include @@ -17,19 +17,14 @@ #include #include -#include "absl/types/optional.h" #include "api/units/timestamp.h" #include "api/video/video_frame.h" #include "api/video/video_sink_interface.h" #include "api/video/video_source_interface.h" #include "rtc_base/ref_counted_object.h" #include "rtc_base/task_queue_for_test.h" -#include "rtc_base/time_utils.h" #include "system_wrappers/include/clock.h" -#include "test/logging/log_writer.h" -#include "test/scenario/quality_info.h" -#include "test/scenario/scenario_config.h" -#include "test/statistics.h" +#include "test/scenario/performance_stats.h" namespace webrtc { namespace test { @@ -72,21 +67,7 @@ class VideoFrameMatcher { rtc::scoped_refptr last_decode; int next_decoded_id = 1; }; - void HandleMatch(CapturedFrame& captured, int layer_id) { - VideoFramePair frame_pair; - frame_pair.layer_id = layer_id; - frame_pair.captured = captured.frame; - frame_pair.capture_id = captured.id; - if (captured.best_decode) { - frame_pair.decode_id = captured.best_decode->id; - frame_pair.capture_time = captured.capture_time; - frame_pair.decoded = captured.best_decode->frame; - frame_pair.render_time = captured.best_decode->render_time; - frame_pair.repeated = captured.best_decode->repeat_count++; - } - for (auto& handler : frame_pair_handlers_) - handler(frame_pair); - } + void HandleMatch(CapturedFrame captured, int layer_id); void Finalize(); int next_capture_id_ = 1; std::vector> frame_pair_handlers_; @@ -133,35 +114,6 @@ class DecodedFrameTap : public rtc::VideoSinkInterface { VideoFrameMatcher* const matcher_; int layer_id_; }; -struct VideoQualityAnalyzerConfig { - double psnr_coverage = 1; -}; -struct VideoQualityStats { - int captures_count = 0; - int valid_count = 0; - int lost_count = 0; - Statistics end_to_end_seconds; - Statistics frame_size; - Statistics psnr; -}; - -class VideoQualityAnalyzer { - public: - explicit VideoQualityAnalyzer( - VideoQualityAnalyzerConfig config = VideoQualityAnalyzerConfig(), - std::unique_ptr writer = nullptr); - ~VideoQualityAnalyzer(); - void HandleFramePair(VideoFramePair sample); - VideoQualityStats stats() const; - void PrintHeaders(); - void PrintFrameInfo(const VideoFramePair& sample); - std::function Handler(); - - private: - const VideoQualityAnalyzerConfig config_; - VideoQualityStats stats_; - const std::unique_ptr writer_; -}; } // namespace test } // namespace webrtc -#endif // TEST_SCENARIO_QUALITY_STATS_H_ +#endif // TEST_SCENARIO_VIDEO_FRAME_MATCHER_H_ diff --git a/test/scenario/video_stream.h b/test/scenario/video_stream.h index 66e8e909a7..cac5861011 100644 --- a/test/scenario/video_stream.h +++ b/test/scenario/video_stream.h @@ -21,8 +21,8 @@ #include "test/scenario/call_client.h" #include "test/scenario/column_printer.h" #include "test/scenario/network_node.h" -#include "test/scenario/quality_stats.h" #include "test/scenario/scenario_config.h" +#include "test/scenario/video_frame_matcher.h" #include "test/test_video_capturer.h" namespace webrtc {