diff --git a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc index 07f3d63b8b..b8f1740e46 100644 --- a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc @@ -12,7 +12,6 @@ #include "api/stats_types.h" #include "rtc_base/logging.h" -#include "test/testsupport/perf_test.h" namespace webrtc { namespace webrtc_pc_e2e { @@ -100,18 +99,21 @@ std::string DefaultAudioQualityAnalyzer::GetTestCaseName( } void DefaultAudioQualityAnalyzer::Stop() { + using ::webrtc::test::ImproveDirection; rtc::CritScope crit(&lock_); for (auto& item : streams_stats_) { - ReportResult("expand_rate", item.first, item.second.expand_rate, - "unitless"); + ReportResult("expand_rate", item.first, item.second.expand_rate, "unitless", + ImproveDirection::kSmallerIsBetter); ReportResult("accelerate_rate", item.first, item.second.accelerate_rate, - "unitless"); + "unitless", ImproveDirection::kSmallerIsBetter); ReportResult("preemptive_rate", item.first, item.second.preemptive_rate, - "unitless"); + "unitless", ImproveDirection::kSmallerIsBetter); ReportResult("speech_expand_rate", item.first, - item.second.speech_expand_rate, "unitless"); + item.second.speech_expand_rate, "unitless", + ImproveDirection::kSmallerIsBetter); ReportResult("preferred_buffer_size_ms", item.first, - item.second.preferred_buffer_size_ms, "ms"); + item.second.preferred_buffer_size_ms, "ms", + ImproveDirection::kNone); } } @@ -125,12 +127,13 @@ void DefaultAudioQualityAnalyzer::ReportResult( const std::string& metric_name, const std::string& stream_label, const SamplesStatsCounter& counter, - const std::string& unit) const { + const std::string& unit, + webrtc::test::ImproveDirection improve_direction) const { test::PrintResultMeanAndError( metric_name, /*modifier=*/"", GetTestCaseName(stream_label), counter.IsEmpty() ? 0 : counter.GetAverage(), counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), unit, - /*important=*/false); + /*important=*/false, improve_direction); } } // namespace webrtc_pc_e2e diff --git a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h index ee34ed345b..33aaefd4c3 100644 --- a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h +++ b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h @@ -19,6 +19,7 @@ #include "api/test/track_id_stream_label_map.h" #include "rtc_base/critical_section.h" #include "rtc_base/numerics/samples_stats_counter.h" +#include "test/testsupport/perf_test.h" namespace webrtc { namespace webrtc_pc_e2e { @@ -51,7 +52,8 @@ class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface { void ReportResult(const std::string& metric_name, const std::string& stream_label, const SamplesStatsCounter& counter, - const std::string& unit) const; + const std::string& unit, + webrtc::test::ImproveDirection improve_direction) const; std::string test_case_name_; TrackIdStreamLabelMap* analyzer_helper_; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index cd3a95d075..2605461f77 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -17,7 +17,6 @@ #include "api/units/time_delta.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "rtc_base/logging.h" -#include "test/testsupport/perf_test.h" namespace webrtc { namespace webrtc_pc_e2e { @@ -633,49 +632,64 @@ void DefaultVideoQualityAnalyzer::ReportResults( const std::string& test_case_name, const StreamStats& stats, const FrameCounters& frame_counters) { - ReportResult("psnr", test_case_name, stats.psnr, "dB"); - ReportResult("ssim", test_case_name, stats.ssim, "unitless"); - ReportResult("transport_time", test_case_name, stats.transport_time_ms, "ms"); + using ::webrtc::test::ImproveDirection; + + ReportResult("psnr", test_case_name, stats.psnr, "dB", + ImproveDirection::kBiggerIsBetter); + ReportResult("ssim", test_case_name, stats.ssim, "unitless", + ImproveDirection::kBiggerIsBetter); + ReportResult("transport_time", test_case_name, stats.transport_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); ReportResult("total_delay_incl_transport", test_case_name, - stats.total_delay_incl_transport_ms, "ms"); + stats.total_delay_incl_transport_ms, "ms", + ImproveDirection::kSmallerIsBetter); ReportResult("time_between_rendered_frames", test_case_name, - stats.time_between_rendered_frames_ms, "ms"); + stats.time_between_rendered_frames_ms, "ms", + ImproveDirection::kSmallerIsBetter); test::PrintResult("encode_frame_rate", "", test_case_name, stats.encode_frame_rate.IsEmpty() ? 0 : stats.encode_frame_rate.GetEventsPerSecond(), - "fps", /*important=*/false); - ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms"); + "fps", /*important=*/false, + ImproveDirection::kBiggerIsBetter); + ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); ReportResult("time_between_freezes", test_case_name, - stats.time_between_freezes_ms, "ms"); - ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms"); + stats.time_between_freezes_ms, "ms", + ImproveDirection::kBiggerIsBetter); + ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); ReportResult("pixels_per_frame", test_case_name, - stats.resolution_of_rendered_frame, "count"); + stats.resolution_of_rendered_frame, "count", + ImproveDirection::kBiggerIsBetter); test::PrintResult("min_psnr", "", test_case_name, stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB", - /*important=*/false); - ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms"); + /*important=*/false, ImproveDirection::kBiggerIsBetter); + ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); ReportResult("receive_to_render_time", test_case_name, - stats.receive_to_render_time_ms, "ms"); + stats.receive_to_render_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); test::PrintResult("dropped_frames", "", test_case_name, frame_counters.dropped, "count", - /*important=*/false); + /*important=*/false, ImproveDirection::kSmallerIsBetter); test::PrintResult("frames_in_flight", "", test_case_name, frame_counters.captured - frame_counters.rendered - frame_counters.dropped, "count", - /*important=*/false); + /*important=*/false, ImproveDirection::kSmallerIsBetter); ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered, - "count"); + "count", ImproveDirection::kSmallerIsBetter); } void DefaultVideoQualityAnalyzer::ReportResult( const std::string& metric_name, const std::string& test_case_name, const SamplesStatsCounter& counter, - const std::string& unit) { + const std::string& unit, + webrtc::test::ImproveDirection improve_direction) { test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit, - /*important=*/false); + /*important=*/false, improve_direction); } std::string DefaultVideoQualityAnalyzer::GetTestCaseName( diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index 5da4cdad65..d99e6ac303 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -28,6 +28,7 @@ #include "rtc_base/numerics/samples_stats_counter.h" #include "rtc_base/platform_thread.h" #include "system_wrappers/include/clock.h" +#include "test/testsupport/perf_test.h" namespace webrtc { namespace webrtc_pc_e2e { @@ -248,7 +249,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { static void ReportResult(const std::string& metric_name, const std::string& test_case_name, const SamplesStatsCounter& counter, - const std::string& unit); + const std::string& unit, + webrtc::test::ImproveDirection improve_direction = + webrtc::test::ImproveDirection::kNone); // Returns name of current test case for reporting. std::string GetTestCaseName(const std::string& stream_label) const; Timestamp Now(); diff --git a/test/testsupport/perf_test.cc b/test/testsupport/perf_test.cc index 4f7f5181c4..07f5feea2e 100644 --- a/test/testsupport/perf_test.cc +++ b/test/testsupport/perf_test.cc @@ -55,11 +55,12 @@ class PerfResultsLogger { const std::string& trace_name, const webrtc::SamplesStatsCounter& counter, const std::string& units, - const bool important) { + const bool important, + webrtc::test::ImproveDirection improve_direction) { LogResultMeanAndError( graph_name, trace_name, counter.IsEmpty() ? 0 : counter.GetAverage(), counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), units, - important); + important, improve_direction); rtc::CritScope lock(&crit_); plottable_counters_.push_back({graph_name, trace_name, counter, units}); @@ -68,7 +69,8 @@ class PerfResultsLogger { const std::string& trace_name, const double value, const std::string& units, - const bool important) { + const bool important, + webrtc::test::ImproveDirection improve_direction) { RTC_CHECK(std::isfinite(value)) << "Expected finite value for graph " << graph_name << ", trace name " << trace_name << ", units " << units << ", got " << value; @@ -76,14 +78,15 @@ class PerfResultsLogger { std::ostringstream value_stream; value_stream.precision(8); value_stream << value; - LogResultsImpl(graph_name, trace_name, value_stream.str(), units, - important); + LogResultsImpl(graph_name, trace_name, value_stream.str(), units, important, + improve_direction); std::ostringstream json_stream; json_stream << '"' << trace_name << R"(":{)"; json_stream << R"("type":"scalar",)"; json_stream << R"("value":)" << value << ','; - json_stream << R"("units":")" << units << R"("})"; + json_stream << R"("units":")" << UnitWithDirection(units, improve_direction) + << R"("})"; rtc::CritScope lock(&crit_); graphs_[graph_name].push_back(json_stream.str()); } @@ -92,22 +95,24 @@ class PerfResultsLogger { const double mean, const double error, const std::string& units, - const bool important) { + const bool important, + webrtc::test::ImproveDirection improve_direction) { RTC_CHECK(std::isfinite(mean)); RTC_CHECK(std::isfinite(error)); std::ostringstream value_stream; value_stream.precision(8); value_stream << '{' << mean << ',' << error << '}'; - LogResultsImpl(graph_name, trace_name, value_stream.str(), units, - important); + LogResultsImpl(graph_name, trace_name, value_stream.str(), units, important, + improve_direction); std::ostringstream json_stream; json_stream << '"' << trace_name << R"(":{)"; json_stream << R"("type":"list_of_scalar_values",)"; json_stream << R"("values":[)" << mean << "],"; json_stream << R"("std":)" << error << ','; - json_stream << R"("units":")" << units << R"("})"; + json_stream << R"("units":")" << UnitWithDirection(units, improve_direction) + << R"("})"; rtc::CritScope lock(&crit_); graphs_[graph_name].push_back(json_stream.str()); } @@ -115,7 +120,8 @@ class PerfResultsLogger { const std::string& trace_name, const rtc::ArrayView values, const std::string& units, - const bool important) { + const bool important, + webrtc::test::ImproveDirection improve_direction) { for (double v : values) { RTC_CHECK(std::isfinite(v)); } @@ -125,14 +131,15 @@ class PerfResultsLogger { value_stream << '['; OutputListToStream(&value_stream, values); value_stream << ']'; - LogResultsImpl(graph_name, trace_name, value_stream.str(), units, - important); + LogResultsImpl(graph_name, trace_name, value_stream.str(), units, important, + improve_direction); std::ostringstream json_stream; json_stream << '"' << trace_name << R"(":{)"; json_stream << R"("type":"list_of_scalar_values",)"; json_stream << R"("values":)" << value_stream.str() << ','; - json_stream << R"("units":")" << units << R"("})"; + json_stream << R"("units":")" << UnitWithDirection(units, improve_direction) + << R"("})"; rtc::CritScope lock(&crit_); graphs_[graph_name].push_back(json_stream.str()); } @@ -178,7 +185,8 @@ class PerfResultsLogger { const std::string& trace, const std::string& values, const std::string& units, - bool important) { + bool important, + webrtc::test::ImproveDirection improve_direction) { // <*>RESULT : = // <*>RESULT : = {, } // <*>RESULT : = [,value,value,...,] @@ -188,7 +196,21 @@ class PerfResultsLogger { fprintf(output_, "*"); } fprintf(output_, "RESULT %s: %s= %s %s\n", graph_name.c_str(), - trace.c_str(), values.c_str(), units.c_str()); + trace.c_str(), values.c_str(), + UnitWithDirection(units, improve_direction).c_str()); + } + + std::string UnitWithDirection( + const std::string& units, + webrtc::test::ImproveDirection improve_direction) { + switch (improve_direction) { + case webrtc::test::ImproveDirection::kNone: + return units; + case webrtc::test::ImproveDirection::kSmallerIsBetter: + return units + "_smallerIsBetter"; + case webrtc::test::ImproveDirection::kBiggerIsBetter: + return units + "_biggerIsBetter"; + } } rtc::CriticalSection crit_; @@ -254,9 +276,10 @@ void PrintResult(const std::string& measurement, const std::string& trace, const double value, const std::string& units, - bool important) { + bool important, + ImproveDirection improve_direction) { GetPerfResultsLogger().LogResult(measurement + modifier, trace, value, units, - important); + important, improve_direction); } void PrintResult(const std::string& measurement, @@ -264,9 +287,10 @@ void PrintResult(const std::string& measurement, const std::string& trace, const SamplesStatsCounter& counter, const std::string& units, - const bool important) { + const bool important, + ImproveDirection improve_direction) { GetPerfResultsLogger().LogResult(measurement + modifier, trace, counter, - units, important); + units, important, improve_direction); } void PrintResultMeanAndError(const std::string& measurement, @@ -275,9 +299,11 @@ void PrintResultMeanAndError(const std::string& measurement, const double mean, const double error, const std::string& units, - bool important) { + bool important, + ImproveDirection improve_direction) { GetPerfResultsLogger().LogResultMeanAndError(measurement + modifier, trace, - mean, error, units, important); + mean, error, units, important, + improve_direction); } void PrintResultList(const std::string& measurement, @@ -285,9 +311,10 @@ void PrintResultList(const std::string& measurement, const std::string& trace, const rtc::ArrayView values, const std::string& units, - bool important) { + bool important, + ImproveDirection improve_direction) { GetPerfResultsLogger().LogResultList(measurement + modifier, trace, values, - units, important); + units, important, improve_direction); } } // namespace test diff --git a/test/testsupport/perf_test.h b/test/testsupport/perf_test.h index 2b5b9f52d0..0550e44231 100644 --- a/test/testsupport/perf_test.h +++ b/test/testsupport/perf_test.h @@ -21,6 +21,16 @@ namespace webrtc { namespace test { +// Metrics improver direction. +enum class ImproveDirection { + // Direction is undefined. + kNone, + // Smaller value is better. + kSmallerIsBetter, + // Bigger value is better. + kBiggerIsBetter, +}; + // Prints numerical information to stdout in a controlled format, for // post-processing. |measurement| is a description of the quantity being // measured, e.g. "vm_peak"; |modifier| is provided as a convenience and @@ -39,29 +49,34 @@ void PrintResult(const std::string& measurement, const std::string& trace, const double value, const std::string& units, - bool important); + bool important, + ImproveDirection improve_direction = ImproveDirection::kNone); // Like PrintResult(), but prints a (mean, standard deviation) result pair. // The || should be two comma-separated numbers, the mean and // standard deviation (or other error metric) of the measurement. -void PrintResultMeanAndError(const std::string& measurement, - const std::string& modifier, - const std::string& trace, - const double mean, - const double error, - const std::string& units, - bool important); +void PrintResultMeanAndError( + const std::string& measurement, + const std::string& modifier, + const std::string& trace, + const double mean, + const double error, + const std::string& units, + bool important, + ImproveDirection improve_direction = ImproveDirection::kNone); // Like PrintResult(), but prints an entire list of results. The |values| // will generally be a list of comma-separated numbers. A typical // post-processing step might produce plots of their mean and standard // deviation. -void PrintResultList(const std::string& measurement, - const std::string& modifier, - const std::string& trace, - rtc::ArrayView values, - const std::string& units, - bool important); +void PrintResultList( + const std::string& measurement, + const std::string& modifier, + const std::string& trace, + rtc::ArrayView values, + const std::string& units, + bool important, + ImproveDirection improve_direction = ImproveDirection::kNone); // Like PrintResult(), but prints a (mean, standard deviation) from stats // counter. Also add specified metric to the plotable metrics output. @@ -70,7 +85,8 @@ void PrintResult(const std::string& measurement, const std::string& trace, const SamplesStatsCounter& counter, const std::string& units, - const bool important); + const bool important, + ImproveDirection improve_direction = ImproveDirection::kNone); // Returns all perf results to date in a JSON string formatted as described in // https://github.com/catapult-project/catapult/blob/master/dashboard/docs/data-format.md diff --git a/video/video_analyzer.cc b/video/video_analyzer.cc index 2e79c2489f..0430faa5dd 100644 --- a/video/video_analyzer.cc +++ b/video/video_analyzer.cc @@ -622,6 +622,8 @@ bool VideoAnalyzer::FrameProcessed() { } void VideoAnalyzer::PrintResults() { + using ::webrtc::test::ImproveDirection; + StopMeasuringCpuProcessTime(); int dropped_frames_diff; { @@ -630,24 +632,36 @@ void VideoAnalyzer::PrintResults() { dropped_frames_before_rendering_ + frames_.size(); } rtc::CritScope crit(&comparison_lock_); - PrintResult("psnr", psnr_, "dB"); - PrintResult("ssim", ssim_, "unitless"); - PrintResult("sender_time", sender_time_, "ms"); - PrintResult("receiver_time", receiver_time_, "ms"); - PrintResult("network_time", network_time_, "ms"); - PrintResult("total_delay_incl_network", end_to_end_, "ms"); - PrintResult("time_between_rendered_frames", rendered_delta_, "ms"); - PrintResult("encode_frame_rate", encode_frame_rate_, "fps"); - PrintResult("encode_time", encode_time_ms_, "ms"); - PrintResult("media_bitrate", media_bitrate_bps_, "bps"); - PrintResult("fec_bitrate", fec_bitrate_bps_, "bps"); - PrintResult("send_bandwidth", send_bandwidth_bps_, "bps"); - PrintResult("pixels_per_frame", pixels_, "count"); + PrintResult("psnr", psnr_, "dB", ImproveDirection::kBiggerIsBetter); + PrintResult("ssim", ssim_, "unitless", ImproveDirection::kBiggerIsBetter); + PrintResult("sender_time", sender_time_, "ms", + ImproveDirection::kSmallerIsBetter); + PrintResult("receiver_time", receiver_time_, "ms", + ImproveDirection::kSmallerIsBetter); + PrintResult("network_time", network_time_, "ms", + ImproveDirection::kSmallerIsBetter); + PrintResult("total_delay_incl_network", end_to_end_, "ms", + ImproveDirection::kSmallerIsBetter); + PrintResult("time_between_rendered_frames", rendered_delta_, "ms", + ImproveDirection::kSmallerIsBetter); + PrintResult("encode_frame_rate", encode_frame_rate_, "fps", + ImproveDirection::kBiggerIsBetter); + PrintResult("encode_time", encode_time_ms_, "ms", + ImproveDirection::kSmallerIsBetter); + PrintResult("media_bitrate", media_bitrate_bps_, "bps", + ImproveDirection::kNone); + PrintResult("fec_bitrate", fec_bitrate_bps_, "bps", ImproveDirection::kNone); + PrintResult("send_bandwidth", send_bandwidth_bps_, "bps", + ImproveDirection::kNone); + PrintResult("pixels_per_frame", pixels_, "count", + ImproveDirection::kBiggerIsBetter); test::PrintResult("decode_frame_rate", "", test_label_.c_str(), - decode_frame_rate_, "fps", false); + decode_frame_rate_, "fps", false, + ImproveDirection::kBiggerIsBetter); test::PrintResult("render_frame_rate", "", test_label_.c_str(), - render_frame_rate_, "fps", false); + render_frame_rate_, "fps", false, + ImproveDirection::kBiggerIsBetter); // Record the time from the last freeze until the last rendered frame to // ensure we cover the full timespan of the session. Otherwise the metric @@ -655,7 +669,8 @@ void VideoAnalyzer::PrintResults() { time_between_freezes_.AddSample(last_render_time_ - last_unfreeze_time_ms_); // Freeze metrics. - PrintResult("time_between_freezes", time_between_freezes_, "ms"); + PrintResult("time_between_freezes", time_between_freezes_, "ms", + ImproveDirection::kBiggerIsBetter); const double freeze_count_double = static_cast(freeze_count_); const double total_freezes_duration_ms_double = @@ -667,7 +682,7 @@ void VideoAnalyzer::PrintResults() { test::PrintResult( "freeze_duration_ratio", "", test_label_.c_str(), total_freezes_duration_ms_double / total_frames_duration_ms_double, - "unitless", false); + "unitless", false, ImproveDirection::kSmallerIsBetter); RTC_DCHECK_LE(total_freezes_duration_ms_double, total_frames_duration_ms_double); @@ -677,7 +692,7 @@ void VideoAnalyzer::PrintResults() { if (total_frames_duration_min > 0) { test::PrintResult("freeze_count_per_minute", "", test_label_.c_str(), freeze_count_double / total_frames_duration_min, - "unitless", false); + "unitless", false, ImproveDirection::kSmallerIsBetter); } } @@ -685,35 +700,37 @@ void VideoAnalyzer::PrintResults() { freeze_count_double > 0 ? total_freezes_duration_ms_double / freeze_count_double : 0, - "ms", false); + "ms", false, ImproveDirection::kSmallerIsBetter); if (1000 * sum_squared_frame_durations_ > 0) { test::PrintResult( "harmonic_frame_rate", "", test_label_.c_str(), total_frames_duration_ms_double / (1000 * sum_squared_frame_durations_), - "fps", false); + "fps", false, ImproveDirection::kBiggerIsBetter); } if (worst_frame_) { test::PrintResult("min_psnr", "", test_label_.c_str(), worst_frame_->psnr, - "dB", false); + "dB", false, ImproveDirection::kBiggerIsBetter); } if (receive_stream_ != nullptr) { PrintResultWithExternalMean("decode_time", mean_decode_time_ms_, - decode_time_ms_, "ms"); + decode_time_ms_, "ms", + ImproveDirection::kSmallerIsBetter); } dropped_frames_ += dropped_frames_diff; test::PrintResult("dropped_frames", "", test_label_.c_str(), dropped_frames_, - "count", false); + "count", false, ImproveDirection::kSmallerIsBetter); test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(), - "%", false); + "%", false, ImproveDirection::kSmallerIsBetter); #if defined(WEBRTC_WIN) // On Linux and Mac in Resident Set some unused pages may be counted. // Therefore this metric will depend on order in which tests are run and // will be flaky. - PrintResult("memory_usage", memory_usage_, "sizeInBytes"); + PrintResult("memory_usage", memory_usage_, "sizeInBytes", + ImproveDirection::kSmallerIsBetter); #endif // Saving only the worst frame for manual analysis. Intention here is to @@ -731,9 +748,12 @@ void VideoAnalyzer::PrintResults() { } if (audio_receive_stream_ != nullptr) { - PrintResult("audio_expand_rate", audio_expand_rate_, "unitless"); - PrintResult("audio_accelerate_rate", audio_accelerate_rate_, "unitless"); - PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, "ms"); + PrintResult("audio_expand_rate", audio_expand_rate_, "unitless", + ImproveDirection::kSmallerIsBetter); + PrintResult("audio_accelerate_rate", audio_accelerate_rate_, "unitless", + ImproveDirection::kSmallerIsBetter); + PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, "ms", + ImproveDirection::kNone); } // Disable quality check for quick test, as quality checks may fail @@ -813,18 +833,22 @@ void VideoAnalyzer::PerformFrameComparison( encoded_frame_size_.AddSample(comparison.encoded_frame_size); } -void VideoAnalyzer::PrintResult(const char* result_type, - Statistics stats, - const char* unit) { +void VideoAnalyzer::PrintResult( + const char* result_type, + Statistics stats, + const char* unit, + webrtc::test::ImproveDirection improve_direction) { test::PrintResultMeanAndError( result_type, "", test_label_.c_str(), stats.GetMean().value_or(0), - stats.GetStandardDeviation().value_or(0), unit, false); + stats.GetStandardDeviation().value_or(0), unit, false, improve_direction); } -void VideoAnalyzer::PrintResultWithExternalMean(const char* result_type, - double mean, - Statistics stats, - const char* unit) { +void VideoAnalyzer::PrintResultWithExternalMean( + const char* result_type, + double mean, + Statistics stats, + const char* unit, + webrtc::test::ImproveDirection improve_direction) { // If the true mean is different than the sample mean, the sample variance is // too low. The sample variance given a known mean is obtained by adding the // squared error between the true mean and the sample mean. @@ -833,7 +857,8 @@ void VideoAnalyzer::PrintResultWithExternalMean(const char* result_type, ? *stats.GetVariance() + pow(mean - *stats.GetMean(), 2.0) : 0.0; test::PrintResultMeanAndError(result_type, "", test_label_.c_str(), mean, - std::sqrt(compensated_variance), unit, false); + std::sqrt(compensated_variance), unit, false, + improve_direction); } void VideoAnalyzer::PrintSamplesToFile() { diff --git a/video/video_analyzer.h b/video/video_analyzer.h index 952d925052..03ee87e676 100644 --- a/video/video_analyzer.h +++ b/video/video_analyzer.h @@ -21,6 +21,7 @@ #include "rtc_base/time_utils.h" #include "test/layer_filtering_transport.h" #include "test/rtp_file_writer.h" +#include "test/testsupport/perf_test.h" namespace webrtc { @@ -192,11 +193,16 @@ class VideoAnalyzer : public PacketReceiver, bool FrameProcessed(); void PrintResults(); void PerformFrameComparison(const FrameComparison& comparison); - void PrintResult(const char* result_type, Statistics stats, const char* unit); - void PrintResultWithExternalMean(const char* result_type, - double mean, - Statistics stats, - const char* unit); + void PrintResult(const char* result_type, + Statistics stats, + const char* unit, + webrtc::test::ImproveDirection improve_direction); + void PrintResultWithExternalMean( + const char* result_type, + double mean, + Statistics stats, + const char* unit, + webrtc::test::ImproveDirection improve_direction); void PrintSamplesToFile(void); void AddCapturedFrameForComparison(const VideoFrame& video_frame);