admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add improvement directions to PC and Call framework metrics

Browse Source 
Bug: webrtc:10138
Change-Id: Ib957950df6e7490a15da0345fcd73e037c1a5b19
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/153892
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Ivo Creusen <ivoc@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29278}
This commit is contained in:
Artem Titov 2019-09-23 17:55:52 +02:00 committed by Commit Bot
parent 834a554962
commit 82ce384801
8 changed files with 207 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc
View File

@ -12,7 +12,6 @@
#include "api/stats_types.h"
#include "rtc_base/logging.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -100,18 +99,21 @@ std::string DefaultAudioQualityAnalyzer::GetTestCaseName(
}
void DefaultAudioQualityAnalyzer::Stop() {
using ::webrtc::test::ImproveDirection;
rtc::CritScope crit(&lock_);
for (auto& item : streams_stats_) {
ReportResult("expand_rate", item.first, item.second.expand_rate,
"unitless");
ReportResult("expand_rate", item.first, item.second.expand_rate, "unitless",
ImproveDirection::kSmallerIsBetter);
ReportResult("accelerate_rate", item.first, item.second.accelerate_rate,
"unitless");
"unitless", ImproveDirection::kSmallerIsBetter);
ReportResult("preemptive_rate", item.first, item.second.preemptive_rate,
"unitless");
"unitless", ImproveDirection::kSmallerIsBetter);
ReportResult("speech_expand_rate", item.first,
item.second.speech_expand_rate, "unitless");
item.second.speech_expand_rate, "unitless",
ImproveDirection::kSmallerIsBetter);
ReportResult("preferred_buffer_size_ms", item.first,
item.second.preferred_buffer_size_ms, "ms");
item.second.preferred_buffer_size_ms, "ms",
ImproveDirection::kNone);
}
}
@ -125,12 +127,13 @@ void DefaultAudioQualityAnalyzer::ReportResult(
const std::string& metric_name,
const std::string& stream_label,
const SamplesStatsCounter& counter,
const std::string& unit) const {
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) const {
test::PrintResultMeanAndError(
metric_name, /*modifier=*/"", GetTestCaseName(stream_label),
counter.IsEmpty() ? 0 : counter.GetAverage(),
counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), unit,
/*important=*/false);
/*important=*/false, improve_direction);
}
} // namespace webrtc_pc_e2e

4
test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h
View File

@ -19,6 +19,7 @@
#include "api/test/track_id_stream_label_map.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/numerics/samples_stats_counter.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -51,7 +52,8 @@ class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface {
void ReportResult(const std::string& metric_name,
const std::string& stream_label,
const SamplesStatsCounter& counter,
const std::string& unit) const;
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) const;
std::string test_case_name_;
TrackIdStreamLabelMap* analyzer_helper_;

52
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View File

@ -17,7 +17,6 @@
#include "api/units/time_delta.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_base/logging.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -633,49 +632,64 @@ void DefaultVideoQualityAnalyzer::ReportResults(
const std::string& test_case_name,
const StreamStats& stats,
const FrameCounters& frame_counters) {
ReportResult("psnr", test_case_name, stats.psnr, "dB");
ReportResult("ssim", test_case_name, stats.ssim, "unitless");
ReportResult("transport_time", test_case_name, stats.transport_time_ms, "ms");
using ::webrtc::test::ImproveDirection;
ReportResult("psnr", test_case_name, stats.psnr, "dB",
ImproveDirection::kBiggerIsBetter);
ReportResult("ssim", test_case_name, stats.ssim, "unitless",
ImproveDirection::kBiggerIsBetter);
ReportResult("transport_time", test_case_name, stats.transport_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("total_delay_incl_transport", test_case_name,
stats.total_delay_incl_transport_ms, "ms");
stats.total_delay_incl_transport_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("time_between_rendered_frames", test_case_name,
stats.time_between_rendered_frames_ms, "ms");
stats.time_between_rendered_frames_ms, "ms",
ImproveDirection::kSmallerIsBetter);
test::PrintResult("encode_frame_rate", "", test_case_name,
stats.encode_frame_rate.IsEmpty()
? 0
: stats.encode_frame_rate.GetEventsPerSecond(),
"fps", /*important=*/false);
ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms");
"fps", /*important=*/false,
ImproveDirection::kBiggerIsBetter);
ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("time_between_freezes", test_case_name,
stats.time_between_freezes_ms, "ms");
ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms");
stats.time_between_freezes_ms, "ms",
ImproveDirection::kBiggerIsBetter);
ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("pixels_per_frame", test_case_name,
stats.resolution_of_rendered_frame, "count");
stats.resolution_of_rendered_frame, "count",
ImproveDirection::kBiggerIsBetter);
test::PrintResult("min_psnr", "", test_case_name,
stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB",
/*important=*/false);
ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms");
/*important=*/false, ImproveDirection::kBiggerIsBetter);
ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("receive_to_render_time", test_case_name,
stats.receive_to_render_time_ms, "ms");
stats.receive_to_render_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
test::PrintResult("dropped_frames", "", test_case_name,
frame_counters.dropped, "count",
/*important=*/false);
/*important=*/false, ImproveDirection::kSmallerIsBetter);
test::PrintResult("frames_in_flight", "", test_case_name,
frame_counters.captured - frame_counters.rendered -
frame_counters.dropped,
"count",
/*important=*/false);
/*important=*/false, ImproveDirection::kSmallerIsBetter);
ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered,
"count");
"count", ImproveDirection::kSmallerIsBetter);
}
void DefaultVideoQualityAnalyzer::ReportResult(
const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit) {
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) {
test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit,
/*important=*/false);
/*important=*/false, improve_direction);
}
std::string DefaultVideoQualityAnalyzer::GetTestCaseName(

5
test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
View File

@ -28,6 +28,7 @@
#include "rtc_base/numerics/samples_stats_counter.h"
#include "rtc_base/platform_thread.h"
#include "system_wrappers/include/clock.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -248,7 +249,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit);
const std::string& unit,
webrtc::test::ImproveDirection improve_direction =
webrtc::test::ImproveDirection::kNone);
// Returns name of current test case for reporting.
std::string GetTestCaseName(const std::string& stream_label) const;
Timestamp Now();

75
test/testsupport/perf_test.cc
View File

@ -55,11 +55,12 @@ class PerfResultsLogger {
const std::string& trace_name,
const webrtc::SamplesStatsCounter& counter,
const std::string& units,
const bool important) {
const bool important,
webrtc::test::ImproveDirection improve_direction) {
LogResultMeanAndError(
graph_name, trace_name, counter.IsEmpty() ? 0 : counter.GetAverage(),
counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), units,
important);
important, improve_direction);
rtc::CritScope lock(&crit_);
plottable_counters_.push_back({graph_name, trace_name, counter, units});
@ -68,7 +69,8 @@ class PerfResultsLogger {
const std::string& trace_name,
const double value,
const std::string& units,
const bool important) {
const bool important,
webrtc::test::ImproveDirection improve_direction) {
RTC_CHECK(std::isfinite(value))
<< "Expected finite value for graph " << graph_name << ", trace name "
<< trace_name << ", units " << units << ", got " << value;
@ -76,14 +78,15 @@ class PerfResultsLogger {
std::ostringstream value_stream;
value_stream.precision(8);
value_stream << value;
LogResultsImpl(graph_name, trace_name, value_stream.str(), units,
important);
LogResultsImpl(graph_name, trace_name, value_stream.str(), units, important,
improve_direction);
std::ostringstream json_stream;
json_stream << '"' << trace_name << R"(":{)";
json_stream << R"("type":"scalar",)";
json_stream << R"("value":)" << value << ',';
json_stream << R"("units":")" << units << R"("})";
json_stream << R"("units":")" << UnitWithDirection(units, improve_direction)
<< R"("})";
rtc::CritScope lock(&crit_);
graphs_[graph_name].push_back(json_stream.str());
}
@ -92,22 +95,24 @@ class PerfResultsLogger {
const double mean,
const double error,
const std::string& units,
const bool important) {
const bool important,
webrtc::test::ImproveDirection improve_direction) {
RTC_CHECK(std::isfinite(mean));
RTC_CHECK(std::isfinite(error));
std::ostringstream value_stream;
value_stream.precision(8);
value_stream << '{' << mean << ',' << error << '}';
LogResultsImpl(graph_name, trace_name, value_stream.str(), units,
important);
LogResultsImpl(graph_name, trace_name, value_stream.str(), units, important,
improve_direction);
std::ostringstream json_stream;
json_stream << '"' << trace_name << R"(":{)";
json_stream << R"("type":"list_of_scalar_values",)";
json_stream << R"("values":[)" << mean << "],";
json_stream << R"("std":)" << error << ',';
json_stream << R"("units":")" << units << R"("})";
json_stream << R"("units":")" << UnitWithDirection(units, improve_direction)
<< R"("})";
rtc::CritScope lock(&crit_);
graphs_[graph_name].push_back(json_stream.str());
}
@ -115,7 +120,8 @@ class PerfResultsLogger {
const std::string& trace_name,
const rtc::ArrayView<const double> values,
const std::string& units,
const bool important) {
const bool important,
webrtc::test::ImproveDirection improve_direction) {
for (double v : values) {
RTC_CHECK(std::isfinite(v));
}
@ -125,14 +131,15 @@ class PerfResultsLogger {
value_stream << '[';
OutputListToStream(&value_stream, values);
value_stream << ']';
LogResultsImpl(graph_name, trace_name, value_stream.str(), units,
important);
LogResultsImpl(graph_name, trace_name, value_stream.str(), units, important,
improve_direction);
std::ostringstream json_stream;
json_stream << '"' << trace_name << R"(":{)";
json_stream << R"("type":"list_of_scalar_values",)";
json_stream << R"("values":)" << value_stream.str() << ',';
json_stream << R"("units":")" << units << R"("})";
json_stream << R"("units":")" << UnitWithDirection(units, improve_direction)
<< R"("})";
rtc::CritScope lock(&crit_);
graphs_[graph_name].push_back(json_stream.str());
}
@ -178,7 +185,8 @@ class PerfResultsLogger {
const std::string& trace,
const std::string& values,
const std::string& units,
bool important) {
bool important,
webrtc::test::ImproveDirection improve_direction) {
// <*>RESULT <graph_name>: <trace_name>= <value> <units>
// <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} <units>
// <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...,] <units>
@ -188,7 +196,21 @@ class PerfResultsLogger {
fprintf(output_, "*");
}
fprintf(output_, "RESULT %s: %s= %s %s\n", graph_name.c_str(),
trace.c_str(), values.c_str(), units.c_str());
trace.c_str(), values.c_str(),
UnitWithDirection(units, improve_direction).c_str());
}
std::string UnitWithDirection(
const std::string& units,
webrtc::test::ImproveDirection improve_direction) {
switch (improve_direction) {
case webrtc::test::ImproveDirection::kNone:
return units;
case webrtc::test::ImproveDirection::kSmallerIsBetter:
return units + "_smallerIsBetter";
case webrtc::test::ImproveDirection::kBiggerIsBetter:
return units + "_biggerIsBetter";
}
}
rtc::CriticalSection crit_;
@ -254,9 +276,10 @@ void PrintResult(const std::string& measurement,
const std::string& trace,
const double value,
const std::string& units,
bool important) {
bool important,
ImproveDirection improve_direction) {
GetPerfResultsLogger().LogResult(measurement + modifier, trace, value, units,
important);
important, improve_direction);
}
void PrintResult(const std::string& measurement,
@ -264,9 +287,10 @@ void PrintResult(const std::string& measurement,
const std::string& trace,
const SamplesStatsCounter& counter,
const std::string& units,
const bool important) {
const bool important,
ImproveDirection improve_direction) {
GetPerfResultsLogger().LogResult(measurement + modifier, trace, counter,
units, important);
units, important, improve_direction);
}
void PrintResultMeanAndError(const std::string& measurement,
@ -275,9 +299,11 @@ void PrintResultMeanAndError(const std::string& measurement,
const double mean,
const double error,
const std::string& units,
bool important) {
bool important,
ImproveDirection improve_direction) {
GetPerfResultsLogger().LogResultMeanAndError(measurement + modifier, trace,
mean, error, units, important);
mean, error, units, important,
improve_direction);
}
void PrintResultList(const std::string& measurement,
@ -285,9 +311,10 @@ void PrintResultList(const std::string& measurement,
const std::string& trace,
const rtc::ArrayView<const double> values,
const std::string& units,
bool important) {
bool important,
ImproveDirection improve_direction) {
GetPerfResultsLogger().LogResultList(measurement + modifier, trace, values,
units, important);
units, important, improve_direction);
}
} // namespace test

46
test/testsupport/perf_test.h
View File

@ -21,6 +21,16 @@
namespace webrtc {
namespace test {
// Metrics improver direction.
enum class ImproveDirection {
// Direction is undefined.
kNone,
// Smaller value is better.
kSmallerIsBetter,
// Bigger value is better.
kBiggerIsBetter,
};
// Prints numerical information to stdout in a controlled format, for
// post-processing. |measurement| is a description of the quantity being
// measured, e.g. "vm_peak"; |modifier| is provided as a convenience and
@ -39,29 +49,34 @@ void PrintResult(const std::string& measurement,
const std::string& trace,
const double value,
const std::string& units,
bool important);
bool important,
ImproveDirection improve_direction = ImproveDirection::kNone);
// Like PrintResult(), but prints a (mean, standard deviation) result pair.
// The |<values>| should be two comma-separated numbers, the mean and
// standard deviation (or other error metric) of the measurement.
void PrintResultMeanAndError(const std::string& measurement,
const std::string& modifier,
const std::string& trace,
const double mean,
const double error,
const std::string& units,
bool important);
void PrintResultMeanAndError(
const std::string& measurement,
const std::string& modifier,
const std::string& trace,
const double mean,
const double error,
const std::string& units,
bool important,
ImproveDirection improve_direction = ImproveDirection::kNone);
// Like PrintResult(), but prints an entire list of results. The |values|
// will generally be a list of comma-separated numbers. A typical
// post-processing step might produce plots of their mean and standard
// deviation.
void PrintResultList(const std::string& measurement,
const std::string& modifier,
const std::string& trace,
rtc::ArrayView<const double> values,
const std::string& units,
bool important);
void PrintResultList(
const std::string& measurement,
const std::string& modifier,
const std::string& trace,
rtc::ArrayView<const double> values,
const std::string& units,
bool important,
ImproveDirection improve_direction = ImproveDirection::kNone);
// Like PrintResult(), but prints a (mean, standard deviation) from stats
// counter. Also add specified metric to the plotable metrics output.
@ -70,7 +85,8 @@ void PrintResult(const std::string& measurement,
const std::string& trace,
const SamplesStatsCounter& counter,
const std::string& units,
const bool important);
const bool important,
ImproveDirection improve_direction = ImproveDirection::kNone);
// Returns all perf results to date in a JSON string formatted as described in
// https://github.com/catapult-project/catapult/blob/master/dashboard/docs/data-format.md

99
video/video_analyzer.cc
View File

@ -622,6 +622,8 @@ bool VideoAnalyzer::FrameProcessed() {
}
void VideoAnalyzer::PrintResults() {
using ::webrtc::test::ImproveDirection;
StopMeasuringCpuProcessTime();
int dropped_frames_diff;
{
@ -630,24 +632,36 @@ void VideoAnalyzer::PrintResults() {
dropped_frames_before_rendering_ + frames_.size();
}
rtc::CritScope crit(&comparison_lock_);
PrintResult("psnr", psnr_, "dB");
PrintResult("ssim", ssim_, "unitless");
PrintResult("sender_time", sender_time_, "ms");
PrintResult("receiver_time", receiver_time_, "ms");
PrintResult("network_time", network_time_, "ms");
PrintResult("total_delay_incl_network", end_to_end_, "ms");
PrintResult("time_between_rendered_frames", rendered_delta_, "ms");
PrintResult("encode_frame_rate", encode_frame_rate_, "fps");
PrintResult("encode_time", encode_time_ms_, "ms");
PrintResult("media_bitrate", media_bitrate_bps_, "bps");
PrintResult("fec_bitrate", fec_bitrate_bps_, "bps");
PrintResult("send_bandwidth", send_bandwidth_bps_, "bps");
PrintResult("pixels_per_frame", pixels_, "count");
PrintResult("psnr", psnr_, "dB", ImproveDirection::kBiggerIsBetter);
PrintResult("ssim", ssim_, "unitless", ImproveDirection::kBiggerIsBetter);
PrintResult("sender_time", sender_time_, "ms",
ImproveDirection::kSmallerIsBetter);
PrintResult("receiver_time", receiver_time_, "ms",
ImproveDirection::kSmallerIsBetter);
PrintResult("network_time", network_time_, "ms",
ImproveDirection::kSmallerIsBetter);
PrintResult("total_delay_incl_network", end_to_end_, "ms",
ImproveDirection::kSmallerIsBetter);
PrintResult("time_between_rendered_frames", rendered_delta_, "ms",
ImproveDirection::kSmallerIsBetter);
PrintResult("encode_frame_rate", encode_frame_rate_, "fps",
ImproveDirection::kBiggerIsBetter);
PrintResult("encode_time", encode_time_ms_, "ms",
ImproveDirection::kSmallerIsBetter);
PrintResult("media_bitrate", media_bitrate_bps_, "bps",
ImproveDirection::kNone);
PrintResult("fec_bitrate", fec_bitrate_bps_, "bps", ImproveDirection::kNone);
PrintResult("send_bandwidth", send_bandwidth_bps_, "bps",
ImproveDirection::kNone);
PrintResult("pixels_per_frame", pixels_, "count",
ImproveDirection::kBiggerIsBetter);
test::PrintResult("decode_frame_rate", "", test_label_.c_str(),
decode_frame_rate_, "fps", false);
decode_frame_rate_, "fps", false,
ImproveDirection::kBiggerIsBetter);
test::PrintResult("render_frame_rate", "", test_label_.c_str(),
render_frame_rate_, "fps", false);
render_frame_rate_, "fps", false,
ImproveDirection::kBiggerIsBetter);
// Record the time from the last freeze until the last rendered frame to
// ensure we cover the full timespan of the session. Otherwise the metric
@ -655,7 +669,8 @@ void VideoAnalyzer::PrintResults() {
time_between_freezes_.AddSample(last_render_time_ - last_unfreeze_time_ms_);
// Freeze metrics.
PrintResult("time_between_freezes", time_between_freezes_, "ms");
PrintResult("time_between_freezes", time_between_freezes_, "ms",
ImproveDirection::kBiggerIsBetter);
const double freeze_count_double = static_cast<double>(freeze_count_);
const double total_freezes_duration_ms_double =
@ -667,7 +682,7 @@ void VideoAnalyzer::PrintResults() {
test::PrintResult(
"freeze_duration_ratio", "", test_label_.c_str(),
total_freezes_duration_ms_double / total_frames_duration_ms_double,
"unitless", false);
"unitless", false, ImproveDirection::kSmallerIsBetter);
RTC_DCHECK_LE(total_freezes_duration_ms_double,
total_frames_duration_ms_double);
@ -677,7 +692,7 @@ void VideoAnalyzer::PrintResults() {
if (total_frames_duration_min > 0) {
test::PrintResult("freeze_count_per_minute", "", test_label_.c_str(),
freeze_count_double / total_frames_duration_min,
"unitless", false);
"unitless", false, ImproveDirection::kSmallerIsBetter);
}
}
@ -685,35 +700,37 @@ void VideoAnalyzer::PrintResults() {
freeze_count_double > 0
? total_freezes_duration_ms_double / freeze_count_double
: 0,
"ms", false);
"ms", false, ImproveDirection::kSmallerIsBetter);
if (1000 * sum_squared_frame_durations_ > 0) {
test::PrintResult(
"harmonic_frame_rate", "", test_label_.c_str(),
total_frames_duration_ms_double / (1000 * sum_squared_frame_durations_),
"fps", false);
"fps", false, ImproveDirection::kBiggerIsBetter);
}
if (worst_frame_) {
test::PrintResult("min_psnr", "", test_label_.c_str(), worst_frame_->psnr,
"dB", false);
"dB", false, ImproveDirection::kBiggerIsBetter);
}
if (receive_stream_ != nullptr) {
PrintResultWithExternalMean("decode_time", mean_decode_time_ms_,
decode_time_ms_, "ms");
decode_time_ms_, "ms",
ImproveDirection::kSmallerIsBetter);
}
dropped_frames_ += dropped_frames_diff;
test::PrintResult("dropped_frames", "", test_label_.c_str(), dropped_frames_,
"count", false);
"count", false, ImproveDirection::kSmallerIsBetter);
test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(),
"%", false);
"%", false, ImproveDirection::kSmallerIsBetter);
#if defined(WEBRTC_WIN)
// On Linux and Mac in Resident Set some unused pages may be counted.
// Therefore this metric will depend on order in which tests are run and
// will be flaky.
PrintResult("memory_usage", memory_usage_, "sizeInBytes");
PrintResult("memory_usage", memory_usage_, "sizeInBytes",
ImproveDirection::kSmallerIsBetter);
#endif
// Saving only the worst frame for manual analysis. Intention here is to
@ -731,9 +748,12 @@ void VideoAnalyzer::PrintResults() {
}
if (audio_receive_stream_ != nullptr) {
PrintResult("audio_expand_rate", audio_expand_rate_, "unitless");
PrintResult("audio_accelerate_rate", audio_accelerate_rate_, "unitless");
PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, "ms");
PrintResult("audio_expand_rate", audio_expand_rate_, "unitless",
ImproveDirection::kSmallerIsBetter);
PrintResult("audio_accelerate_rate", audio_accelerate_rate_, "unitless",
ImproveDirection::kSmallerIsBetter);
PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, "ms",
ImproveDirection::kNone);
}
// Disable quality check for quick test, as quality checks may fail
@ -813,18 +833,22 @@ void VideoAnalyzer::PerformFrameComparison(
encoded_frame_size_.AddSample(comparison.encoded_frame_size);
}
void VideoAnalyzer::PrintResult(const char* result_type,
Statistics stats,
const char* unit) {
void VideoAnalyzer::PrintResult(
const char* result_type,
Statistics stats,
const char* unit,
webrtc::test::ImproveDirection improve_direction) {
test::PrintResultMeanAndError(
result_type, "", test_label_.c_str(), stats.GetMean().value_or(0),
stats.GetStandardDeviation().value_or(0), unit, false);
stats.GetStandardDeviation().value_or(0), unit, false, improve_direction);
}
void VideoAnalyzer::PrintResultWithExternalMean(const char* result_type,
double mean,
Statistics stats,
const char* unit) {
void VideoAnalyzer::PrintResultWithExternalMean(
const char* result_type,
double mean,
Statistics stats,
const char* unit,
webrtc::test::ImproveDirection improve_direction) {
// If the true mean is different than the sample mean, the sample variance is
// too low. The sample variance given a known mean is obtained by adding the
// squared error between the true mean and the sample mean.
@ -833,7 +857,8 @@ void VideoAnalyzer::PrintResultWithExternalMean(const char* result_type,
? *stats.GetVariance() + pow(mean - *stats.GetMean(), 2.0)
: 0.0;
test::PrintResultMeanAndError(result_type, "", test_label_.c_str(), mean,
std::sqrt(compensated_variance), unit, false);
std::sqrt(compensated_variance), unit, false,
improve_direction);
}
void VideoAnalyzer::PrintSamplesToFile() {

16
video/video_analyzer.h
View File

@ -21,6 +21,7 @@
#include "rtc_base/time_utils.h"
#include "test/layer_filtering_transport.h"
#include "test/rtp_file_writer.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
@ -192,11 +193,16 @@ class VideoAnalyzer : public PacketReceiver,
bool FrameProcessed();
void PrintResults();
void PerformFrameComparison(const FrameComparison& comparison);
void PrintResult(const char* result_type, Statistics stats, const char* unit);
void PrintResultWithExternalMean(const char* result_type,
double mean,
Statistics stats,
const char* unit);
void PrintResult(const char* result_type,
Statistics stats,
const char* unit,
webrtc::test::ImproveDirection improve_direction);
void PrintResultWithExternalMean(
const char* result_type,
double mean,
Statistics stats,
const char* unit,
webrtc::test::ImproveDirection improve_direction);
void PrintSamplesToFile(void);
void AddCapturedFrameForComparison(const VideoFrame& video_frame);