admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract results line plotting.

Browse Source 
This will make RESULT lines still come out after we add a second JSON
writer implementation.

Bug: chromium:1029452
Change-Id: I5cba3151c21df2901f19305e9b71bc5c9638a0ae
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/165399
Reviewed-by: Artem Titov <titovartem@webrtc.org>
Commit-Queue: Patrik Höglund <phoglund@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30208}
This commit is contained in:
Patrik Höglund 2020-01-10 11:11:40 +01:00 committed by Commit Bot
parent 56452dd17b
commit 6ddbe2c5b0
1 changed files with 130 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

212
test/testsupport/perf_test.cc
View File

@ -22,6 +22,9 @@
#include "rtc_base/checks.h"
#include "rtc_base/critical_section.h"
namespace webrtc {
namespace test {
namespace {
template <typename Container>
@ -33,6 +36,19 @@ void OutputListToStream(std::ostream* ostream, const Container& values) {
}
}
std::string UnitWithDirection(
const std::string& units,
webrtc::test::ImproveDirection improve_direction) {
switch (improve_direction) {
case webrtc::test::ImproveDirection::kNone:
return units;
case webrtc::test::ImproveDirection::kSmallerIsBetter:
return units + "_smallerIsBetter";
case webrtc::test::ImproveDirection::kBiggerIsBetter:
return units + "_biggerIsBetter";
}
}
struct PlottableCounter {
std::string graph_name;
std::string trace_name;
@ -103,33 +119,97 @@ PlottableCounterPrinter& GetPlottableCounterPrinter() {
return *printer_;
}
class PerfResultsLogger {
class ResultsLinePrinter {
public:
PerfResultsLogger() : crit_(), output_(stdout), graphs_() {}
void ClearResults() {
rtc::CritScope lock(&crit_);
graphs_.clear();
}
ResultsLinePrinter() : output_(stdout) {}
void SetOutput(FILE* output) {
rtc::CritScope lock(&crit_);
output_ = output;
}
void PrintResult(const std::string& graph_name,
const std::string& trace_name,
const double value,
const std::string& units,
bool important,
ImproveDirection improve_direction) {
std::ostringstream value_stream;
value_stream.precision(8);
value_stream << value;
PrintResultImpl(graph_name, trace_name, value_stream.str(), std::string(),
std::string(), UnitWithDirection(units, improve_direction),
important);
}
void PrintResultMeanAndError(const std::string& graph_name,
const std::string& trace_name,
const double mean,
const double error,
const std::string& units,
bool important,
ImproveDirection improve_direction) {
std::ostringstream value_stream;
value_stream.precision(8);
value_stream << mean << ',' << error;
PrintResultImpl(graph_name, trace_name, value_stream.str(), "{", "}",
UnitWithDirection(units, improve_direction), important);
}
void PrintResultList(const std::string& graph_name,
const std::string& trace_name,
const rtc::ArrayView<const double> values,
const std::string& units,
const bool important,
webrtc::test::ImproveDirection improve_direction) {
std::ostringstream value_stream;
value_stream.precision(8);
OutputListToStream(&value_stream, values);
PrintResultImpl(graph_name, trace_name, value_stream.str(), "[", "]", units,
important);
}
private:
void PrintResultImpl(const std::string& graph_name,
const std::string& trace_name,
const std::string& values,
const std::string& prefix,
const std::string& suffix,
const std::string& units,
bool important) {
rtc::CritScope lock(&crit_);
// <*>RESULT <graph_name>: <trace_name>= <value> <units>
// <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} <units>
// <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...,] <units>
fprintf(output_, "%sRESULT %s: %s= %s%s%s %s\n", important ? "*" : "",
graph_name.c_str(), trace_name.c_str(), prefix.c_str(),
values.c_str(), suffix.c_str(), units.c_str());
}
rtc::CriticalSection crit_;
FILE* output_ RTC_GUARDED_BY(&crit_);
};
ResultsLinePrinter& GetResultsLinePrinter() {
static ResultsLinePrinter* const printer_ = new ResultsLinePrinter();
return *printer_;
}
class PerfResultsLogger {
public:
PerfResultsLogger() : crit_(), graphs_() {}
void ClearResults() {
rtc::CritScope lock(&crit_);
graphs_.clear();
}
void LogResult(const std::string& graph_name,
const std::string& trace_name,
const double value,
const std::string& units,
const bool important,
webrtc::test::ImproveDirection improve_direction) {
RTC_CHECK(std::isfinite(value))
<< "Expected finite value for graph " << graph_name << ", trace name "
<< trace_name << ", units " << units << ", got " << value;
std::ostringstream value_stream;
value_stream.precision(8);
value_stream << value;
LogResultsImpl(graph_name, trace_name, value_stream.str(), units, important,
improve_direction);
std::ostringstream json_stream;
json_stream << '"' << trace_name << R"(":{)";
json_stream << R"("type":"scalar",)";
@ -146,15 +226,6 @@ class PerfResultsLogger {
const std::string& units,
const bool important,
webrtc::test::ImproveDirection improve_direction) {
RTC_CHECK(std::isfinite(mean));
RTC_CHECK(std::isfinite(error));
std::ostringstream value_stream;
value_stream.precision(8);
value_stream << '{' << mean << ',' << error << '}';
LogResultsImpl(graph_name, trace_name, value_stream.str(), units, important,
improve_direction);
std::ostringstream json_stream;
json_stream << '"' << trace_name << R"(":{)";
json_stream << R"("type":"list_of_scalar_values",)";
@ -171,17 +242,11 @@ class PerfResultsLogger {
const std::string& units,
const bool important,
webrtc::test::ImproveDirection improve_direction) {
for (double v : values) {
RTC_CHECK(std::isfinite(v));
}
std::ostringstream value_stream;
value_stream.precision(8);
value_stream << '[';
OutputListToStream(&value_stream, values);
value_stream << ']';
LogResultsImpl(graph_name, trace_name, value_stream.str(), units, important,
improve_direction);
std::ostringstream json_stream;
json_stream << '"' << trace_name << R"(":{)";
@ -195,40 +260,7 @@ class PerfResultsLogger {
std::string ToJSON() const;
private:
void LogResultsImpl(const std::string& graph_name,
const std::string& trace,
const std::string& values,
const std::string& units,
bool important,
webrtc::test::ImproveDirection improve_direction) {
// <*>RESULT <graph_name>: <trace_name>= <value> <units>
// <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} <units>
// <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...,] <units>
rtc::CritScope lock(&crit_);
if (important) {
fprintf(output_, "*");
}
fprintf(output_, "RESULT %s: %s= %s %s\n", graph_name.c_str(),
trace.c_str(), values.c_str(),
UnitWithDirection(units, improve_direction).c_str());
}
std::string UnitWithDirection(
const std::string& units,
webrtc::test::ImproveDirection improve_direction) {
switch (improve_direction) {
case webrtc::test::ImproveDirection::kNone:
return units;
case webrtc::test::ImproveDirection::kSmallerIsBetter:
return units + "_smallerIsBetter";
case webrtc::test::ImproveDirection::kBiggerIsBetter:
return units + "_biggerIsBetter";
}
}
rtc::CriticalSection crit_;
FILE* output_ RTC_GUARDED_BY(&crit_);
std::map<std::string, std::vector<std::string>> graphs_
RTC_GUARDED_BY(&crit_);
};
@ -258,16 +290,13 @@ PerfResultsLogger& GetPerfResultsLogger() {
} // namespace
namespace webrtc {
namespace test {
void ClearPerfResults() {
GetPerfResultsLogger().ClearResults();
}
void SetPerfResultsOutput(FILE* output) {
GetPerfResultsLogger().SetOutput(output);
GetPlottableCounterPrinter().SetOutput(output);
GetResultsLinePrinter().SetOutput(output);
}
std::string GetPerfResultsJSON() {
@ -292,8 +321,15 @@ void PrintResult(const std::string& measurement,
const std::string& units,
bool important,
ImproveDirection improve_direction) {
GetPerfResultsLogger().LogResult(measurement + modifier, trace, value, units,
important, improve_direction);
std::string graph_name = measurement + modifier;
RTC_CHECK(std::isfinite(value))
<< "Expected finite value for graph " << graph_name << ", trace name "
<< trace << ", units " << units << ", got " << value;
GetPerfResultsLogger().LogResult(graph_name, trace, value, units, important,
improve_direction);
GetResultsLinePrinter().PrintResult(graph_name, trace, value, units,
important, improve_direction);
}
void PrintResult(const std::string& measurement,
@ -303,13 +339,13 @@ void PrintResult(const std::string& measurement,
const std::string& units,
const bool important,
ImproveDirection improve_direction) {
PrintResultMeanAndError(
measurement, modifier, trace,
counter.IsEmpty() ? 0 : counter.GetAverage(),
counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), units, important,
improve_direction);
GetPlottableCounterPrinter().AddCounter(measurement + modifier, trace,
counter, units);
std::string graph_name = measurement + modifier;
GetPlottableCounterPrinter().AddCounter(graph_name, trace, counter, units);
double mean = counter.IsEmpty() ? 0 : counter.GetAverage();
double error = counter.IsEmpty() ? 0 : counter.GetStandardDeviation();
PrintResultMeanAndError(measurement, modifier, trace, mean, error, units,
important, improve_direction);
}
void PrintResultMeanAndError(const std::string& measurement,
@ -320,9 +356,14 @@ void PrintResultMeanAndError(const std::string& measurement,
const std::string& units,
bool important,
ImproveDirection improve_direction) {
GetPerfResultsLogger().LogResultMeanAndError(measurement + modifier, trace,
mean, error, units, important,
improve_direction);
RTC_CHECK(std::isfinite(mean));
RTC_CHECK(std::isfinite(error));
std::string graph_name = measurement + modifier;
GetPerfResultsLogger().LogResultMeanAndError(
graph_name, trace, mean, error, units, important, improve_direction);
GetResultsLinePrinter().PrintResultMeanAndError(
graph_name, trace, mean, error, units, important, improve_direction);
}
void PrintResultList(const std::string& measurement,
@ -332,8 +373,15 @@ void PrintResultList(const std::string& measurement,
const std::string& units,
bool important,
ImproveDirection improve_direction) {
GetPerfResultsLogger().LogResultList(measurement + modifier, trace, values,
units, important, improve_direction);
for (double v : values) {
RTC_CHECK(std::isfinite(v));
}
std::string graph_name = measurement + modifier;
GetPerfResultsLogger().LogResultList(graph_name, trace, values, units,
important, improve_direction);
GetResultsLinePrinter().PrintResultList(graph_name, trace, values, units,
important, improve_direction);
}
} // namespace test