From 936dfb1cb297027447dbe96aa032c59a8a5b1832 Mon Sep 17 00:00:00 2001 From: Edward Lemur Date: Thu, 30 Nov 2017 11:43:42 +0100 Subject: [PATCH] Add a function to report perf results in JSON format. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support to report perf results in the JSON format specified in [1]. [1] https://github.com/catapult-project/catapult/blob/master/dashboard/docs/data-format.md Bug: webrtc:8566 Change-Id: I25f829a4b012b3e2a3d56d61582a674f780148d0 Reviewed-on: https://webrtc-review.googlesource.com/26031 Reviewed-by: Patrik Höglund Reviewed-by: Karl Wiberg Commit-Queue: Edward Lemur Cr-Commit-Position: refs/heads/master@{#20950} --- test/BUILD.gn | 1 + test/testsupport/perf_test.cc | 150 ++++++++++++++++++++----- test/testsupport/perf_test.h | 17 +-- test/testsupport/perf_test_unittest.cc | 91 +++++++++++++-- 4 files changed, 212 insertions(+), 47 deletions(-) diff --git a/test/BUILD.gn b/test/BUILD.gn index 9a50e1cc79..7eb896da85 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -148,6 +148,7 @@ rtc_source_set("test_support") { deps = [ "..:webrtc_common", + "../api:array_view", "../common_video", "../rtc_base:gtest_prod", "../rtc_base:rtc_base_approved", diff --git a/test/testsupport/perf_test.cc b/test/testsupport/perf_test.cc index b7578d101e..affd45815f 100644 --- a/test/testsupport/perf_test.cc +++ b/test/testsupport/perf_test.cc @@ -8,14 +8,12 @@ * be found in the AUTHORS file in the root of the source tree. */ -// A stripped-down version of Chromium's chrome/test/perf/perf_test.cc. -// ResultsToString(), PrintResult(size_t value) and AppendResult(size_t value) -// have been modified. The remainder are identical to the Chromium version. - #include "test/testsupport/perf_test.h" +#include "rtc_base/criticalsection.h" -#include #include +#include +#include #include namespace { @@ -36,21 +34,131 @@ void PrintResultsImpl(const std::string& graph_name, values.c_str(), units.c_str()); } +template +void OutputListToStream(std::ostream* ostream, const Container& values) { + const char* sep = ""; + for (const auto& v : values) { + (*ostream) << sep << v; + sep = ","; + } +} + +class PerfResultsLogger { + public: + void ClearResults() { + rtc::CritScope lock(&crit_); + graphs_.clear(); + } + void LogResult(const std::string& graph_name, + const std::string& trace_name, + const double value, + const std::string& units, + const bool important) { + std::ostringstream value_stream; + value_stream << value; + PrintResultsImpl(graph_name, trace_name, value_stream.str(), units, + important); + + std::ostringstream json_stream; + json_stream << '"' << trace_name << R"(":{)"; + json_stream << R"("type":"scalar",)"; + json_stream << R"("value":)" << value << ','; + json_stream << R"("units":")" << units << R"("})"; + rtc::CritScope lock(&crit_); + graphs_[graph_name].push_back(json_stream.str()); + } + void LogResultMeanAndError(const std::string& graph_name, + const std::string& trace_name, + const double mean, + const double error, + const std::string& units, + const bool important) { + std::ostringstream value_stream; + value_stream << '{' << mean << ',' << error << '}'; + PrintResultsImpl(graph_name, trace_name, value_stream.str(), units, + important); + + std::ostringstream json_stream; + json_stream << '"' << trace_name << R"(":{)"; + json_stream << R"("type":"list_of_scalars",)"; + json_stream << R"("values":[)" << mean << "],"; + json_stream << R"("std":)" << error << ','; + json_stream << R"("units":")" << units << R"("})"; + rtc::CritScope lock(&crit_); + graphs_[graph_name].push_back(json_stream.str()); + } + void LogResultList(const std::string& graph_name, + const std::string& trace_name, + const rtc::ArrayView values, + const std::string& units, + const bool important) { + std::ostringstream value_stream; + value_stream << '['; + OutputListToStream(&value_stream, values); + value_stream << ']'; + PrintResultsImpl(graph_name, trace_name, value_stream.str(), units, + important); + + std::ostringstream json_stream; + json_stream << '"' << trace_name << R"(":{)"; + json_stream << R"("type":"list_of_scalars",)"; + json_stream << R"("values":)" << value_stream.str() << ','; + json_stream << R"("units":")" << units << R"("})"; + rtc::CritScope lock(&crit_); + graphs_[graph_name].push_back(json_stream.str()); + } + std::string ToJSON() const; + + private: + rtc::CriticalSection crit_; + std::map> graphs_ + RTC_GUARDED_BY(&crit_); +}; + +std::string PerfResultsLogger::ToJSON() const { + std::ostringstream json_stream; + json_stream << R"({"format_version":"1.0",)"; + json_stream << R"("charts":{)"; + rtc::CritScope lock(&crit_); + for (auto graphs_it = graphs_.begin(); graphs_it != graphs_.end(); + ++graphs_it) { + if (graphs_it != graphs_.begin()) + json_stream << ','; + json_stream << '"' << graphs_it->first << "\":"; + json_stream << '{'; + OutputListToStream(&json_stream, graphs_it->second); + json_stream << '}'; + } + json_stream << "}}"; + return json_stream.str(); +} + +PerfResultsLogger& GetPerfResultsLogger() { + static PerfResultsLogger* const logger_ = new PerfResultsLogger(); + return *logger_; +} + } // namespace namespace webrtc { namespace test { +void ClearPerfResults() { + GetPerfResultsLogger().ClearResults(); +} + +std::string GetPerfResultsJSON() { + return GetPerfResultsLogger().ToJSON(); +} + void PrintResult(const std::string& measurement, const std::string& modifier, const std::string& trace, const double value, const std::string& units, bool important) { - std::ostringstream value_stream; - value_stream << value; - PrintResultsImpl(measurement + modifier, trace, value_stream.str(), units, - important); + GetPerfResultsLogger().LogResult(measurement + modifier, trace, value, units, + important); } void PrintResultMeanAndError(const std::string& measurement, @@ -60,32 +168,18 @@ void PrintResultMeanAndError(const std::string& measurement, const double error, const std::string& units, bool important) { - std::ostringstream value_stream; - value_stream << '{' << mean << ',' << error << '}'; - PrintResultsImpl(measurement + modifier, trace, value_stream.str(), units, - important); + GetPerfResultsLogger().LogResultMeanAndError(measurement + modifier, trace, + mean, error, units, important); } void PrintResultList(const std::string& measurement, const std::string& modifier, const std::string& trace, - const std::vector& values, + const rtc::ArrayView values, const std::string& units, bool important) { - std::ostringstream value_stream; - value_stream << '['; - if (!values.empty()) { - auto it = values.begin(); - while (true) { - value_stream << *it; - if (++it == values.end()) - break; - value_stream << ','; - } - } - value_stream << ']'; - PrintResultsImpl(measurement + modifier, trace, value_stream.str(), units, - important); + GetPerfResultsLogger().LogResultList(measurement + modifier, trace, values, + units, important); } } // namespace test diff --git a/test/testsupport/perf_test.h b/test/testsupport/perf_test.h index 31b7b1345c..1a72c042fe 100644 --- a/test/testsupport/perf_test.h +++ b/test/testsupport/perf_test.h @@ -8,16 +8,13 @@ * be found in the AUTHORS file in the root of the source tree. */ -// A stripped-down version of Chromium's chrome/test/perf/perf_test.h. -// Several functions have been removed; the prototypes of the remainder have -// not been changed. - #ifndef TEST_TESTSUPPORT_PERF_TEST_H_ #define TEST_TESTSUPPORT_PERF_TEST_H_ +#include "api/array_view.h" + #include #include -#include namespace webrtc { namespace test { @@ -53,7 +50,6 @@ void PrintResultMeanAndError(const std::string& measurement, const std::string& units, bool important); - // Like PrintResult(), but prints an entire list of results. The |values| // will generally be a list of comma-separated numbers. A typical // post-processing step might produce plots of their mean and standard @@ -61,10 +57,17 @@ void PrintResultMeanAndError(const std::string& measurement, void PrintResultList(const std::string& measurement, const std::string& modifier, const std::string& trace, - const std::vector& values, + rtc::ArrayView values, const std::string& units, bool important); +// Get all perf results to date in a JSON format as described in +// https://github.com/catapult-project/catapult/blob/master/dashboard/docs/data-format.md +std::string GetPerfResultsJSON(); + +// You shouldn't use this function. It's only used to test the functions above. +void ClearPerfResults(); + } // namespace test } // namespace webrtc diff --git a/test/testsupport/perf_test_unittest.cc b/test/testsupport/perf_test_unittest.cc index 5c0f54333d..f80bcf8bd3 100644 --- a/test/testsupport/perf_test_unittest.cc +++ b/test/testsupport/perf_test_unittest.cc @@ -14,30 +14,97 @@ #include "test/gtest.h" +namespace { + +const char* kJsonExpected = R"({ + "format_version":"1.0", + "charts":{ + "foobar":{ + "baz_v":{ + "type":"scalar", + "value":7, + "units":"widgets" + }, + "baz_me":{ + "type":"list_of_scalars", + "values":[1], + "std":2, + "units":"lemurs" + }, + "baz_vl":{ + "type":"list_of_scalars", + "values":[1,2,3], + "units":"units" + } + }, + "measurementmodifier":{ + "trace":{ + "type":"scalar", + "value":42, + "units":"units" + } + } + } +})"; + +std::string RemoveSpaces(std::string s) { + s.erase(std::remove(s.begin(), s.end(), ' '), s.end()); + s.erase(std::remove(s.begin(), s.end(), '\n'), s.end()); + return s; +} + +} // namespace + namespace webrtc { namespace test { +class PerfTest : public ::testing::Test { + protected: + void TearDown() override { + ClearPerfResults(); + } +}; + + #if defined(WEBRTC_IOS) -#define MAYBE_AppendResult DISABLED_AppendResult +#define MAYBE_TestPrintResult DISABLED_TestPrintResult #else -#define MAYBE_AppendResult AppendResult +#define MAYBE_TestPrintResult TestPrintResult #endif -TEST(PerfTest, MAYBE_AppendResult) { +TEST_F(PerfTest, MAYBE_TestPrintResult) { testing::internal::CaptureStdout(); - std::string expected = "RESULT measurementmodifier: trace= 42 units\n"; + std::string expected; + + expected += "RESULT measurementmodifier: trace= 42 units\n"; PrintResult("measurement", "modifier", "trace", 42, "units", false); - expected += "*RESULT foobar: baz= 7 widgets\n"; - PrintResult("foo", "bar", "baz", 7, "widgets", true); + expected += "*RESULT foobar: baz_v= 7 widgets\n"; + PrintResult("foo", "bar", "baz_v", 7, "widgets", true); - expected += "RESULT foobar: baz= {1,2} lemurs\n"; - PrintResultMeanAndError("foo", "bar", "baz", 1, 2, "lemurs", false); + expected += "RESULT foobar: baz_me= {1,2} lemurs\n"; + PrintResultMeanAndError("foo", "bar", "baz_me", 1, 2, "lemurs", false); - expected += "RESULT foobar: baz= [1,2,3] units\n"; - PrintResultList("foo", "bar", "baz", {1, 2, 3}, "units", false); + const double kListOfScalars[] = {1, 2, 3}; + expected += "RESULT foobar: baz_vl= [1,2,3] units\n"; + PrintResultList("foo", "bar", "baz_vl", kListOfScalars, "units", false); - std::string output = testing::internal::GetCapturedStdout(); - EXPECT_EQ(expected, output); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST_F(PerfTest, TestGetPerfResultsJSON) { + PrintResult("measurement", "modifier", "trace", 42, "units", false); + PrintResult("foo", "bar", "baz_v", 7, "widgets", true); + PrintResultMeanAndError("foo", "bar", "baz_me", 1, 2, "lemurs", false); + const double kListOfScalars[] = {1, 2, 3}; + PrintResultList("foo", "bar", "baz_vl", kListOfScalars, "units", false); + + EXPECT_EQ(RemoveSpaces(kJsonExpected), GetPerfResultsJSON()); +} + +TEST_F(PerfTest, TestClearPerfResults) { + PrintResult("measurement", "modifier", "trace", 42, "units", false); + ClearPerfResults(); + EXPECT_EQ(R"({"format_version":"1.0","charts":{}})", GetPerfResultsJSON()); } } // namespace test