diff --git a/api/test/metrics/BUILD.gn b/api/test/metrics/BUILD.gn index a7283cd5f6..43698002da 100644 --- a/api/test/metrics/BUILD.gn +++ b/api/test/metrics/BUILD.gn @@ -16,6 +16,7 @@ group("metrics") { ":global_metrics_logger_and_exporter", ":metric", ":metrics_exporter", + ":metrics_logger", ":metrics_logger_and_exporter", ":stdout_metrics_exporter", ] @@ -27,6 +28,7 @@ if (rtc_include_tests) { deps = [ ":metrics_logger_and_exporter_test", + ":metrics_logger_test", ":print_result_proxy_metrics_exporter_test", ":stdout_metrics_exporter_test", ] @@ -50,6 +52,22 @@ rtc_library("metric") { deps = [ "../../../api/units:timestamp" ] } +rtc_library("metrics_logger") { + visibility = [ "*" ] + sources = [ + "metrics_logger.cc", + "metrics_logger.h", + ] + deps = [ + ":metric", + "../..:array_view", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers", + "../../numerics", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + rtc_library("metrics_exporter") { visibility = [ "*" ] sources = [ "metrics_exporter.h" ] @@ -147,6 +165,7 @@ rtc_library("metrics_logger_and_exporter") { deps = [ ":metric", ":metrics_exporter", + ":metrics_logger", "../../../rtc_base:checks", "../../../rtc_base:logging", "../../../rtc_base/synchronization:mutex", @@ -175,6 +194,19 @@ rtc_library("global_metrics_logger_and_exporter") { } if (rtc_include_tests) { + rtc_library("metrics_logger_test") { + testonly = true + sources = [ "metrics_logger_test.cc" ] + deps = [ + ":metric", + ":metrics_logger", + "../../../system_wrappers", + "../../../test:test_support", + "../../numerics", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + rtc_library("stdout_metrics_exporter_test") { testonly = true sources = [ "stdout_metrics_exporter_test.cc" ] diff --git a/api/test/metrics/DEPS b/api/test/metrics/DEPS index d0def604b7..a1c845b629 100644 --- a/api/test/metrics/DEPS +++ b/api/test/metrics/DEPS @@ -3,4 +3,8 @@ specific_include_rules = { "+rtc_base/synchronization/mutex.h", "+system_wrappers/include/clock.h", ], + "metrics_logger\.h": [ + "+rtc_base/synchronization/mutex.h", + "+system_wrappers/include/clock.h", + ], } diff --git a/api/test/metrics/metrics_logger.cc b/api/test/metrics/metrics_logger.cc new file mode 100644 index 0000000000..c207505847 --- /dev/null +++ b/api/test/metrics/metrics_logger.cc @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/metrics_logger.h" + +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace test { +namespace { + +Metric::Stats ToStats(const SamplesStatsCounter& values) { + if (values.IsEmpty()) { + return Metric::Stats(); + } + return Metric::Stats{.mean = values.GetAverage(), + .stddev = values.GetStandardDeviation(), + .min = values.GetMin(), + .max = values.GetMax()}; +} + +} // namespace + +void DefaultMetricsLogger::LogSingleValueMetric( + absl::string_view name, + absl::string_view test_case_name, + double value, + Unit unit, + ImprovementDirection improvement_direction, + std::map metadata) { + MutexLock lock(&mutex_); + metrics_.push_back(Metric{ + .name = std::string(name), + .unit = unit, + .improvement_direction = improvement_direction, + .test_case = std::string(test_case_name), + .metric_metadata = std::move(metadata), + .time_series = + Metric::TimeSeries{.samples = std::vector{Metric::TimeSeries::Sample{ + .timestamp = Now(), .value = value}}}, + .stats = Metric::Stats{ + .mean = value, .stddev = absl::nullopt, .min = value, .max = value}}); +} + +void DefaultMetricsLogger::LogMetric( + absl::string_view name, + absl::string_view test_case_name, + const SamplesStatsCounter& values, + Unit unit, + ImprovementDirection improvement_direction, + std::map metadata) { + MutexLock lock(&mutex_); + Metric::TimeSeries time_series; + for (const SamplesStatsCounter::StatsSample& sample : + values.GetTimedSamples()) { + time_series.samples.push_back( + Metric::TimeSeries::Sample{.timestamp = sample.time, + .value = sample.value, + .sample_metadata = sample.metadata}); + } + + metrics_.push_back(Metric{.name = std::string(name), + .unit = unit, + .improvement_direction = improvement_direction, + .test_case = std::string(test_case_name), + .metric_metadata = std::move(metadata), + .time_series = std::move(time_series), + .stats = ToStats(values)}); +} + +void DefaultMetricsLogger::LogMetric( + absl::string_view name, + absl::string_view test_case_name, + const Metric::Stats& metric_stats, + Unit unit, + ImprovementDirection improvement_direction, + std::map metadata) { + MutexLock lock(&mutex_); + metrics_.push_back(Metric{.name = std::string(name), + .unit = unit, + .improvement_direction = improvement_direction, + .test_case = std::string(test_case_name), + .metric_metadata = std::move(metadata), + .time_series = Metric::TimeSeries{.samples = {}}, + .stats = std::move(metric_stats)}); +} + +Timestamp DefaultMetricsLogger::Now() { + return clock_->CurrentTime(); +} + +} // namespace test +} // namespace webrtc diff --git a/api/test/metrics/metrics_logger.h b/api/test/metrics/metrics_logger.h new file mode 100644 index 0000000000..6839bad19d --- /dev/null +++ b/api/test/metrics/metrics_logger.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_METRICS_LOGGER_H_ +#define API_TEST_METRICS_METRICS_LOGGER_H_ + +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "rtc_base/synchronization/mutex.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace test { + +// Provides API to log and collect performance metrics. +class MetricsLogger { + public: + virtual ~MetricsLogger() = default; + + // Adds a metric with a single value. + // `metadata` - metric's level metadata to add. + virtual void LogSingleValueMetric( + absl::string_view name, + absl::string_view test_case_name, + double value, + Unit unit, + ImprovementDirection improvement_direction, + std::map metadata = {}) = 0; + + // Adds metrics with a time series created based on the provided `values`. + // `metadata` - metric's level metadata to add. + virtual void LogMetric(absl::string_view name, + absl::string_view test_case_name, + const SamplesStatsCounter& values, + Unit unit, + ImprovementDirection improvement_direction, + std::map metadata = {}) = 0; + + // Adds metric with a time series with only stats object and without actual + // collected values. + // `metadata` - metric's level metadata to add. + virtual void LogMetric(absl::string_view name, + absl::string_view test_case_name, + const Metric::Stats& metric_stats, + Unit unit, + ImprovementDirection improvement_direction, + std::map metadata = {}) = 0; + + // Returns all metrics collected by this logger. + virtual std::vector GetCollectedMetrics() const = 0; +}; + +class DefaultMetricsLogger : public MetricsLogger { + public: + explicit DefaultMetricsLogger(webrtc::Clock* clock) : clock_(clock) {} + ~DefaultMetricsLogger() override = default; + + void LogSingleValueMetric( + absl::string_view name, + absl::string_view test_case_name, + double value, + Unit unit, + ImprovementDirection improvement_direction, + std::map metadata = {}) override; + + void LogMetric(absl::string_view name, + absl::string_view test_case_name, + const SamplesStatsCounter& values, + Unit unit, + ImprovementDirection improvement_direction, + std::map metadata = {}) override; + + void LogMetric(absl::string_view name, + absl::string_view test_case_name, + const Metric::Stats& metric_stats, + Unit unit, + ImprovementDirection improvement_direction, + std::map metadata = {}) override; + + // Returns all metrics collected by this logger. + std::vector GetCollectedMetrics() const override { + MutexLock lock(&mutex_); + return metrics_; + } + + private: + webrtc::Timestamp Now(); + + webrtc::Clock* const clock_; + + mutable Mutex mutex_; + std::vector metrics_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_METRICS_LOGGER_H_ diff --git a/api/test/metrics/metrics_logger_and_exporter.h b/api/test/metrics/metrics_logger_and_exporter.h index 454179028c..562aa6e264 100644 --- a/api/test/metrics/metrics_logger_and_exporter.h +++ b/api/test/metrics/metrics_logger_and_exporter.h @@ -21,6 +21,7 @@ #include "api/numerics/samples_stats_counter.h" #include "api/test/metrics/metric.h" #include "api/test/metrics/metrics_exporter.h" +#include "api/test/metrics/metrics_logger.h" #include "rtc_base/synchronization/mutex.h" #include "system_wrappers/include/clock.h" @@ -29,7 +30,7 @@ namespace test { // Combines metrics logging and exporting to provide simple API to automatically // export metrics at the end of the scope. -class MetricsLoggerAndExporter { +class MetricsLoggerAndExporter : public MetricsLogger { public: // `crash_on_export_failure` - makes MetricsLoggerAndExporter to crash if // any of exporters failed to export data. @@ -40,16 +41,17 @@ class MetricsLoggerAndExporter { : clock_(clock), crash_on_export_failure_(crash_on_export_failure), exporters_(std::move(exporters)) {} - ~MetricsLoggerAndExporter(); + ~MetricsLoggerAndExporter() override; // Adds a metric with a single value. // `metadata` - metric's level metadata to add. - void LogSingleValueMetric(absl::string_view name, - absl::string_view test_case_name, - double value, - Unit unit, - ImprovementDirection improvement_direction, - std::map metadata = {}); + void LogSingleValueMetric( + absl::string_view name, + absl::string_view test_case_name, + double value, + Unit unit, + ImprovementDirection improvement_direction, + std::map metadata = {}) override; // Adds metrics with a time series created based on the provided `values`. // `metadata` - metric's level metadata to add. @@ -58,7 +60,7 @@ class MetricsLoggerAndExporter { const SamplesStatsCounter& values, Unit unit, ImprovementDirection improvement_direction, - std::map metadata = {}); + std::map metadata = {}) override; // Adds metric with a time series with only stats object and without actual // collected values. @@ -68,10 +70,10 @@ class MetricsLoggerAndExporter { const Metric::Stats& metric_stats, Unit unit, ImprovementDirection improvement_direction, - std::map metadata = {}); + std::map metadata = {}) override; // Returns all metrics collected by this logger. - std::vector GetCollectedMetrics() const { + std::vector GetCollectedMetrics() const override { MutexLock lock(&mutex_); return metrics_; } diff --git a/api/test/metrics/metrics_logger_test.cc b/api/test/metrics/metrics_logger_test.cc new file mode 100644 index 0000000000..9a594462bd --- /dev/null +++ b/api/test/metrics/metrics_logger_test.cc @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/metrics_logger.h" + +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { +namespace { + +using ::testing::Eq; +using ::testing::IsEmpty; + +std::map DefaultMetadata() { + return std::map{{"key", "value"}}; +} + +TEST(DefaultMetricsLoggerTest, LogSingleValueMetricRecordsMetric) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + logger.LogSingleValueMetric( + "metric_name", "test_case_name", + /*value=*/10, Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + std::map{{"key", "value"}}); + + std::vector metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics.size(), Eq(1lu)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map{{"key", "value"}})); + ASSERT_THAT(metric.time_series.samples.size(), Eq(1lu)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map{})); + ASSERT_THAT(metric.stats.mean, absl::optional(10.0)); + ASSERT_THAT(metric.stats.stddev, absl::nullopt); + ASSERT_THAT(metric.stats.min, absl::optional(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional(10.0)); +} + +TEST(DefaultMetricsLoggerTest, LogMetricWithSamplesStatsCounterRecordsMetric) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + + SamplesStatsCounter values; + values.AddSample(SamplesStatsCounter::StatsSample{ + .value = 10, + .time = Clock::GetRealTimeClock()->CurrentTime(), + .metadata = + std::map{{"point_key1", "value1"}}}); + values.AddSample(SamplesStatsCounter::StatsSample{ + .value = 20, + .time = Clock::GetRealTimeClock()->CurrentTime(), + .metadata = + std::map{{"point_key2", "value2"}}}); + logger.LogMetric("metric_name", "test_case_name", values, Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + std::map{{"key", "value"}}); + + std::vector metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics.size(), Eq(1lu)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map{{"key", "value"}})); + ASSERT_THAT(metric.time_series.samples.size(), Eq(2lu)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map{{"point_key1", "value1"}})); + EXPECT_THAT(metric.time_series.samples[1].value, Eq(20.0)); + EXPECT_THAT(metric.time_series.samples[1].sample_metadata, + Eq(std::map{{"point_key2", "value2"}})); + ASSERT_THAT(metric.stats.mean, absl::optional(15.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional(5.0)); + ASSERT_THAT(metric.stats.min, absl::optional(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional(20.0)); +} + +TEST(DefaultMetricsLoggerTest, + LogMetricWithEmptySamplesStatsCounterRecordsEmptyMetric) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + SamplesStatsCounter values; + logger.LogMetric("metric_name", "test_case_name", values, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, DefaultMetadata()); + + std::vector metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics.size(), Eq(1lu)); + EXPECT_THAT(metrics[0].name, Eq("metric_name")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name")); + EXPECT_THAT(metrics[0].time_series.samples, IsEmpty()); + ASSERT_THAT(metrics[0].stats.mean, Eq(absl::nullopt)); + ASSERT_THAT(metrics[0].stats.stddev, Eq(absl::nullopt)); + ASSERT_THAT(metrics[0].stats.min, Eq(absl::nullopt)); + ASSERT_THAT(metrics[0].stats.max, Eq(absl::nullopt)); +} + +TEST(DefaultMetricsLoggerTest, LogMetricWithStatsRecordsMetric) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20}; + logger.LogMetric("metric_name", "test_case_name", metric_stats, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + std::map{{"key", "value"}}); + + std::vector metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics.size(), Eq(1lu)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map{{"key", "value"}})); + ASSERT_THAT(metric.time_series.samples.size(), Eq(0lu)); + ASSERT_THAT(metric.stats.mean, absl::optional(15.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional(5.0)); + ASSERT_THAT(metric.stats.min, absl::optional(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional(20.0)); +} + +TEST(DefaultMetricsLoggerTest, LogSingleValueMetricRecordsMultipleMetrics) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + + logger.LogSingleValueMetric("metric_name1", "test_case_name1", + /*value=*/10, Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + logger.LogSingleValueMetric("metric_name2", "test_case_name2", + /*value=*/10, Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + + std::vector metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics.size(), Eq(2lu)); + EXPECT_THAT(metrics[0].name, Eq("metric_name1")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[1].name, Eq("metric_name2")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); +} + +TEST(DefaultMetricsLoggerTest, + LogMetricWithSamplesStatsCounterRecordsMultipleMetrics) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + SamplesStatsCounter values; + values.AddSample(SamplesStatsCounter::StatsSample{ + .value = 10, + .time = Clock::GetRealTimeClock()->CurrentTime(), + .metadata = DefaultMetadata()}); + values.AddSample(SamplesStatsCounter::StatsSample{ + .value = 20, + .time = Clock::GetRealTimeClock()->CurrentTime(), + .metadata = DefaultMetadata()}); + + logger.LogMetric("metric_name1", "test_case_name1", values, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + logger.LogMetric("metric_name2", "test_case_name2", values, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + + std::vector metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics.size(), Eq(2lu)); + EXPECT_THAT(metrics[0].name, Eq("metric_name1")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[1].name, Eq("metric_name2")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); +} + +TEST(DefaultMetricsLoggerTest, LogMetricWithStatsRecordsMultipleMetrics) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20}; + + logger.LogMetric("metric_name1", "test_case_name1", metric_stats, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + logger.LogMetric("metric_name2", "test_case_name2", metric_stats, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + + std::vector metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics.size(), Eq(2lu)); + EXPECT_THAT(metrics[0].name, Eq("metric_name1")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[1].name, Eq("metric_name2")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); +} + +TEST(DefaultMetricsLoggerTest, + LogMetricThroughtAllMethodsAccumulateAllMetrics) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + SamplesStatsCounter values; + values.AddSample(SamplesStatsCounter::StatsSample{ + .value = 10, + .time = Clock::GetRealTimeClock()->CurrentTime(), + .metadata = DefaultMetadata()}); + values.AddSample(SamplesStatsCounter::StatsSample{ + .value = 20, + .time = Clock::GetRealTimeClock()->CurrentTime(), + .metadata = DefaultMetadata()}); + Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20}; + + logger.LogSingleValueMetric("metric_name1", "test_case_name1", + /*value=*/10, Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + logger.LogMetric("metric_name2", "test_case_name2", values, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + logger.LogMetric("metric_name3", "test_case_name3", metric_stats, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + + std::vector metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics.size(), Eq(3lu)); + EXPECT_THAT(metrics[0].name, Eq("metric_name1")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[1].name, Eq("metric_name2")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); + EXPECT_THAT(metrics[2].name, Eq("metric_name3")); + EXPECT_THAT(metrics[2].test_case, Eq("test_case_name3")); +} + +} // namespace +} // namespace test +} // namespace webrtc