Introduce MetricsLogger to separate logging and export logic

Bug: b/246095034
Change-Id: If870016b87126feefb9c63b1544091f0855e169f
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276600
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38180}
This commit is contained in:
Artem Titov 2022-09-23 14:57:51 +02:00 committed by WebRTC LUCI CQ
parent 3ab76d945c
commit 7b0f4a211a
6 changed files with 516 additions and 11 deletions

View File

@ -16,6 +16,7 @@ group("metrics") {
":global_metrics_logger_and_exporter",
":metric",
":metrics_exporter",
":metrics_logger",
":metrics_logger_and_exporter",
":stdout_metrics_exporter",
]
@ -27,6 +28,7 @@ if (rtc_include_tests) {
deps = [
":metrics_logger_and_exporter_test",
":metrics_logger_test",
":print_result_proxy_metrics_exporter_test",
":stdout_metrics_exporter_test",
]
@ -50,6 +52,22 @@ rtc_library("metric") {
deps = [ "../../../api/units:timestamp" ]
}
rtc_library("metrics_logger") {
visibility = [ "*" ]
sources = [
"metrics_logger.cc",
"metrics_logger.h",
]
deps = [
":metric",
"../..:array_view",
"../../../rtc_base/synchronization:mutex",
"../../../system_wrappers",
"../../numerics",
]
absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
}
rtc_library("metrics_exporter") {
visibility = [ "*" ]
sources = [ "metrics_exporter.h" ]
@ -147,6 +165,7 @@ rtc_library("metrics_logger_and_exporter") {
deps = [
":metric",
":metrics_exporter",
":metrics_logger",
"../../../rtc_base:checks",
"../../../rtc_base:logging",
"../../../rtc_base/synchronization:mutex",
@ -175,6 +194,19 @@ rtc_library("global_metrics_logger_and_exporter") {
}
if (rtc_include_tests) {
rtc_library("metrics_logger_test") {
testonly = true
sources = [ "metrics_logger_test.cc" ]
deps = [
":metric",
":metrics_logger",
"../../../system_wrappers",
"../../../test:test_support",
"../../numerics",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_library("stdout_metrics_exporter_test") {
testonly = true
sources = [ "stdout_metrics_exporter_test.cc" ]

View File

@ -3,4 +3,8 @@ specific_include_rules = {
"+rtc_base/synchronization/mutex.h",
"+system_wrappers/include/clock.h",
],
"metrics_logger\.h": [
"+rtc_base/synchronization/mutex.h",
"+system_wrappers/include/clock.h",
],
}

View File

@ -0,0 +1,107 @@
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/test/metrics/metrics_logger.h"
#include <map>
#include <string>
#include <utility>
#include <vector>
#include "absl/strings/string_view.h"
#include "api/numerics/samples_stats_counter.h"
#include "api/test/metrics/metric.h"
#include "rtc_base/synchronization/mutex.h"
namespace webrtc {
namespace test {
namespace {
Metric::Stats ToStats(const SamplesStatsCounter& values) {
if (values.IsEmpty()) {
return Metric::Stats();
}
return Metric::Stats{.mean = values.GetAverage(),
.stddev = values.GetStandardDeviation(),
.min = values.GetMin(),
.max = values.GetMax()};
}
} // namespace
void DefaultMetricsLogger::LogSingleValueMetric(
absl::string_view name,
absl::string_view test_case_name,
double value,
Unit unit,
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata) {
MutexLock lock(&mutex_);
metrics_.push_back(Metric{
.name = std::string(name),
.unit = unit,
.improvement_direction = improvement_direction,
.test_case = std::string(test_case_name),
.metric_metadata = std::move(metadata),
.time_series =
Metric::TimeSeries{.samples = std::vector{Metric::TimeSeries::Sample{
.timestamp = Now(), .value = value}}},
.stats = Metric::Stats{
.mean = value, .stddev = absl::nullopt, .min = value, .max = value}});
}
void DefaultMetricsLogger::LogMetric(
absl::string_view name,
absl::string_view test_case_name,
const SamplesStatsCounter& values,
Unit unit,
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata) {
MutexLock lock(&mutex_);
Metric::TimeSeries time_series;
for (const SamplesStatsCounter::StatsSample& sample :
values.GetTimedSamples()) {
time_series.samples.push_back(
Metric::TimeSeries::Sample{.timestamp = sample.time,
.value = sample.value,
.sample_metadata = sample.metadata});
}
metrics_.push_back(Metric{.name = std::string(name),
.unit = unit,
.improvement_direction = improvement_direction,
.test_case = std::string(test_case_name),
.metric_metadata = std::move(metadata),
.time_series = std::move(time_series),
.stats = ToStats(values)});
}
void DefaultMetricsLogger::LogMetric(
absl::string_view name,
absl::string_view test_case_name,
const Metric::Stats& metric_stats,
Unit unit,
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata) {
MutexLock lock(&mutex_);
metrics_.push_back(Metric{.name = std::string(name),
.unit = unit,
.improvement_direction = improvement_direction,
.test_case = std::string(test_case_name),
.metric_metadata = std::move(metadata),
.time_series = Metric::TimeSeries{.samples = {}},
.stats = std::move(metric_stats)});
}
Timestamp DefaultMetricsLogger::Now() {
return clock_->CurrentTime();
}
} // namespace test
} // namespace webrtc

View File

@ -0,0 +1,111 @@
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef API_TEST_METRICS_METRICS_LOGGER_H_
#define API_TEST_METRICS_METRICS_LOGGER_H_
#include <map>
#include <string>
#include <utility>
#include <vector>
#include "absl/strings/string_view.h"
#include "api/numerics/samples_stats_counter.h"
#include "api/test/metrics/metric.h"
#include "rtc_base/synchronization/mutex.h"
#include "system_wrappers/include/clock.h"
namespace webrtc {
namespace test {
// Provides API to log and collect performance metrics.
class MetricsLogger {
public:
virtual ~MetricsLogger() = default;
// Adds a metric with a single value.
// `metadata` - metric's level metadata to add.
virtual void LogSingleValueMetric(
absl::string_view name,
absl::string_view test_case_name,
double value,
Unit unit,
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata = {}) = 0;
// Adds metrics with a time series created based on the provided `values`.
// `metadata` - metric's level metadata to add.
virtual void LogMetric(absl::string_view name,
absl::string_view test_case_name,
const SamplesStatsCounter& values,
Unit unit,
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata = {}) = 0;
// Adds metric with a time series with only stats object and without actual
// collected values.
// `metadata` - metric's level metadata to add.
virtual void LogMetric(absl::string_view name,
absl::string_view test_case_name,
const Metric::Stats& metric_stats,
Unit unit,
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata = {}) = 0;
// Returns all metrics collected by this logger.
virtual std::vector<Metric> GetCollectedMetrics() const = 0;
};
class DefaultMetricsLogger : public MetricsLogger {
public:
explicit DefaultMetricsLogger(webrtc::Clock* clock) : clock_(clock) {}
~DefaultMetricsLogger() override = default;
void LogSingleValueMetric(
absl::string_view name,
absl::string_view test_case_name,
double value,
Unit unit,
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata = {}) override;
void LogMetric(absl::string_view name,
absl::string_view test_case_name,
const SamplesStatsCounter& values,
Unit unit,
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata = {}) override;
void LogMetric(absl::string_view name,
absl::string_view test_case_name,
const Metric::Stats& metric_stats,
Unit unit,
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata = {}) override;
// Returns all metrics collected by this logger.
std::vector<Metric> GetCollectedMetrics() const override {
MutexLock lock(&mutex_);
return metrics_;
}
private:
webrtc::Timestamp Now();
webrtc::Clock* const clock_;
mutable Mutex mutex_;
std::vector<Metric> metrics_ RTC_GUARDED_BY(mutex_);
};
} // namespace test
} // namespace webrtc
#endif // API_TEST_METRICS_METRICS_LOGGER_H_

View File

@ -21,6 +21,7 @@
#include "api/numerics/samples_stats_counter.h"
#include "api/test/metrics/metric.h"
#include "api/test/metrics/metrics_exporter.h"
#include "api/test/metrics/metrics_logger.h"
#include "rtc_base/synchronization/mutex.h"
#include "system_wrappers/include/clock.h"
@ -29,7 +30,7 @@ namespace test {
// Combines metrics logging and exporting to provide simple API to automatically
// export metrics at the end of the scope.
class MetricsLoggerAndExporter {
class MetricsLoggerAndExporter : public MetricsLogger {
public:
// `crash_on_export_failure` - makes MetricsLoggerAndExporter to crash if
// any of exporters failed to export data.
@ -40,16 +41,17 @@ class MetricsLoggerAndExporter {
: clock_(clock),
crash_on_export_failure_(crash_on_export_failure),
exporters_(std::move(exporters)) {}
~MetricsLoggerAndExporter();
~MetricsLoggerAndExporter() override;
// Adds a metric with a single value.
// `metadata` - metric's level metadata to add.
void LogSingleValueMetric(absl::string_view name,
absl::string_view test_case_name,
double value,
Unit unit,
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata = {});
void LogSingleValueMetric(
absl::string_view name,
absl::string_view test_case_name,
double value,
Unit unit,
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata = {}) override;
// Adds metrics with a time series created based on the provided `values`.
// `metadata` - metric's level metadata to add.
@ -58,7 +60,7 @@ class MetricsLoggerAndExporter {
const SamplesStatsCounter& values,
Unit unit,
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata = {});
std::map<std::string, std::string> metadata = {}) override;
// Adds metric with a time series with only stats object and without actual
// collected values.
@ -68,10 +70,10 @@ class MetricsLoggerAndExporter {
const Metric::Stats& metric_stats,
Unit unit,
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata = {});
std::map<std::string, std::string> metadata = {}) override;
// Returns all metrics collected by this logger.
std::vector<Metric> GetCollectedMetrics() const {
std::vector<Metric> GetCollectedMetrics() const override {
MutexLock lock(&mutex_);
return metrics_;
}

View File

@ -0,0 +1,249 @@
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/test/metrics/metrics_logger.h"
#include <map>
#include <memory>
#include <string>
#include <vector>
#include "absl/types/optional.h"
#include "api/numerics/samples_stats_counter.h"
#include "api/test/metrics/metric.h"
#include "system_wrappers/include/clock.h"
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
namespace test {
namespace {
using ::testing::Eq;
using ::testing::IsEmpty;
std::map<std::string, std::string> DefaultMetadata() {
return std::map<std::string, std::string>{{"key", "value"}};
}
TEST(DefaultMetricsLoggerTest, LogSingleValueMetricRecordsMetric) {
DefaultMetricsLogger logger(Clock::GetRealTimeClock());
logger.LogSingleValueMetric(
"metric_name", "test_case_name",
/*value=*/10, Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
std::map<std::string, std::string>{{"key", "value"}});
std::vector<Metric> metrics = logger.GetCollectedMetrics();
ASSERT_THAT(metrics.size(), Eq(1lu));
const Metric& metric = metrics[0];
EXPECT_THAT(metric.name, Eq("metric_name"));
EXPECT_THAT(metric.test_case, Eq("test_case_name"));
EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds));
EXPECT_THAT(metric.improvement_direction,
Eq(ImprovementDirection::kBiggerIsBetter));
EXPECT_THAT(metric.metric_metadata,
Eq(std::map<std::string, std::string>{{"key", "value"}}));
ASSERT_THAT(metric.time_series.samples.size(), Eq(1lu));
EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
Eq(std::map<std::string, std::string>{}));
ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0));
ASSERT_THAT(metric.stats.stddev, absl::nullopt);
ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0));
}
TEST(DefaultMetricsLoggerTest, LogMetricWithSamplesStatsCounterRecordsMetric) {
DefaultMetricsLogger logger(Clock::GetRealTimeClock());
SamplesStatsCounter values;
values.AddSample(SamplesStatsCounter::StatsSample{
.value = 10,
.time = Clock::GetRealTimeClock()->CurrentTime(),
.metadata =
std::map<std::string, std::string>{{"point_key1", "value1"}}});
values.AddSample(SamplesStatsCounter::StatsSample{
.value = 20,
.time = Clock::GetRealTimeClock()->CurrentTime(),
.metadata =
std::map<std::string, std::string>{{"point_key2", "value2"}}});
logger.LogMetric("metric_name", "test_case_name", values, Unit::kMilliseconds,
ImprovementDirection::kBiggerIsBetter,
std::map<std::string, std::string>{{"key", "value"}});
std::vector<Metric> metrics = logger.GetCollectedMetrics();
ASSERT_THAT(metrics.size(), Eq(1lu));
const Metric& metric = metrics[0];
EXPECT_THAT(metric.name, Eq("metric_name"));
EXPECT_THAT(metric.test_case, Eq("test_case_name"));
EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds));
EXPECT_THAT(metric.improvement_direction,
Eq(ImprovementDirection::kBiggerIsBetter));
EXPECT_THAT(metric.metric_metadata,
Eq(std::map<std::string, std::string>{{"key", "value"}}));
ASSERT_THAT(metric.time_series.samples.size(), Eq(2lu));
EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
Eq(std::map<std::string, std::string>{{"point_key1", "value1"}}));
EXPECT_THAT(metric.time_series.samples[1].value, Eq(20.0));
EXPECT_THAT(metric.time_series.samples[1].sample_metadata,
Eq(std::map<std::string, std::string>{{"point_key2", "value2"}}));
ASSERT_THAT(metric.stats.mean, absl::optional<double>(15.0));
ASSERT_THAT(metric.stats.stddev, absl::optional<double>(5.0));
ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
ASSERT_THAT(metric.stats.max, absl::optional<double>(20.0));
}
TEST(DefaultMetricsLoggerTest,
LogMetricWithEmptySamplesStatsCounterRecordsEmptyMetric) {
DefaultMetricsLogger logger(Clock::GetRealTimeClock());
SamplesStatsCounter values;
logger.LogMetric("metric_name", "test_case_name", values, Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter, DefaultMetadata());
std::vector<Metric> metrics = logger.GetCollectedMetrics();
ASSERT_THAT(metrics.size(), Eq(1lu));
EXPECT_THAT(metrics[0].name, Eq("metric_name"));
EXPECT_THAT(metrics[0].test_case, Eq("test_case_name"));
EXPECT_THAT(metrics[0].time_series.samples, IsEmpty());
ASSERT_THAT(metrics[0].stats.mean, Eq(absl::nullopt));
ASSERT_THAT(metrics[0].stats.stddev, Eq(absl::nullopt));
ASSERT_THAT(metrics[0].stats.min, Eq(absl::nullopt));
ASSERT_THAT(metrics[0].stats.max, Eq(absl::nullopt));
}
TEST(DefaultMetricsLoggerTest, LogMetricWithStatsRecordsMetric) {
DefaultMetricsLogger logger(Clock::GetRealTimeClock());
Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20};
logger.LogMetric("metric_name", "test_case_name", metric_stats,
Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
std::map<std::string, std::string>{{"key", "value"}});
std::vector<Metric> metrics = logger.GetCollectedMetrics();
ASSERT_THAT(metrics.size(), Eq(1lu));
const Metric& metric = metrics[0];
EXPECT_THAT(metric.name, Eq("metric_name"));
EXPECT_THAT(metric.test_case, Eq("test_case_name"));
EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds));
EXPECT_THAT(metric.improvement_direction,
Eq(ImprovementDirection::kBiggerIsBetter));
EXPECT_THAT(metric.metric_metadata,
Eq(std::map<std::string, std::string>{{"key", "value"}}));
ASSERT_THAT(metric.time_series.samples.size(), Eq(0lu));
ASSERT_THAT(metric.stats.mean, absl::optional<double>(15.0));
ASSERT_THAT(metric.stats.stddev, absl::optional<double>(5.0));
ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
ASSERT_THAT(metric.stats.max, absl::optional<double>(20.0));
}
TEST(DefaultMetricsLoggerTest, LogSingleValueMetricRecordsMultipleMetrics) {
DefaultMetricsLogger logger(Clock::GetRealTimeClock());
logger.LogSingleValueMetric("metric_name1", "test_case_name1",
/*value=*/10, Unit::kMilliseconds,
ImprovementDirection::kBiggerIsBetter,
DefaultMetadata());
logger.LogSingleValueMetric("metric_name2", "test_case_name2",
/*value=*/10, Unit::kMilliseconds,
ImprovementDirection::kBiggerIsBetter,
DefaultMetadata());
std::vector<Metric> metrics = logger.GetCollectedMetrics();
ASSERT_THAT(metrics.size(), Eq(2lu));
EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2"));
}
TEST(DefaultMetricsLoggerTest,
LogMetricWithSamplesStatsCounterRecordsMultipleMetrics) {
DefaultMetricsLogger logger(Clock::GetRealTimeClock());
SamplesStatsCounter values;
values.AddSample(SamplesStatsCounter::StatsSample{
.value = 10,
.time = Clock::GetRealTimeClock()->CurrentTime(),
.metadata = DefaultMetadata()});
values.AddSample(SamplesStatsCounter::StatsSample{
.value = 20,
.time = Clock::GetRealTimeClock()->CurrentTime(),
.metadata = DefaultMetadata()});
logger.LogMetric("metric_name1", "test_case_name1", values,
Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
DefaultMetadata());
logger.LogMetric("metric_name2", "test_case_name2", values,
Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
DefaultMetadata());
std::vector<Metric> metrics = logger.GetCollectedMetrics();
ASSERT_THAT(metrics.size(), Eq(2lu));
EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2"));
}
TEST(DefaultMetricsLoggerTest, LogMetricWithStatsRecordsMultipleMetrics) {
DefaultMetricsLogger logger(Clock::GetRealTimeClock());
Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20};
logger.LogMetric("metric_name1", "test_case_name1", metric_stats,
Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
DefaultMetadata());
logger.LogMetric("metric_name2", "test_case_name2", metric_stats,
Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
DefaultMetadata());
std::vector<Metric> metrics = logger.GetCollectedMetrics();
ASSERT_THAT(metrics.size(), Eq(2lu));
EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2"));
}
TEST(DefaultMetricsLoggerTest,
LogMetricThroughtAllMethodsAccumulateAllMetrics) {
DefaultMetricsLogger logger(Clock::GetRealTimeClock());
SamplesStatsCounter values;
values.AddSample(SamplesStatsCounter::StatsSample{
.value = 10,
.time = Clock::GetRealTimeClock()->CurrentTime(),
.metadata = DefaultMetadata()});
values.AddSample(SamplesStatsCounter::StatsSample{
.value = 20,
.time = Clock::GetRealTimeClock()->CurrentTime(),
.metadata = DefaultMetadata()});
Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20};
logger.LogSingleValueMetric("metric_name1", "test_case_name1",
/*value=*/10, Unit::kMilliseconds,
ImprovementDirection::kBiggerIsBetter,
DefaultMetadata());
logger.LogMetric("metric_name2", "test_case_name2", values,
Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
DefaultMetadata());
logger.LogMetric("metric_name3", "test_case_name3", metric_stats,
Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
DefaultMetadata());
std::vector<Metric> metrics = logger.GetCollectedMetrics();
ASSERT_THAT(metrics.size(), Eq(3lu));
EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2"));
EXPECT_THAT(metrics[2].name, Eq("metric_name3"));
EXPECT_THAT(metrics[2].test_case, Eq("test_case_name3"));
}
} // namespace
} // namespace test
} // namespace webrtc