diff --git a/api/test/metrics/BUILD.gn b/api/test/metrics/BUILD.gn index 64e7c1c44f..b635cf2943 100644 --- a/api/test/metrics/BUILD.gn +++ b/api/test/metrics/BUILD.gn @@ -15,6 +15,7 @@ group("metrics") { deps = [ ":global_metrics_logger_and_exporter", ":metric", + ":metrics_accumulator", ":metrics_exporter", ":metrics_logger", ":metrics_logger_and_exporter", @@ -28,6 +29,7 @@ if (rtc_include_tests) { deps = [ ":global_metrics_logger_and_exporter_test", + ":metrics_accumulator_test", ":metrics_logger_and_exporter_test", ":metrics_logger_test", ":print_result_proxy_metrics_exporter_test", @@ -61,6 +63,7 @@ rtc_library("metrics_logger") { ] deps = [ ":metric", + ":metrics_accumulator", "../..:array_view", "../../../rtc_base/synchronization:mutex", "../../../system_wrappers", @@ -69,6 +72,22 @@ rtc_library("metrics_logger") { absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } +rtc_library("metrics_accumulator") { + visibility = [ "*" ] + sources = [ + "metrics_accumulator.cc", + "metrics_accumulator.h", + ] + deps = [ + ":metric", + "../../../rtc_base:macromagic", + "../../../rtc_base/synchronization:mutex", + "../../numerics", + "../../units:timestamp", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + rtc_library("metrics_exporter") { visibility = [ "*" ] sources = [ "metrics_exporter.h" ] @@ -209,6 +228,18 @@ if (rtc_include_tests) { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } + rtc_library("metrics_accumulator_test") { + testonly = true + sources = [ "metrics_accumulator_test.cc" ] + deps = [ + ":metric", + ":metrics_accumulator", + "../../../test:test_support", + "../../units:timestamp", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + rtc_library("stdout_metrics_exporter_test") { testonly = true sources = [ "stdout_metrics_exporter_test.cc" ] diff --git a/api/test/metrics/DEPS b/api/test/metrics/DEPS index a1c845b629..74889c61c7 100644 --- a/api/test/metrics/DEPS +++ b/api/test/metrics/DEPS @@ -7,4 +7,8 @@ specific_include_rules = { "+rtc_base/synchronization/mutex.h", "+system_wrappers/include/clock.h", ], + "metrics_accumulator\.h": [ + "+rtc_base/synchronization/mutex.h", + "+rtc_base/thread_annotations.h", + ], } diff --git a/api/test/metrics/global_metrics_logger_and_exporter.cc b/api/test/metrics/global_metrics_logger_and_exporter.cc index 915dcac883..9c3c8978f5 100644 --- a/api/test/metrics/global_metrics_logger_and_exporter.cc +++ b/api/test/metrics/global_metrics_logger_and_exporter.cc @@ -22,8 +22,8 @@ namespace webrtc { namespace test { -MetricsLogger* GetGlobalMetricsLogger() { - static MetricsLogger* logger_ = +DefaultMetricsLogger* GetGlobalMetricsLogger() { + static DefaultMetricsLogger* logger_ = new DefaultMetricsLogger(Clock::GetRealTimeClock()); return logger_; } diff --git a/api/test/metrics/global_metrics_logger_and_exporter.h b/api/test/metrics/global_metrics_logger_and_exporter.h index 3f9bcec0e6..42bdf93c12 100644 --- a/api/test/metrics/global_metrics_logger_and_exporter.h +++ b/api/test/metrics/global_metrics_logger_and_exporter.h @@ -21,7 +21,7 @@ namespace webrtc { namespace test { // Returns non-null global `MetricsLogger` to log metrics. -MetricsLogger* GetGlobalMetricsLogger(); +DefaultMetricsLogger* GetGlobalMetricsLogger(); bool ExportPerfMetric(MetricsLogger& logger, std::vector> exporters); diff --git a/api/test/metrics/metrics_accumulator.cc b/api/test/metrics/metrics_accumulator.cc new file mode 100644 index 0000000000..c34396be97 --- /dev/null +++ b/api/test/metrics/metrics_accumulator.cc @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/metrics_accumulator.h" + +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace test { +namespace { + +Metric::Stats ToStats(const SamplesStatsCounter& values) { + if (values.IsEmpty()) { + return Metric::Stats(); + } + return Metric::Stats{.mean = values.GetAverage(), + .stddev = values.GetStandardDeviation(), + .min = values.GetMin(), + .max = values.GetMax()}; +} + +Metric SetTimeseries(const Metric& prototype, + const SamplesStatsCounter& counter) { + Metric output(prototype); + Metric::TimeSeries time_series; + for (const SamplesStatsCounter::StatsSample& sample : + counter.GetTimedSamples()) { + time_series.samples.push_back( + Metric::TimeSeries::Sample{.timestamp = sample.time, + .value = sample.value, + .sample_metadata = sample.metadata}); + } + output.time_series = std::move(time_series); + output.stats = ToStats(counter); + return output; +} + +} // namespace + +bool operator<(const MetricsAccumulator::MetricKey& a, + const MetricsAccumulator::MetricKey& b) { + if (a.test_case_name < b.test_case_name) { + return true; + } else if (a.test_case_name > b.test_case_name) { + return false; + } else { + return a.metric_name < b.metric_name; + } +} + +bool MetricsAccumulator::AddSample( + absl::string_view metric_name, + absl::string_view test_case_name, + double value, + Timestamp timestamp, + std::map point_metadata) { + MutexLock lock(&mutex_); + bool created; + MetricValue* metric_value = + GetOrCreateMetric(metric_name, test_case_name, &created); + metric_value->counter.AddSample( + SamplesStatsCounter::StatsSample{.value = value, + .time = timestamp, + .metadata = std::move(point_metadata)}); + return created; +} + +bool MetricsAccumulator::AddMetricMetadata( + absl::string_view metric_name, + absl::string_view test_case_name, + Unit unit, + ImprovementDirection improvement_direction, + std::map metric_metadata) { + MutexLock lock(&mutex_); + bool created; + MetricValue* metric_value = + GetOrCreateMetric(metric_name, test_case_name, &created); + metric_value->metric.unit = unit; + metric_value->metric.improvement_direction = improvement_direction; + metric_value->metric.metric_metadata = std::move(metric_metadata); + return created; +} + +std::vector MetricsAccumulator::GetCollectedMetrics() const { + MutexLock lock(&mutex_); + std::vector out; + out.reserve(metrics_.size()); + for (const auto& [unused_key, metric_value] : metrics_) { + out.push_back(SetTimeseries(metric_value.metric, metric_value.counter)); + } + return out; +} + +MetricsAccumulator::MetricValue* MetricsAccumulator::GetOrCreateMetric( + absl::string_view metric_name, + absl::string_view test_case_name, + bool* created) { + MetricKey key(metric_name, test_case_name); + auto it = metrics_.find(key); + if (it != metrics_.end()) { + *created = false; + return &it->second; + } + *created = true; + + Metric metric{ + .name = key.metric_name, + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .test_case = key.test_case_name, + }; + return &metrics_.emplace(key, MetricValue{.metric = std::move(metric)}) + .first->second; +} + +} // namespace test +} // namespace webrtc diff --git a/api/test/metrics/metrics_accumulator.h b/api/test/metrics/metrics_accumulator.h new file mode 100644 index 0000000000..c75bd9429c --- /dev/null +++ b/api/test/metrics/metrics_accumulator.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_METRICS_ACCUMULATOR_H_ +#define API_TEST_METRICS_METRICS_ACCUMULATOR_H_ + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { +namespace test { + +// Accumulates metrics' samples internally and provides API to get collected +// ones. +// +// This object is thread safe. +class MetricsAccumulator { + public: + MetricsAccumulator() = default; + + // Adds sample for the specified `metric_name` within specified + // `test_case_name`. If it is the first time when this combination of + // `metric_name` and `test_case_name` is used, creates a new Metric to collect + // samples, otherwise adds a sample to the previously created Metric. + // + // By default metric will use `Unit::kUnitless` and + // `ImprovementDirection::kNeitherIsBetter`. + // + // `point_metadata` - the metadata to be added to the single data point that + // this method adds to the Metric (it is not a metric global metadata). + // + // Returns true if a new metric was created and false otherwise. + bool AddSample(absl::string_view metric_name, + absl::string_view test_case_name, + double value, + Timestamp timestamp, + std::map point_metadata = {}); + + // Adds metadata to the metric specified by `metric_name` within specified + // `test_case_name`. If such a metric doesn't exist, creates a new one, + // otherwise overrides previously recorded values. + // + // Returns true if a new metric was created and false otherwise. + bool AddMetricMetadata( + absl::string_view metric_name, + absl::string_view test_case_name, + Unit unit, + ImprovementDirection improvement_direction, + std::map metric_metadata = {}); + + // Returns all metrics collected by this accumulator. No order guarantees + // provided. + std::vector GetCollectedMetrics() const; + + private: + struct MetricKey { + MetricKey(absl::string_view metric_name, absl::string_view test_case_name) + : metric_name(metric_name), test_case_name(test_case_name) {} + + std::string metric_name; + std::string test_case_name; + }; + friend bool operator<(const MetricKey& a, const MetricKey& b); + + struct MetricValue { + SamplesStatsCounter counter; + Metric metric; + }; + + // Gets existing metrics or creates a new one. If metric was created `created` + // will be set to true. + MetricValue* GetOrCreateMetric(absl::string_view metric_name, + absl::string_view test_case_name, + bool* created) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + mutable Mutex mutex_; + std::map metrics_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_METRICS_ACCUMULATOR_H_ diff --git a/api/test/metrics/metrics_accumulator_test.cc b/api/test/metrics/metrics_accumulator_test.cc new file mode 100644 index 0000000000..677f523339 --- /dev/null +++ b/api/test/metrics/metrics_accumulator_test.cc @@ -0,0 +1,315 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/metrics_accumulator.h" + +#include +#include + +#include "api/test/metrics/metric.h" +#include "api/units/timestamp.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { +namespace { + +using ::testing::Eq; +using ::testing::IsEmpty; +using ::testing::SizeIs; + +TEST(MetricsAccumulatorTest, AddSampleToTheNewMetricWillCreateOne) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddSample( + "metric_name", "test_case_name", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/std::map{{"key", "value"}})); + + std::vector metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kUnitless)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kNeitherIsBetter)); + EXPECT_THAT(metric.metric_metadata, IsEmpty()); + ASSERT_THAT(metric.time_series.samples, SizeIs(1)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map{{"key", "value"}})); + ASSERT_THAT(metric.stats.mean, absl::optional(10.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional(0.0)); + ASSERT_THAT(metric.stats.min, absl::optional(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional(10.0)); +} + +TEST(MetricsAccumulatorTest, AddSamplesToExistingMetricWontCreateNewOne) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddSample( + "metric_name", "test_case_name", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/ + std::map{{"key1", "value1"}})); + ASSERT_FALSE(accumulator.AddSample( + "metric_name", "test_case_name", + /*value=*/20, Timestamp::Seconds(2), + /*point_metadata=*/ + std::map{{"key2", "value2"}})); + + std::vector metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kUnitless)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kNeitherIsBetter)); + EXPECT_THAT(metric.metric_metadata, IsEmpty()); + ASSERT_THAT(metric.time_series.samples, SizeIs(2)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map{{"key1", "value1"}})); + EXPECT_THAT(metric.time_series.samples[1].value, Eq(20.0)); + EXPECT_THAT(metric.time_series.samples[1].timestamp, + Eq(Timestamp::Seconds(2))); + EXPECT_THAT(metric.time_series.samples[1].sample_metadata, + Eq(std::map{{"key2", "value2"}})); + ASSERT_THAT(metric.stats.mean, absl::optional(15.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional(5.0)); + ASSERT_THAT(metric.stats.min, absl::optional(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional(20.0)); +} + +TEST(MetricsAccumulatorTest, AddSampleToDifferentMetricsWillCreateBoth) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddSample( + "metric_name1", "test_case_name1", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/ + std::map{{"key1", "value1"}})); + ASSERT_TRUE(accumulator.AddSample( + "metric_name2", "test_case_name2", + /*value=*/20, Timestamp::Seconds(2), + /*point_metadata=*/ + std::map{{"key2", "value2"}})); + + std::vector metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(2)); + EXPECT_THAT(metrics[0].name, Eq("metric_name1")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[0].unit, Eq(Unit::kUnitless)); + EXPECT_THAT(metrics[0].improvement_direction, + Eq(ImprovementDirection::kNeitherIsBetter)); + EXPECT_THAT(metrics[0].metric_metadata, IsEmpty()); + ASSERT_THAT(metrics[0].time_series.samples, SizeIs(1)); + EXPECT_THAT(metrics[0].time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metrics[0].time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metrics[0].time_series.samples[0].sample_metadata, + Eq(std::map{{"key1", "value1"}})); + ASSERT_THAT(metrics[0].stats.mean, absl::optional(10.0)); + ASSERT_THAT(metrics[0].stats.stddev, absl::optional(0.0)); + ASSERT_THAT(metrics[0].stats.min, absl::optional(10.0)); + ASSERT_THAT(metrics[0].stats.max, absl::optional(10.0)); + EXPECT_THAT(metrics[1].name, Eq("metric_name2")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); + EXPECT_THAT(metrics[1].unit, Eq(Unit::kUnitless)); + EXPECT_THAT(metrics[1].improvement_direction, + Eq(ImprovementDirection::kNeitherIsBetter)); + EXPECT_THAT(metrics[1].metric_metadata, IsEmpty()); + ASSERT_THAT(metrics[1].time_series.samples, SizeIs(1)); + EXPECT_THAT(metrics[1].time_series.samples[0].value, Eq(20.0)); + EXPECT_THAT(metrics[1].time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(2))); + EXPECT_THAT(metrics[1].time_series.samples[0].sample_metadata, + Eq(std::map{{"key2", "value2"}})); + ASSERT_THAT(metrics[1].stats.mean, absl::optional(20.0)); + ASSERT_THAT(metrics[1].stats.stddev, absl::optional(0.0)); + ASSERT_THAT(metrics[1].stats.min, absl::optional(20.0)); + ASSERT_THAT(metrics[1].stats.max, absl::optional(20.0)); +} + +TEST(MetricsAccumulatorTest, AddMetadataToTheNewMetricWillCreateOne) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddMetricMetadata( + "metric_name", "test_case_name", Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + /*metric_metadata=*/ + std::map{{"key", "value"}})); + + std::vector metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map{{"key", "value"}})); + ASSERT_THAT(metric.time_series.samples, IsEmpty()); + ASSERT_THAT(metric.stats.mean, absl::nullopt); + ASSERT_THAT(metric.stats.stddev, absl::nullopt); + ASSERT_THAT(metric.stats.min, absl::nullopt); + ASSERT_THAT(metric.stats.max, absl::nullopt); +} + +TEST(MetricsAccumulatorTest, + AddMetadataToTheExistingMetricWillOverwriteValues) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddMetricMetadata( + "metric_name", "test_case_name", Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + /*metric_metadata=*/ + std::map{{"key1", "value1"}})); + + ASSERT_FALSE(accumulator.AddMetricMetadata( + "metric_name", "test_case_name", Unit::kBytes, + ImprovementDirection::kSmallerIsBetter, + /*metric_metadata=*/ + std::map{{"key2", "value2"}})); + + std::vector metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kBytes)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kSmallerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map{{"key2", "value2"}})); + ASSERT_THAT(metric.time_series.samples, IsEmpty()); + ASSERT_THAT(metric.stats.mean, absl::nullopt); + ASSERT_THAT(metric.stats.stddev, absl::nullopt); + ASSERT_THAT(metric.stats.min, absl::nullopt); + ASSERT_THAT(metric.stats.max, absl::nullopt); +} + +TEST(MetricsAccumulatorTest, AddMetadataToDifferentMetricsWillCreateBoth) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddMetricMetadata( + "metric_name1", "test_case_name1", Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + /*metric_metadata=*/ + std::map{{"key1", "value1"}})); + + ASSERT_TRUE(accumulator.AddMetricMetadata( + "metric_name2", "test_case_name2", Unit::kBytes, + ImprovementDirection::kSmallerIsBetter, + /*metric_metadata=*/ + std::map{{"key2", "value2"}})); + + std::vector metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(2)); + EXPECT_THAT(metrics[0].name, Eq("metric_name1")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[0].unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metrics[0].improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metrics[0].metric_metadata, + Eq(std::map{{"key1", "value1"}})); + ASSERT_THAT(metrics[0].time_series.samples, IsEmpty()); + ASSERT_THAT(metrics[0].stats.mean, absl::nullopt); + ASSERT_THAT(metrics[0].stats.stddev, absl::nullopt); + ASSERT_THAT(metrics[0].stats.min, absl::nullopt); + ASSERT_THAT(metrics[0].stats.max, absl::nullopt); + EXPECT_THAT(metrics[1].name, Eq("metric_name2")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); + EXPECT_THAT(metrics[1].unit, Eq(Unit::kBytes)); + EXPECT_THAT(metrics[1].improvement_direction, + Eq(ImprovementDirection::kSmallerIsBetter)); + EXPECT_THAT(metrics[1].metric_metadata, + Eq(std::map{{"key2", "value2"}})); + ASSERT_THAT(metrics[1].time_series.samples, IsEmpty()); + ASSERT_THAT(metrics[1].stats.mean, absl::nullopt); + ASSERT_THAT(metrics[1].stats.stddev, absl::nullopt); + ASSERT_THAT(metrics[1].stats.min, absl::nullopt); + ASSERT_THAT(metrics[1].stats.max, absl::nullopt); +} + +TEST(MetricsAccumulatorTest, AddMetadataAfterAddingSampleWontCreateNewMetric) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddSample( + "metric_name", "test_case_name", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/ + std::map{{"key_s", "value_s"}})); + ASSERT_FALSE(accumulator.AddMetricMetadata( + "metric_name", "test_case_name", Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + /*metric_metadata=*/ + std::map{{"key_m", "value_m"}})); + + std::vector metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map{{"key_m", "value_m"}})); + ASSERT_THAT(metric.time_series.samples, SizeIs(1)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map{{"key_s", "value_s"}})); + ASSERT_THAT(metric.stats.mean, absl::optional(10.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional(0.0)); + ASSERT_THAT(metric.stats.min, absl::optional(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional(10.0)); +} + +TEST(MetricsAccumulatorTest, AddSampleAfterAddingMetadataWontCreateNewMetric) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddMetricMetadata( + "metric_name", "test_case_name", Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + /*metric_metadata=*/ + std::map{{"key_m", "value_m"}})); + ASSERT_FALSE(accumulator.AddSample( + "metric_name", "test_case_name", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/ + std::map{{"key_s", "value_s"}})); + + std::vector metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map{{"key_m", "value_m"}})); + ASSERT_THAT(metric.time_series.samples, SizeIs(1)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map{{"key_s", "value_s"}})); + ASSERT_THAT(metric.stats.mean, absl::optional(10.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional(0.0)); + ASSERT_THAT(metric.stats.min, absl::optional(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional(10.0)); +} + +} // namespace +} // namespace test +} // namespace webrtc diff --git a/api/test/metrics/metrics_logger.cc b/api/test/metrics/metrics_logger.cc index c207505847..1e24400367 100644 --- a/api/test/metrics/metrics_logger.cc +++ b/api/test/metrics/metrics_logger.cc @@ -99,6 +99,13 @@ void DefaultMetricsLogger::LogMetric( .stats = std::move(metric_stats)}); } +std::vector DefaultMetricsLogger::GetCollectedMetrics() const { + std::vector out = metrics_accumulator_.GetCollectedMetrics(); + MutexLock lock(&mutex_); + out.insert(out.end(), metrics_.begin(), metrics_.end()); + return out; +} + Timestamp DefaultMetricsLogger::Now() { return clock_->CurrentTime(); } diff --git a/api/test/metrics/metrics_logger.h b/api/test/metrics/metrics_logger.h index 6839bad19d..66f9e55b95 100644 --- a/api/test/metrics/metrics_logger.h +++ b/api/test/metrics/metrics_logger.h @@ -19,6 +19,7 @@ #include "absl/strings/string_view.h" #include "api/numerics/samples_stats_counter.h" #include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_accumulator.h" #include "rtc_base/synchronization/mutex.h" #include "system_wrappers/include/clock.h" @@ -90,16 +91,16 @@ class DefaultMetricsLogger : public MetricsLogger { ImprovementDirection improvement_direction, std::map metadata = {}) override; - // Returns all metrics collected by this logger. - std::vector GetCollectedMetrics() const override { - MutexLock lock(&mutex_); - return metrics_; - } + // Returns all metrics collected by this logger and its `MetricsAccumulator`. + std::vector GetCollectedMetrics() const override; + + MetricsAccumulator* GetMetricsAccumulator() { return &metrics_accumulator_; } private: webrtc::Timestamp Now(); webrtc::Clock* const clock_; + MetricsAccumulator metrics_accumulator_; mutable Mutex mutex_; std::vector metrics_ RTC_GUARDED_BY(mutex_); diff --git a/api/test/metrics/metrics_logger_test.cc b/api/test/metrics/metrics_logger_test.cc index 9a594462bd..de4501ca36 100644 --- a/api/test/metrics/metrics_logger_test.cc +++ b/api/test/metrics/metrics_logger_test.cc @@ -27,6 +27,7 @@ namespace { using ::testing::Eq; using ::testing::IsEmpty; +using ::testing::SizeIs; std::map DefaultMetadata() { return std::map{{"key", "value"}}; @@ -40,7 +41,7 @@ TEST(DefaultMetricsLoggerTest, LogSingleValueMetricRecordsMetric) { std::map{{"key", "value"}}); std::vector metrics = logger.GetCollectedMetrics(); - ASSERT_THAT(metrics.size(), Eq(1lu)); + ASSERT_THAT(metrics, SizeIs(1)); const Metric& metric = metrics[0]; EXPECT_THAT(metric.name, Eq("metric_name")); EXPECT_THAT(metric.test_case, Eq("test_case_name")); @@ -49,7 +50,7 @@ TEST(DefaultMetricsLoggerTest, LogSingleValueMetricRecordsMetric) { Eq(ImprovementDirection::kBiggerIsBetter)); EXPECT_THAT(metric.metric_metadata, Eq(std::map{{"key", "value"}})); - ASSERT_THAT(metric.time_series.samples.size(), Eq(1lu)); + ASSERT_THAT(metric.time_series.samples, SizeIs(1)); EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); EXPECT_THAT(metric.time_series.samples[0].sample_metadata, Eq(std::map{})); @@ -78,7 +79,7 @@ TEST(DefaultMetricsLoggerTest, LogMetricWithSamplesStatsCounterRecordsMetric) { std::map{{"key", "value"}}); std::vector metrics = logger.GetCollectedMetrics(); - ASSERT_THAT(metrics.size(), Eq(1lu)); + ASSERT_THAT(metrics, SizeIs(1)); const Metric& metric = metrics[0]; EXPECT_THAT(metric.name, Eq("metric_name")); EXPECT_THAT(metric.test_case, Eq("test_case_name")); @@ -87,7 +88,7 @@ TEST(DefaultMetricsLoggerTest, LogMetricWithSamplesStatsCounterRecordsMetric) { Eq(ImprovementDirection::kBiggerIsBetter)); EXPECT_THAT(metric.metric_metadata, Eq(std::map{{"key", "value"}})); - ASSERT_THAT(metric.time_series.samples.size(), Eq(2lu)); + ASSERT_THAT(metric.time_series.samples, SizeIs(2)); EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); EXPECT_THAT(metric.time_series.samples[0].sample_metadata, Eq(std::map{{"point_key1", "value1"}})); @@ -108,7 +109,7 @@ TEST(DefaultMetricsLoggerTest, ImprovementDirection::kBiggerIsBetter, DefaultMetadata()); std::vector metrics = logger.GetCollectedMetrics(); - ASSERT_THAT(metrics.size(), Eq(1lu)); + ASSERT_THAT(metrics, SizeIs(1)); EXPECT_THAT(metrics[0].name, Eq("metric_name")); EXPECT_THAT(metrics[0].test_case, Eq("test_case_name")); EXPECT_THAT(metrics[0].time_series.samples, IsEmpty()); @@ -126,7 +127,7 @@ TEST(DefaultMetricsLoggerTest, LogMetricWithStatsRecordsMetric) { std::map{{"key", "value"}}); std::vector metrics = logger.GetCollectedMetrics(); - ASSERT_THAT(metrics.size(), Eq(1lu)); + ASSERT_THAT(metrics, SizeIs(1)); const Metric& metric = metrics[0]; EXPECT_THAT(metric.name, Eq("metric_name")); EXPECT_THAT(metric.test_case, Eq("test_case_name")); @@ -135,7 +136,7 @@ TEST(DefaultMetricsLoggerTest, LogMetricWithStatsRecordsMetric) { Eq(ImprovementDirection::kBiggerIsBetter)); EXPECT_THAT(metric.metric_metadata, Eq(std::map{{"key", "value"}})); - ASSERT_THAT(metric.time_series.samples.size(), Eq(0lu)); + ASSERT_THAT(metric.time_series.samples, IsEmpty()); ASSERT_THAT(metric.stats.mean, absl::optional(15.0)); ASSERT_THAT(metric.stats.stddev, absl::optional(5.0)); ASSERT_THAT(metric.stats.min, absl::optional(10.0)); @@ -155,7 +156,7 @@ TEST(DefaultMetricsLoggerTest, LogSingleValueMetricRecordsMultipleMetrics) { DefaultMetadata()); std::vector metrics = logger.GetCollectedMetrics(); - ASSERT_THAT(metrics.size(), Eq(2lu)); + ASSERT_THAT(metrics, SizeIs(2)); EXPECT_THAT(metrics[0].name, Eq("metric_name1")); EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); EXPECT_THAT(metrics[1].name, Eq("metric_name2")); @@ -183,7 +184,7 @@ TEST(DefaultMetricsLoggerTest, DefaultMetadata()); std::vector metrics = logger.GetCollectedMetrics(); - ASSERT_THAT(metrics.size(), Eq(2lu)); + ASSERT_THAT(metrics, SizeIs(2)); EXPECT_THAT(metrics[0].name, Eq("metric_name1")); EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); EXPECT_THAT(metrics[1].name, Eq("metric_name2")); @@ -202,7 +203,7 @@ TEST(DefaultMetricsLoggerTest, LogMetricWithStatsRecordsMultipleMetrics) { DefaultMetadata()); std::vector metrics = logger.GetCollectedMetrics(); - ASSERT_THAT(metrics.size(), Eq(2lu)); + ASSERT_THAT(metrics, SizeIs(2)); EXPECT_THAT(metrics[0].name, Eq("metric_name1")); EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); EXPECT_THAT(metrics[1].name, Eq("metric_name2")); @@ -244,6 +245,82 @@ TEST(DefaultMetricsLoggerTest, EXPECT_THAT(metrics[2].test_case, Eq("test_case_name3")); } +TEST(DefaultMetricsLoggerTest, AccumulatedMetricsReturnedInCollectedMetrics) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + logger.GetMetricsAccumulator()->AddSample( + "metric_name", "test_case_name", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/std::map{{"key", "value"}}); + + std::vector metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kUnitless)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kNeitherIsBetter)); + EXPECT_THAT(metric.metric_metadata, IsEmpty()); + ASSERT_THAT(metric.time_series.samples, SizeIs(1)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map{{"key", "value"}})); + ASSERT_THAT(metric.stats.mean, absl::optional(10.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional(0.0)); + ASSERT_THAT(metric.stats.min, absl::optional(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional(10.0)); +} + +TEST(DefaultMetricsLoggerTest, + AccumulatedMetricsReturnedTogetherWithLoggedMetrics) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + logger.LogSingleValueMetric( + "metric_name1", "test_case_name1", + /*value=*/10, Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + std::map{{"key_m", "value_m"}}); + logger.GetMetricsAccumulator()->AddSample( + "metric_name2", "test_case_name2", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/ + std::map{{"key_s", "value_s"}}); + + std::vector metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(2)); + EXPECT_THAT(metrics[0].name, Eq("metric_name2")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name2")); + EXPECT_THAT(metrics[0].unit, Eq(Unit::kUnitless)); + EXPECT_THAT(metrics[0].improvement_direction, + Eq(ImprovementDirection::kNeitherIsBetter)); + EXPECT_THAT(metrics[0].metric_metadata, IsEmpty()); + ASSERT_THAT(metrics[0].time_series.samples, SizeIs(1)); + EXPECT_THAT(metrics[0].time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metrics[0].time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metrics[0].time_series.samples[0].sample_metadata, + Eq(std::map{{"key_s", "value_s"}})); + ASSERT_THAT(metrics[0].stats.mean, absl::optional(10.0)); + ASSERT_THAT(metrics[0].stats.stddev, absl::optional(0.0)); + ASSERT_THAT(metrics[0].stats.min, absl::optional(10.0)); + ASSERT_THAT(metrics[0].stats.max, absl::optional(10.0)); + EXPECT_THAT(metrics[1].name, Eq("metric_name1")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[1].unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metrics[1].improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metrics[1].metric_metadata, + Eq(std::map{{"key_m", "value_m"}})); + ASSERT_THAT(metrics[1].time_series.samples, SizeIs(1)); + EXPECT_THAT(metrics[1].time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metrics[1].time_series.samples[0].sample_metadata, + Eq(std::map{})); + ASSERT_THAT(metrics[1].stats.mean, absl::optional(10.0)); + ASSERT_THAT(metrics[1].stats.stddev, absl::nullopt); + ASSERT_THAT(metrics[1].stats.min, absl::optional(10.0)); + ASSERT_THAT(metrics[1].stats.max, absl::optional(10.0)); +} + } // namespace } // namespace test } // namespace webrtc