From 9b526180c9e9722d3fc7f8689da6ec094fc7fc0a Mon Sep 17 00:00:00 2001 From: Andrey Logvin Date: Mon, 15 Jun 2020 16:14:07 +0000 Subject: [PATCH] Migrate pc level test metrics to new getStart API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:10430 Change-Id: I7555cb967f2e341da43338cb0f8652490992bd31 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/176857 Commit-Queue: Andrey Logvin Reviewed-by: Karl Wiberg Reviewed-by: Artem Titov Reviewed-by: Henrik Boström Cr-Commit-Position: refs/heads/master@{#31526} --- api/BUILD.gn | 8 +- api/test/stats_observer_interface.h | 10 +- api/test/video_quality_analyzer_interface.h | 5 +- test/pc/e2e/BUILD.gn | 13 +- .../audio/default_audio_quality_analyzer.cc | 130 ++++++++++-------- .../audio/default_audio_quality_analyzer.h | 22 ++- .../video/default_video_quality_analyzer.h | 5 +- ...video_quality_analyzer_injection_helper.cc | 6 +- .../video_quality_analyzer_injection_helper.h | 5 +- .../video/video_quality_metrics_reporter.cc | 114 +++++++++------ .../video/video_quality_metrics_reporter.h | 24 +++- .../e2e/network_quality_metrics_reporter.cc | 52 +++---- .../pc/e2e/network_quality_metrics_reporter.h | 10 +- test/pc/e2e/peer_connection_quality_test.cc | 2 +- test/pc/e2e/stats_poller.cc | 9 +- test/pc/e2e/stats_poller.h | 6 +- 16 files changed, 253 insertions(+), 168 deletions(-) diff --git a/api/BUILD.gn b/api/BUILD.gn index 30e414cddb..06a5736e76 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -359,11 +359,9 @@ rtc_source_set("stats_observer_interface") { testonly = true sources = [ "test/stats_observer_interface.h" ] - deps = [ - # For api/stats_types.h - ":libjingle_peerconnection_api", - ":rtp_parameters", - ] + deps = [ ":rtc_stats_api" ] + + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } rtc_source_set("peer_connection_quality_test_fixture_api") { diff --git a/api/test/stats_observer_interface.h b/api/test/stats_observer_interface.h index 98c8dd937f..ea4d6c23db 100644 --- a/api/test/stats_observer_interface.h +++ b/api/test/stats_observer_interface.h @@ -11,9 +11,8 @@ #ifndef API_TEST_STATS_OBSERVER_INTERFACE_H_ #define API_TEST_STATS_OBSERVER_INTERFACE_H_ -#include - -#include "api/stats_types.h" +#include "absl/strings/string_view.h" +#include "api/stats/rtc_stats_report.h" namespace webrtc { namespace webrtc_pc_e2e { @@ -25,8 +24,9 @@ class StatsObserverInterface { // Method called when stats reports are available for the PeerConnection // identified by |pc_label|. - virtual void OnStatsReports(const std::string& pc_label, - const StatsReports& reports) = 0; + virtual void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr& report) = 0; }; } // namespace webrtc_pc_e2e diff --git a/api/test/video_quality_analyzer_interface.h b/api/test/video_quality_analyzer_interface.h index 22114beb5e..c5370a7089 100644 --- a/api/test/video_quality_analyzer_interface.h +++ b/api/test/video_quality_analyzer_interface.h @@ -134,8 +134,9 @@ class VideoQualityAnalyzerInterface : public StatsObserverInterface { int32_t error_code) {} // Will be called every time new stats reports are available for the // Peer Connection identified by |pc_label|. - void OnStatsReports(const std::string& pc_label, - const StatsReports& stats_reports) override {} + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr& report) override {} // Tells analyzer that analysis complete and it should calculate final // statistics. diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 1671a82ecd..a1add95ed6 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -495,6 +495,7 @@ if (!build_with_chromium) { deps = [ ":test_peer", "../../../api:libjingle_peerconnection_api", + "../../../api:rtc_stats_api", "../../../api:stats_observer_interface", "../../../rtc_base:logging", ] @@ -552,9 +553,11 @@ if (!build_with_chromium) { deps = [ "../..:perf_test", "../../../api:audio_quality_analyzer_api", - "../../../api:libjingle_peerconnection_api", + "../../../api:rtc_stats_api", "../../../api:stats_observer_interface", "../../../api:track_id_stream_label_map", + "../../../api/units:time_delta", + "../../../api/units:timestamp", "../../../rtc_base:criticalsection", "../../../rtc_base:logging", "../../../rtc_base:rtc_numerics", @@ -591,6 +594,11 @@ if (!build_with_chromium) { deps = [ "../..:perf_test", "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:rtc_stats_api", + "../../../api/units:data_rate", + "../../../api/units:data_size", + "../../../api/units:time_delta", + "../../../api/units:timestamp", "../../../rtc_base:criticalsection", "../../../rtc_base:rtc_numerics", ] @@ -637,9 +645,10 @@ if (!build_with_chromium) { ] deps = [ "../..:perf_test", - "../../../api:libjingle_peerconnection_api", "../../../api:network_emulation_manager_api", "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:rtc_stats_api", + "../../../api/units:data_size", "../../../rtc_base:criticalsection", "../../../rtc_base:rtc_event", "../../../system_wrappers:field_trial", diff --git a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc index b8f1740e46..b890233521 100644 --- a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc @@ -10,16 +10,12 @@ #include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" -#include "api/stats_types.h" +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" #include "rtc_base/logging.h" namespace webrtc { namespace webrtc_pc_e2e { -namespace { - -static const char kStatsAudioMediaType[] = "audio"; - -} // namespace void DefaultAudioQualityAnalyzer::Start( std::string test_case_name, @@ -29,68 +25,82 @@ void DefaultAudioQualityAnalyzer::Start( } void DefaultAudioQualityAnalyzer::OnStatsReports( - const std::string& pc_label, - const StatsReports& stats_reports) { - for (const StatsReport* stats_report : stats_reports) { - // NetEq stats are only present in kStatsReportTypeSsrc reports, so all - // other reports are just ignored. - if (stats_report->type() != StatsReport::StatsType::kStatsReportTypeSsrc) { - continue; - } - // Ignoring stats reports of "video" SSRC. - const webrtc::StatsReport::Value* media_type = stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameMediaType); - RTC_CHECK(media_type); - if (strcmp(media_type->static_string_val(), kStatsAudioMediaType) != 0) { - continue; - } - if (stats_report->FindValue( - webrtc::StatsReport::kStatsValueNameBytesSent)) { - // If kStatsValueNameBytesSent is present, it means it's a send stream, - // but we need audio metrics for receive stream, so skip it. + absl::string_view pc_label, + const rtc::scoped_refptr& report) { + // TODO(https://crbug.com/webrtc/11683): use "inbound-rtp" instead of "track" + // stats when required audio metrics moved there + auto stats = report->GetStatsOfType(); + + for (auto& stat : stats) { + if (!stat->kind.is_defined() || + !(*stat->kind == RTCMediaStreamTrackKind::kAudio) || + !*stat->remote_source) { continue; } - const webrtc::StatsReport::Value* expand_rate = stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameExpandRate); - const webrtc::StatsReport::Value* accelerate_rate = stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameAccelerateRate); - const webrtc::StatsReport::Value* preemptive_rate = stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNamePreemptiveExpandRate); - const webrtc::StatsReport::Value* speech_expand_rate = - stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameSpeechExpandRate); - const webrtc::StatsReport::Value* preferred_buffer_size_ms = - stats_report->FindValue(StatsReport::StatsValueName:: - kStatsValueNamePreferredJitterBufferMs); - RTC_CHECK(expand_rate); - RTC_CHECK(accelerate_rate); - RTC_CHECK(preemptive_rate); - RTC_CHECK(speech_expand_rate); - RTC_CHECK(preferred_buffer_size_ms); + StatsSample sample; + sample.total_samples_received = + stat->total_samples_received.ValueOrDefault(0ul); + sample.concealed_samples = stat->concealed_samples.ValueOrDefault(0ul); + sample.removed_samples_for_acceleration = + stat->removed_samples_for_acceleration.ValueOrDefault(0ul); + sample.inserted_samples_for_deceleration = + stat->inserted_samples_for_deceleration.ValueOrDefault(0ul); + sample.silent_concealed_samples = + stat->silent_concealed_samples.ValueOrDefault(0ul); + sample.jitter_buffer_target_delay = + TimeDelta::Seconds(stat->jitter_buffer_target_delay.ValueOrDefault(0.)); + sample.jitter_buffer_emitted_count = + stat->jitter_buffer_emitted_count.ValueOrDefault(0ul); const std::string& stream_label = - GetStreamLabelFromStatsReport(stats_report); + analyzer_helper_->GetStreamLabelFromTrackId(*stat->track_identifier); rtc::CritScope crit(&lock_); - AudioStreamStats& audio_stream_stats = streams_stats_[stream_label]; - audio_stream_stats.expand_rate.AddSample(expand_rate->float_val()); - audio_stream_stats.accelerate_rate.AddSample(accelerate_rate->float_val()); - audio_stream_stats.preemptive_rate.AddSample(preemptive_rate->float_val()); - audio_stream_stats.speech_expand_rate.AddSample( - speech_expand_rate->float_val()); - audio_stream_stats.preferred_buffer_size_ms.AddSample( - preferred_buffer_size_ms->int_val()); - } -} + StatsSample prev_sample = last_stats_sample_[stream_label]; + RTC_CHECK_GE(sample.total_samples_received, + prev_sample.total_samples_received); + double total_samples_diff = static_cast( + sample.total_samples_received - prev_sample.total_samples_received); + if (total_samples_diff == 0) { + return; + } -const std::string& DefaultAudioQualityAnalyzer::GetStreamLabelFromStatsReport( - const StatsReport* stats_report) const { - const webrtc::StatsReport::Value* report_track_id = stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameTrackId); - RTC_CHECK(report_track_id); - return analyzer_helper_->GetStreamLabelFromTrackId( - report_track_id->string_val()); + AudioStreamStats& audio_stream_stats = streams_stats_[stream_label]; + audio_stream_stats.expand_rate.AddSample( + (sample.concealed_samples - prev_sample.concealed_samples) / + total_samples_diff); + audio_stream_stats.accelerate_rate.AddSample( + (sample.removed_samples_for_acceleration - + prev_sample.removed_samples_for_acceleration) / + total_samples_diff); + audio_stream_stats.preemptive_rate.AddSample( + (sample.inserted_samples_for_deceleration - + prev_sample.inserted_samples_for_deceleration) / + total_samples_diff); + + int64_t speech_concealed_samples = + sample.concealed_samples - sample.silent_concealed_samples; + int64_t prev_speech_concealed_samples = + prev_sample.concealed_samples - prev_sample.silent_concealed_samples; + audio_stream_stats.speech_expand_rate.AddSample( + (speech_concealed_samples - prev_speech_concealed_samples) / + total_samples_diff); + + int64_t jitter_buffer_emitted_count_diff = + sample.jitter_buffer_emitted_count - + prev_sample.jitter_buffer_emitted_count; + if (jitter_buffer_emitted_count_diff > 0) { + TimeDelta jitter_buffer_target_delay_diff = + sample.jitter_buffer_target_delay - + prev_sample.jitter_buffer_target_delay; + audio_stream_stats.preferred_buffer_size_ms.AddSample( + jitter_buffer_target_delay_diff.ms() / + jitter_buffer_emitted_count_diff); + } + + last_stats_sample_[stream_label] = sample; + } } std::string DefaultAudioQualityAnalyzer::GetTestCaseName( diff --git a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h index 33aaefd4c3..c990e4f357 100644 --- a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h +++ b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h @@ -14,9 +14,9 @@ #include #include -#include "api/stats_types.h" #include "api/test/audio_quality_analyzer_interface.h" #include "api/test/track_id_stream_label_map.h" +#include "api/units/time_delta.h" #include "rtc_base/critical_section.h" #include "rtc_base/numerics/samples_stats_counter.h" #include "test/testsupport/perf_test.h" @@ -32,22 +32,29 @@ struct AudioStreamStats { SamplesStatsCounter preferred_buffer_size_ms; }; -// TODO(bugs.webrtc.org/10430): Migrate to the new GetStats as soon as -// bugs.webrtc.org/10428 is fixed. class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface { public: void Start(std::string test_case_name, TrackIdStreamLabelMap* analyzer_helper) override; - void OnStatsReports(const std::string& pc_label, - const StatsReports& stats_reports) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr& report) override; void Stop() override; // Returns audio quality stats per stream label. std::map GetAudioStreamsStats() const; private: - const std::string& GetStreamLabelFromStatsReport( - const StatsReport* stats_report) const; + struct StatsSample { + uint64_t total_samples_received = 0; + uint64_t concealed_samples = 0; + uint64_t removed_samples_for_acceleration = 0; + uint64_t inserted_samples_for_deceleration = 0; + uint64_t silent_concealed_samples = 0; + TimeDelta jitter_buffer_target_delay = TimeDelta::Zero(); + uint64_t jitter_buffer_emitted_count = 0; + }; + std::string GetTestCaseName(const std::string& stream_label) const; void ReportResult(const std::string& metric_name, const std::string& stream_label, @@ -60,6 +67,7 @@ class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface { rtc::CriticalSection lock_; std::map streams_stats_ RTC_GUARDED_BY(lock_); + std::map last_stats_sample_ RTC_GUARDED_BY(lock_); }; } // namespace webrtc_pc_e2e diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index f37e0401fa..95049b1054 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -204,8 +204,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { int32_t error_code) override; void Stop() override; std::string GetStreamLabel(uint16_t frame_id) override; - void OnStatsReports(const std::string& pc_label, - const StatsReports& stats_reports) override {} + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr& report) override {} // Returns set of stream labels, that were met during test call. std::set GetKnownVideoStreams() const; diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc index d96fde0ce3..1948777848 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc @@ -152,9 +152,9 @@ void VideoQualityAnalyzerInjectionHelper::Start( } void VideoQualityAnalyzerInjectionHelper::OnStatsReports( - const std::string& pc_label, - const StatsReports& stats_reports) { - analyzer_->OnStatsReports(pc_label, stats_reports); + absl::string_view pc_label, + const rtc::scoped_refptr& report) { + analyzer_->OnStatsReports(pc_label, report); } void VideoQualityAnalyzerInjectionHelper::Stop() { diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h index 58701820a9..ca5243484d 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h @@ -79,8 +79,9 @@ class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface { // Forwards |stats_reports| for Peer Connection |pc_label| to // |analyzer_|. - void OnStatsReports(const std::string& pc_label, - const StatsReports& stats_reports) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr& report) override; // Stops VideoQualityAnalyzerInterface to populate final data and metrics. // Should be invoked after analyzed video tracks are disposed. diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc index 754a0a468f..baf973f277 100644 --- a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc +++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc @@ -10,51 +10,88 @@ #include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h" +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "api/units/data_rate.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" + namespace webrtc { namespace webrtc_pc_e2e { -namespace { - -constexpr int kBitsInByte = 8; - -} // namespace void VideoQualityMetricsReporter::Start(absl::string_view test_case_name) { test_case_name_ = std::string(test_case_name); + start_time_ = Now(); } -// TODO(bugs.webrtc.org/10430): Migrate to the new GetStats as soon as -// bugs.webrtc.org/10428 is fixed. void VideoQualityMetricsReporter::OnStatsReports( - const std::string& pc_label, - const StatsReports& stats_reports) { - for (const StatsReport* stats_report : stats_reports) { - // The only stats collected by this analyzer are present in - // kStatsReportTypeBwe reports, so all other reports are just ignored. - if (stats_report->type() != StatsReport::StatsType::kStatsReportTypeBwe) { + absl::string_view pc_label, + const rtc::scoped_refptr& report) { + RTC_CHECK(start_time_) + << "Please invoke Start(...) method before calling OnStatsReports(...)"; + + auto transport_stats = report->GetStatsOfType(); + if (transport_stats.size() == 0u || + !transport_stats[0]->selected_candidate_pair_id.is_defined()) { + return; + } + RTC_DCHECK_EQ(transport_stats.size(), 1); + std::string selected_ice_id = + transport_stats[0]->selected_candidate_pair_id.ValueToString(); + // Use the selected ICE candidate pair ID to get the appropriate ICE stats. + const RTCIceCandidatePairStats ice_candidate_pair_stats = + report->Get(selected_ice_id)->cast_to(); + + auto outbound_rtp_stats = report->GetStatsOfType(); + StatsSample sample; + for (auto& s : outbound_rtp_stats) { + if (!s->media_type.is_defined()) { continue; } - const webrtc::StatsReport::Value* available_send_bandwidth = - stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameAvailableSendBandwidth); - const webrtc::StatsReport::Value* retransmission_bitrate = - stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameRetransmitBitrate); - const webrtc::StatsReport::Value* transmission_bitrate = - stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameTransmitBitrate); - RTC_CHECK(available_send_bandwidth); - RTC_CHECK(retransmission_bitrate); - RTC_CHECK(transmission_bitrate); - - rtc::CritScope crit(&video_bwe_stats_lock_); - VideoBweStats& video_bwe_stats = video_bwe_stats_[pc_label]; - video_bwe_stats.available_send_bandwidth.AddSample( - available_send_bandwidth->int_val()); - video_bwe_stats.transmission_bitrate.AddSample( - transmission_bitrate->int_val()); - video_bwe_stats.retransmission_bitrate.AddSample( - retransmission_bitrate->int_val()); + if (!(*s->media_type == RTCMediaStreamTrackKind::kVideo)) { + continue; + } + if (s->timestamp_us() > sample.sample_time.us()) { + sample.sample_time = Timestamp::Micros(s->timestamp_us()); + } + sample.retransmitted_bytes_sent += + DataSize::Bytes(s->retransmitted_bytes_sent.ValueOrDefault(0ul)); + sample.bytes_sent += DataSize::Bytes(s->bytes_sent.ValueOrDefault(0ul)); + sample.header_bytes_sent += + DataSize::Bytes(s->header_bytes_sent.ValueOrDefault(0ul)); } + + rtc::CritScope crit(&video_bwe_stats_lock_); + VideoBweStats& video_bwe_stats = video_bwe_stats_[std::string(pc_label)]; + if (ice_candidate_pair_stats.available_outgoing_bitrate.is_defined()) { + video_bwe_stats.available_send_bandwidth.AddSample( + DataRate::BitsPerSec( + *ice_candidate_pair_stats.available_outgoing_bitrate) + .bytes_per_sec()); + } + + StatsSample prev_sample = last_stats_sample_[std::string(pc_label)]; + if (prev_sample.sample_time.IsZero()) { + prev_sample.sample_time = start_time_.value(); + } + last_stats_sample_[std::string(pc_label)] = sample; + + TimeDelta time_between_samples = sample.sample_time - prev_sample.sample_time; + if (time_between_samples.IsZero()) { + return; + } + + DataRate retransmission_bitrate = + (sample.retransmitted_bytes_sent - prev_sample.retransmitted_bytes_sent) / + time_between_samples; + video_bwe_stats.retransmission_bitrate.AddSample( + retransmission_bitrate.bytes_per_sec()); + DataRate transmission_bitrate = + (sample.bytes_sent + sample.header_bytes_sent - prev_sample.bytes_sent - + prev_sample.header_bytes_sent) / + time_between_samples; + video_bwe_stats.transmission_bitrate.AddSample( + transmission_bitrate.bytes_per_sec()); } void VideoQualityMetricsReporter::StopAndReportResults() { @@ -73,14 +110,11 @@ void VideoQualityMetricsReporter::ReportVideoBweResults( const std::string& test_case_name, const VideoBweStats& video_bwe_stats) { ReportResult("available_send_bandwidth", test_case_name, - video_bwe_stats.available_send_bandwidth / kBitsInByte, - "bytesPerSecond"); + video_bwe_stats.available_send_bandwidth, "bytesPerSecond"); ReportResult("transmission_bitrate", test_case_name, - video_bwe_stats.transmission_bitrate / kBitsInByte, - "bytesPerSecond"); + video_bwe_stats.transmission_bitrate, "bytesPerSecond"); ReportResult("retransmission_bitrate", test_case_name, - video_bwe_stats.retransmission_bitrate / kBitsInByte, - "bytesPerSecond"); + video_bwe_stats.retransmission_bitrate, "bytesPerSecond"); } void VideoQualityMetricsReporter::ReportResult( diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h index 1688a7b6fc..a990b2f3d1 100644 --- a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h +++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h @@ -15,6 +15,8 @@ #include #include "api/test/peerconnection_quality_test_fixture.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" #include "rtc_base/critical_section.h" #include "rtc_base/numerics/samples_stats_counter.h" #include "test/testsupport/perf_test.h" @@ -31,15 +33,25 @@ struct VideoBweStats { class VideoQualityMetricsReporter : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { public: - VideoQualityMetricsReporter() = default; + VideoQualityMetricsReporter() : clock_(Clock::GetRealTimeClock()) {} + VideoQualityMetricsReporter(Clock* const clock) : clock_(clock) {} ~VideoQualityMetricsReporter() override = default; void Start(absl::string_view test_case_name) override; - void OnStatsReports(const std::string& pc_label, - const StatsReports& reports) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr& report) override; void StopAndReportResults() override; private: + struct StatsSample { + DataSize bytes_sent = DataSize::Zero(); + DataSize header_bytes_sent = DataSize::Zero(); + DataSize retransmitted_bytes_sent = DataSize::Zero(); + + Timestamp sample_time = Timestamp::Zero(); + }; + std::string GetTestCaseName(const std::string& stream_label) const; static void ReportVideoBweResults(const std::string& test_case_name, const VideoBweStats& video_bwe_stats); @@ -50,14 +62,20 @@ class VideoQualityMetricsReporter const std::string& unit, webrtc::test::ImproveDirection improve_direction = webrtc::test::ImproveDirection::kNone); + Timestamp Now() const { return clock_->CurrentTime(); } + + Clock* const clock_; std::string test_case_name_; + absl::optional start_time_; rtc::CriticalSection video_bwe_stats_lock_; // Map between a peer connection label (provided by the framework) and // its video BWE stats. std::map video_bwe_stats_ RTC_GUARDED_BY(video_bwe_stats_lock_); + std::map last_stats_sample_ + RTC_GUARDED_BY(video_bwe_stats_lock_); }; } // namespace webrtc_pc_e2e diff --git a/test/pc/e2e/network_quality_metrics_reporter.cc b/test/pc/e2e/network_quality_metrics_reporter.cc index 56f0337037..3b232fdc71 100644 --- a/test/pc/e2e/network_quality_metrics_reporter.cc +++ b/test/pc/e2e/network_quality_metrics_reporter.cc @@ -11,7 +11,8 @@ #include -#include "api/stats_types.h" +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" #include "rtc_base/event.h" #include "system_wrappers/include/field_trial.h" #include "test/testsupport/perf_test.h" @@ -40,28 +41,29 @@ void NetworkQualityMetricsReporter::Start(absl::string_view test_case_name) { } void NetworkQualityMetricsReporter::OnStatsReports( - const std::string& pc_label, - const StatsReports& reports) { - rtc::CritScope cs(&lock_); - int64_t payload_bytes_received = 0; - int64_t payload_bytes_sent = 0; - for (const StatsReport* report : reports) { - if (report->type() == StatsReport::kStatsReportTypeSsrc) { - const auto* received = - report->FindValue(StatsReport::kStatsValueNameBytesReceived); - if (received) { - payload_bytes_received += received->int64_val(); - } - const auto* sent = - report->FindValue(StatsReport::kStatsValueNameBytesSent); - if (sent) { - payload_bytes_sent += sent->int64_val(); - } - } + absl::string_view pc_label, + const rtc::scoped_refptr& report) { + DataSize payload_received = DataSize::Zero(); + DataSize payload_sent = DataSize::Zero(); + + auto inbound_stats = report->GetStatsOfType(); + for (const auto& stat : inbound_stats) { + payload_received += + DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul) + + stat->header_bytes_received.ValueOrDefault(0ul)); } - PCStats& stats = pc_stats_[pc_label]; - stats.payload_bytes_received = payload_bytes_received; - stats.payload_bytes_sent = payload_bytes_sent; + + auto outbound_stats = report->GetStatsOfType(); + for (const auto& stat : outbound_stats) { + payload_sent += + DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul) + + stat->header_bytes_sent.ValueOrDefault(0ul)); + } + + rtc::CritScope cs(&lock_); + PCStats& stats = pc_stats_[std::string(pc_label)]; + stats.payload_received = payload_received; + stats.payload_sent = payload_sent; } void NetworkQualityMetricsReporter::StopAndReportResults() { @@ -125,9 +127,9 @@ void NetworkQualityMetricsReporter::ReportStats( void NetworkQualityMetricsReporter::ReportPCStats(const std::string& pc_label, const PCStats& stats) { - ReportResult("payload_bytes_received", pc_label, stats.payload_bytes_received, - "sizeInBytes"); - ReportResult("payload_bytes_sent", pc_label, stats.payload_bytes_sent, + ReportResult("payload_bytes_received", pc_label, + stats.payload_received.bytes(), "sizeInBytes"); + ReportResult("payload_bytes_sent", pc_label, stats.payload_sent.bytes(), "sizeInBytes"); } diff --git a/test/pc/e2e/network_quality_metrics_reporter.h b/test/pc/e2e/network_quality_metrics_reporter.h index 6454f17526..932e03140b 100644 --- a/test/pc/e2e/network_quality_metrics_reporter.h +++ b/test/pc/e2e/network_quality_metrics_reporter.h @@ -15,6 +15,7 @@ #include "api/test/network_emulation_manager.h" #include "api/test/peerconnection_quality_test_fixture.h" +#include "api/units/data_size.h" #include "rtc_base/critical_section.h" namespace webrtc { @@ -30,16 +31,17 @@ class NetworkQualityMetricsReporter // Network stats must be empty when this method will be invoked. void Start(absl::string_view test_case_name) override; - void OnStatsReports(const std::string& pc_label, - const StatsReports& reports) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr& report) override; void StopAndReportResults() override; private: struct PCStats { // TODO(nisse): Separate audio and video counters. Depends on standard stat // counters, enabled by field trial "WebRTC-UseStandardBytesStats". - int64_t payload_bytes_received = 0; - int64_t payload_bytes_sent = 0; + DataSize payload_received = DataSize::Zero(); + DataSize payload_sent = DataSize::Zero(); }; static EmulatedNetworkStats PopulateStats( diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index 362c7c01c9..a23d2248f3 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -250,7 +250,7 @@ void PeerConnectionE2EQualityTest::Run(RunParams run_params) { std::min(video_analyzer_threads, kMaxVideoAnalyzerThreads); RTC_LOG(INFO) << "video_analyzer_threads=" << video_analyzer_threads; quality_metrics_reporters_.push_back( - std::make_unique()); + std::make_unique(clock_)); video_quality_analyzer_injection_helper_->Start( test_case_name_, diff --git a/test/pc/e2e/stats_poller.cc b/test/pc/e2e/stats_poller.cc index 987f26e7e8..e6973e6af1 100644 --- a/test/pc/e2e/stats_poller.cc +++ b/test/pc/e2e/stats_poller.cc @@ -18,14 +18,13 @@ namespace webrtc { namespace webrtc_pc_e2e { void InternalStatsObserver::PollStats() { - peer_->pc()->GetStats(this, nullptr, - webrtc::PeerConnectionInterface::StatsOutputLevel:: - kStatsOutputLevelStandard); + peer_->pc()->GetStats(this); } -void InternalStatsObserver::OnComplete(const StatsReports& reports) { +void InternalStatsObserver::OnStatsDelivered( + const rtc::scoped_refptr& report) { for (auto* observer : observers_) { - observer->OnStatsReports(pc_label_, reports); + observer->OnStatsReports(pc_label_, report); } } diff --git a/test/pc/e2e/stats_poller.h b/test/pc/e2e/stats_poller.h index 3d0c2d6801..157a147834 100644 --- a/test/pc/e2e/stats_poller.h +++ b/test/pc/e2e/stats_poller.h @@ -17,6 +17,7 @@ #include #include "api/peer_connection_interface.h" +#include "api/stats/rtc_stats_collector_callback.h" #include "api/test/stats_observer_interface.h" #include "test/pc/e2e/test_peer.h" @@ -25,7 +26,7 @@ namespace webrtc_pc_e2e { // Helper class that will notify all the webrtc::test::StatsObserverInterface // objects subscribed. -class InternalStatsObserver : public StatsObserver { +class InternalStatsObserver : public RTCStatsCollectorCallback { public: InternalStatsObserver(std::string pc_label, TestPeer* peer, @@ -36,7 +37,8 @@ class InternalStatsObserver : public StatsObserver { void PollStats(); - void OnComplete(const StatsReports& reports) override; + void OnStatsDelivered( + const rtc::scoped_refptr& report) override; private: std::string pc_label_;