diff --git a/test/scenario/BUILD.gn b/test/scenario/BUILD.gn index aafafa54c4..1918a2b414 100644 --- a/test/scenario/BUILD.gn +++ b/test/scenario/BUILD.gn @@ -59,6 +59,7 @@ if (rtc_include_tests) { "hardware_codecs.h", "network_node.cc", "network_node.h", + "performance_stats.cc", "performance_stats.h", "scenario.cc", "scenario.h", @@ -128,6 +129,7 @@ if (rtc_include_tests) { "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_base_tests_utils", + "../../rtc_base:rtc_numerics", "../../rtc_base:rtc_task_queue", "../../rtc_base:safe_minmax", "../../rtc_base:task_queue_for_test", diff --git a/test/scenario/performance_stats.cc b/test/scenario/performance_stats.cc new file mode 100644 index 0000000000..3ff66cd10c --- /dev/null +++ b/test/scenario/performance_stats.cc @@ -0,0 +1,193 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/scenario/performance_stats.h" + +#include + +namespace webrtc { +namespace test { +void EventRateCounter::AddEvent(Timestamp event_time) { + if (first_time_.IsInfinite()) { + first_time_ = event_time; + } else { + RTC_DCHECK(event_time >= last_time_); + interval_.AddSample(event_time - last_time_); + } + last_time_ = event_time; + event_count_++; +} + +void EventRateCounter::AddEvents(EventRateCounter other) { + first_time_ = std::min(first_time_, other.first_time_); + last_time_ = std::max(last_time_, other.last_time_); + event_count_ += other.event_count_; + interval_.AddSamples(other.interval_); +} + +bool EventRateCounter::IsEmpty() const { + return first_time_ == last_time_; +} + +double EventRateCounter::Rate() const { + if (event_count_ == 0) + return 0; + if (event_count_ == 1) + return NAN; + return (event_count_ - 1) / (last_time_ - first_time_).seconds(); +} + +double SampleStats::Max() { + if (IsEmpty()) + return INFINITY; + return GetMax(); +} + +double SampleStats::Mean() { + if (IsEmpty()) + return 0; + return GetAverage(); +} + +double SampleStats::Median() { + return Quantile(0.5); +} + +double SampleStats::Quantile(double quantile) { + if (IsEmpty()) + return 0; + return GetPercentile(quantile); +} + +double SampleStats::Min() { + if (IsEmpty()) + return -INFINITY; + return GetMin(); +} + +double SampleStats::Variance() { + if (IsEmpty()) + return 0; + return GetVariance(); +} + +double SampleStats::StandardDeviation() { + return sqrt(Variance()); +} + +void SampleStats::AddSample(TimeDelta delta) { + RTC_DCHECK(delta.IsFinite()); + stats_.AddSample(delta.seconds()); +} + +void SampleStats::AddSampleMs(double delta_ms) { + AddSample(TimeDelta::ms(delta_ms)); +} +void SampleStats::AddSamples(const SampleStats& other) { + stats_.AddSamples(other.stats_); +} + +TimeDelta SampleStats::Max() { + return TimeDelta::seconds(stats_.Max()); +} + +TimeDelta SampleStats::Mean() { + return TimeDelta::seconds(stats_.Mean()); +} + +TimeDelta SampleStats::Median() { + return Quantile(0.5); +} + +TimeDelta SampleStats::Quantile(double quantile) { + return TimeDelta::seconds(stats_.Quantile(quantile)); +} + +TimeDelta SampleStats::Min() { + return TimeDelta::seconds(stats_.Min()); +} + +TimeDelta SampleStats::Variance() { + return TimeDelta::seconds(stats_.Variance()); +} + +TimeDelta SampleStats::StandardDeviation() { + return TimeDelta::seconds(stats_.StandardDeviation()); +} + +void SampleStats::AddSample(DataRate sample) { + stats_.AddSample(sample.bps()); +} + +void SampleStats::AddSampleBps(double rate_bps) { + stats_.AddSample(rate_bps); +} + +void SampleStats::AddSamples(const SampleStats& other) { + stats_.AddSamples(other.stats_); +} + +DataRate SampleStats::Max() { + return DataRate::bps(stats_.Max()); +} + +DataRate SampleStats::Mean() { + return DataRate::bps(stats_.Mean()); +} + +DataRate SampleStats::Median() { + return Quantile(0.5); +} + +DataRate SampleStats::Quantile(double quantile) { + return DataRate::bps(stats_.Quantile(quantile)); +} + +DataRate SampleStats::Min() { + return DataRate::bps(stats_.Min()); +} + +DataRate SampleStats::Variance() { + return DataRate::bps(stats_.Variance()); +} + +DataRate SampleStats::StandardDeviation() { + return DataRate::bps(stats_.StandardDeviation()); +} + +void VideoFramesStats::AddFrameInfo(const VideoFrameBuffer& frame, + Timestamp at_time) { + ++count; + RTC_DCHECK(at_time.IsFinite()); + pixels.AddSample(frame.width() * frame.height()); + resolution.AddSample(std::max(frame.width(), frame.height())); + frames.AddEvent(at_time); +} + +void VideoFramesStats::AddStats(const VideoFramesStats& other) { + count += other.count; + pixels.AddSamples(other.pixels); + resolution.AddSamples(other.resolution); + frames.AddEvents(other.frames); +} + +void VideoQualityStats::AddStats(const VideoQualityStats& other) { + capture.AddStats(other.capture); + render.AddStats(other.render); + lost_count += other.lost_count; + freeze_count += other.freeze_count; + end_to_end_delay.AddSamples(other.end_to_end_delay); + psnr.AddSamples(other.psnr); + skipped_between_rendered.AddSamples(other.skipped_between_rendered); + freeze_duration.AddSamples(other.freeze_duration); + time_between_freezes.AddSamples(other.time_between_freezes); +} + +} // namespace test +} // namespace webrtc diff --git a/test/scenario/performance_stats.h b/test/scenario/performance_stats.h index e58dab37bf..19798760b4 100644 --- a/test/scenario/performance_stats.h +++ b/test/scenario/performance_stats.h @@ -14,7 +14,7 @@ #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "api/video/video_frame_buffer.h" -#include "test/statistics.h" +#include "rtc_base/numerics/samples_stats_counter.h" namespace webrtc { namespace test { @@ -35,13 +35,125 @@ struct VideoFramePair { int repeated = 0; }; +template +class SampleStats; + +template <> +class SampleStats : public SamplesStatsCounter { + public: + double Max(); + double Mean(); + double Median(); + double Quantile(double quantile); + double Min(); + double Variance(); + double StandardDeviation(); +}; + +template <> +class SampleStats { + public: + void AddSample(TimeDelta delta); + void AddSampleMs(double delta_ms); + void AddSamples(const SampleStats& other); + TimeDelta Max(); + TimeDelta Mean(); + TimeDelta Median(); + TimeDelta Quantile(double quantile); + TimeDelta Min(); + TimeDelta Variance(); + TimeDelta StandardDeviation(); + + private: + SampleStats stats_; +}; + +template <> +class SampleStats { + public: + void AddSample(DataRate rate); + void AddSampleBps(double rate_bps); + void AddSamples(const SampleStats& other); + DataRate Max(); + DataRate Mean(); + DataRate Median(); + DataRate Quantile(double quantile); + DataRate Min(); + DataRate Variance(); + DataRate StandardDeviation(); + + private: + SampleStats stats_; +}; + +class EventRateCounter { + public: + void AddEvent(Timestamp event_time); + void AddEvents(EventRateCounter other); + bool IsEmpty() const; + double Rate() const; + SampleStats& interval() { return interval_; } + + private: + Timestamp first_time_ = Timestamp::PlusInfinity(); + Timestamp last_time_ = Timestamp::MinusInfinity(); + int64_t event_count_ = 0; + SampleStats interval_; +}; + +struct VideoFramesStats { + int count = 0; + SampleStats pixels; + SampleStats resolution; + EventRateCounter frames; + void AddFrameInfo(const VideoFrameBuffer& frame, Timestamp at_time); + void AddStats(const VideoFramesStats& other); +}; + struct VideoQualityStats { - int captures_count = 0; - int valid_count = 0; int lost_count = 0; - Statistics end_to_end_seconds; - Statistics frame_size; - Statistics psnr; + int freeze_count = 0; + VideoFramesStats capture; + VideoFramesStats render; + // Time from frame was captured on device to time frame was displayed on + // device. + SampleStats end_to_end_delay; + SampleStats psnr; + // Frames skipped between two nearest. + SampleStats skipped_between_rendered; + // In the next 2 metrics freeze is a pause that is longer, than maximum: + // 1. 150ms + // 2. 3 * average time between two sequential frames. + // Item 1 will cover high fps video and is a duration, that is noticeable by + // human eye. Item 2 will cover low fps video like screen sharing. + SampleStats freeze_duration; + // Mean time between one freeze end and next freeze start. + SampleStats time_between_freezes; + void AddStats(const VideoQualityStats& other); +}; + +struct CollectedCallStats { + SampleStats target_rate; + SampleStats memory_usage; +}; + +struct CollectedAudioReceiveStats { + SampleStats expand_rate; + SampleStats accelerate_rate; + SampleStats jitter_buffer; +}; +struct CollectedVideoSendStats { + SampleStats encode_frame_rate; + SampleStats encode_time; + SampleStats encode_usage; + SampleStats media_bitrate; + SampleStats fec_bitrate; +}; +struct CollectedVideoReceiveStats { + SampleStats decode_time; + SampleStats decode_time_max; + SampleStats decode_pixels; + SampleStats resolution; }; } // namespace test diff --git a/test/scenario/stats_collection.cc b/test/scenario/stats_collection.cc index 77b2ae0438..104c824c56 100644 --- a/test/scenario/stats_collection.cc +++ b/test/scenario/stats_collection.cc @@ -9,10 +9,8 @@ */ #include "test/scenario/stats_collection.h" - -#include - #include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/memory_usage.h" namespace webrtc { namespace test { @@ -39,20 +37,47 @@ std::function VideoQualityAnalyzer::Handler() { } void VideoQualityAnalyzer::HandleFramePair(VideoFramePair sample) { + layer_analyzers_[sample.layer_id].HandleFramePair(sample, writer_.get()); + cached_.reset(); +} + +std::vector VideoQualityAnalyzer::layer_stats() const { + std::vector res; + for (auto& layer : layer_analyzers_) + res.push_back(layer.second.stats_); + return res; +} + +VideoQualityStats& VideoQualityAnalyzer::stats() { + if (!cached_) { + cached_ = VideoQualityStats(); + for (auto& layer : layer_analyzers_) + cached_->AddStats(layer.second.stats_); + } + return *cached_; +} + +void VideoLayerAnalyzer::HandleFramePair(VideoFramePair sample, + RtcEventLogOutput* writer) { double psnr = NAN; RTC_CHECK(sample.captured); - ++stats_.captures_count; + HandleCapturedFrame(sample); if (!sample.decoded) { ++stats_.lost_count; + ++skip_count_; } else { psnr = I420PSNR(*sample.captured->ToI420(), *sample.decoded->ToI420()); - ++stats_.valid_count; - stats_.end_to_end_seconds.AddSample( - (sample.render_time - sample.capture_time).seconds()); + stats_.end_to_end_delay.AddSample(sample.render_time - sample.capture_time); stats_.psnr.AddSample(psnr); + if (sample.repeated) { + ++stats_.freeze_count; + ++skip_count_; + } else { + HandleRenderedFrame(sample); + } } - if (writer_) { - LogWriteFormat(writer_.get(), "%.3f %.3f %.3f %i %i %i %i %.3f\n", + if (writer) { + LogWriteFormat(writer, "%.3f %.3f %.3f %i %i %i %i %.3f\n", sample.capture_time.seconds(), sample.render_time.seconds(), sample.captured->width(), sample.captured->height(), @@ -60,9 +85,78 @@ void VideoQualityAnalyzer::HandleFramePair(VideoFramePair sample) { } } -VideoQualityStats VideoQualityAnalyzer::stats() const { - return stats_; +void VideoLayerAnalyzer::HandleCapturedFrame(const VideoFramePair& sample) { + stats_.capture.AddFrameInfo(*sample.captured, sample.capture_time); + if (last_freeze_time_.IsInfinite()) + last_freeze_time_ = sample.capture_time; } +void VideoLayerAnalyzer::HandleRenderedFrame(const VideoFramePair& sample) { + stats_.render.AddFrameInfo(*sample.decoded, sample.render_time); + stats_.skipped_between_rendered.AddSample(skip_count_); + skip_count_ = 0; + + if (last_render_time_.IsFinite()) { + RTC_DCHECK(sample.render_time.IsFinite()); + TimeDelta render_interval = sample.render_time - last_render_time_; + TimeDelta mean_interval = stats_.render.frames.interval().Mean(); + if (render_interval > TimeDelta::ms(150) + mean_interval || + render_interval > 3 * mean_interval) { + stats_.freeze_duration.AddSample(render_interval); + stats_.time_between_freezes.AddSample(last_render_time_ - + last_freeze_time_); + last_freeze_time_ = sample.render_time; + } + } + last_render_time_ = sample.render_time; +} + +void CallStatsCollector::AddStats(Call::Stats sample) { + stats_.target_rate.AddSampleBps(sample.send_bandwidth_bps); + stats_.memory_usage.AddSample(rtc::GetProcessResidentSizeBytes()); +} + +void AudioReceiveStatsCollector::AddStats(AudioReceiveStream::Stats sample) { + stats_.expand_rate.AddSample(sample.expand_rate); + stats_.accelerate_rate.AddSample(sample.accelerate_rate); + stats_.jitter_buffer.AddSampleMs(sample.jitter_buffer_ms); +} + +void VideoSendStatsCollector::AddStats(VideoSendStream::Stats sample, + Timestamp at_time) { + // It's not certain that we yet have estimates for any of these stats. + // Check that they are positive before mixing them in. + if (sample.encode_frame_rate <= 0) + return; + + stats_.encode_frame_rate.AddSample(sample.encode_frame_rate); + stats_.encode_time.AddSampleMs(sample.avg_encode_time_ms); + stats_.encode_usage.AddSample(sample.encode_usage_percent / 100.0); + stats_.media_bitrate.AddSampleBps(sample.media_bitrate_bps); + + size_t fec_bytes = 0; + for (const auto& kv : sample.substreams) { + fec_bytes += kv.second.rtp_stats.fec.payload_bytes + + kv.second.rtp_stats.fec.padding_bytes; + } + if (last_update_.IsFinite()) { + auto fec_delta = DataSize::bytes(fec_bytes - last_fec_bytes_); + auto time_delta = at_time - last_update_; + stats_.fec_bitrate.AddSample(fec_delta / time_delta); + } + last_fec_bytes_ = fec_bytes; + last_update_ = at_time; +} + +void VideoReceiveStatsCollector::AddStats(VideoReceiveStream::Stats sample) { + if (sample.decode_ms > 0) + stats_.decode_time.AddSampleMs(sample.decode_ms); + if (sample.max_decode_ms > 0) + stats_.decode_time_max.AddSampleMs(sample.max_decode_ms); + if (sample.width > 0 && sample.height > 0) { + stats_.decode_pixels.AddSample(sample.width * sample.height); + stats_.resolution.AddSample(sample.height); + } +} } // namespace test } // namespace webrtc diff --git a/test/scenario/stats_collection.h b/test/scenario/stats_collection.h index d1b46e4c78..0b8b4a327f 100644 --- a/test/scenario/stats_collection.h +++ b/test/scenario/stats_collection.h @@ -10,8 +10,11 @@ #ifndef TEST_SCENARIO_STATS_COLLECTION_H_ #define TEST_SCENARIO_STATS_COLLECTION_H_ +#include #include +#include "absl/types/optional.h" +#include "call/call.h" #include "test/logging/log_writer.h" #include "test/scenario/performance_stats.h" @@ -22,6 +25,18 @@ struct VideoQualityAnalyzerConfig { double psnr_coverage = 1; }; +class VideoLayerAnalyzer { + public: + void HandleCapturedFrame(const VideoFramePair& sample); + void HandleRenderedFrame(const VideoFramePair& sample); + void HandleFramePair(VideoFramePair sample, RtcEventLogOutput* writer); + VideoQualityStats stats_; + Timestamp last_capture_time_ = Timestamp::MinusInfinity(); + Timestamp last_render_time_ = Timestamp::MinusInfinity(); + Timestamp last_freeze_time_ = Timestamp::MinusInfinity(); + int skip_count_ = 0; +}; + class VideoQualityAnalyzer { public: explicit VideoQualityAnalyzer( @@ -29,15 +44,59 @@ class VideoQualityAnalyzer { std::unique_ptr writer = nullptr); ~VideoQualityAnalyzer(); void HandleFramePair(VideoFramePair sample); - VideoQualityStats stats() const; + std::vector layer_stats() const; + VideoQualityStats& stats(); void PrintHeaders(); void PrintFrameInfo(const VideoFramePair& sample); std::function Handler(); private: const VideoQualityAnalyzerConfig config_; - VideoQualityStats stats_; + std::map layer_analyzers_; const std::unique_ptr writer_; + absl::optional cached_; +}; + +class CallStatsCollector { + public: + void AddStats(Call::Stats sample); + CollectedCallStats& stats() { return stats_; } + + private: + CollectedCallStats stats_; +}; +class AudioReceiveStatsCollector { + public: + void AddStats(AudioReceiveStream::Stats sample); + CollectedAudioReceiveStats& stats() { return stats_; } + + private: + CollectedAudioReceiveStats stats_; +}; +class VideoSendStatsCollector { + public: + void AddStats(VideoSendStream::Stats sample, Timestamp at_time); + CollectedVideoSendStats& stats() { return stats_; } + + private: + CollectedVideoSendStats stats_; + Timestamp last_update_ = Timestamp::MinusInfinity(); + size_t last_fec_bytes_ = 0; +}; +class VideoReceiveStatsCollector { + public: + void AddStats(VideoReceiveStream::Stats sample); + CollectedVideoReceiveStats& stats() { return stats_; } + + private: + CollectedVideoReceiveStats stats_; +}; + +struct CallStatsCollectors { + CallStatsCollector call; + AudioReceiveStatsCollector audio_receive; + VideoSendStatsCollector video_send; + VideoReceiveStatsCollector video_receive; }; } // namespace test diff --git a/test/scenario/stats_collection_unittest.cc b/test/scenario/stats_collection_unittest.cc index ee01ad53c2..48a4bc32e7 100644 --- a/test/scenario/stats_collection_unittest.cc +++ b/test/scenario/stats_collection_unittest.cc @@ -16,47 +16,71 @@ namespace test { namespace { void CreateAnalyzedStream(Scenario* s, NetworkNodeConfig network_config, - VideoQualityAnalyzer* analyzer) { + VideoQualityAnalyzer* analyzer, + CallStatsCollectors* collectors) { VideoStreamConfig config; config.encoder.codec = VideoStreamConfig::Encoder::Codec::kVideoCodecVP8; config.encoder.implementation = VideoStreamConfig::Encoder::Implementation::kSoftware; config.hooks.frame_pair_handlers = {analyzer->Handler()}; - auto route = s->CreateRoutes(s->CreateClient("caller", CallClientConfig()), - {s->CreateSimulationNode(network_config)}, - s->CreateClient("callee", CallClientConfig()), - {s->CreateSimulationNode(NetworkNodeConfig())}); - s->CreateVideoStream(route->forward(), config); + auto* caller = s->CreateClient("caller", CallClientConfig()); + auto route = + s->CreateRoutes(caller, {s->CreateSimulationNode(network_config)}, + s->CreateClient("callee", CallClientConfig()), + {s->CreateSimulationNode(NetworkNodeConfig())}); + auto* video = s->CreateVideoStream(route->forward(), config); + auto* audio = s->CreateAudioStream(route->forward(), AudioStreamConfig()); + if (collectors) { + s->Every(TimeDelta::seconds(1), [=] { + collectors->call.AddStats(caller->GetStats()); + collectors->audio_receive.AddStats(audio->receive()->GetStats()); + collectors->video_send.AddStats(video->send()->GetStats(), s->Now()); + collectors->video_receive.AddStats(video->receive()->GetStats()); + }); + } } } // namespace TEST(ScenarioAnalyzerTest, PsnrIsHighWhenNetworkIsGood) { VideoQualityAnalyzer analyzer; + CallStatsCollectors stats; { - Scenario s("", /*real_time*/ false); + Scenario s; NetworkNodeConfig good_network; good_network.simulation.bandwidth = DataRate::kbps(1000); - CreateAnalyzedStream(&s, good_network, &analyzer); - s.RunFor(TimeDelta::seconds(1)); + CreateAnalyzedStream(&s, good_network, &analyzer, &stats); + s.RunFor(TimeDelta::seconds(3)); } - // This is mainty a regression test, the target is based on previous runs and - // might change due to changes in configuration and encoder etc. - EXPECT_GT(analyzer.stats().psnr.Mean(), 40); + // This is a change detecting test, the targets are based on previous runs and + // might change due to changes in configuration and encoder etc. The main + // purpose is to show how the stats can be used. To avoid being overly + // sensistive to change, the ranges are chosen to be quite large. + EXPECT_NEAR(analyzer.stats().psnr.Mean(), 43, 10); + EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 700, 300); + EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 500, 200); + EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10); + EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 40, 20); } TEST(ScenarioAnalyzerTest, PsnrIsLowWhenNetworkIsBad) { VideoQualityAnalyzer analyzer; + CallStatsCollectors stats; { - Scenario s("", /*real_time*/ false); + Scenario s; NetworkNodeConfig bad_network; bad_network.simulation.bandwidth = DataRate::kbps(100); bad_network.simulation.loss_rate = 0.02; - CreateAnalyzedStream(&s, bad_network, &analyzer); - s.RunFor(TimeDelta::seconds(1)); + CreateAnalyzedStream(&s, bad_network, &analyzer, &stats); + s.RunFor(TimeDelta::seconds(3)); } - // This is mainty a regression test, the target is based on previous runs and + // This is a change detecting test, the targets are based on previous runs and // might change due to changes in configuration and encoder etc. - EXPECT_LT(analyzer.stats().psnr.Mean(), 30); + EXPECT_NEAR(analyzer.stats().psnr.Mean(), 16, 10); + EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 75, 50); + EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 100, 50); + EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10); + EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 45, 20); } + } // namespace test } // namespace webrtc