From a40672a120d83d5b244d7284adf79ff975010edd Mon Sep 17 00:00:00 2001 From: palmkvist Date: Fri, 13 Jan 2017 05:58:34 -0800 Subject: [PATCH] Add UMA stats to bad call detection. Just simple "percentage of call that was bad" stats. BUG=webrtc:6814 Review-Url: https://codereview.webrtc.org/2578213003 Cr-Commit-Position: refs/heads/master@{#16049} --- webrtc/video/quality_threshold.cc | 20 ++++++++++- webrtc/video/quality_threshold.h | 3 ++ webrtc/video/quality_threshold_unittest.cc | 36 +++++++++++++++++++ webrtc/video/receive_statistics_proxy.cc | 36 ++++++++++++++++++- webrtc/video/receive_statistics_proxy.h | 2 ++ .../receive_statistics_proxy_unittest.cc | 29 +++++++++++++++ 6 files changed, 124 insertions(+), 2 deletions(-) diff --git a/webrtc/video/quality_threshold.cc b/webrtc/video/quality_threshold.cc index 3f2d835e80..c1d3b101eb 100644 --- a/webrtc/video/quality_threshold.cc +++ b/webrtc/video/quality_threshold.cc @@ -28,7 +28,9 @@ QualityThreshold::QualityThreshold(int low_threshold, next_index_(0), sum_(0), count_low_(0), - count_high_(0) { + count_high_(0), + num_high_states_(0), + num_certain_states_(0) { RTC_CHECK_GT(fraction, 0.5f); RTC_CHECK_GT(max_measurements, 1); RTC_CHECK_LT(low_threshold, high_threshold); @@ -64,6 +66,12 @@ void QualityThreshold::AddMeasurement(int measurement) { if (until_full_ > 0) --until_full_; + + if (is_high_) { + if (*is_high_) + ++num_high_states_; + ++num_certain_states_; + } } rtc::Optional QualityThreshold::IsHigh() const { @@ -83,4 +91,14 @@ rtc::Optional QualityThreshold::CalculateVariance() const { return rtc::Optional(variance / (max_measurements_ - 1)); } +rtc::Optional QualityThreshold::FractionHigh( + int min_required_samples) const { + RTC_DCHECK_GT(min_required_samples, 0); + if (num_certain_states_ < min_required_samples) + return rtc::Optional(); + + return rtc::Optional(static_cast(num_high_states_) / + num_certain_states_); +} + } // namespace webrtc diff --git a/webrtc/video/quality_threshold.h b/webrtc/video/quality_threshold.h index 9bd0aaee62..95e11d2016 100644 --- a/webrtc/video/quality_threshold.h +++ b/webrtc/video/quality_threshold.h @@ -29,6 +29,7 @@ class QualityThreshold { void AddMeasurement(int measurement); rtc::Optional IsHigh() const; rtc::Optional CalculateVariance() const; + rtc::Optional FractionHigh(int min_required_samples) const; private: const std::unique_ptr buffer_; @@ -42,6 +43,8 @@ class QualityThreshold { int sum_; int count_low_; int count_high_; + int num_high_states_; + int num_certain_states_; }; } // namespace webrtc diff --git a/webrtc/video/quality_threshold_unittest.cc b/webrtc/video/quality_threshold_unittest.cc index dcd094c221..6aa70c57ba 100644 --- a/webrtc/video/quality_threshold_unittest.cc +++ b/webrtc/video/quality_threshold_unittest.cc @@ -94,4 +94,40 @@ TEST(QualityThresholdTest, BetweenThresholds) { EXPECT_FALSE(thresh.IsHigh()); } +TEST(QualityThresholdTest, FractionHigh) { + const int kLowThreshold = 0; + const int kHighThreshold = 2; + const float kFraction = 0.75f; + const int kMaxMeasurements = 10; + + const int kBetweenThresholds = (kLowThreshold + kHighThreshold) / 2; + const int kNeededMeasurements = + static_cast(kFraction * kMaxMeasurements + 1); + + QualityThreshold thresh(kLowThreshold, kHighThreshold, kFraction, + kMaxMeasurements); + + for (int i = 0; i < kMaxMeasurements; ++i) { + EXPECT_FALSE(thresh.FractionHigh(1)); + thresh.AddMeasurement(kBetweenThresholds); + } + + for (int i = 0; i < kNeededMeasurements; i++) { + EXPECT_FALSE(thresh.FractionHigh(1)); + thresh.AddMeasurement(kHighThreshold); + } + EXPECT_FALSE(thresh.FractionHigh(2)); + ASSERT_TRUE(thresh.FractionHigh(1)); + EXPECT_NEAR(*thresh.FractionHigh(1), 1, 0.001); + + for (int i = 0; i < kNeededMeasurements; i++) { + EXPECT_NEAR(*thresh.FractionHigh(1), 1, 0.001); + thresh.AddMeasurement(kLowThreshold); + } + EXPECT_NEAR( + *thresh.FractionHigh(1), + static_cast(kNeededMeasurements) / (kNeededMeasurements + 1), + 0.001); +} + } // namespace webrtc diff --git a/webrtc/video/receive_statistics_proxy.cc b/webrtc/video/receive_statistics_proxy.cc index 3f23c0047a..673fabb7bb 100644 --- a/webrtc/video/receive_statistics_proxy.cc +++ b/webrtc/video/receive_statistics_proxy.cc @@ -25,6 +25,7 @@ namespace { const int64_t kFreqOffsetProcessIntervalMs = 40000; // Configuration for bad call detection. +const int kBadCallMinRequiredSamples = 10; const int kMinSampleLengthMs = 990; const int kNumMeasurements = 10; const int kNumMeasurementsVariance = kNumMeasurements * 1.5; @@ -60,6 +61,8 @@ ReceiveStatisticsProxy::ReceiveStatisticsProxy( kHighVarianceThreshold, kBadFraction, kNumMeasurementsVariance), + num_bad_states_(0), + num_certain_states_(0), // 1000ms window, scale 1000 for ms to s. decode_fps_estimator_(1000, 1000), renders_fps_estimator_(1000, 1000), @@ -201,6 +204,29 @@ void ReceiveStatisticsProxy::UpdateHistograms() { counters.UniqueNackRequestsInPercent()); } } + + if (num_certain_states_ >= kBadCallMinRequiredSamples) { + RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.BadCall.Any", + 100 * num_bad_states_ / num_certain_states_); + } + rtc::Optional fps_fraction = + fps_threshold_.FractionHigh(kBadCallMinRequiredSamples); + if (fps_fraction) { + RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.BadCall.FrameRate", + static_cast(100 * (1 - *fps_fraction))); + } + rtc::Optional variance_fraction = + variance_threshold_.FractionHigh(kBadCallMinRequiredSamples); + if (variance_fraction) { + RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.BadCall.FrameRateVariance", + static_cast(100 * *variance_fraction)); + } + rtc::Optional qp_fraction = + qp_threshold_.FractionHigh(kBadCallMinRequiredSamples); + if (qp_fraction) { + RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.BadCall.Qp", + static_cast(100 * *qp_fraction)); + } } void ReceiveStatisticsProxy::QualitySample() { @@ -262,6 +288,13 @@ void ReceiveStatisticsProxy::QualitySample() { last_sample_time_ = now; qp_sample_.Reset(); + + if (fps_threshold_.IsHigh() || variance_threshold_.IsHigh() || + qp_threshold_.IsHigh()) { + if (any_bad) + ++num_bad_states_; + ++num_certain_states_; + } } VideoReceiveStream::Stats ReceiveStatisticsProxy::GetStats() const { @@ -282,7 +315,8 @@ void ReceiveStatisticsProxy::OnDecoderImplementationName( void ReceiveStatisticsProxy::OnIncomingRate(unsigned int framerate, unsigned int bitrate_bps) { rtc::CritScope lock(&crit_); - QualitySample(); + if (stats_.rtp_stats.first_packet_time_ms != -1) + QualitySample(); stats_.network_frame_rate = framerate; stats_.total_bitrate_bps = bitrate_bps; } diff --git a/webrtc/video/receive_statistics_proxy.h b/webrtc/video/receive_statistics_proxy.h index 6b21a01510..e54f53aac1 100644 --- a/webrtc/video/receive_statistics_proxy.h +++ b/webrtc/video/receive_statistics_proxy.h @@ -117,6 +117,8 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, QualityThreshold qp_threshold_ GUARDED_BY(crit_); QualityThreshold variance_threshold_ GUARDED_BY(crit_); SampleCounter qp_sample_ GUARDED_BY(crit_); + int num_bad_states_ GUARDED_BY(crit_); + int num_certain_states_ GUARDED_BY(crit_); VideoReceiveStream::Stats stats_ GUARDED_BY(crit_); RateStatistics decode_fps_estimator_ GUARDED_BY(crit_); RateStatistics renders_fps_estimator_ GUARDED_BY(crit_); diff --git a/webrtc/video/receive_statistics_proxy_unittest.cc b/webrtc/video/receive_statistics_proxy_unittest.cc index dc944d59e3..a485edae02 100644 --- a/webrtc/video/receive_statistics_proxy_unittest.cc +++ b/webrtc/video/receive_statistics_proxy_unittest.cc @@ -189,6 +189,35 @@ TEST_F(ReceiveStatisticsProxyTest, LifetimeHistogramIsUpdated) { kTimeSec)); } +TEST_F(ReceiveStatisticsProxyTest, BadCallHistogramsAreUpdated) { + // Based on the tuning parameters this will produce 7 uncertain states, + // then 10 certainly bad states. There has to be 10 certain states before + // any histograms are recorded. + const int kNumBadSamples = 17; + + StreamDataCounters counters; + counters.first_packet_time_ms = fake_clock_.TimeInMilliseconds(); + statistics_proxy_->DataCountersUpdated(counters, config_.rtp.remote_ssrc); + + for (int i = 0; i < kNumBadSamples; ++i) { + // Since OnRenderedFrame is never called the fps in each sample will be 0, + // i.e. bad + fake_clock_.AdvanceTimeMilliseconds(1000); + statistics_proxy_->OnIncomingRate(0, 0); + } + // Histograms are updated when the statistics_proxy_ is deleted. + statistics_proxy_.reset(); + EXPECT_EQ(1, metrics::NumSamples("WebRTC.Video.BadCall.Any")); + EXPECT_EQ(1, metrics::NumEvents("WebRTC.Video.BadCall.Any", 100)); + + EXPECT_EQ(1, metrics::NumSamples("WebRTC.Video.BadCall.FrameRate")); + EXPECT_EQ(1, metrics::NumEvents("WebRTC.Video.BadCall.FrameRate", 100)); + + EXPECT_EQ(0, metrics::NumSamples("WebRTC.Video.BadCall.FrameRateVariance")); + + EXPECT_EQ(0, metrics::NumSamples("WebRTC.Video.BadCall.Qp")); +} + TEST_F(ReceiveStatisticsProxyTest, PacketLossHistogramIsUpdated) { const uint32_t kCumLost1 = 1; const uint32_t kExtSeqNum1 = 10;