Logging basic bad call detection

BUG=webrtc:6814

Review-Url: https://codereview.webrtc.org/2474913002
Cr-Commit-Position: refs/heads/master@{#15568}

webrtc/video/BUILD.gn

@@ -18,6 +18,8 @@ rtc_static_library("video") {
     "overuse_frame_detector.h",
     "payload_router.cc",
     "payload_router.h",
+    "quality_threshold.cc",
+    "quality_threshold.h",
     "receive_statistics_proxy.cc",
     "receive_statistics_proxy.h",
     "report_block_stats.cc",
@@ -151,6 +153,7 @@ if (rtc_include_tests) {
       "end_to_end_tests.cc",
       "overuse_frame_detector_unittest.cc",
       "payload_router_unittest.cc",
+      "quality_threshold_unittest.cc",
       "receive_statistics_proxy_unittest.cc",
       "report_block_stats_unittest.cc",
       "send_delay_stats_unittest.cc",

webrtc/video/quality_threshold.cc

@@ -0,0 +1,86 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/video/quality_threshold.h"
+
+#include "webrtc/base/checks.h"
+#include "webrtc/base/logging.h"
+
+namespace webrtc {
+
+QualityThreshold::QualityThreshold(int low_threshold,
+                                   int high_threshold,
+                                   float fraction,
+                                   int max_measurements)
+    : buffer_(new int[max_measurements]),
+      max_measurements_(max_measurements),
+      fraction_(fraction),
+      low_threshold_(low_threshold),
+      high_threshold_(high_threshold),
+      until_full_(max_measurements),
+      next_index_(0),
+      sum_(0),
+      count_low_(0),
+      count_high_(0) {
+  RTC_CHECK_GT(fraction, 0.5f);
+  RTC_CHECK_GT(max_measurements, 1);
+  RTC_CHECK_LT(low_threshold, high_threshold);
+}
+
+void QualityThreshold::AddMeasurement(int measurement) {
+  int prev_val = until_full_ > 0 ? 0 : buffer_[next_index_];
+  buffer_[next_index_] = measurement;
+  next_index_ = (next_index_ + 1) % max_measurements_;
+
+  sum_ += measurement - prev_val;
+
+  if (until_full_ == 0) {
+    if (prev_val <= low_threshold_) {
+      --count_low_;
+    } else if (prev_val >= high_threshold_) {
+      --count_high_;
+    }
+  }
+
+  if (measurement <= low_threshold_) {
+    ++count_low_;
+  } else if (measurement >= high_threshold_) {
+    ++count_high_;
+  }
+
+  float sufficient_majority = fraction_ * max_measurements_;
+  if (count_high_ >= sufficient_majority) {
+    is_high_ = rtc::Optional<bool>(true);
+  } else if (count_low_ >= sufficient_majority) {
+    is_high_ = rtc::Optional<bool>(false);
+  }
+
+  if (until_full_ > 0)
+    --until_full_;
+}
+
+rtc::Optional<bool> QualityThreshold::IsHigh() const {
+  return is_high_;
+}
+
+rtc::Optional<double> QualityThreshold::CalculateVariance() const {
+  if (until_full_ > 0) {
+    return rtc::Optional<double>();
+  }
+
+  double variance = 0;
+  double mean = static_cast<double>(sum_) / max_measurements_;
+  for (int i = 0; i < max_measurements_; ++i) {
+    variance += (buffer_[i] - mean) * (buffer_[i] - mean);
+  }
+  return rtc::Optional<double>(variance / (max_measurements_ - 1));
+}
+
+}  // namespace webrtc

webrtc/video/quality_threshold.h

@@ -0,0 +1,49 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_VIDEO_QUALITY_THRESHOLD_H_
+#define WEBRTC_VIDEO_QUALITY_THRESHOLD_H_
+
+#include <memory>
+
+#include "webrtc/base/optional.h"
+
+namespace webrtc {
+
+class QualityThreshold {
+ public:
+  // Both thresholds are inclusive, i.e. measurement >= high signifies a high
+  // state, while measurement <= low signifies a low state.
+  QualityThreshold(int low_threshold,
+                   int high_threshold,
+                   float fraction,
+                   int max_measurements);
+
+  void AddMeasurement(int measurement);
+  rtc::Optional<bool> IsHigh() const;
+  rtc::Optional<double> CalculateVariance() const;
+
+ private:
+  const std::unique_ptr<int[]> buffer_;
+  const int max_measurements_;
+  const float fraction_;
+  const int low_threshold_;
+  const int high_threshold_;
+  int until_full_;
+  int next_index_;
+  rtc::Optional<bool> is_high_;
+  int sum_;
+  int count_low_;
+  int count_high_;
+};
+
+}  // namespace webrtc
+
+#endif  // WEBRTC_VIDEO_QUALITY_THRESHOLD_H_

webrtc/video/quality_threshold_unittest.cc

@@ -0,0 +1,97 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/video/quality_threshold.h"
+
+#include "webrtc/test/gtest.h"
+
+namespace webrtc {
+
+TEST(QualityThresholdTest, BackAndForth) {
+  const int kLowThreshold = 0;
+  const int kHighThreshold = 1;
+  const float kFraction = 0.75f;
+  const int kMaxMeasurements = 10;
+
+  QualityThreshold thresh(kLowThreshold, kHighThreshold, kFraction,
+                          kMaxMeasurements);
+
+  const int kNeededMeasurements =
+      static_cast<int>(kFraction * kMaxMeasurements + 1);
+  for (int i = 0; i < kNeededMeasurements; ++i) {
+    EXPECT_FALSE(thresh.IsHigh());
+    thresh.AddMeasurement(kLowThreshold);
+  }
+  ASSERT_TRUE(thresh.IsHigh());
+  for (int i = 0; i < kNeededMeasurements; ++i) {
+    EXPECT_FALSE(*thresh.IsHigh());
+    thresh.AddMeasurement(kHighThreshold);
+  }
+  EXPECT_TRUE(*thresh.IsHigh());
+
+  for (int i = 0; i < kNeededMeasurements; ++i) {
+    EXPECT_TRUE(*thresh.IsHigh());
+    thresh.AddMeasurement(kLowThreshold);
+  }
+  EXPECT_FALSE(*thresh.IsHigh());
+}
+
+TEST(QualityThresholdTest, Variance) {
+  const int kLowThreshold = 0;
+  const int kHighThreshold = 1;
+  const float kFraction = 0.8f;
+  const int kMaxMeasurements = 10;
+  const double kMaxError = 0.01;
+
+  // Previously randomly generated values...
+  int values[] = {51, 79, 80, 56, 19, 20, 48, 57, 48, 25, 2, 25, 38, 37, 25};
+  // ...with precomputed variances.
+  double variances[] = {476.9, 687.6, 552, 336.4, 278.767, 265.167};
+
+  QualityThreshold thresh(kLowThreshold, kHighThreshold, kFraction,
+                          kMaxMeasurements);
+
+  for (int i = 0; i < kMaxMeasurements; ++i) {
+    EXPECT_FALSE(thresh.CalculateVariance());
+    thresh.AddMeasurement(values[i]);
+  }
+
+  ASSERT_TRUE(thresh.CalculateVariance());
+  EXPECT_NEAR(variances[0], *thresh.CalculateVariance(), kMaxError);
+  for (unsigned int i = 1; i < sizeof(variances) / sizeof(double); ++i) {
+    thresh.AddMeasurement(values[i + kMaxMeasurements - 1]);
+    EXPECT_NEAR(variances[i], *thresh.CalculateVariance(), kMaxError);
+  }
+
+  for (int i = 0; i < kMaxMeasurements; ++i) {
+    thresh.AddMeasurement(42);
+  }
+  EXPECT_NEAR(0, *thresh.CalculateVariance(), kMaxError);
+}
+
+TEST(QualityThresholdTest, BetweenThresholds) {
+  const int kLowThreshold = 0;
+  const int kHighThreshold = 2;
+  const float kFraction = 0.6f;
+  const int kMaxMeasurements = 10;
+
+  const int kBetweenThresholds = (kLowThreshold + kHighThreshold) / 2;
+
+  QualityThreshold thresh(kLowThreshold, kHighThreshold, kFraction,
+                          kMaxMeasurements);
+
+  for (int i = 0; i < 2 * kMaxMeasurements; ++i) {
+    EXPECT_FALSE(thresh.IsHigh());
+    thresh.AddMeasurement(kBetweenThresholds);
+  }
+  EXPECT_FALSE(thresh.IsHigh());
+}
+
+}  // namespace webrtc

webrtc/video/receive_statistics_proxy.cc

@@ -13,6 +13,7 @@
 #include <cmath>
 
 #include "webrtc/base/checks.h"
+#include "webrtc/base/logging.h"
 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
 #include "webrtc/system_wrappers/include/clock.h"
 #include "webrtc/system_wrappers/include/field_trial.h"
@@ -22,6 +23,22 @@ namespace webrtc {
 namespace {
 // Periodic time interval for processing samples for |freq_offset_counter_|.
 const int64_t kFreqOffsetProcessIntervalMs = 40000;
+
+// Configuration for bad call detection.
+const int kMinSampleLengthMs = 990;
+const int kNumMeasurements = 10;
+const int kNumMeasurementsVariance = kNumMeasurements * 1.5;
+const float kBadFraction = 0.8f;
+// For fps:
+// Low means low enough to be bad, high means high enough to be good
+const int kLowFpsThreshold = 12;
+const int kHighFpsThreshold = 14;
+// For qp and fps variance:
+// Low means low enough to be good, high means high enough to be bad
+const int kLowQpThresholdVp8 = 60;
+const int kHighQpThresholdVp8 = 70;
+const int kLowVarianceThreshold = 1;
+const int kHighVarianceThreshold = 2;
 }  // namespace
 
 ReceiveStatisticsProxy::ReceiveStatisticsProxy(
@@ -30,6 +47,19 @@ ReceiveStatisticsProxy::ReceiveStatisticsProxy(
     : clock_(clock),
       config_(*config),
       start_ms_(clock->TimeInMilliseconds()),
+      last_sample_time_(clock->TimeInMilliseconds()),
+      fps_threshold_(kLowFpsThreshold,
+                     kHighFpsThreshold,
+                     kBadFraction,
+                     kNumMeasurements),
+      qp_threshold_(kLowQpThresholdVp8,
+                    kHighQpThresholdVp8,
+                    kBadFraction,
+                    kNumMeasurements),
+      variance_threshold_(kLowVarianceThreshold,
+                          kHighVarianceThreshold,
+                          kBadFraction,
+                          kNumMeasurementsVariance),
       // 1000ms window, scale 1000 for ms to s.
       decode_fps_estimator_(1000, 1000),
       renders_fps_estimator_(1000, 1000),
@@ -173,6 +203,67 @@ void ReceiveStatisticsProxy::UpdateHistograms() {
   }
 }
 
+void ReceiveStatisticsProxy::QualitySample() {
+  int64_t now = clock_->TimeInMilliseconds();
+  if (last_sample_time_ + kMinSampleLengthMs > now)
+    return;
+
+  double fps =
+      render_fps_tracker_.ComputeRateForInterval(now - last_sample_time_);
+  int qp = qp_sample_.Avg(1);
+
+  bool prev_fps_bad = !fps_threshold_.IsHigh().value_or(true);
+  bool prev_qp_bad = qp_threshold_.IsHigh().value_or(false);
+  bool prev_variance_bad = variance_threshold_.IsHigh().value_or(false);
+  bool prev_any_bad = prev_fps_bad || prev_qp_bad || prev_variance_bad;
+
+  fps_threshold_.AddMeasurement(static_cast<int>(fps));
+  if (qp != -1)
+    qp_threshold_.AddMeasurement(qp);
+  rtc::Optional<double> fps_variance_opt = fps_threshold_.CalculateVariance();
+  double fps_variance = fps_variance_opt.value_or(0);
+  if (fps_variance_opt) {
+    variance_threshold_.AddMeasurement(static_cast<int>(fps_variance));
+  }
+
+  bool fps_bad = !fps_threshold_.IsHigh().value_or(true);
+  bool qp_bad = qp_threshold_.IsHigh().value_or(false);
+  bool variance_bad = variance_threshold_.IsHigh().value_or(false);
+  bool any_bad = fps_bad || qp_bad || variance_bad;
+
+  if (!prev_any_bad && any_bad) {
+    LOG(LS_WARNING) << "Bad call (any) start: " << now;
+  } else if (prev_any_bad && !any_bad) {
+    LOG(LS_WARNING) << "Bad call (any) end: " << now;
+  }
+
+  if (!prev_fps_bad && fps_bad) {
+    LOG(LS_WARNING) << "Bad call (fps) start: " << now;
+  } else if (prev_fps_bad && !fps_bad) {
+    LOG(LS_WARNING) << "Bad call (fps) end: " << now;
+  }
+
+  if (!prev_qp_bad && qp_bad) {
+    LOG(LS_WARNING) << "Bad call (qp) start: " << now;
+  } else if (prev_qp_bad && !qp_bad) {
+    LOG(LS_WARNING) << "Bad call (qp) end: " << now;
+  }
+
+  if (!prev_variance_bad && variance_bad) {
+    LOG(LS_WARNING) << "Bad call (variance) start: " << now;
+  } else if (prev_variance_bad && !variance_bad) {
+    LOG(LS_WARNING) << "Bad call (variance) end: " << now;
+  }
+
+  LOG(LS_INFO) << "SAMPLE: sample_length: " << (now - last_sample_time_)
+               << " fps: " << fps << " fps_bad: " << fps_bad << " qp: " << qp
+               << " qp_bad: " << qp_bad << " variance_bad: " << variance_bad
+               << " fps_variance: " << fps_variance;
+
+  last_sample_time_ = now;
+  qp_sample_.Reset();
+}
+
 VideoReceiveStream::Stats ReceiveStatisticsProxy::GetStats() const {
   rtc::CritScope lock(&crit_);
   return stats_;
@@ -191,6 +282,7 @@ void ReceiveStatisticsProxy::OnDecoderImplementationName(
 void ReceiveStatisticsProxy::OnIncomingRate(unsigned int framerate,
                                             unsigned int bitrate_bps) {
   rtc::CritScope lock(&crit_);
+  QualitySample();
   stats_.network_frame_rate = framerate;
   stats_.total_bitrate_bps = bitrate_bps;
 }
@@ -340,6 +432,8 @@ void ReceiveStatisticsProxy::OnPreDecode(
   }
   if (codec_specific_info->codecType == kVideoCodecVP8) {
     qp_counters_.vp8.Add(encoded_image.qp_);
+    rtc::CritScope lock(&crit_);
+    qp_sample_.Add(encoded_image.qp_);
   }
 }
 
@@ -354,4 +448,9 @@ int ReceiveStatisticsProxy::SampleCounter::Avg(int min_required_samples) const {
   return sum / num_samples;
 }
 
+void ReceiveStatisticsProxy::SampleCounter::Reset() {
+  num_samples = 0;
+  sum = 0;
+}
+
 }  // namespace webrtc

webrtc/video/receive_statistics_proxy.h

@@ -21,6 +21,7 @@
 #include "webrtc/common_types.h"
 #include "webrtc/common_video/include/frame_callback.h"
 #include "webrtc/modules/video_coding/include/video_coding_defines.h"
+#include "webrtc/video/quality_threshold.h"
 #include "webrtc/video/report_block_stats.h"
 #include "webrtc/video/stats_counter.h"
 #include "webrtc/video/video_stream_decoder.h"
@@ -85,6 +86,7 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback,
     SampleCounter() : sum(0), num_samples(0) {}
     void Add(int sample);
     int Avg(int min_required_samples) const;
+    void Reset();
 
    private:
     int sum;
@@ -96,6 +98,8 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback,
 
   void UpdateHistograms() EXCLUSIVE_LOCKS_REQUIRED(crit_);
 
+  void QualitySample() EXCLUSIVE_LOCKS_REQUIRED(crit_);
+
   Clock* const clock_;
   // Ownership of this object lies with the owner of the ReceiveStatisticsProxy
   // instance.  Lifetime is guaranteed to outlive |this|.
@@ -108,6 +112,11 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback,
   const int64_t start_ms_;
 
   rtc::CriticalSection crit_;
+  int64_t last_sample_time_ GUARDED_BY(crit_);
+  QualityThreshold fps_threshold_ GUARDED_BY(crit_);
+  QualityThreshold qp_threshold_ GUARDED_BY(crit_);
+  QualityThreshold variance_threshold_ GUARDED_BY(crit_);
+  SampleCounter qp_sample_ GUARDED_BY(crit_);
   VideoReceiveStream::Stats stats_ GUARDED_BY(crit_);
   RateStatistics decode_fps_estimator_ GUARDED_BY(crit_);
   RateStatistics renders_fps_estimator_ GUARDED_BY(crit_);