BlankDetectorDesktopCapturerWrapper to detect a blank DesktopFrame

DXGI capturer highly depends on video adapter and its driver, as well as Windows
itself. I recently found it cannot work on my virtualbox instance any more,
which indicates it may not work well on some specific systems. What worse is,
the APIs do not return a failure in such case.

So this change adds a BlankDetectorDesktopCapturerWrapper, which samples several
pixels in the frame returned by a DesktopCapturer implementation. If all the
pixels selected are blank, this wrapper returns a failure. A typical usage is to
combine BlankDetectorDesktopCapturerWrapper with FallbackDesktopCapturerWrapper,
and use GDI capturer in case of failure.

Usually less than 500 pixels are checked, so the
BlankDetectorDesktopCapturerWrapper should not impact the capturer performance.

This change is expected to resolve bug 682112 in another dimension.

BUG=682112

Review-Url: https://codereview.webrtc.org/2709523003
Cr-Commit-Position: refs/heads/master@{#16984}

webrtc/modules/desktop_capture/BUILD.gn

@@ -58,6 +58,7 @@ if (rtc_include_tests) {
   rtc_source_set("desktop_capture_unittests") {
     testonly = true
     sources = [
+      "blank_detector_desktop_capturer_wrapper_unittest.cc",
       "desktop_and_cursor_composer_unittest.cc",
       "desktop_capturer_differ_wrapper_unittest.cc",
       "desktop_frame_rotation_unittest.cc",
@@ -77,7 +78,6 @@ if (rtc_include_tests) {
       ":desktop_capture",
       ":desktop_capture_mock",
       ":primitives",
-      ":rgba_color",
       "../..:webrtc_common",
       "../../base:rtc_base_approved",
       "../../system_wrappers:system_wrappers",
@@ -99,29 +99,11 @@ if (rtc_include_tests) {
     }
   }
 
-  source_set("rgba_color") {
-    testonly = true
-
-    public_deps = [
-      ":desktop_capture",
-    ]
-
-    sources = [
-      "rgba_color.cc",
-      "rgba_color.h",
-    ]
-
-    deps = [
-      ":primitives",
-      "../..:webrtc_common",
-    ]
-  }
-
   source_set("screen_drawer") {
     testonly = true
 
     public_deps = [
-      ":rgba_color",
+      ":desktop_capture",
     ]
 
     sources = [
@@ -144,7 +126,6 @@ if (rtc_include_tests) {
 
     public_deps = [
       ":desktop_capture",
-      ":rgba_color",
       "//testing/gmock",
     ]
 
@@ -167,6 +148,8 @@ if (rtc_include_tests) {
 
 rtc_static_library("desktop_capture") {
   sources = [
+    "blank_detector_desktop_capturer_wrapper.cc",
+    "blank_detector_desktop_capturer_wrapper.h",
     "cropped_desktop_frame.cc",
     "cropped_desktop_frame.h",
     "cropping_window_capturer.cc",
@@ -205,6 +188,8 @@ rtc_static_library("desktop_capture") {
     "mouse_cursor_monitor_win.cc",
     "resolution_change_detector.cc",
     "resolution_change_detector.h",
+    "rgba_color.cc",
+    "rgba_color.h",
     "screen_capture_frame_queue.h",
     "screen_capturer_helper.cc",
     "screen_capturer_helper.h",

webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc

@@ -0,0 +1,118 @@
+/*
+ *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h"
+
+#include <algorithm>
+#include <utility>
+
+#include "webrtc/base/checks.h"
+#include "webrtc/modules/desktop_capture/desktop_geometry.h"
+#include "webrtc/system_wrappers/include/metrics.h"
+
+namespace webrtc {
+
+BlankDetectorDesktopCapturerWrapper::BlankDetectorDesktopCapturerWrapper(
+    std::unique_ptr<DesktopCapturer> capturer,
+    RgbaColor blank_pixel)
+    : capturer_(std::move(capturer)),
+      blank_pixel_(blank_pixel) {
+  RTC_DCHECK(capturer_);
+}
+
+BlankDetectorDesktopCapturerWrapper::~BlankDetectorDesktopCapturerWrapper() =
+    default;
+
+void BlankDetectorDesktopCapturerWrapper::Start(
+    DesktopCapturer::Callback* callback) {
+  capturer_->Start(this);
+  callback_ = callback;
+}
+
+void BlankDetectorDesktopCapturerWrapper::SetSharedMemoryFactory(
+    std::unique_ptr<SharedMemoryFactory> shared_memory_factory) {
+  capturer_->SetSharedMemoryFactory(std::move(shared_memory_factory));
+}
+
+void BlankDetectorDesktopCapturerWrapper::CaptureFrame() {
+  RTC_DCHECK(callback_);
+  capturer_->CaptureFrame();
+}
+
+void BlankDetectorDesktopCapturerWrapper::SetExcludedWindow(WindowId window) {
+  capturer_->SetExcludedWindow(window);
+}
+
+bool BlankDetectorDesktopCapturerWrapper::GetSourceList(SourceList* sources) {
+  return capturer_->GetSourceList(sources);
+}
+
+bool BlankDetectorDesktopCapturerWrapper::SelectSource(SourceId id) {
+  return capturer_->SelectSource(id);
+}
+
+bool BlankDetectorDesktopCapturerWrapper::FocusOnSelectedSource() {
+  return capturer_->FocusOnSelectedSource();
+}
+
+void BlankDetectorDesktopCapturerWrapper::OnCaptureResult(
+    Result result,
+    std::unique_ptr<DesktopFrame> frame) {
+  RTC_DCHECK(callback_);
+  if (result != Result::SUCCESS || non_blank_frame_received_) {
+    callback_->OnCaptureResult(result, std::move(frame));
+    return;
+  }
+
+  RTC_DCHECK(frame);
+
+  // If nothing has been changed in current frame, we do not need to check it
+  // again.
+  if (!frame->updated_region().is_empty() || is_first_frame_) {
+    last_frame_is_blank_ = IsBlankFrame(*frame);
+    is_first_frame_ = false;
+  }
+  RTC_HISTOGRAM_BOOLEAN("WebRTC.DesktopCapture.BlankFrameDetected",
+                        last_frame_is_blank_);
+  if (!last_frame_is_blank_) {
+    non_blank_frame_received_ = true;
+    callback_->OnCaptureResult(Result::SUCCESS, std::move(frame));
+    return;
+  }
+
+  callback_->OnCaptureResult(Result::ERROR_TEMPORARY,
+                             std::unique_ptr<DesktopFrame>());
+}
+
+bool BlankDetectorDesktopCapturerWrapper::IsBlankFrame(
+    const DesktopFrame& frame) const {
+  // We will check 7489 pixels for a frame with 1024 x 768 resolution.
+  for (int i = 0; i < frame.size().width() * frame.size().height(); i += 105) {
+    const int x = i % frame.size().width();
+    const int y = i / frame.size().width();
+    if (!IsBlankPixel(frame, x, y)) {
+      return false;
+    }
+  }
+
+  // We are verifying the pixel in the center as well.
+  return IsBlankPixel(frame, frame.size().width() / 2,
+                      frame.size().height() / 2);
+}
+
+bool BlankDetectorDesktopCapturerWrapper::IsBlankPixel(
+    const DesktopFrame& frame,
+    int x,
+    int y) const {
+  uint8_t* pixel_data = frame.GetFrameDataAtPos(DesktopVector(x, y));
+  return RgbaColor(pixel_data) == blank_pixel_;
+}
+
+}  // namespace webrtc

webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h

@@ -0,0 +1,74 @@
+/*
+ *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_DESKTOP_CAPTURE_BLANK_DETECTOR_DESKTOP_CAPTURER_WRAPPER_H_
+#define WEBRTC_MODULES_DESKTOP_CAPTURE_BLANK_DETECTOR_DESKTOP_CAPTURER_WRAPPER_H_
+
+#include <memory>
+
+#include "webrtc/modules/desktop_capture/desktop_capturer.h"
+#include "webrtc/modules/desktop_capture/rgba_color.h"
+
+namespace webrtc {
+
+// A DesktopCapturer wrapper detects the return value of its owned
+// DesktopCapturer implementation. If sampled pixels returned by the
+// DesktopCapturer implementation all equal to the blank pixel, this wrapper
+// returns ERROR_TEMPORARY. If the DesktopCapturer implementation fails for too
+// many times, this wrapper returns ERROR_PERMANENT.
+class BlankDetectorDesktopCapturerWrapper final
+    : public DesktopCapturer,
+      public DesktopCapturer::Callback {
+ public:
+  // Creates BlankDetectorDesktopCapturerWrapper. BlankDesktopCapturerWrapper
+  // takes ownership of |capturer|. The |blank_pixel| is the unmodified color
+  // returned by the |capturer|.
+  BlankDetectorDesktopCapturerWrapper(std::unique_ptr<DesktopCapturer> capturer,
+                                      RgbaColor blank_pixel);
+  ~BlankDetectorDesktopCapturerWrapper() override;
+
+  // DesktopCapturer interface.
+  void Start(DesktopCapturer::Callback* callback) override;
+  void SetSharedMemoryFactory(
+      std::unique_ptr<SharedMemoryFactory> shared_memory_factory) override;
+  void CaptureFrame() override;
+  void SetExcludedWindow(WindowId window) override;
+  bool GetSourceList(SourceList* sources) override;
+  bool SelectSource(SourceId id) override;
+  bool FocusOnSelectedSource() override;
+
+ private:
+  // DesktopCapturer::Callback interface.
+  void OnCaptureResult(Result result,
+                       std::unique_ptr<DesktopFrame> frame) override;
+
+  bool IsBlankFrame(const DesktopFrame& frame) const;
+
+  // Detects whether pixel at (x, y) equals to |blank_pixel_|.
+  bool IsBlankPixel(const DesktopFrame& frame, int x, int y) const;
+
+  const std::unique_ptr<DesktopCapturer> capturer_;
+  const RgbaColor blank_pixel_;
+
+  // Whether a non-blank frame has been received.
+  bool non_blank_frame_received_ = false;
+
+  // Whether the last frame is blank.
+  bool last_frame_is_blank_ = false;
+
+  // Whether current frame is the first frame.
+  bool is_first_frame_ = true;
+
+  DesktopCapturer::Callback* callback_ = nullptr;
+};
+
+}  // namespace webrtc
+
+#endif  // WEBRTC_MODULES_DESKTOP_CAPTURE_BLANK_DETECTOR_DESKTOP_CAPTURER_WRAPPER_H_

webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper_unittest.cc

@@ -0,0 +1,163 @@
+/*
+ *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h"
+
+#include <memory>
+#include <utility>
+
+#include "webrtc/modules/desktop_capture/desktop_capturer.h"
+#include "webrtc/modules/desktop_capture/desktop_frame.h"
+#include "webrtc/modules/desktop_capture/desktop_frame_generator.h"
+#include "webrtc/modules/desktop_capture/fake_desktop_capturer.h"
+#include "webrtc/test/gtest.h"
+
+namespace webrtc {
+
+class BlankDetectorDesktopCapturerWrapperTest
+    : public testing::Test,
+      public DesktopCapturer::Callback {
+ public:
+  BlankDetectorDesktopCapturerWrapperTest();
+  ~BlankDetectorDesktopCapturerWrapperTest() override;
+
+ protected:
+  void PerfTest(DesktopCapturer* capturer);
+
+  const int frame_width_ = 1024;
+  const int frame_height_ = 768;
+  std::unique_ptr<BlankDetectorDesktopCapturerWrapper> wrapper_;
+  DesktopCapturer* capturer_ = nullptr;
+  BlackWhiteDesktopFramePainter painter_;
+  int num_frames_captured_ = 0;
+  DesktopCapturer::Result last_result_ = DesktopCapturer::Result::SUCCESS;
+  std::unique_ptr<DesktopFrame> last_frame_;
+
+ private:
+  // DesktopCapturer::Callback interface.
+  void OnCaptureResult(DesktopCapturer::Result result,
+                       std::unique_ptr<DesktopFrame> frame) override;
+
+  PainterDesktopFrameGenerator frame_generator_;
+};
+
+BlankDetectorDesktopCapturerWrapperTest::
+BlankDetectorDesktopCapturerWrapperTest() {
+  frame_generator_.size()->set(frame_width_, frame_height_);
+  frame_generator_.set_desktop_frame_painter(&painter_);
+  std::unique_ptr<DesktopCapturer> capturer(new FakeDesktopCapturer());
+  FakeDesktopCapturer* fake_capturer =
+      static_cast<FakeDesktopCapturer*>(capturer.get());
+  fake_capturer->set_frame_generator(&frame_generator_);
+  capturer_ = fake_capturer;
+  wrapper_.reset(new BlankDetectorDesktopCapturerWrapper(
+      std::move(capturer), RgbaColor(0, 0, 0, 0)));
+  wrapper_->Start(this);
+}
+
+BlankDetectorDesktopCapturerWrapperTest::
+~BlankDetectorDesktopCapturerWrapperTest() = default;
+
+void BlankDetectorDesktopCapturerWrapperTest::OnCaptureResult(
+    DesktopCapturer::Result result,
+    std::unique_ptr<DesktopFrame> frame) {
+  last_result_ = result;
+  last_frame_ = std::move(frame);
+  num_frames_captured_++;
+}
+
+void BlankDetectorDesktopCapturerWrapperTest::PerfTest(
+    DesktopCapturer* capturer) {
+  for (int i = 0; i < 10000; i++) {
+    capturer->CaptureFrame();
+    ASSERT_EQ(num_frames_captured_, i + 1);
+  }
+}
+
+TEST_F(BlankDetectorDesktopCapturerWrapperTest, ShouldDetectBlankFrame) {
+  wrapper_->CaptureFrame();
+  ASSERT_EQ(num_frames_captured_, 1);
+  ASSERT_EQ(last_result_, DesktopCapturer::Result::ERROR_TEMPORARY);
+  ASSERT_FALSE(last_frame_);
+}
+
+TEST_F(BlankDetectorDesktopCapturerWrapperTest, ShouldPassBlankDetection) {
+  painter_.updated_region()->AddRect(DesktopRect::MakeXYWH(0, 0, 100, 100));
+  wrapper_->CaptureFrame();
+  ASSERT_EQ(num_frames_captured_, 1);
+  ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS);
+  ASSERT_TRUE(last_frame_);
+
+  painter_.updated_region()->AddRect(
+      DesktopRect::MakeXYWH(frame_width_ - 100, frame_height_ - 100, 100, 100));
+  wrapper_->CaptureFrame();
+  ASSERT_EQ(num_frames_captured_, 2);
+  ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS);
+  ASSERT_TRUE(last_frame_);
+
+  painter_.updated_region()->AddRect(
+      DesktopRect::MakeXYWH(0, frame_height_ - 100, 100, 100));
+  wrapper_->CaptureFrame();
+  ASSERT_EQ(num_frames_captured_, 3);
+  ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS);
+  ASSERT_TRUE(last_frame_);
+
+  painter_.updated_region()->AddRect(
+      DesktopRect::MakeXYWH(frame_width_ - 100, 0, 100, 100));
+  wrapper_->CaptureFrame();
+  ASSERT_EQ(num_frames_captured_, 4);
+  ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS);
+  ASSERT_TRUE(last_frame_);
+
+  painter_.updated_region()->AddRect(DesktopRect::MakeXYWH(
+      (frame_width_ >> 1) - 50, (frame_height_ >> 1) - 50, 100, 100));
+  wrapper_->CaptureFrame();
+  ASSERT_EQ(num_frames_captured_, 5);
+  ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS);
+  ASSERT_TRUE(last_frame_);
+}
+
+TEST_F(BlankDetectorDesktopCapturerWrapperTest,
+       ShouldNotCheckAfterANonBlankFrameReceived) {
+  wrapper_->CaptureFrame();
+  ASSERT_EQ(num_frames_captured_, 1);
+  ASSERT_EQ(last_result_, DesktopCapturer::Result::ERROR_TEMPORARY);
+  ASSERT_FALSE(last_frame_);
+
+  painter_.updated_region()->AddRect(
+      DesktopRect::MakeXYWH(frame_width_ - 100, 0, 100, 100));
+  wrapper_->CaptureFrame();
+  ASSERT_EQ(num_frames_captured_, 2);
+  ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS);
+  ASSERT_TRUE(last_frame_);
+
+  for (int i = 0; i < 100; i++) {
+    wrapper_->CaptureFrame();
+    ASSERT_EQ(num_frames_captured_, i + 3);
+    ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS);
+    ASSERT_TRUE(last_frame_);
+  }
+}
+
+// There is no perceptible impact by using BlankDetectorDesktopCapturerWrapper.
+// i.e. less than 0.2ms per frame.
+// [ OK ] DISABLED_Performance (10210 ms)
+// [ OK ] DISABLED_PerformanceComparison (8791 ms)
+TEST_F(BlankDetectorDesktopCapturerWrapperTest, DISABLED_Performance) {
+  PerfTest(wrapper_.get());
+}
+
+TEST_F(BlankDetectorDesktopCapturerWrapperTest,
+       DISABLED_PerformanceComparison) {
+  capturer_->Start(this);
+  PerfTest(capturer_);
+}
+
+}  // namespace webrtc

webrtc/modules/desktop_capture/rgba_color.h

@@ -21,9 +21,6 @@ namespace webrtc {
 // provides functions to be created from uint8_t array, say,
 // DesktopFrame::data(). It always uses BGRA order for internal storage to match
 // DesktopFrame::data().
-//
-// This struct is for testing purpose only, and should not be used in production
-// logic.
 struct RgbaColor final {
   // Creates a color with BGRA channels.
   RgbaColor(uint8_t blue, uint8_t green, uint8_t red, uint8_t alpha);

webrtc/modules/desktop_capture/screen_capturer_win.cc

@@ -11,24 +11,38 @@
 #include <memory>
 #include <utility>
 
+#include "webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h"
 #include "webrtc/modules/desktop_capture/desktop_capturer.h"
 #include "webrtc/modules/desktop_capture/desktop_capture_options.h"
 #include "webrtc/modules/desktop_capture/fallback_desktop_capturer_wrapper.h"
+#include "webrtc/modules/desktop_capture/rgba_color.h"
 #include "webrtc/modules/desktop_capture/win/screen_capturer_win_directx.h"
 #include "webrtc/modules/desktop_capture/win/screen_capturer_win_gdi.h"
 #include "webrtc/modules/desktop_capture/win/screen_capturer_win_magnifier.h"
 
 namespace webrtc {
 
+namespace {
+
+std::unique_ptr<DesktopCapturer> CreateScreenCapturerWinDirectx(
+    const DesktopCaptureOptions& options) {
+  std::unique_ptr<DesktopCapturer> capturer(
+      new ScreenCapturerWinDirectx(options));
+  capturer.reset(new BlankDetectorDesktopCapturerWrapper(
+      std::move(capturer), RgbaColor(0, 0, 0, 0)));
+  return capturer;
+}
+
+}  // namespace
+
 // static
 std::unique_ptr<DesktopCapturer> DesktopCapturer::CreateRawScreenCapturer(
     const DesktopCaptureOptions& options) {
-  std::unique_ptr<DesktopCapturer> capturer;
+  std::unique_ptr<DesktopCapturer> capturer(new ScreenCapturerWinGdi(options));
   if (options.allow_directx_capturer() &&
       ScreenCapturerWinDirectx::IsSupported()) {
-    capturer.reset(new ScreenCapturerWinDirectx(options));
-  } else {
-    capturer.reset(new ScreenCapturerWinGdi(options));
+    capturer.reset(new FallbackDesktopCapturerWrapper(
+        CreateScreenCapturerWinDirectx(options), std::move(capturer)));
   }
 
   if (options.allow_use_magnification_api()) {