diff --git a/webrtc/modules/desktop_capture/BUILD.gn b/webrtc/modules/desktop_capture/BUILD.gn index 151627de39..3f3130dd4d 100644 --- a/webrtc/modules/desktop_capture/BUILD.gn +++ b/webrtc/modules/desktop_capture/BUILD.gn @@ -58,6 +58,7 @@ if (rtc_include_tests) { rtc_source_set("desktop_capture_unittests") { testonly = true sources = [ + "blank_detector_desktop_capturer_wrapper_unittest.cc", "desktop_and_cursor_composer_unittest.cc", "desktop_capturer_differ_wrapper_unittest.cc", "desktop_frame_rotation_unittest.cc", @@ -77,7 +78,6 @@ if (rtc_include_tests) { ":desktop_capture", ":desktop_capture_mock", ":primitives", - ":rgba_color", "../..:webrtc_common", "../../base:rtc_base_approved", "../../system_wrappers:system_wrappers", @@ -99,29 +99,11 @@ if (rtc_include_tests) { } } - source_set("rgba_color") { - testonly = true - - public_deps = [ - ":desktop_capture", - ] - - sources = [ - "rgba_color.cc", - "rgba_color.h", - ] - - deps = [ - ":primitives", - "../..:webrtc_common", - ] - } - source_set("screen_drawer") { testonly = true public_deps = [ - ":rgba_color", + ":desktop_capture", ] sources = [ @@ -144,7 +126,6 @@ if (rtc_include_tests) { public_deps = [ ":desktop_capture", - ":rgba_color", "//testing/gmock", ] @@ -167,6 +148,8 @@ if (rtc_include_tests) { rtc_static_library("desktop_capture") { sources = [ + "blank_detector_desktop_capturer_wrapper.cc", + "blank_detector_desktop_capturer_wrapper.h", "cropped_desktop_frame.cc", "cropped_desktop_frame.h", "cropping_window_capturer.cc", @@ -205,6 +188,8 @@ rtc_static_library("desktop_capture") { "mouse_cursor_monitor_win.cc", "resolution_change_detector.cc", "resolution_change_detector.h", + "rgba_color.cc", + "rgba_color.h", "screen_capture_frame_queue.h", "screen_capturer_helper.cc", "screen_capturer_helper.h", diff --git a/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc b/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc new file mode 100644 index 0000000000..b2cc2848c4 --- /dev/null +++ b/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h" + +#include +#include + +#include "webrtc/base/checks.h" +#include "webrtc/modules/desktop_capture/desktop_geometry.h" +#include "webrtc/system_wrappers/include/metrics.h" + +namespace webrtc { + +BlankDetectorDesktopCapturerWrapper::BlankDetectorDesktopCapturerWrapper( + std::unique_ptr capturer, + RgbaColor blank_pixel) + : capturer_(std::move(capturer)), + blank_pixel_(blank_pixel) { + RTC_DCHECK(capturer_); +} + +BlankDetectorDesktopCapturerWrapper::~BlankDetectorDesktopCapturerWrapper() = + default; + +void BlankDetectorDesktopCapturerWrapper::Start( + DesktopCapturer::Callback* callback) { + capturer_->Start(this); + callback_ = callback; +} + +void BlankDetectorDesktopCapturerWrapper::SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory) { + capturer_->SetSharedMemoryFactory(std::move(shared_memory_factory)); +} + +void BlankDetectorDesktopCapturerWrapper::CaptureFrame() { + RTC_DCHECK(callback_); + capturer_->CaptureFrame(); +} + +void BlankDetectorDesktopCapturerWrapper::SetExcludedWindow(WindowId window) { + capturer_->SetExcludedWindow(window); +} + +bool BlankDetectorDesktopCapturerWrapper::GetSourceList(SourceList* sources) { + return capturer_->GetSourceList(sources); +} + +bool BlankDetectorDesktopCapturerWrapper::SelectSource(SourceId id) { + return capturer_->SelectSource(id); +} + +bool BlankDetectorDesktopCapturerWrapper::FocusOnSelectedSource() { + return capturer_->FocusOnSelectedSource(); +} + +void BlankDetectorDesktopCapturerWrapper::OnCaptureResult( + Result result, + std::unique_ptr frame) { + RTC_DCHECK(callback_); + if (result != Result::SUCCESS || non_blank_frame_received_) { + callback_->OnCaptureResult(result, std::move(frame)); + return; + } + + RTC_DCHECK(frame); + + // If nothing has been changed in current frame, we do not need to check it + // again. + if (!frame->updated_region().is_empty() || is_first_frame_) { + last_frame_is_blank_ = IsBlankFrame(*frame); + is_first_frame_ = false; + } + RTC_HISTOGRAM_BOOLEAN("WebRTC.DesktopCapture.BlankFrameDetected", + last_frame_is_blank_); + if (!last_frame_is_blank_) { + non_blank_frame_received_ = true; + callback_->OnCaptureResult(Result::SUCCESS, std::move(frame)); + return; + } + + callback_->OnCaptureResult(Result::ERROR_TEMPORARY, + std::unique_ptr()); +} + +bool BlankDetectorDesktopCapturerWrapper::IsBlankFrame( + const DesktopFrame& frame) const { + // We will check 7489 pixels for a frame with 1024 x 768 resolution. + for (int i = 0; i < frame.size().width() * frame.size().height(); i += 105) { + const int x = i % frame.size().width(); + const int y = i / frame.size().width(); + if (!IsBlankPixel(frame, x, y)) { + return false; + } + } + + // We are verifying the pixel in the center as well. + return IsBlankPixel(frame, frame.size().width() / 2, + frame.size().height() / 2); +} + +bool BlankDetectorDesktopCapturerWrapper::IsBlankPixel( + const DesktopFrame& frame, + int x, + int y) const { + uint8_t* pixel_data = frame.GetFrameDataAtPos(DesktopVector(x, y)); + return RgbaColor(pixel_data) == blank_pixel_; +} + +} // namespace webrtc diff --git a/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h b/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h new file mode 100644 index 0000000000..0501cae14e --- /dev/null +++ b/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_DESKTOP_CAPTURE_BLANK_DETECTOR_DESKTOP_CAPTURER_WRAPPER_H_ +#define WEBRTC_MODULES_DESKTOP_CAPTURE_BLANK_DETECTOR_DESKTOP_CAPTURER_WRAPPER_H_ + +#include + +#include "webrtc/modules/desktop_capture/desktop_capturer.h" +#include "webrtc/modules/desktop_capture/rgba_color.h" + +namespace webrtc { + +// A DesktopCapturer wrapper detects the return value of its owned +// DesktopCapturer implementation. If sampled pixels returned by the +// DesktopCapturer implementation all equal to the blank pixel, this wrapper +// returns ERROR_TEMPORARY. If the DesktopCapturer implementation fails for too +// many times, this wrapper returns ERROR_PERMANENT. +class BlankDetectorDesktopCapturerWrapper final + : public DesktopCapturer, + public DesktopCapturer::Callback { + public: + // Creates BlankDetectorDesktopCapturerWrapper. BlankDesktopCapturerWrapper + // takes ownership of |capturer|. The |blank_pixel| is the unmodified color + // returned by the |capturer|. + BlankDetectorDesktopCapturerWrapper(std::unique_ptr capturer, + RgbaColor blank_pixel); + ~BlankDetectorDesktopCapturerWrapper() override; + + // DesktopCapturer interface. + void Start(DesktopCapturer::Callback* callback) override; + void SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory) override; + void CaptureFrame() override; + void SetExcludedWindow(WindowId window) override; + bool GetSourceList(SourceList* sources) override; + bool SelectSource(SourceId id) override; + bool FocusOnSelectedSource() override; + + private: + // DesktopCapturer::Callback interface. + void OnCaptureResult(Result result, + std::unique_ptr frame) override; + + bool IsBlankFrame(const DesktopFrame& frame) const; + + // Detects whether pixel at (x, y) equals to |blank_pixel_|. + bool IsBlankPixel(const DesktopFrame& frame, int x, int y) const; + + const std::unique_ptr capturer_; + const RgbaColor blank_pixel_; + + // Whether a non-blank frame has been received. + bool non_blank_frame_received_ = false; + + // Whether the last frame is blank. + bool last_frame_is_blank_ = false; + + // Whether current frame is the first frame. + bool is_first_frame_ = true; + + DesktopCapturer::Callback* callback_ = nullptr; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_DESKTOP_CAPTURE_BLANK_DETECTOR_DESKTOP_CAPTURER_WRAPPER_H_ diff --git a/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper_unittest.cc b/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper_unittest.cc new file mode 100644 index 0000000000..bce82ddc25 --- /dev/null +++ b/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper_unittest.cc @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h" + +#include +#include + +#include "webrtc/modules/desktop_capture/desktop_capturer.h" +#include "webrtc/modules/desktop_capture/desktop_frame.h" +#include "webrtc/modules/desktop_capture/desktop_frame_generator.h" +#include "webrtc/modules/desktop_capture/fake_desktop_capturer.h" +#include "webrtc/test/gtest.h" + +namespace webrtc { + +class BlankDetectorDesktopCapturerWrapperTest + : public testing::Test, + public DesktopCapturer::Callback { + public: + BlankDetectorDesktopCapturerWrapperTest(); + ~BlankDetectorDesktopCapturerWrapperTest() override; + + protected: + void PerfTest(DesktopCapturer* capturer); + + const int frame_width_ = 1024; + const int frame_height_ = 768; + std::unique_ptr wrapper_; + DesktopCapturer* capturer_ = nullptr; + BlackWhiteDesktopFramePainter painter_; + int num_frames_captured_ = 0; + DesktopCapturer::Result last_result_ = DesktopCapturer::Result::SUCCESS; + std::unique_ptr last_frame_; + + private: + // DesktopCapturer::Callback interface. + void OnCaptureResult(DesktopCapturer::Result result, + std::unique_ptr frame) override; + + PainterDesktopFrameGenerator frame_generator_; +}; + +BlankDetectorDesktopCapturerWrapperTest:: +BlankDetectorDesktopCapturerWrapperTest() { + frame_generator_.size()->set(frame_width_, frame_height_); + frame_generator_.set_desktop_frame_painter(&painter_); + std::unique_ptr capturer(new FakeDesktopCapturer()); + FakeDesktopCapturer* fake_capturer = + static_cast(capturer.get()); + fake_capturer->set_frame_generator(&frame_generator_); + capturer_ = fake_capturer; + wrapper_.reset(new BlankDetectorDesktopCapturerWrapper( + std::move(capturer), RgbaColor(0, 0, 0, 0))); + wrapper_->Start(this); +} + +BlankDetectorDesktopCapturerWrapperTest:: +~BlankDetectorDesktopCapturerWrapperTest() = default; + +void BlankDetectorDesktopCapturerWrapperTest::OnCaptureResult( + DesktopCapturer::Result result, + std::unique_ptr frame) { + last_result_ = result; + last_frame_ = std::move(frame); + num_frames_captured_++; +} + +void BlankDetectorDesktopCapturerWrapperTest::PerfTest( + DesktopCapturer* capturer) { + for (int i = 0; i < 10000; i++) { + capturer->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, i + 1); + } +} + +TEST_F(BlankDetectorDesktopCapturerWrapperTest, ShouldDetectBlankFrame) { + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 1); + ASSERT_EQ(last_result_, DesktopCapturer::Result::ERROR_TEMPORARY); + ASSERT_FALSE(last_frame_); +} + +TEST_F(BlankDetectorDesktopCapturerWrapperTest, ShouldPassBlankDetection) { + painter_.updated_region()->AddRect(DesktopRect::MakeXYWH(0, 0, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 1); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + painter_.updated_region()->AddRect( + DesktopRect::MakeXYWH(frame_width_ - 100, frame_height_ - 100, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 2); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + painter_.updated_region()->AddRect( + DesktopRect::MakeXYWH(0, frame_height_ - 100, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 3); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + painter_.updated_region()->AddRect( + DesktopRect::MakeXYWH(frame_width_ - 100, 0, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 4); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + painter_.updated_region()->AddRect(DesktopRect::MakeXYWH( + (frame_width_ >> 1) - 50, (frame_height_ >> 1) - 50, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 5); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); +} + +TEST_F(BlankDetectorDesktopCapturerWrapperTest, + ShouldNotCheckAfterANonBlankFrameReceived) { + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 1); + ASSERT_EQ(last_result_, DesktopCapturer::Result::ERROR_TEMPORARY); + ASSERT_FALSE(last_frame_); + + painter_.updated_region()->AddRect( + DesktopRect::MakeXYWH(frame_width_ - 100, 0, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 2); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + for (int i = 0; i < 100; i++) { + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, i + 3); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + } +} + +// There is no perceptible impact by using BlankDetectorDesktopCapturerWrapper. +// i.e. less than 0.2ms per frame. +// [ OK ] DISABLED_Performance (10210 ms) +// [ OK ] DISABLED_PerformanceComparison (8791 ms) +TEST_F(BlankDetectorDesktopCapturerWrapperTest, DISABLED_Performance) { + PerfTest(wrapper_.get()); +} + +TEST_F(BlankDetectorDesktopCapturerWrapperTest, + DISABLED_PerformanceComparison) { + capturer_->Start(this); + PerfTest(capturer_); +} + +} // namespace webrtc diff --git a/webrtc/modules/desktop_capture/rgba_color.h b/webrtc/modules/desktop_capture/rgba_color.h index 1236d03ee6..11e8d44f5f 100644 --- a/webrtc/modules/desktop_capture/rgba_color.h +++ b/webrtc/modules/desktop_capture/rgba_color.h @@ -21,9 +21,6 @@ namespace webrtc { // provides functions to be created from uint8_t array, say, // DesktopFrame::data(). It always uses BGRA order for internal storage to match // DesktopFrame::data(). -// -// This struct is for testing purpose only, and should not be used in production -// logic. struct RgbaColor final { // Creates a color with BGRA channels. RgbaColor(uint8_t blue, uint8_t green, uint8_t red, uint8_t alpha); diff --git a/webrtc/modules/desktop_capture/screen_capturer_win.cc b/webrtc/modules/desktop_capture/screen_capturer_win.cc index 602600be89..d8aecb15c7 100644 --- a/webrtc/modules/desktop_capture/screen_capturer_win.cc +++ b/webrtc/modules/desktop_capture/screen_capturer_win.cc @@ -11,24 +11,38 @@ #include #include +#include "webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h" #include "webrtc/modules/desktop_capture/desktop_capturer.h" #include "webrtc/modules/desktop_capture/desktop_capture_options.h" #include "webrtc/modules/desktop_capture/fallback_desktop_capturer_wrapper.h" +#include "webrtc/modules/desktop_capture/rgba_color.h" #include "webrtc/modules/desktop_capture/win/screen_capturer_win_directx.h" #include "webrtc/modules/desktop_capture/win/screen_capturer_win_gdi.h" #include "webrtc/modules/desktop_capture/win/screen_capturer_win_magnifier.h" namespace webrtc { +namespace { + +std::unique_ptr CreateScreenCapturerWinDirectx( + const DesktopCaptureOptions& options) { + std::unique_ptr capturer( + new ScreenCapturerWinDirectx(options)); + capturer.reset(new BlankDetectorDesktopCapturerWrapper( + std::move(capturer), RgbaColor(0, 0, 0, 0))); + return capturer; +} + +} // namespace + // static std::unique_ptr DesktopCapturer::CreateRawScreenCapturer( const DesktopCaptureOptions& options) { - std::unique_ptr capturer; + std::unique_ptr capturer(new ScreenCapturerWinGdi(options)); if (options.allow_directx_capturer() && ScreenCapturerWinDirectx::IsSupported()) { - capturer.reset(new ScreenCapturerWinDirectx(options)); - } else { - capturer.reset(new ScreenCapturerWinGdi(options)); + capturer.reset(new FallbackDesktopCapturerWrapper( + CreateScreenCapturerWinDirectx(options), std::move(capturer))); } if (options.allow_use_magnification_api()) {