From 347f9b07b97366af5172aff24cdd8eb45ec81ba6 Mon Sep 17 00:00:00 2001 From: Sunggook Chue Date: Mon, 14 Mar 2022 11:49:25 -0700 Subject: [PATCH] getDisplayMedia shows black window on Youtube PiP in Windows. getDisplayMedia capture the view of the screens and windows in the capture dialog, but the issue is that captured view of the Youtube somehow is blank. It repros only in certain circumstances, for example, Canary channel. If user reinstall the Canary as fresh new, we observed that it doesn't repro. Cause: We aren't sure what's cause of this one yet. Solution: We decided to provide fallback WGC capturer when the main capturer (GDI) shows blank. WGC could show yellow outline in prior Win11 OS, but yellow outline looks better than blank. The blank detector and fallback capturer are what screen capturer already supported. So, the solution will follow similar pattern in the window capturer. Bug: webrtc:13726 Change-Id: I620c817d259d7bb5c295adab11c4444349ab1c6c Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/252625 Reviewed-by: Alexander Cooper Commit-Queue: Alexander Cooper Cr-Commit-Position: refs/heads/main@{#36224} --- ...blank_detector_desktop_capturer_wrapper.cc | 14 ++++++++-- .../blank_detector_desktop_capturer_wrapper.h | 7 ++++- .../desktop_capture/desktop_capture_options.h | 12 +++++++++ .../desktop_capture/win/wgc_capturer_win.cc | 24 +++++++++++++---- .../desktop_capture/win/wgc_capturer_win.h | 11 ++++++-- .../desktop_capture/window_capturer_win.cc | 27 ++++++++++++++++++- 6 files changed, 84 insertions(+), 11 deletions(-) diff --git a/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc b/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc index ca3a89f49b..c20843414b 100644 --- a/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc +++ b/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc @@ -23,8 +23,11 @@ namespace webrtc { BlankDetectorDesktopCapturerWrapper::BlankDetectorDesktopCapturerWrapper( std::unique_ptr capturer, - RgbaColor blank_pixel) - : capturer_(std::move(capturer)), blank_pixel_(blank_pixel) { + RgbaColor blank_pixel, + bool check_per_capture) + : capturer_(std::move(capturer)), + blank_pixel_(blank_pixel), + check_per_capture_(check_per_capture) { RTC_DCHECK(capturer_); } @@ -56,6 +59,13 @@ bool BlankDetectorDesktopCapturerWrapper::GetSourceList(SourceList* sources) { } bool BlankDetectorDesktopCapturerWrapper::SelectSource(SourceId id) { + if (check_per_capture_) { + // If we start capturing a new source, we must reset these members + // so we don't short circuit the blank detection logic. + is_first_frame_ = true; + non_blank_frame_received_ = false; + } + return capturer_->SelectSource(id); } diff --git a/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h b/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h index f5c2ce201b..d10f9cf725 100644 --- a/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h +++ b/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h @@ -34,7 +34,8 @@ class BlankDetectorDesktopCapturerWrapper final // takes ownership of `capturer`. The `blank_pixel` is the unmodified color // returned by the `capturer`. BlankDetectorDesktopCapturerWrapper(std::unique_ptr capturer, - RgbaColor blank_pixel); + RgbaColor blank_pixel, + bool check_per_capture = false); ~BlankDetectorDesktopCapturerWrapper() override; // DesktopCapturer interface. @@ -70,6 +71,10 @@ class BlankDetectorDesktopCapturerWrapper final // Whether current frame is the first frame. bool is_first_frame_ = true; + // Blank inspection is made per capture instead of once for all + // screens or windows. + bool check_per_capture_ = false; + DesktopCapturer::Callback* callback_ = nullptr; }; diff --git a/modules/desktop_capture/desktop_capture_options.h b/modules/desktop_capture/desktop_capture_options.h index c6bc52f8a7..dd3cde0145 100644 --- a/modules/desktop_capture/desktop_capture_options.h +++ b/modules/desktop_capture/desktop_capture_options.h @@ -163,6 +163,17 @@ class RTC_EXPORT DesktopCaptureOptions { // precedence over the cropping, directx, and magnification flags. bool allow_wgc_capturer() const { return allow_wgc_capturer_; } void set_allow_wgc_capturer(bool allow) { allow_wgc_capturer_ = allow; } + + // This flag enables the WGC capturer for fallback capturer. + // The flag is useful when the first capturer (eg. WindowCapturerWinGdi) is + // unreliable in certain devices where WGC is supported, but not used by + // default. + bool allow_wgc_capturer_fallback() const { + return allow_wgc_capturer_fallback_; + } + void set_allow_wgc_capturer_fallback(bool allow) { + allow_wgc_capturer_fallback_ = allow; + } #endif // defined(RTC_ENABLE_WIN_WGC) #endif // defined(WEBRTC_WIN) @@ -203,6 +214,7 @@ class RTC_EXPORT DesktopCaptureOptions { bool allow_cropping_window_capturer_ = false; #if defined(RTC_ENABLE_WIN_WGC) bool allow_wgc_capturer_ = false; + bool allow_wgc_capturer_fallback_ = false; #endif #endif #if defined(WEBRTC_USE_X11) diff --git a/modules/desktop_capture/win/wgc_capturer_win.cc b/modules/desktop_capture/win/wgc_capturer_win.cc index 9b2cce4a8e..35d7bd1cec 100644 --- a/modules/desktop_capture/win/wgc_capturer_win.cc +++ b/modules/desktop_capture/win/wgc_capturer_win.cc @@ -47,18 +47,22 @@ void RecordWgcCapturerResult(WgcCapturerResult error) { WgcCapturerWin::WgcCapturerWin( std::unique_ptr source_factory, - std::unique_ptr source_enumerator) + std::unique_ptr source_enumerator, + bool allow_delayed_capturable_check) : source_factory_(std::move(source_factory)), - source_enumerator_(std::move(source_enumerator)) {} + source_enumerator_(std::move(source_enumerator)), + allow_delayed_capturable_check_(allow_delayed_capturable_check) {} WgcCapturerWin::~WgcCapturerWin() = default; // static std::unique_ptr WgcCapturerWin::CreateRawWindowCapturer( - const DesktopCaptureOptions& options) { + const DesktopCaptureOptions& options, + bool allow_delayed_capturable_check) { return std::make_unique( std::make_unique(), std::make_unique( - options.enumerate_current_process_windows())); + options.enumerate_current_process_windows()), + allow_delayed_capturable_check); } // static @@ -66,7 +70,7 @@ std::unique_ptr WgcCapturerWin::CreateRawScreenCapturer( const DesktopCaptureOptions& options) { return std::make_unique( std::make_unique(), - std::make_unique()); + std::make_unique(), false); } bool WgcCapturerWin::GetSourceList(SourceList* sources) { @@ -75,6 +79,9 @@ bool WgcCapturerWin::GetSourceList(SourceList* sources) { bool WgcCapturerWin::SelectSource(DesktopCapturer::SourceId id) { capture_source_ = source_factory_->CreateCaptureSource(id); + if (allow_delayed_capturable_check_) + return true; + return capture_source_->IsCapturable(); } @@ -135,6 +142,13 @@ void WgcCapturerWin::CaptureFrame() { return; } + if (allow_delayed_capturable_check_ && !capture_source_->IsCapturable()) { + RTC_LOG(LS_ERROR) << "Source is not capturable."; + callback_->OnCaptureResult(DesktopCapturer::Result::ERROR_PERMANENT, + /*frame=*/nullptr); + return; + } + int64_t capture_start_time_nanos = rtc::TimeNanos(); HRESULT hr; diff --git a/modules/desktop_capture/win/wgc_capturer_win.h b/modules/desktop_capture/win/wgc_capturer_win.h index 34e6874dc0..0eef5283c7 100644 --- a/modules/desktop_capture/win/wgc_capturer_win.h +++ b/modules/desktop_capture/win/wgc_capturer_win.h @@ -80,7 +80,8 @@ class ScreenEnumerator final : public SourceEnumerator { class WgcCapturerWin : public DesktopCapturer { public: WgcCapturerWin(std::unique_ptr source_factory, - std::unique_ptr source_enumerator); + std::unique_ptr source_enumerator, + bool allow_delayed_capturable_check); WgcCapturerWin(const WgcCapturerWin&) = delete; WgcCapturerWin& operator=(const WgcCapturerWin&) = delete; @@ -88,7 +89,8 @@ class WgcCapturerWin : public DesktopCapturer { ~WgcCapturerWin() override; static std::unique_ptr CreateRawWindowCapturer( - const DesktopCaptureOptions& options); + const DesktopCaptureOptions& options, + bool allow_delayed_capturable_check = false); static std::unique_ptr CreateRawScreenCapturer( const DesktopCaptureOptions& options); @@ -128,6 +130,11 @@ class WgcCapturerWin : public DesktopCapturer { // returns. Callback* callback_ = nullptr; + // WgcCaptureSource::IsCapturable is expensive to run. So, caller can + // delay capturable check till capture frame is called if the WgcCapturerWin + // is used as a fallback capturer. + bool allow_delayed_capturable_check_ = false; + // A Direct3D11 device that is shared amongst the WgcCaptureSessions, who // require one to perform the capture. Microsoft::WRL::ComPtr<::ID3D11Device> d3d11_device_; diff --git a/modules/desktop_capture/window_capturer_win.cc b/modules/desktop_capture/window_capturer_win.cc index 4bfa09f4d6..7f7bea6eff 100644 --- a/modules/desktop_capture/window_capturer_win.cc +++ b/modules/desktop_capture/window_capturer_win.cc @@ -12,12 +12,37 @@ #include "modules/desktop_capture/desktop_capturer.h" #include "modules/desktop_capture/win/window_capturer_win_gdi.h" +#if defined(RTC_ENABLE_WIN_WGC) +#include "modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h" +#include "modules/desktop_capture/fallback_desktop_capturer_wrapper.h" +#include "modules/desktop_capture/win/wgc_capturer_win.h" +#include "rtc_base/win/windows_version.h" +#endif // defined(RTC_ENABLE_WIN_WGC) + namespace webrtc { // static std::unique_ptr DesktopCapturer::CreateRawWindowCapturer( const DesktopCaptureOptions& options) { - return WindowCapturerWinGdi::CreateRawWindowCapturer(options); + std::unique_ptr capturer( + WindowCapturerWinGdi::CreateRawWindowCapturer(options)); +#if defined(RTC_ENABLE_WIN_WGC) + if (options.allow_wgc_capturer_fallback() && + rtc::rtc_win::GetVersion() >= rtc::rtc_win::Version::VERSION_WIN10_RS5) { + // BlankDectector capturer will send an error when it detects a failed + // GDI rendering, then Fallback capturer will try to capture it again with + // WGC. + capturer = std::make_unique( + std::move(capturer), RgbaColor(0, 0, 0, 0), + /*check_per_capture*/ true); + + capturer = std::make_unique( + std::move(capturer), + WgcCapturerWin::CreateRawWindowCapturer( + options, /*allow_delayed_capturable_check*/ true)); + } +#endif // defined(RTC_ENABLE_WIN_WGC) + return capturer; } } // namespace webrtc