From dcac9fe3d1646465e6dbc2c6bf1d465e79ba5414 Mon Sep 17 00:00:00 2001 From: Austin Orion Date: Thu, 15 Apr 2021 16:15:52 -0700 Subject: [PATCH] Add may_contain_cursor property to DesktopFrame to avoid double capture This CL adds a new property to the DesktopFrame interface to indicate that the capturer supports cursor capture and the frame may contain an image of the cursor (if the cursor was over the window or screen being captured). This allows the DesktopAndCursorComposer to avoid compositing another image of the cursor on the frame. This is preferred because natively capturing the cursor will likely be more efficient, and for WGC the API to disable cursor capture is only availabe on later versions of Win10, reducing the number of users that could use it. Bug: webrtc:12654 Change-Id: I992804ff2a65eb423fb8ecc66e066408dc05e849 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/215341 Reviewed-by: Jamie Walch Commit-Queue: Austin Orion Cr-Commit-Position: refs/heads/master@{#33780} --- .../desktop_and_cursor_composer.cc | 4 +- .../desktop_and_cursor_composer_unittest.cc | 58 ++++++++++++++++++- modules/desktop_capture/desktop_frame.h | 10 ++++ .../desktop_capture/win/wgc_capturer_win.cc | 1 + 4 files changed, 69 insertions(+), 4 deletions(-) diff --git a/modules/desktop_capture/desktop_and_cursor_composer.cc b/modules/desktop_capture/desktop_and_cursor_composer.cc index f282c1d500..69b8b40c73 100644 --- a/modules/desktop_capture/desktop_and_cursor_composer.cc +++ b/modules/desktop_capture/desktop_and_cursor_composer.cc @@ -207,7 +207,8 @@ void DesktopAndCursorComposer::OnCaptureResult( DesktopCapturer::Result result, std::unique_ptr frame) { if (frame && cursor_) { - if (frame->rect().Contains(cursor_position_) && + if (!frame->may_contain_cursor() && + frame->rect().Contains(cursor_position_) && !desktop_capturer_->IsOccluded(cursor_position_)) { DesktopVector relative_position = cursor_position_.subtract(frame->top_left()); @@ -228,6 +229,7 @@ void DesktopAndCursorComposer::OnCaptureResult( previous_cursor_rect_ = frame_with_cursor->cursor_rect(); cursor_changed_ = false; frame = std::move(frame_with_cursor); + frame->set_may_contain_cursor(true); } } diff --git a/modules/desktop_capture/desktop_and_cursor_composer_unittest.cc b/modules/desktop_capture/desktop_and_cursor_composer_unittest.cc index c9cb56d8c2..00253d38e2 100644 --- a/modules/desktop_capture/desktop_and_cursor_composer_unittest.cc +++ b/modules/desktop_capture/desktop_and_cursor_composer_unittest.cc @@ -27,6 +27,8 @@ namespace webrtc { namespace { +const int kFrameXCoord = 100; +const int kFrameYCoord = 200; const int kScreenWidth = 100; const int kScreenHeight = 100; const int kCursorWidth = 10; @@ -249,11 +251,61 @@ TEST_F(DesktopAndCursorComposerTest, CursorShouldBeIgnoredIfNoFrameCaptured) { } } +TEST_F(DesktopAndCursorComposerTest, CursorShouldBeIgnoredIfFrameMayContainIt) { + // We can't use a shared frame because we need to detect modifications + // compared to a control. + std::unique_ptr control_frame(CreateTestFrame()); + control_frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord)); + + struct { + int x; + int y; + bool may_contain_cursor; + } tests[] = { + {100, 200, true}, + {100, 200, false}, + {150, 250, true}, + {150, 250, false}, + }; + + for (size_t i = 0; i < arraysize(tests); i++) { + SCOPED_TRACE(i); + + std::unique_ptr frame(CreateTestFrame()); + frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord)); + frame->set_may_contain_cursor(tests[i].may_contain_cursor); + fake_screen_->SetNextFrame(std::move(frame)); + + const DesktopVector abs_pos(tests[i].x, tests[i].y); + fake_cursor_->SetState(MouseCursorMonitor::INSIDE, abs_pos); + blender_.CaptureFrame(); + + // If the frame may already have contained the cursor, then |CaptureFrame()| + // should not have modified it, so it should be the same as the control. + EXPECT_TRUE(frame_); + const DesktopVector rel_pos(abs_pos.subtract(control_frame->top_left())); + if (tests[i].may_contain_cursor) { + EXPECT_EQ( + *reinterpret_cast(frame_->GetFrameDataAtPos(rel_pos)), + *reinterpret_cast( + control_frame->GetFrameDataAtPos(rel_pos))); + + } else { + // |CaptureFrame()| should have modified the frame to have the cursor. + EXPECT_NE( + *reinterpret_cast(frame_->GetFrameDataAtPos(rel_pos)), + *reinterpret_cast( + control_frame->GetFrameDataAtPos(rel_pos))); + EXPECT_TRUE(frame_->may_contain_cursor()); + } + } +} + TEST_F(DesktopAndCursorComposerTest, CursorShouldBeIgnoredIfItIsOutOfDesktopFrame) { std::unique_ptr frame( SharedDesktopFrame::Wrap(CreateTestFrame())); - frame->set_top_left(DesktopVector(100, 200)); + frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord)); // The frame covers (100, 200) - (200, 300). struct { @@ -279,7 +331,7 @@ TEST_F(DesktopAndCursorComposerTest, TEST_F(DesktopAndCursorComposerTest, IsOccludedShouldBeConsidered) { std::unique_ptr frame( SharedDesktopFrame::Wrap(CreateTestFrame())); - frame->set_top_left(DesktopVector(100, 200)); + frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord)); // The frame covers (100, 200) - (200, 300). struct { @@ -304,7 +356,7 @@ TEST_F(DesktopAndCursorComposerTest, IsOccludedShouldBeConsidered) { TEST_F(DesktopAndCursorComposerTest, CursorIncluded) { std::unique_ptr frame( SharedDesktopFrame::Wrap(CreateTestFrame())); - frame->set_top_left(DesktopVector(100, 200)); + frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord)); // The frame covers (100, 200) - (200, 300). struct { diff --git a/modules/desktop_capture/desktop_frame.h b/modules/desktop_capture/desktop_frame.h index 4ee3680670..bc47cc50f2 100644 --- a/modules/desktop_capture/desktop_frame.h +++ b/modules/desktop_capture/desktop_frame.h @@ -72,6 +72,15 @@ class RTC_EXPORT DesktopFrame { const DesktopVector& dpi() const { return dpi_; } void set_dpi(const DesktopVector& dpi) { dpi_ = dpi; } + // Indicates if this frame may have the mouse cursor in it. Capturers that + // support cursor capture may set this to true. If the cursor was + // outside of the captured area, this may be true even though the cursor is + // not in the image. + bool may_contain_cursor() const { return may_contain_cursor_; } + void set_may_contain_cursor(bool may_contain_cursor) { + may_contain_cursor_ = may_contain_cursor; + } + // Time taken to capture the frame in milliseconds. int64_t capture_time_ms() const { return capture_time_ms_; } void set_capture_time_ms(int64_t time_ms) { capture_time_ms_ = time_ms; } @@ -150,6 +159,7 @@ class RTC_EXPORT DesktopFrame { DesktopRegion updated_region_; DesktopVector top_left_; DesktopVector dpi_; + bool may_contain_cursor_ = false; int64_t capture_time_ms_; uint32_t capturer_id_; std::vector icc_profile_; diff --git a/modules/desktop_capture/win/wgc_capturer_win.cc b/modules/desktop_capture/win/wgc_capturer_win.cc index 0b4d9740d2..0d4848e3e2 100644 --- a/modules/desktop_capture/win/wgc_capturer_win.cc +++ b/modules/desktop_capture/win/wgc_capturer_win.cc @@ -191,6 +191,7 @@ void WgcCapturerWin::CaptureFrame() { capture_time_ms); frame->set_capture_time_ms(capture_time_ms); frame->set_capturer_id(DesktopCapturerId::kWgcCapturerWin); + frame->set_may_contain_cursor(true); RecordWgcCapturerResult(WgcCapturerResult::kSuccess); callback_->OnCaptureResult(DesktopCapturer::Result::SUCCESS, std::move(frame));