Add may_contain_cursor property to DesktopFrame to avoid double capture

This CL adds a new property to the DesktopFrame interface to indicate
that the capturer supports cursor capture and the frame may contain
an image of the cursor (if the cursor was over the window or screen
being captured). This allows the DesktopAndCursorComposer to avoid
compositing another image of the cursor on the frame.

This is preferred because natively capturing the cursor will likely
be more efficient, and for WGC the API to disable cursor capture
is only availabe on later versions of Win10, reducing the number
of users that could use it.

Bug: webrtc:12654
Change-Id: I992804ff2a65eb423fb8ecc66e066408dc05e849
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/215341
Reviewed-by: Jamie Walch <jamiewalch@chromium.org>
Commit-Queue: Austin Orion <auorion@microsoft.com>
Cr-Commit-Position: refs/heads/master@{#33780}

modules/desktop_capture/desktop_and_cursor_composer.cc

@@ -207,7 +207,8 @@ void DesktopAndCursorComposer::OnCaptureResult(
     DesktopCapturer::Result result,
     std::unique_ptr<DesktopFrame> frame) {
   if (frame && cursor_) {
-    if (frame->rect().Contains(cursor_position_) &&
+    if (!frame->may_contain_cursor() &&
+        frame->rect().Contains(cursor_position_) &&
         !desktop_capturer_->IsOccluded(cursor_position_)) {
       DesktopVector relative_position =
           cursor_position_.subtract(frame->top_left());
@@ -228,6 +229,7 @@ void DesktopAndCursorComposer::OnCaptureResult(
       previous_cursor_rect_ = frame_with_cursor->cursor_rect();
       cursor_changed_ = false;
       frame = std::move(frame_with_cursor);
+      frame->set_may_contain_cursor(true);
     }
   }
 

modules/desktop_capture/desktop_and_cursor_composer_unittest.cc

@@ -27,6 +27,8 @@ namespace webrtc {
 
 namespace {
 
+const int kFrameXCoord = 100;
+const int kFrameYCoord = 200;
 const int kScreenWidth = 100;
 const int kScreenHeight = 100;
 const int kCursorWidth = 10;
@@ -249,11 +251,61 @@ TEST_F(DesktopAndCursorComposerTest, CursorShouldBeIgnoredIfNoFrameCaptured) {
   }
 }
 
+TEST_F(DesktopAndCursorComposerTest, CursorShouldBeIgnoredIfFrameMayContainIt) {
+  // We can't use a shared frame because we need to detect modifications
+  // compared to a control.
+  std::unique_ptr<DesktopFrame> control_frame(CreateTestFrame());
+  control_frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
+
+  struct {
+    int x;
+    int y;
+    bool may_contain_cursor;
+  } tests[] = {
+      {100, 200, true},
+      {100, 200, false},
+      {150, 250, true},
+      {150, 250, false},
+  };
+
+  for (size_t i = 0; i < arraysize(tests); i++) {
+    SCOPED_TRACE(i);
+
+    std::unique_ptr<DesktopFrame> frame(CreateTestFrame());
+    frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
+    frame->set_may_contain_cursor(tests[i].may_contain_cursor);
+    fake_screen_->SetNextFrame(std::move(frame));
+
+    const DesktopVector abs_pos(tests[i].x, tests[i].y);
+    fake_cursor_->SetState(MouseCursorMonitor::INSIDE, abs_pos);
+    blender_.CaptureFrame();
+
+    // If the frame may already have contained the cursor, then |CaptureFrame()|
+    // should not have modified it, so it should be the same as the control.
+    EXPECT_TRUE(frame_);
+    const DesktopVector rel_pos(abs_pos.subtract(control_frame->top_left()));
+    if (tests[i].may_contain_cursor) {
+      EXPECT_EQ(
+          *reinterpret_cast<uint32_t*>(frame_->GetFrameDataAtPos(rel_pos)),
+          *reinterpret_cast<uint32_t*>(
+              control_frame->GetFrameDataAtPos(rel_pos)));
+
+    } else {
+      // |CaptureFrame()| should have modified the frame to have the cursor.
+      EXPECT_NE(
+          *reinterpret_cast<uint32_t*>(frame_->GetFrameDataAtPos(rel_pos)),
+          *reinterpret_cast<uint32_t*>(
+              control_frame->GetFrameDataAtPos(rel_pos)));
+      EXPECT_TRUE(frame_->may_contain_cursor());
+    }
+  }
+}
+
 TEST_F(DesktopAndCursorComposerTest,
        CursorShouldBeIgnoredIfItIsOutOfDesktopFrame) {
   std::unique_ptr<SharedDesktopFrame> frame(
       SharedDesktopFrame::Wrap(CreateTestFrame()));
-  frame->set_top_left(DesktopVector(100, 200));
+  frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
   // The frame covers (100, 200) - (200, 300).
 
   struct {
@@ -279,7 +331,7 @@ TEST_F(DesktopAndCursorComposerTest,
 TEST_F(DesktopAndCursorComposerTest, IsOccludedShouldBeConsidered) {
   std::unique_ptr<SharedDesktopFrame> frame(
       SharedDesktopFrame::Wrap(CreateTestFrame()));
-  frame->set_top_left(DesktopVector(100, 200));
+  frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
   // The frame covers (100, 200) - (200, 300).
 
   struct {
@@ -304,7 +356,7 @@ TEST_F(DesktopAndCursorComposerTest, IsOccludedShouldBeConsidered) {
 TEST_F(DesktopAndCursorComposerTest, CursorIncluded) {
   std::unique_ptr<SharedDesktopFrame> frame(
       SharedDesktopFrame::Wrap(CreateTestFrame()));
-  frame->set_top_left(DesktopVector(100, 200));
+  frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
   // The frame covers (100, 200) - (200, 300).
 
   struct {

modules/desktop_capture/desktop_frame.h

@@ -72,6 +72,15 @@ class RTC_EXPORT DesktopFrame {
   const DesktopVector& dpi() const { return dpi_; }
   void set_dpi(const DesktopVector& dpi) { dpi_ = dpi; }
 
+  // Indicates if this frame may have the mouse cursor in it. Capturers that
+  // support cursor capture may set this to true. If the cursor was
+  // outside of the captured area, this may be true even though the cursor is
+  // not in the image.
+  bool may_contain_cursor() const { return may_contain_cursor_; }
+  void set_may_contain_cursor(bool may_contain_cursor) {
+    may_contain_cursor_ = may_contain_cursor;
+  }
+
   // Time taken to capture the frame in milliseconds.
   int64_t capture_time_ms() const { return capture_time_ms_; }
   void set_capture_time_ms(int64_t time_ms) { capture_time_ms_ = time_ms; }
@@ -150,6 +159,7 @@ class RTC_EXPORT DesktopFrame {
   DesktopRegion updated_region_;
   DesktopVector top_left_;
   DesktopVector dpi_;
+  bool may_contain_cursor_ = false;
   int64_t capture_time_ms_;
   uint32_t capturer_id_;
   std::vector<uint8_t> icc_profile_;

modules/desktop_capture/win/wgc_capturer_win.cc

@@ -191,6 +191,7 @@ void WgcCapturerWin::CaptureFrame() {
                             capture_time_ms);
   frame->set_capture_time_ms(capture_time_ms);
   frame->set_capturer_id(DesktopCapturerId::kWgcCapturerWin);
+  frame->set_may_contain_cursor(true);
   RecordWgcCapturerResult(WgcCapturerResult::kSuccess);
   callback_->OnCaptureResult(DesktopCapturer::Result::SUCCESS,
                              std::move(frame));