New macOS screen-capturer which uses ScreenCaptureKit.

This supports:
* Full-screen capture from any display, via SelectSource().
* Changing the display, via SelectSource(), while capture is running.
* Handling screen-resolution changes while capture is running.
* Capturing from high-DPI displays at their native resolution.
* Basic damage-tracking: the frame's updated-region is either set to
  empty, or the full frame area.

It currently does not support:
* Window capture.
* Excluded windows.
* Full-desktop capture across all displays.
* More detailed damage-tracking.

The capturer is not yet enabled. Followup CLs will add a
DesktopCaptureOption to enable this capturer on supported versions of
macOS.

Bug: chromium:327458809
Change-Id: Ie619f6c6c1d6edf0fb9320d4fece578754a732dc
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/352544
Reviewed-by: Johannes Kron <kron@webrtc.org>
Reviewed-by: Alexander Cooper <alcooper@chromium.org>
Commit-Queue: Lambros Lambrou <lambroslambrou@chromium.org>
Cr-Commit-Position: refs/heads/main@{#42510}

modules/desktop_capture/BUILD.gn

@@ -628,6 +628,8 @@ if (is_mac) {
       "mac/desktop_frame_provider.mm",
       "mac/screen_capturer_mac.h",
       "mac/screen_capturer_mac.mm",
+      "mac/screen_capturer_sck.h",
+      "mac/screen_capturer_sck.mm",
       "mac/window_list_utils.h",
       "mouse_cursor.h",
       "mouse_cursor_monitor.h",
@@ -657,9 +659,11 @@ if (is_mac) {
     ]
     frameworks = [
       "AppKit.framework",
+      "CoreVideo.framework",
       "IOKit.framework",
       "IOSurface.framework",
     ]
+    weak_frameworks = [ "ScreenCaptureKit.framework" ]  # macOS 12.3
   }
 }
 

modules/desktop_capture/DEPS

@@ -16,4 +16,7 @@ specific_include_rules = {
   "screen_capturer_mac\.mm": [
     "+sdk/objc",
   ],
+  "screen_capturer_sck\.mm": [
+    "+sdk/objc",
+  ],
 }

modules/desktop_capture/mac/screen_capturer_sck.h

@@ -0,0 +1,27 @@
+/*
+ *  Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_DESKTOP_CAPTURE_MAC_SCREEN_CAPTURER_SCK_H_
+#define MODULES_DESKTOP_CAPTURE_MAC_SCREEN_CAPTURER_SCK_H_
+
+#include <memory>
+
+#include "modules/desktop_capture/desktop_capture_options.h"
+#include "modules/desktop_capture/desktop_capturer.h"
+
+namespace webrtc {
+
+// A DesktopCapturer implementation that uses ScreenCaptureKit.
+std::unique_ptr<DesktopCapturer> CreateScreenCapturerSck(
+    const DesktopCaptureOptions& options);
+
+}  // namespace webrtc
+
+#endif  // MODULES_DESKTOP_CAPTURE_MAC_SCREEN_CAPTURER_SCK_H_

modules/desktop_capture/mac/screen_capturer_sck.mm

@@ -0,0 +1,409 @@
+/*
+ *  Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/desktop_capture/mac/screen_capturer_sck.h"
+
+#import <ScreenCaptureKit/ScreenCaptureKit.h>
+
+#include <atomic>
+
+#include "modules/desktop_capture/mac/desktop_frame_iosurface.h"
+#include "modules/desktop_capture/shared_desktop_frame.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/thread_annotations.h"
+#include "sdk/objc/helpers/scoped_cftyperef.h"
+
+using webrtc::DesktopFrameIOSurface;
+
+namespace webrtc {
+class ScreenCapturerSck;
+}  // namespace webrtc
+
+// The ScreenCaptureKit API was available in macOS 12.3, but full-screen capture was reported to be
+// broken before macOS 13 - see http://crbug.com/40234870.
+API_AVAILABLE(macos(13.0))
+@interface SckHelper : NSObject <SCStreamDelegate, SCStreamOutput>
+
+- (instancetype)initWithCapturer:(webrtc::ScreenCapturerSck*)capturer;
+
+- (void)onShareableContentCreated:(SCShareableContent*)content;
+
+// Called just before the capturer is destroyed. This avoids a dangling pointer, and prevents any
+// new calls into a deleted capturer. If any method-call on the capturer is currently running on a
+// different thread, this blocks until it completes.
+- (void)releaseCapturer;
+
+@end
+
+namespace webrtc {
+
+class API_AVAILABLE(macos(13.0)) ScreenCapturerSck final : public DesktopCapturer {
+ public:
+  explicit ScreenCapturerSck(const DesktopCaptureOptions& options);
+
+  ScreenCapturerSck(const ScreenCapturerSck&) = delete;
+  ScreenCapturerSck& operator=(const ScreenCapturerSck&) = delete;
+
+  ~ScreenCapturerSck() override;
+
+  // DesktopCapturer interface. All these methods run on the caller's thread.
+  void Start(DesktopCapturer::Callback* callback) override;
+  void SetMaxFrameRate(uint32_t max_frame_rate) override;
+  void CaptureFrame() override;
+  bool SelectSource(SourceId id) override;
+
+  // Called by SckHelper when shareable content is returned by ScreenCaptureKit. `content` will be
+  // nil if an error occurred. May run on an arbitrary thread.
+  void OnShareableContentCreated(SCShareableContent* content);
+
+  // Called by SckHelper to notify of a newly captured frame. May run on an arbitrary thread.
+  void OnNewIOSurface(IOSurfaceRef io_surface, CFDictionaryRef attachment);
+
+ private:
+  // Called when starting the capturer or the configuration has changed (either from a
+  // SelectSource() call, or the screen-resolution has changed). This tells SCK to fetch new
+  // shareable content, and the completion-handler will either start a new stream, or reconfigure
+  // the existing stream. Runs on the caller's thread.
+  void StartOrReconfigureCapturer();
+
+  // Helper object to receive Objective-C callbacks from ScreenCaptureKit and call into this C++
+  // object. The helper may outlive this C++ instance, if a completion-handler is passed to
+  // ScreenCaptureKit APIs and the C++ object is deleted before the handler executes.
+  SckHelper* __strong helper_;
+
+  // Callback for returning captured frames, or errors, to the caller. Only used on the caller's
+  // thread.
+  Callback* callback_ = nullptr;
+
+  // Options passed to the constructor. May be accessed on any thread, but the options are
+  // unchanged during the capturer's lifetime.
+  DesktopCaptureOptions capture_options_;
+
+  // Signals that a permanent error occurred. This may be set on any thread, and is read by
+  // CaptureFrame() which runs on the caller's thread.
+  std::atomic<bool> permanent_error_ = false;
+
+  // Guards some variables that may be accessed on different threads.
+  Mutex lock_;
+
+  // Provides captured desktop frames.
+  SCStream* __strong stream_ RTC_GUARDED_BY(lock_);
+
+  // Currently selected display, or 0 if the full desktop is selected. This capturer does not
+  // support full-desktop capture, and will fall back to the first display.
+  CGDirectDisplayID current_display_ RTC_GUARDED_BY(lock_) = 0;
+
+  // Used by CaptureFrame() to detect if the screen configuration has changed. Only used on the
+  // caller's thread.
+  MacDesktopConfiguration desktop_config_;
+
+  Mutex latest_frame_lock_;
+  std::unique_ptr<SharedDesktopFrame> latest_frame_ RTC_GUARDED_BY(latest_frame_lock_);
+
+  // Tracks whether the latest frame contains new data since it was returned to the caller. This is
+  // used to set the DesktopFrame's `updated_region` property. The flag is cleared after the frame
+  // is sent to OnCaptureResult(), and is set when SCK reports a new frame with non-empty "dirty"
+  // rectangles.
+  // TODO: crbug.com/327458809 - Replace this flag with ScreenCapturerHelper to more accurately
+  // track the dirty rectangles from the SCStreamFrameInfoDirtyRects attachment.
+  bool frame_is_dirty_ RTC_GUARDED_BY(latest_frame_lock_) = true;
+};
+
+ScreenCapturerSck::ScreenCapturerSck(const DesktopCaptureOptions& options)
+    : capture_options_(options) {
+  helper_ = [[SckHelper alloc] initWithCapturer:this];
+}
+
+ScreenCapturerSck::~ScreenCapturerSck() {
+  [stream_ stopCaptureWithCompletionHandler:nil];
+  [helper_ releaseCapturer];
+}
+
+void ScreenCapturerSck::Start(DesktopCapturer::Callback* callback) {
+  callback_ = callback;
+  desktop_config_ = capture_options_.configuration_monitor()->desktop_configuration();
+  StartOrReconfigureCapturer();
+}
+
+void ScreenCapturerSck::SetMaxFrameRate(uint32_t max_frame_rate) {
+  // TODO: crbug.com/327458809 - Implement this.
+}
+
+void ScreenCapturerSck::CaptureFrame() {
+  if (permanent_error_) {
+    callback_->OnCaptureResult(Result::ERROR_PERMANENT, nullptr);
+    return;
+  }
+
+  MacDesktopConfiguration new_config =
+      capture_options_.configuration_monitor()->desktop_configuration();
+  if (!desktop_config_.Equals(new_config)) {
+    desktop_config_ = new_config;
+    StartOrReconfigureCapturer();
+  }
+
+  std::unique_ptr<DesktopFrame> frame;
+  {
+    MutexLock lock(&latest_frame_lock_);
+    if (latest_frame_) {
+      frame = latest_frame_->Share();
+      if (frame_is_dirty_) {
+        frame->mutable_updated_region()->AddRect(DesktopRect::MakeSize(frame->size()));
+        frame_is_dirty_ = false;
+      }
+    }
+  }
+
+  if (frame) {
+    callback_->OnCaptureResult(Result::SUCCESS, std::move(frame));
+  } else {
+    callback_->OnCaptureResult(Result::ERROR_TEMPORARY, nullptr);
+  }
+}
+
+bool ScreenCapturerSck::SelectSource(SourceId id) {
+  bool stream_started = false;
+  {
+    MutexLock lock(&lock_);
+    current_display_ = id;
+
+    if (stream_) {
+      stream_started = true;
+    }
+  }
+
+  // If the capturer was already started, reconfigure it. Otherwise, wait until Start() gets called.
+  if (stream_started) {
+    StartOrReconfigureCapturer();
+  }
+
+  return true;
+}
+
+void ScreenCapturerSck::OnShareableContentCreated(SCShareableContent* content) {
+  if (!content) {
+    RTC_LOG(LS_ERROR) << "getShareableContent failed.";
+    permanent_error_ = true;
+    return;
+  }
+
+  if (!content.displays.count) {
+    RTC_LOG(LS_ERROR) << "getShareableContent returned no displays.";
+    permanent_error_ = true;
+    return;
+  }
+
+  SCDisplay* captured_display;
+  {
+    MutexLock lock(&lock_);
+    for (SCDisplay* display in content.displays) {
+      if (current_display_ == display.displayID) {
+        captured_display = display;
+        break;
+      }
+    }
+    if (!captured_display) {
+      if (current_display_ == static_cast<CGDirectDisplayID>(kFullDesktopScreenId)) {
+        RTC_LOG(LS_WARNING)
+            << "Full screen capture is not supported, falling back to first display.";
+      } else {
+        RTC_LOG(LS_WARNING) << "Display " << current_display_
+                            << " not found, falling back to first display.";
+      }
+      captured_display = content.displays.firstObject;
+    }
+  }
+
+  SCContentFilter* filter = [[SCContentFilter alloc] initWithDisplay:captured_display
+                                                    excludingWindows:@[]];
+  SCStreamConfiguration* config = [[SCStreamConfiguration alloc] init];
+  config.pixelFormat = kCVPixelFormatType_32BGRA;
+  config.showsCursor = capture_options_.prefer_cursor_embedded();
+  config.width = filter.contentRect.size.width * filter.pointPixelScale;
+  config.height = filter.contentRect.size.height * filter.pointPixelScale;
+
+  if (@available(macOS 14.0, *)) {
+    config.captureResolution = SCCaptureResolutionNominal;
+  }
+
+  MutexLock lock(&lock_);
+
+  if (stream_) {
+    RTC_LOG(LS_INFO) << "Updating stream configuration.";
+    [stream_ updateContentFilter:filter completionHandler:nil];
+    [stream_ updateConfiguration:config completionHandler:nil];
+  } else {
+    stream_ = [[SCStream alloc] initWithFilter:filter configuration:config delegate:helper_];
+
+    // TODO: crbug.com/327458809 - Choose an appropriate sampleHandlerQueue for best performance.
+    NSError* add_stream_output_error;
+    bool add_stream_output_result = [stream_ addStreamOutput:helper_
+                                                        type:SCStreamOutputTypeScreen
+                                          sampleHandlerQueue:nil
+                                                       error:&add_stream_output_error];
+    if (!add_stream_output_result) {
+      stream_ = nil;
+      RTC_LOG(LS_ERROR) << "addStreamOutput failed.";
+      permanent_error_ = true;
+      return;
+    }
+
+    auto handler = ^(NSError* error) {
+      if (error) {
+        // It should be safe to access `this` here, because the C++ destructor calls
+        // stopCaptureWithCompletionHandler on the stream, which cancels this handler.
+        permanent_error_ = true;
+        RTC_LOG(LS_ERROR) << "startCaptureWithCompletionHandler failed.";
+      } else {
+        RTC_LOG(LS_INFO) << "Capture started.";
+      }
+    };
+
+    [stream_ startCaptureWithCompletionHandler:handler];
+  }
+}
+
+void ScreenCapturerSck::OnNewIOSurface(IOSurfaceRef io_surface, CFDictionaryRef attachment) {
+  rtc::ScopedCFTypeRef<IOSurfaceRef> scoped_io_surface(io_surface, rtc::RetainPolicy::RETAIN);
+  std::unique_ptr<DesktopFrameIOSurface> desktop_frame_io_surface =
+      DesktopFrameIOSurface::Wrap(scoped_io_surface);
+  if (!desktop_frame_io_surface) {
+    RTC_LOG(LS_ERROR) << "Failed to lock IOSurface.";
+    return;
+  }
+
+  std::unique_ptr<SharedDesktopFrame> frame =
+      SharedDesktopFrame::Wrap(std::move(desktop_frame_io_surface));
+
+  bool dirty;
+  {
+    MutexLock lock(&latest_frame_lock_);
+    // Mark the frame as dirty if it has a different size, and ignore any DirtyRects attachment in
+    // this case. This is because SCK does not apply a correct attachment to the frame in the case
+    // where the stream was reconfigured.
+    dirty = !latest_frame_ || !latest_frame_->size().equals(frame->size());
+  }
+
+  if (!dirty) {
+    const void* dirty_rects_ptr =
+        CFDictionaryGetValue(attachment, (__bridge CFStringRef)SCStreamFrameInfoDirtyRects);
+    if (!dirty_rects_ptr) {
+      // This is never expected to happen - SCK attaches a non-empty dirty-rects list to every
+      // frame, even when nothing has changed.
+      return;
+    }
+    if (CFGetTypeID(dirty_rects_ptr) != CFArrayGetTypeID()) {
+      return;
+    }
+
+    CFArrayRef dirty_rects_array = static_cast<CFArrayRef>(dirty_rects_ptr);
+    int size = CFArrayGetCount(dirty_rects_array);
+    for (int i = 0; i < size; i++) {
+      const void* rect_ptr = CFArrayGetValueAtIndex(dirty_rects_array, i);
+      if (CFGetTypeID(rect_ptr) != CFDictionaryGetTypeID()) {
+        // This is never expected to happen - the dirty-rects attachment should always be an array
+        // of dictionaries.
+        return;
+      }
+      CGRect rect{};
+      CGRectMakeWithDictionaryRepresentation(static_cast<CFDictionaryRef>(rect_ptr), &rect);
+      if (!CGRectIsEmpty(rect)) {
+        dirty = true;
+        break;
+      }
+    }
+  }
+
+  if (dirty) {
+    MutexLock lock(&latest_frame_lock_);
+    frame_is_dirty_ = true;
+    std::swap(latest_frame_, frame);
+  }
+}
+
+void ScreenCapturerSck::StartOrReconfigureCapturer() {
+  // The copy is needed to avoid capturing `this` in the Objective-C block. Accessing `helper_`
+  // inside the block is equivalent to `this->helper_` and would crash (UAF) if `this` is
+  // deleted before the block is executed.
+  SckHelper* local_helper = helper_;
+  auto handler = ^(SCShareableContent* content, NSError* error) {
+    [local_helper onShareableContentCreated:content];
+  };
+
+  [SCShareableContent getShareableContentWithCompletionHandler:handler];
+}
+
+std::unique_ptr<DesktopCapturer> CreateScreenCapturerSck(const DesktopCaptureOptions& options) {
+  if (@available(macOS 13.0, *)) {
+    return std::make_unique<ScreenCapturerSck>(options);
+  } else {
+    return nullptr;
+  }
+}
+
+}  // namespace webrtc
+
+@implementation SckHelper {
+  // This lock is to prevent the capturer being destroyed while an instance method is still running
+  // on another thread.
+  webrtc::Mutex _capturer_lock;
+  webrtc::ScreenCapturerSck* _capturer;
+}
+
+- (instancetype)initWithCapturer:(webrtc::ScreenCapturerSck*)capturer {
+  if (self = [super init]) {
+    _capturer = capturer;
+  }
+  return self;
+}
+
+- (void)onShareableContentCreated:(SCShareableContent*)content {
+  webrtc::MutexLock lock(&_capturer_lock);
+  if (_capturer) {
+    _capturer->OnShareableContentCreated(content);
+  }
+}
+
+- (void)stream:(SCStream*)stream
+    didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+                   ofType:(SCStreamOutputType)type {
+  CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+  if (!pixelBuffer) {
+    return;
+  }
+
+  IOSurfaceRef ioSurface = CVPixelBufferGetIOSurface(pixelBuffer);
+  if (!ioSurface) {
+    return;
+  }
+
+  CFArrayRef attachmentsArray =
+      CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, /*createIfNecessary=*/false);
+  if (!attachmentsArray || CFArrayGetCount(attachmentsArray) <= 0) {
+    RTC_LOG(LS_ERROR) << "Discarding frame with no attachments.";
+    return;
+  }
+
+  CFDictionaryRef attachment =
+      static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachmentsArray, 0));
+
+  webrtc::MutexLock lock(&_capturer_lock);
+  if (_capturer) {
+    _capturer->OnNewIOSurface(ioSurface, attachment);
+  }
+}
+
+- (void)releaseCapturer {
+  webrtc::MutexLock lock(&_capturer_lock);
+  _capturer = nullptr;
+}
+
+@end