From 079e93de17e5126676c729309adcf9d483486092 Mon Sep 17 00:00:00 2001 From: Michael Olbrich Date: Tue, 6 Dec 2022 16:27:44 +0100 Subject: [PATCH] Add callback for raw frames for video capture This is needed for Chromium. The video capture API in Chromium expects the raw frames and it will always convert or copy the frame. With the existing API that would mean copying the frame twice. Bug: webrtc:13177 Change-Id: I71f6e2dc6d5a812c3641ac691b75d50178fa0de7 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/264548 Commit-Queue: Ilya Nikolaevskiy Reviewed-by: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/main@{#39095} --- modules/video_capture/BUILD.gn | 1 + .../video_capture/raw_video_sink_interface.h | 34 +++++++++++++++++++ modules/video_capture/video_capture.h | 3 ++ modules/video_capture/video_capture_impl.cc | 24 +++++++++++++ modules/video_capture/video_capture_impl.h | 7 ++++ 5 files changed, 69 insertions(+) create mode 100644 modules/video_capture/raw_video_sink_interface.h diff --git a/modules/video_capture/BUILD.gn b/modules/video_capture/BUILD.gn index ad2d85e3fe..cfa5184d71 100644 --- a/modules/video_capture/BUILD.gn +++ b/modules/video_capture/BUILD.gn @@ -17,6 +17,7 @@ rtc_library("video_capture_module") { sources = [ "device_info_impl.cc", "device_info_impl.h", + "raw_video_sink_interface.h", "video_capture.h", "video_capture_config.h", "video_capture_defines.h", diff --git a/modules/video_capture/raw_video_sink_interface.h b/modules/video_capture/raw_video_sink_interface.h new file mode 100644 index 0000000000..094e9e20bd --- /dev/null +++ b/modules/video_capture/raw_video_sink_interface.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file contains interfaces used for creating the VideoCaptureModule +// and DeviceInfo. + +#ifndef MODULES_VIDEO_CAPTURE_RAW_VIDEO_SINK_INTERFACE_H_ +#define MODULES_VIDEO_CAPTURE_RAW_VIDEO_SINK_INTERFACE_H_ + +#include "modules/video_capture/video_capture_defines.h" + +namespace webrtc { + +class RawVideoSinkInterface { + public: + virtual ~RawVideoSinkInterface() = default; + + virtual int32_t OnRawFrame(uint8_t* videoFrame, + size_t videoFrameLength, + const webrtc::VideoCaptureCapability& frameInfo, + VideoRotation rotation, + int64_t captureTime) = 0; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CAPTURE_RAW_VIDEO_SINK_INTERFACE_H_ diff --git a/modules/video_capture/video_capture.h b/modules/video_capture/video_capture.h index 3bbe217cba..eddc31414a 100644 --- a/modules/video_capture/video_capture.h +++ b/modules/video_capture/video_capture.h @@ -13,6 +13,7 @@ #include "api/video/video_rotation.h" #include "api/video/video_sink_interface.h" +#include "modules/video_capture/raw_video_sink_interface.h" #include "modules/video_capture/video_capture_defines.h" namespace webrtc { @@ -74,6 +75,8 @@ class VideoCaptureModule : public rtc::RefCountInterface { // Register capture data callback virtual void RegisterCaptureDataCallback( rtc::VideoSinkInterface* dataCallback) = 0; + virtual void RegisterCaptureDataCallback( + RawVideoSinkInterface* dataCallback) = 0; // Remove capture data callback virtual void DeRegisterCaptureDataCallback() = 0; diff --git a/modules/video_capture/video_capture_impl.cc b/modules/video_capture/video_capture_impl.cc index 234c2e131e..d539b38264 100644 --- a/modules/video_capture/video_capture_impl.cc +++ b/modules/video_capture/video_capture_impl.cc @@ -77,6 +77,7 @@ VideoCaptureImpl::VideoCaptureImpl() _lastProcessTimeNanos(rtc::TimeNanos()), _lastFrameRateCallbackTimeNanos(rtc::TimeNanos()), _dataCallBack(NULL), + _rawDataCallBack(NULL), _lastProcessFrameTimeNanos(rtc::TimeNanos()), _rotateFrame(kVideoRotation_0), apply_rotation_(false) { @@ -96,12 +97,21 @@ VideoCaptureImpl::~VideoCaptureImpl() { void VideoCaptureImpl::RegisterCaptureDataCallback( rtc::VideoSinkInterface* dataCallBack) { MutexLock lock(&api_lock_); + RTC_DCHECK(!_rawDataCallBack); _dataCallBack = dataCallBack; } +void VideoCaptureImpl::RegisterCaptureDataCallback( + RawVideoSinkInterface* dataCallBack) { + MutexLock lock(&api_lock_); + RTC_DCHECK(!_dataCallBack); + _rawDataCallBack = dataCallBack; +} + void VideoCaptureImpl::DeRegisterCaptureDataCallback() { MutexLock lock(&api_lock_); _dataCallBack = NULL; + _rawDataCallBack = NULL; } int32_t VideoCaptureImpl::DeliverCapturedFrame(VideoFrame& captureFrame) { UpdateFrameCount(); // frame count used for local frame rate callback. @@ -113,6 +123,15 @@ int32_t VideoCaptureImpl::DeliverCapturedFrame(VideoFrame& captureFrame) { return 0; } +void VideoCaptureImpl::DeliverRawFrame(uint8_t* videoFrame, + size_t videoFrameLength, + const VideoCaptureCapability& frameInfo, + int64_t captureTime) { + UpdateFrameCount(); + _rawDataCallBack->OnRawFrame(videoFrame, videoFrameLength, frameInfo, + _rotateFrame, captureTime); +} + int32_t VideoCaptureImpl::IncomingFrame(uint8_t* videoFrame, size_t videoFrameLength, const VideoCaptureCapability& frameInfo, @@ -124,6 +143,11 @@ int32_t VideoCaptureImpl::IncomingFrame(uint8_t* videoFrame, TRACE_EVENT1("webrtc", "VC::IncomingFrame", "capture_time", captureTime); + if (_rawDataCallBack) { + DeliverRawFrame(videoFrame, videoFrameLength, frameInfo, captureTime); + return 0; + } + // Not encoded, convert to I420. if (frameInfo.videoType != VideoType::kMJPEG && CalcBufferSize(frameInfo.videoType, width, abs(height)) != diff --git a/modules/video_capture/video_capture_impl.h b/modules/video_capture/video_capture_impl.h index 9e4afe7ec1..fee93396d3 100644 --- a/modules/video_capture/video_capture_impl.h +++ b/modules/video_capture/video_capture_impl.h @@ -53,6 +53,8 @@ class VideoCaptureImpl : public VideoCaptureModule { // Call backs void RegisterCaptureDataCallback( rtc::VideoSinkInterface* dataCallback) override; + virtual void RegisterCaptureDataCallback( + RawVideoSinkInterface* dataCallback) override; void DeRegisterCaptureDataCallback() override; int32_t SetCaptureRotation(VideoRotation rotation) override; @@ -86,6 +88,10 @@ class VideoCaptureImpl : public VideoCaptureModule { void UpdateFrameCount(); uint32_t CalculateFrameRate(int64_t now_ns); int32_t DeliverCapturedFrame(VideoFrame& captureFrame); + void DeliverRawFrame(uint8_t* videoFrame, + size_t videoFrameLength, + const VideoCaptureCapability& frameInfo, + int64_t captureTime); // last time the module process function was called. int64_t _lastProcessTimeNanos; @@ -93,6 +99,7 @@ class VideoCaptureImpl : public VideoCaptureModule { int64_t _lastFrameRateCallbackTimeNanos; rtc::VideoSinkInterface* _dataCallBack; + RawVideoSinkInterface* _rawDataCallBack; int64_t _lastProcessFrameTimeNanos; // timestamp for local captured frames