admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch WGC to ScreenCaptureFrameQueue

Browse Source 
* Avoids alloc/dealloc for each captured frame.
* Reduces time to capture first frame.
* Improves performance in terms of max FPS.

Bug: chromium:1412584
Change-Id: Ie16519ad788165c9553451ecea5adff12cd15eea
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/293582
Reviewed-by: Alexander Cooper <alcooper@chromium.org>
Commit-Queue: Henrik Andreassson <henrika@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39384}
This commit is contained in:
henrika 2023-02-23 19:28:33 +01:00 committed by WebRTC LUCI CQ
parent 765812902c
commit 274408feab
3 changed files with 114 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
modules/desktop_capture/win/wgc_capture_session.cc
View File

@ -17,6 +17,7 @@
#include <wrl/client.h>
#include <wrl/event.h>
#include <algorithm>
#include <memory>
#include <utility>
#include <vector>
@ -28,6 +29,7 @@
#include "rtc_base/win/create_direct3d_device.h"
#include "rtc_base/win/get_activation_factory.h"
#include "system_wrappers/include/metrics.h"
#include "system_wrappers/include/sleep.h"
using Microsoft::WRL::ComPtr;
namespace WGC = ABI::Windows::Graphics::Capture;
@ -40,11 +42,6 @@ namespace {
constexpr auto kPixelFormat = ABI::Windows::Graphics::DirectX::
DirectXPixelFormat::DirectXPixelFormat_B8G8R8A8UIntNormalized;
// The maximum time `GetFrame` will wait for a frame to arrive, if we don't have
// any in the pool.
constexpr TimeDelta kMaxWaitForFrame = TimeDelta::Millis(50);
constexpr TimeDelta kMaxWaitForFirstFrame = TimeDelta::Millis(500);
// These values are persisted to logs. Entries should not be renumbered and
// numeric values should never be reused.
enum class StartCaptureResult {
@ -101,6 +98,7 @@ WgcCaptureSession::WgcCaptureSession(ComPtr<ID3D11Device> d3d11_device,
: d3d11_device_(std::move(d3d11_device)),
item_(std::move(item)),
size_(size) {}
WgcCaptureSession::~WgcCaptureSession() {
RemoveEventHandlers();
}
@ -168,8 +166,6 @@ HRESULT WgcCaptureSession::StartCapture(const DesktopCaptureOptions& options) {
return hr;
}
frames_in_pool_ = 0;
// Because `WgcCapturerWin` created a `DispatcherQueue`, and we created
// `frame_pool_` via `Create`, the `FrameArrived` event will be delivered on
// the current thread.
@ -209,8 +205,76 @@ HRESULT WgcCaptureSession::StartCapture(const DesktopCaptureOptions& options) {
return hr;
}
HRESULT WgcCaptureSession::GetFrame(
std::unique_ptr<DesktopFrame>* output_frame) {
bool WgcCaptureSession::GetFrame(std::unique_ptr<DesktopFrame>* output_frame) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
// When GetFrame() asks for the first frame it can happen that no frame has
// arrived yet. We therefore try to get a new frame from the frame pool for a
// maximum of 10 times after sleeping for 20ms. We choose 20ms as it's just a
// bit longer than 17ms (for 60fps*) and hopefully avoids unlucky timing
// causing us to wait two frames when we mostly seem to only need to wait for
// one. This approach should ensure that GetFrame() always delivers a valid
// frame with a max latency of 200ms and often after sleeping only once.
// (*) On a modern system, the FPS / monitor refresh rate is usually larger
// than or equal to 60.
const int max_sleep_count = 10;
const int sleep_time_ms = 20;
int sleep_count = 0;
while (!queue_.current_frame() && sleep_count < max_sleep_count) {
sleep_count++;
webrtc::SleepMs(sleep_time_ms);
ProcessFrame();
}
// Return false if we still don't have a valid frame leading to a
// DesktopCapturer::Result::ERROR_PERMANENT posted by the WGC capturer.
if (!queue_.current_frame()) {
RTC_LOG(LS_ERROR) << "GetFrame failed.";
return false;
}
// Emit the current frame.
std::unique_ptr<DesktopFrame> new_frame = queue_.current_frame()->Share();
*output_frame = std::move(new_frame);
return true;
}
HRESULT WgcCaptureSession::CreateMappedTexture(
ComPtr<ID3D11Texture2D> src_texture,
UINT width,
UINT height) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
D3D11_TEXTURE2D_DESC src_desc;
src_texture->GetDesc(&src_desc);
D3D11_TEXTURE2D_DESC map_desc;
map_desc.Width = width == 0 ? src_desc.Width : width;
map_desc.Height = height == 0 ? src_desc.Height : height;
map_desc.MipLevels = src_desc.MipLevels;
map_desc.ArraySize = src_desc.ArraySize;
map_desc.Format = src_desc.Format;
map_desc.SampleDesc = src_desc.SampleDesc;
map_desc.Usage = D3D11_USAGE_STAGING;
map_desc.BindFlags = 0;
map_desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
map_desc.MiscFlags = 0;
return d3d11_device_->CreateTexture2D(&map_desc, nullptr, &mapped_texture_);
}
HRESULT WgcCaptureSession::OnFrameArrived(
WGC::IDirect3D11CaptureFramePool* sender,
IInspectable* event_args) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
HRESULT hr = ProcessFrame();
if (FAILED(hr)) {
RTC_DLOG(LS_WARNING) << "ProcessFrame failed: " << hr;
}
return hr;
}
HRESULT WgcCaptureSession::ProcessFrame() {
RTC_DCHECK_RUN_ON(&sequence_checker_);
if (item_closed_) {
@ -221,9 +285,10 @@ HRESULT WgcCaptureSession::GetFrame(
RTC_DCHECK(is_capture_started_);
if (frames_in_pool_ < 1)
wait_for_frame_event_.Wait(first_frame_ ? kMaxWaitForFirstFrame
: kMaxWaitForFrame);
queue_.MoveToNextFrame();
if (queue_.current_frame() && queue_.current_frame()->IsShared()) {
RTC_DLOG(LS_WARNING) << "Overwriting frame that is still shared.";
}
ComPtr<WGC::IDirect3D11CaptureFrame> capture_frame;
HRESULT hr = frame_pool_->TryGetNextFrame(&capture_frame);
@ -234,13 +299,11 @@ HRESULT WgcCaptureSession::GetFrame(
}
if (!capture_frame) {
RTC_DLOG(LS_WARNING) << "Frame pool was empty.";
RecordGetFrameResult(GetFrameResult::kFrameDropped);
return hr;
return E_FAIL;
}
first_frame_ = false;
--frames_in_pool_;
// We need to get `capture_frame` as an `ID3D11Texture2D` so that we can get
// the raw image data in the format required by the `DesktopFrame` interface.
ComPtr<ABI::Windows::Graphics::DirectX::Direct3D11::IDirect3DSurface>
@ -333,64 +396,37 @@ HRESULT WgcCaptureSession::GetFrame(
return hr;
}
int row_data_length = image_width * DesktopFrame::kBytesPerPixel;
// Allocate the current frame buffer only if it is not already allocated or
// if the size has changed. Note that we can't reallocate other buffers at
// this point, since the caller may still be reading from them. The queue can
// hold up tp two frames.
DesktopSize image_size(image_width, image_height);
if (!queue_.current_frame() ||
!queue_.current_frame()->size().equals(image_size)) {
std::unique_ptr<DesktopFrame> buffer =
std::make_unique<BasicDesktopFrame>(image_size);
queue_.ReplaceCurrentFrame(SharedDesktopFrame::Wrap(std::move(buffer)));
}
// Make a copy of the data pointed to by `map_info.pData` so we are free to
// unmap our texture.
DesktopFrame* current_frame = queue_.current_frame();
// Make a copy of the data pointed to by `map_info.pData` to the preallocated
// `current_frame` so we are free to unmap our texture.
uint8_t* src_data = static_cast<uint8_t*>(map_info.pData);
std::vector<uint8_t> image_data;
image_data.resize(image_height * row_data_length);
uint8_t* image_data_ptr = image_data.data();
uint8_t* dst_data = current_frame->data();
for (int i = 0; i < image_height; i++) {
memcpy(image_data_ptr, src_data, row_data_length);
image_data_ptr += row_data_length;
memcpy(dst_data, src_data, current_frame->stride());
dst_data += current_frame->stride();
src_data += map_info.RowPitch;
}
d3d_context->Unmap(mapped_texture_.Get(), 0);
// Transfer ownership of `image_data` to the output_frame.
DesktopSize size(image_width, image_height);
*output_frame = std::make_unique<WgcDesktopFrame>(size, row_data_length,
std::move(image_data));
size_ = new_size;
RecordGetFrameResult(GetFrameResult::kSuccess);
return hr;
}
HRESULT WgcCaptureSession::CreateMappedTexture(
ComPtr<ID3D11Texture2D> src_texture,
UINT width,
UINT height) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
D3D11_TEXTURE2D_DESC src_desc;
src_texture->GetDesc(&src_desc);
D3D11_TEXTURE2D_DESC map_desc;
map_desc.Width = width == 0 ? src_desc.Width : width;
map_desc.Height = height == 0 ? src_desc.Height : height;
map_desc.MipLevels = src_desc.MipLevels;
map_desc.ArraySize = src_desc.ArraySize;
map_desc.Format = src_desc.Format;
map_desc.SampleDesc = src_desc.SampleDesc;
map_desc.Usage = D3D11_USAGE_STAGING;
map_desc.BindFlags = 0;
map_desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
map_desc.MiscFlags = 0;
return d3d11_device_->CreateTexture2D(&map_desc, nullptr, &mapped_texture_);
}
HRESULT WgcCaptureSession::OnFrameArrived(
WGC::IDirect3D11CaptureFramePool* sender,
IInspectable* event_args) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
RTC_DCHECK_LT(frames_in_pool_, kNumBuffers);
++frames_in_pool_;
wait_for_frame_event_.Set();
return S_OK;
}
HRESULT WgcCaptureSession::OnItemClosed(WGC::IGraphicsCaptureItem* sender,
IInspectable* event_args) {
RTC_DCHECK_RUN_ON(&sequence_checker_);

29
modules/desktop_capture/win/wgc_capture_session.h
View File

@ -20,6 +20,8 @@
#include "api/sequence_checker.h"
#include "modules/desktop_capture/desktop_capture_options.h"
#include "modules/desktop_capture/screen_capture_frame_queue.h"
#include "modules/desktop_capture/shared_desktop_frame.h"
#include "modules/desktop_capture/win/wgc_capture_source.h"
#include "rtc_base/event.h"
@ -41,19 +43,18 @@ class WgcCaptureSession final {
HRESULT StartCapture(const DesktopCaptureOptions& options);
// Returns a frame from the frame pool, if any are present.
HRESULT GetFrame(std::unique_ptr<DesktopFrame>* output_frame);
// Returns a frame from the local frame queue, if any are present.
bool GetFrame(std::unique_ptr<DesktopFrame>* output_frame);
bool IsCaptureStarted() const {
RTC_DCHECK_RUN_ON(&sequence_checker_);
return is_capture_started_;
}
// We keep 2 buffers in the frame pool to balance the staleness of the frame
// with having to wait for frames to arrive too frequently. Too many buffers
// will lead to a high latency, and too few will lead to poor performance.
// We only keep 1 buffer in the internal frame pool to reduce the latency as
// much as possible.
// We make this public for tests.
static constexpr int kNumBuffers = 2;
static constexpr int kNumBuffers = 1;
private:
// Initializes `mapped_texture_` with the properties of the `src_texture`,
@ -76,16 +77,11 @@ class WgcCaptureSession final {
ABI::Windows::Graphics::Capture::IDirect3D11CaptureFramePool* sender,
IInspectable* event_args);
// Process the captured frame and copy it to the `queue_`.
HRESULT ProcessFrame();
void RemoveEventHandlers();
// We wait on this event in `GetFrame` if there are no frames in the pool.
// `OnFrameArrived` will set the event so we can proceed.
rtc::Event wait_for_frame_event_;
int frames_in_pool_;
// We're willing to wait for a frame a little longer if it's the first one.
bool first_frame_ = true;
std::unique_ptr<EventRegistrationToken> frame_arrived_token_;
std::unique_ptr<EventRegistrationToken> item_closed_token_;
@ -126,6 +122,11 @@ class WgcCaptureSession final {
ABI::Windows::Graphics::Capture::IGraphicsCaptureSession>
session_;
// Queue of captured video frames. The queue holds 2 frames and it avoids
// alloc/dealloc per captured frame. Incoming frames from the internal frame
// pool are copied to this queue after required processing in ProcessFrame().
ScreenCaptureFrameQueue<SharedDesktopFrame> queue_;
bool item_closed_ = false;
bool is_capture_started_ = false;

5
modules/desktop_capture/win/wgc_capturer_win.cc
View File

@ -323,9 +323,8 @@ void WgcCapturerWin::CaptureFrame() {
}
std::unique_ptr<DesktopFrame> frame;
hr = capture_session->GetFrame(&frame);
if (FAILED(hr)) {
RTC_LOG(LS_ERROR) << "GetFrame failed: " << hr;
if (!capture_session->GetFrame(&frame)) {
RTC_LOG(LS_ERROR) << "GetFrame failed.";
ongoing_captures_.erase(capture_source_->GetSourceId());
callback_->OnCaptureResult(DesktopCapturer::Result::ERROR_PERMANENT,
/*frame=*/nullptr);