diff --git a/modules/desktop_capture/win/wgc_capture_session.cc b/modules/desktop_capture/win/wgc_capture_session.cc index ea5565c89c..f6d4c9188d 100644 --- a/modules/desktop_capture/win/wgc_capture_session.cc +++ b/modules/desktop_capture/win/wgc_capture_session.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include "rtc_base/win/create_direct3d_device.h" #include "rtc_base/win/get_activation_factory.h" #include "system_wrappers/include/metrics.h" +#include "system_wrappers/include/sleep.h" using Microsoft::WRL::ComPtr; namespace WGC = ABI::Windows::Graphics::Capture; @@ -40,11 +42,6 @@ namespace { constexpr auto kPixelFormat = ABI::Windows::Graphics::DirectX:: DirectXPixelFormat::DirectXPixelFormat_B8G8R8A8UIntNormalized; -// The maximum time `GetFrame` will wait for a frame to arrive, if we don't have -// any in the pool. -constexpr TimeDelta kMaxWaitForFrame = TimeDelta::Millis(50); -constexpr TimeDelta kMaxWaitForFirstFrame = TimeDelta::Millis(500); - // These values are persisted to logs. Entries should not be renumbered and // numeric values should never be reused. enum class StartCaptureResult { @@ -101,6 +98,7 @@ WgcCaptureSession::WgcCaptureSession(ComPtr d3d11_device, : d3d11_device_(std::move(d3d11_device)), item_(std::move(item)), size_(size) {} + WgcCaptureSession::~WgcCaptureSession() { RemoveEventHandlers(); } @@ -168,8 +166,6 @@ HRESULT WgcCaptureSession::StartCapture(const DesktopCaptureOptions& options) { return hr; } - frames_in_pool_ = 0; - // Because `WgcCapturerWin` created a `DispatcherQueue`, and we created // `frame_pool_` via `Create`, the `FrameArrived` event will be delivered on // the current thread. @@ -209,8 +205,76 @@ HRESULT WgcCaptureSession::StartCapture(const DesktopCaptureOptions& options) { return hr; } -HRESULT WgcCaptureSession::GetFrame( - std::unique_ptr* output_frame) { +bool WgcCaptureSession::GetFrame(std::unique_ptr* output_frame) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + + // When GetFrame() asks for the first frame it can happen that no frame has + // arrived yet. We therefore try to get a new frame from the frame pool for a + // maximum of 10 times after sleeping for 20ms. We choose 20ms as it's just a + // bit longer than 17ms (for 60fps*) and hopefully avoids unlucky timing + // causing us to wait two frames when we mostly seem to only need to wait for + // one. This approach should ensure that GetFrame() always delivers a valid + // frame with a max latency of 200ms and often after sleeping only once. + // (*) On a modern system, the FPS / monitor refresh rate is usually larger + // than or equal to 60. + const int max_sleep_count = 10; + const int sleep_time_ms = 20; + + int sleep_count = 0; + while (!queue_.current_frame() && sleep_count < max_sleep_count) { + sleep_count++; + webrtc::SleepMs(sleep_time_ms); + ProcessFrame(); + } + + // Return false if we still don't have a valid frame leading to a + // DesktopCapturer::Result::ERROR_PERMANENT posted by the WGC capturer. + if (!queue_.current_frame()) { + RTC_LOG(LS_ERROR) << "GetFrame failed."; + return false; + } + + // Emit the current frame. + std::unique_ptr new_frame = queue_.current_frame()->Share(); + *output_frame = std::move(new_frame); + + return true; +} + +HRESULT WgcCaptureSession::CreateMappedTexture( + ComPtr src_texture, + UINT width, + UINT height) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + + D3D11_TEXTURE2D_DESC src_desc; + src_texture->GetDesc(&src_desc); + D3D11_TEXTURE2D_DESC map_desc; + map_desc.Width = width == 0 ? src_desc.Width : width; + map_desc.Height = height == 0 ? src_desc.Height : height; + map_desc.MipLevels = src_desc.MipLevels; + map_desc.ArraySize = src_desc.ArraySize; + map_desc.Format = src_desc.Format; + map_desc.SampleDesc = src_desc.SampleDesc; + map_desc.Usage = D3D11_USAGE_STAGING; + map_desc.BindFlags = 0; + map_desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + map_desc.MiscFlags = 0; + return d3d11_device_->CreateTexture2D(&map_desc, nullptr, &mapped_texture_); +} + +HRESULT WgcCaptureSession::OnFrameArrived( + WGC::IDirect3D11CaptureFramePool* sender, + IInspectable* event_args) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + HRESULT hr = ProcessFrame(); + if (FAILED(hr)) { + RTC_DLOG(LS_WARNING) << "ProcessFrame failed: " << hr; + } + return hr; +} + +HRESULT WgcCaptureSession::ProcessFrame() { RTC_DCHECK_RUN_ON(&sequence_checker_); if (item_closed_) { @@ -221,9 +285,10 @@ HRESULT WgcCaptureSession::GetFrame( RTC_DCHECK(is_capture_started_); - if (frames_in_pool_ < 1) - wait_for_frame_event_.Wait(first_frame_ ? kMaxWaitForFirstFrame - : kMaxWaitForFrame); + queue_.MoveToNextFrame(); + if (queue_.current_frame() && queue_.current_frame()->IsShared()) { + RTC_DLOG(LS_WARNING) << "Overwriting frame that is still shared."; + } ComPtr capture_frame; HRESULT hr = frame_pool_->TryGetNextFrame(&capture_frame); @@ -234,13 +299,11 @@ HRESULT WgcCaptureSession::GetFrame( } if (!capture_frame) { + RTC_DLOG(LS_WARNING) << "Frame pool was empty."; RecordGetFrameResult(GetFrameResult::kFrameDropped); - return hr; + return E_FAIL; } - first_frame_ = false; - --frames_in_pool_; - // We need to get `capture_frame` as an `ID3D11Texture2D` so that we can get // the raw image data in the format required by the `DesktopFrame` interface. ComPtr @@ -333,64 +396,37 @@ HRESULT WgcCaptureSession::GetFrame( return hr; } - int row_data_length = image_width * DesktopFrame::kBytesPerPixel; + // Allocate the current frame buffer only if it is not already allocated or + // if the size has changed. Note that we can't reallocate other buffers at + // this point, since the caller may still be reading from them. The queue can + // hold up tp two frames. + DesktopSize image_size(image_width, image_height); + if (!queue_.current_frame() || + !queue_.current_frame()->size().equals(image_size)) { + std::unique_ptr buffer = + std::make_unique(image_size); + queue_.ReplaceCurrentFrame(SharedDesktopFrame::Wrap(std::move(buffer))); + } - // Make a copy of the data pointed to by `map_info.pData` so we are free to - // unmap our texture. + DesktopFrame* current_frame = queue_.current_frame(); + + // Make a copy of the data pointed to by `map_info.pData` to the preallocated + // `current_frame` so we are free to unmap our texture. uint8_t* src_data = static_cast(map_info.pData); - std::vector image_data; - image_data.resize(image_height * row_data_length); - uint8_t* image_data_ptr = image_data.data(); + uint8_t* dst_data = current_frame->data(); for (int i = 0; i < image_height; i++) { - memcpy(image_data_ptr, src_data, row_data_length); - image_data_ptr += row_data_length; + memcpy(dst_data, src_data, current_frame->stride()); + dst_data += current_frame->stride(); src_data += map_info.RowPitch; } d3d_context->Unmap(mapped_texture_.Get(), 0); - // Transfer ownership of `image_data` to the output_frame. - DesktopSize size(image_width, image_height); - *output_frame = std::make_unique(size, row_data_length, - std::move(image_data)); - size_ = new_size; RecordGetFrameResult(GetFrameResult::kSuccess); return hr; } -HRESULT WgcCaptureSession::CreateMappedTexture( - ComPtr src_texture, - UINT width, - UINT height) { - RTC_DCHECK_RUN_ON(&sequence_checker_); - - D3D11_TEXTURE2D_DESC src_desc; - src_texture->GetDesc(&src_desc); - D3D11_TEXTURE2D_DESC map_desc; - map_desc.Width = width == 0 ? src_desc.Width : width; - map_desc.Height = height == 0 ? src_desc.Height : height; - map_desc.MipLevels = src_desc.MipLevels; - map_desc.ArraySize = src_desc.ArraySize; - map_desc.Format = src_desc.Format; - map_desc.SampleDesc = src_desc.SampleDesc; - map_desc.Usage = D3D11_USAGE_STAGING; - map_desc.BindFlags = 0; - map_desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; - map_desc.MiscFlags = 0; - return d3d11_device_->CreateTexture2D(&map_desc, nullptr, &mapped_texture_); -} - -HRESULT WgcCaptureSession::OnFrameArrived( - WGC::IDirect3D11CaptureFramePool* sender, - IInspectable* event_args) { - RTC_DCHECK_RUN_ON(&sequence_checker_); - RTC_DCHECK_LT(frames_in_pool_, kNumBuffers); - ++frames_in_pool_; - wait_for_frame_event_.Set(); - return S_OK; -} - HRESULT WgcCaptureSession::OnItemClosed(WGC::IGraphicsCaptureItem* sender, IInspectable* event_args) { RTC_DCHECK_RUN_ON(&sequence_checker_); diff --git a/modules/desktop_capture/win/wgc_capture_session.h b/modules/desktop_capture/win/wgc_capture_session.h index dfe1fa60bb..27298a3985 100644 --- a/modules/desktop_capture/win/wgc_capture_session.h +++ b/modules/desktop_capture/win/wgc_capture_session.h @@ -20,6 +20,8 @@ #include "api/sequence_checker.h" #include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/screen_capture_frame_queue.h" +#include "modules/desktop_capture/shared_desktop_frame.h" #include "modules/desktop_capture/win/wgc_capture_source.h" #include "rtc_base/event.h" @@ -41,19 +43,18 @@ class WgcCaptureSession final { HRESULT StartCapture(const DesktopCaptureOptions& options); - // Returns a frame from the frame pool, if any are present. - HRESULT GetFrame(std::unique_ptr* output_frame); + // Returns a frame from the local frame queue, if any are present. + bool GetFrame(std::unique_ptr* output_frame); bool IsCaptureStarted() const { RTC_DCHECK_RUN_ON(&sequence_checker_); return is_capture_started_; } - // We keep 2 buffers in the frame pool to balance the staleness of the frame - // with having to wait for frames to arrive too frequently. Too many buffers - // will lead to a high latency, and too few will lead to poor performance. + // We only keep 1 buffer in the internal frame pool to reduce the latency as + // much as possible. // We make this public for tests. - static constexpr int kNumBuffers = 2; + static constexpr int kNumBuffers = 1; private: // Initializes `mapped_texture_` with the properties of the `src_texture`, @@ -76,16 +77,11 @@ class WgcCaptureSession final { ABI::Windows::Graphics::Capture::IDirect3D11CaptureFramePool* sender, IInspectable* event_args); + // Process the captured frame and copy it to the `queue_`. + HRESULT ProcessFrame(); + void RemoveEventHandlers(); - // We wait on this event in `GetFrame` if there are no frames in the pool. - // `OnFrameArrived` will set the event so we can proceed. - rtc::Event wait_for_frame_event_; - int frames_in_pool_; - - // We're willing to wait for a frame a little longer if it's the first one. - bool first_frame_ = true; - std::unique_ptr frame_arrived_token_; std::unique_ptr item_closed_token_; @@ -126,6 +122,11 @@ class WgcCaptureSession final { ABI::Windows::Graphics::Capture::IGraphicsCaptureSession> session_; + // Queue of captured video frames. The queue holds 2 frames and it avoids + // alloc/dealloc per captured frame. Incoming frames from the internal frame + // pool are copied to this queue after required processing in ProcessFrame(). + ScreenCaptureFrameQueue queue_; + bool item_closed_ = false; bool is_capture_started_ = false; diff --git a/modules/desktop_capture/win/wgc_capturer_win.cc b/modules/desktop_capture/win/wgc_capturer_win.cc index 8ec6a29f23..20d4eb938d 100644 --- a/modules/desktop_capture/win/wgc_capturer_win.cc +++ b/modules/desktop_capture/win/wgc_capturer_win.cc @@ -323,9 +323,8 @@ void WgcCapturerWin::CaptureFrame() { } std::unique_ptr frame; - hr = capture_session->GetFrame(&frame); - if (FAILED(hr)) { - RTC_LOG(LS_ERROR) << "GetFrame failed: " << hr; + if (!capture_session->GetFrame(&frame)) { + RTC_LOG(LS_ERROR) << "GetFrame failed."; ongoing_captures_.erase(capture_source_->GetSourceId()); callback_->OnCaptureResult(DesktopCapturer::Result::ERROR_PERMANENT, /*frame=*/nullptr);