From 6c453b7d59145bec2e7d3060ea373a9abcde3ef1 Mon Sep 17 00:00:00 2001 From: henrika Date: Fri, 16 Jun 2023 11:30:54 +0200 Subject: [PATCH] Light-weight detection of static content when using WGC This CL adds a light-weight detection of "frame content has changed since last frame" to an existing pass where bytes are copied from a texture to a DesktopFrame. The resulting boolean can then later be used to bypass a full detection of if the content is static or not. As a result, we only check for static content for a small fraction of all captured WGC frames and this reduces the total load when 0Hz is enabled for WGC. Both WGC and 0Hz support for WGC is still behind a flag. Bug: chromium:1421242 Change-Id: If9e3721c60a244a3919758fe861d56d4b54cb039 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/308821 Reviewed-by: Alexander Cooper Commit-Queue: Henrik Andreassson Cr-Commit-Position: refs/heads/main@{#40299} --- .../win/wgc_capture_session.cc | 84 ++++++++++++++++--- .../desktop_capture/win/wgc_capture_session.h | 2 + 2 files changed, 73 insertions(+), 13 deletions(-) diff --git a/modules/desktop_capture/win/wgc_capture_session.cc b/modules/desktop_capture/win/wgc_capture_session.cc index f439e21068..61138c04de 100644 --- a/modules/desktop_capture/win/wgc_capture_session.cc +++ b/modules/desktop_capture/win/wgc_capture_session.cc @@ -90,6 +90,12 @@ void RecordGetFrameResult(GetFrameResult error) { static_cast(error), static_cast(GetFrameResult::kMaxValue)); } +bool SizeHasChanged(ABI::Windows::Graphics::SizeInt32 size_new, + ABI::Windows::Graphics::SizeInt32 size_old) { + return (size_new.Height != size_old.Height || + size_new.Width != size_old.Width); +} + } // namespace WgcCaptureSession::WgcCaptureSession(ComPtr d3d11_device, @@ -370,7 +376,7 @@ HRESULT WgcCaptureSession::ProcessFrame() { // If the size changed, we must resize `mapped_texture_` and `frame_pool_` to // fit the new size. This must be done before `CopySubresourceRegion` so that // the textures are the same size. - if (size_.Height != new_size.Height || size_.Width != new_size.Width) { + if (SizeHasChanged(new_size, size_)) { hr = CreateMappedTexture(texture_2D, new_size.Width, new_size.Height); if (FAILED(hr)) { RecordGetFrameResult(GetFrameResult::kResizeMappedTextureFailed); @@ -426,21 +432,49 @@ HRESULT WgcCaptureSession::ProcessFrame() { } DesktopFrame* current_frame = queue_.current_frame(); + DesktopFrame* previous_frame = queue_.previous_frame(); + + // Will be set to true while copying the frame data to the `current_frame` if + // we can already determine that the content of the new frame differs from the + // previous. The idea is to get a low-complexity indication of if the content + // is static or not without performing a full/deep memory comparison when + // updating the damaged region. + bool frame_content_has_changed = false; + + // Check if the queue contains two frames whose content can be compared. + const bool frame_content_can_be_compared = FrameContentCanBeCompared(); // Make a copy of the data pointed to by `map_info.pData` to the preallocated - // `current_frame` so we are free to unmap our texture. + // `current_frame` so we are free to unmap our texture. If possible, also + // perform a light-weight scan of the vertical line of pixels in the middle + // of the screen. A comparison is performed between two 32-bit pixels (RGBA); + // one from the current frame and one from the previous, and as soon as a + // difference is detected the scan stops and `frame_content_has_changed` is + // set to true. uint8_t* src_data = static_cast(map_info.pData); uint8_t* dst_data = current_frame->data(); + uint8_t* prev_data = + frame_content_can_be_compared ? previous_frame->data() : nullptr; + RTC_DCHECK_EQ(map_info.RowPitch, current_frame->stride()); + const int width_in_bytes = map_info.RowPitch; + const int middle_pixel_offset = + (image_width / 2) * DesktopFrame::kBytesPerPixel; for (int i = 0; i < image_height; i++) { - memcpy(dst_data, src_data, current_frame->stride()); - dst_data += current_frame->stride(); - src_data += map_info.RowPitch; + memcpy(dst_data, src_data, width_in_bytes); + if (prev_data && !frame_content_has_changed) { + uint8_t* previous_pixel = prev_data + middle_pixel_offset; + uint8_t* current_pixel = dst_data + middle_pixel_offset; + frame_content_has_changed = + memcmp(previous_pixel, current_pixel, DesktopFrame::kBytesPerPixel); + prev_data += width_in_bytes; + } + dst_data += width_in_bytes; + src_data += width_in_bytes; } d3d_context->Unmap(mapped_texture_.Get(), 0); if (allow_zero_hertz()) { - DesktopFrame* previous_frame = queue_.previous_frame(); if (previous_frame) { const int previous_frame_size = previous_frame->stride() * previous_frame->size().height(); @@ -448,15 +482,27 @@ HRESULT WgcCaptureSession::ProcessFrame() { current_frame->stride() * current_frame->size().height(); // Compare the latest frame with the previous and check if the frames are - // equal (both contain the exact same pixel values). + // equal (both contain the exact same pixel values). Avoid full memory + // comparison if indication of a changed frame already exists from the + // stage above. if (current_frame_size == previous_frame_size) { - const bool frames_are_equal = !memcmp( - current_frame->data(), previous_frame->data(), current_frame_size); - if (!frames_are_equal) { - // TODO(https://crbug.com/1421242): If we had an API to report proper - // damage regions we should be doing AddRect() with a SetRect() call - // on a resize. + if (frame_content_has_changed) { + // Mark frame as damaged based on existing light-weight indicator. + // Avoids deep memcmp of complete frame and saves resources. damage_region_.SetRect(DesktopRect::MakeSize(current_frame->size())); + } else { + // Perform full memory comparison for all bytes between the current + // and the previous frames. + const bool frames_are_equal = + !memcmp(current_frame->data(), previous_frame->data(), + current_frame_size); + if (!frames_are_equal) { + // TODO(https://crbug.com/1421242): If we had an API to report + // proper damage regions we should be doing AddRect() with a + // SetRect() call on a resize. + damage_region_.SetRect( + DesktopRect::MakeSize(current_frame->size())); + } } } else { // Mark resized frames as damaged. @@ -497,4 +543,16 @@ void WgcCaptureSession::RemoveEventHandler() { } } +bool WgcCaptureSession::FrameContentCanBeCompared() { + DesktopFrame* current_frame = queue_.current_frame(); + DesktopFrame* previous_frame = queue_.previous_frame(); + if (!current_frame || !previous_frame) { + return false; + } + if (current_frame->stride() != previous_frame->stride()) { + return false; + } + return current_frame->size().equals(previous_frame->size()); +} + } // namespace webrtc diff --git a/modules/desktop_capture/win/wgc_capture_session.h b/modules/desktop_capture/win/wgc_capture_session.h index ee15096eab..499c75ee98 100644 --- a/modules/desktop_capture/win/wgc_capture_session.h +++ b/modules/desktop_capture/win/wgc_capture_session.h @@ -81,6 +81,8 @@ class WgcCaptureSession final { void RemoveEventHandler(); + bool FrameContentCanBeCompared(); + bool allow_zero_hertz() const { return allow_zero_hertz_; } std::unique_ptr item_closed_token_;