admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

desktop_capturer: Support frame rate negotiation via pipewire

Browse Source 
This change adds support for renegotiating the frame rate
via pipewire.

Bug: chromium:1291247
Change-Id: Iacd4a3c924900839a8db75a50b448df6c48c83ab
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/291460
Commit-Queue: Salman Malik <salmanmalik@chromium.org>
Reviewed-by: Alexander Cooper <alcooper@chromium.org>
Cr-Commit-Position: refs/heads/main@{#39216}
This commit is contained in:
Salman 2023-01-27 18:26:05 +00:00 committed by WebRTC LUCI CQ
parent 4c4566a8e6
commit 26340b07a4
11 changed files with 104 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
modules/desktop_capture/desktop_and_cursor_composer.cc
View File

@ -177,6 +177,10 @@ void DesktopAndCursorComposer::Start(DesktopCapturer::Callback* callback) {
desktop_capturer_->Start(this);
}
void DesktopAndCursorComposer::SetMaxFrameRate(uint32_t max_frame_rate) {
desktop_capturer_->SetMaxFrameRate(max_frame_rate);
}
void DesktopAndCursorComposer::SetSharedMemoryFactory(
std::unique_ptr<SharedMemoryFactory> shared_memory_factory) {
desktop_capturer_->SetSharedMemoryFactory(std::move(shared_memory_factory));

1
modules/desktop_capture/desktop_and_cursor_composer.h
View File

@ -61,6 +61,7 @@ class RTC_EXPORT DesktopAndCursorComposer
bool SelectSource(SourceId id) override;
bool FocusOnSelectedSource() override;
bool IsOccluded(const DesktopVector& pos) override;
void SetMaxFrameRate(uint32_t max_frame_rate) override;
#if defined(WEBRTC_USE_GIO)
DesktopCaptureMetadata GetMetadata() override;
#endif // defined(WEBRTC_USE_GIO)

6
modules/desktop_capture/desktop_capturer.h
View File

@ -100,6 +100,12 @@ class RTC_EXPORT DesktopCapturer {
// valid until capturer is destroyed.
virtual void Start(Callback* callback) = 0;
// Sets max frame rate for the capturer. This is best effort and may not be
// supported by all capturers. This will only affect the frequency at which
// new frames are available, not the frequency at which you are allowed to
// capture the frames.
virtual void SetMaxFrameRate(uint32_t max_frame_rate) {}
// Returns a valid pointer if the capturer requires the user to make a
// selection from a source list provided by the capturer.
// Returns nullptr if the capturer does not provide a UI for the user to make

7
modules/desktop_capture/linux/wayland/base_capturer_pipewire.cc
View File

@ -118,6 +118,13 @@ void BaseCapturerPipeWire::UpdateResolution(uint32_t width, uint32_t height) {
}
}
void BaseCapturerPipeWire::SetMaxFrameRate(uint32_t max_frame_rate) {
if (!capturer_failed_) {
options_.screencast_stream()->UpdateScreenCastStreamFrameRate(
max_frame_rate);
}
}
void BaseCapturerPipeWire::Start(Callback* callback) {
RTC_DCHECK(!callback_);
RTC_DCHECK(callback);

1
modules/desktop_capture/linux/wayland/base_capturer_pipewire.h
View File

@ -49,6 +49,7 @@ class RTC_EXPORT BaseCapturerPipeWire
bool GetSourceList(SourceList* sources) override;
bool SelectSource(SourceId id) override;
DelegatedSourceListController* GetDelegatedSourceListController() override;
void SetMaxFrameRate(uint32_t max_frame_rate) override;
// DelegatedSourceListController
void Observe(Observer* observer) override;

16
modules/desktop_capture/linux/wayland/screencast_stream_utils.cc
View File

@ -67,11 +67,11 @@ bool PipeWireVersion::operator<=(const PipeWireVersion& other) {
spa_pod* BuildFormat(spa_pod_builder* builder,
uint32_t format,
const std::vector<uint64_t>& modifiers,
const struct spa_rectangle* resolution) {
const struct spa_rectangle* resolution,
const struct spa_fraction* frame_rate) {
spa_pod_frame frames[2];
spa_rectangle pw_min_screen_bounds = spa_rectangle{1, 1};
spa_rectangle pw_max_screen_bounds = spa_rectangle{UINT32_MAX, UINT32_MAX};
spa_pod_builder_push_object(builder, &frames[0], SPA_TYPE_OBJECT_Format,
SPA_PARAM_EnumFormat);
spa_pod_builder_add(builder, SPA_FORMAT_mediaType,
@ -116,7 +116,17 @@ spa_pod* BuildFormat(spa_pod_builder* builder,
&pw_max_screen_bounds),
0);
}
if (frame_rate) {
static const spa_fraction pw_min_frame_rate = spa_fraction{0, 1};
spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_framerate,
SPA_POD_CHOICE_RANGE_Fraction(
frame_rate, &pw_min_frame_rate, frame_rate),
0);
spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_maxFramerate,
SPA_POD_CHOICE_RANGE_Fraction(
frame_rate, &pw_min_frame_rate, frame_rate),
0);
}
return static_cast<spa_pod*>(spa_pod_builder_pop(builder, &frames[0]));
}

4
modules/desktop_capture/linux/wayland/screencast_stream_utils.h
View File

@ -21,6 +21,7 @@
struct spa_pod;
struct spa_pod_builder;
struct spa_rectangle;
struct spa_fraction;
namespace webrtc {
@ -44,7 +45,8 @@ struct PipeWireVersion {
spa_pod* BuildFormat(spa_pod_builder* builder,
uint32_t format,
const std::vector<uint64_t>& modifiers,
const struct spa_rectangle* resolution);
const struct spa_rectangle* resolution,
const struct spa_fraction* frame_rate);
} // namespace webrtc

58
modules/desktop_capture/linux/wayland/shared_screencast_stream.cc
View File

@ -80,6 +80,7 @@ class SharedScreenCastStreamPrivate {
uint32_t height = 0,
bool is_cursor_embedded = false);
void UpdateScreenCastStreamResolution(uint32_t width, uint32_t height);
void UpdateScreenCastStreamFrameRate(uint32_t frame_rate);
void SetUseDamageRegion(bool use_damage_region) {
use_damage_region_ = use_damage_region;
}
@ -138,9 +139,8 @@ class SharedScreenCastStreamPrivate {
// Resolution parameters.
uint32_t width_ = 0;
uint32_t height_ = 0;
webrtc::Mutex resolution_lock_;
// Resolution changes are processed during buffer processing.
bool pending_resolution_change_ RTC_GUARDED_BY(&resolution_lock_) = false;
// Frame rate.
uint32_t frame_rate_ = 60;
bool use_damage_region_ = true;
@ -256,6 +256,12 @@ void SharedScreenCastStreamPrivate::OnStreamParamChanged(
spa_format_video_raw_parse(format, &that->spa_video_format_);
if (that->observer_ && that->spa_video_format_.max_framerate.denom) {
that->observer_->OnFrameRateChanged(
that->spa_video_format_.max_framerate.num /
that->spa_video_format_.max_framerate.denom);
}
auto width = that->spa_video_format_.size.width;
auto height = that->spa_video_format_.size.height;
auto stride = SPA_ROUND_UP_N(width * kBytesPerPixel, 4);
@ -355,22 +361,22 @@ void SharedScreenCastStreamPrivate::OnRenegotiateFormat(void* data, uint64_t) {
std::vector<const spa_pod*> params;
struct spa_rectangle resolution =
SPA_RECTANGLE(that->width_, that->height_);
struct spa_fraction frame_rate = SPA_FRACTION(that->frame_rate_, 1);
webrtc::MutexLock lock(&that->resolution_lock_);
for (uint32_t format : {SPA_VIDEO_FORMAT_BGRA, SPA_VIDEO_FORMAT_RGBA,
SPA_VIDEO_FORMAT_BGRx, SPA_VIDEO_FORMAT_RGBx}) {
if (!that->modifiers_.empty()) {
params.push_back(BuildFormat(
&builder, format, that->modifiers_,
that->pending_resolution_change_ ? &resolution : nullptr));
params.push_back(
BuildFormat(&builder, format, that->modifiers_,
that->width_ && that->height_ ? &resolution : nullptr,
&frame_rate));
}
params.push_back(BuildFormat(
&builder, format, /*modifiers=*/{},
that->pending_resolution_change_ ? &resolution : nullptr));
that->width_ && that->height_ ? &resolution : nullptr, &frame_rate));
}
pw_stream_update_params(that->pw_stream_, params.data(), params.size());
that->pending_resolution_change_ = false;
}
}
@ -479,6 +485,7 @@ bool SharedScreenCastStreamPrivate::StartScreenCastStream(
resolution = SPA_RECTANGLE(width, height);
set_resolution = true;
}
struct spa_fraction default_frame_rate = SPA_FRACTION(frame_rate_, 1);
for (uint32_t format : {SPA_VIDEO_FORMAT_BGRA, SPA_VIDEO_FORMAT_RGBA,
SPA_VIDEO_FORMAT_BGRx, SPA_VIDEO_FORMAT_RGBx}) {
// Modifiers can be used with PipeWire >= 0.3.33
@ -487,12 +494,14 @@ bool SharedScreenCastStreamPrivate::StartScreenCastStream(
if (!modifiers_.empty()) {
params.push_back(BuildFormat(&builder, format, modifiers_,
set_resolution ? &resolution : nullptr));
set_resolution ? &resolution : nullptr,
&default_frame_rate));
}
}
params.push_back(BuildFormat(&builder, format, /*modifiers=*/{},
set_resolution ? &resolution : nullptr));
set_resolution ? &resolution : nullptr,
&default_frame_rate));
}
if (pw_stream_connect(pw_stream_, PW_DIRECTION_INPUT, pw_stream_node_id_,
@ -528,10 +537,24 @@ void SharedScreenCastStreamPrivate::UpdateScreenCastStreamResolution(
if (width_ != width || height_ != height) {
width_ = width;
height_ = height;
{
webrtc::MutexLock lock(&resolution_lock_);
pending_resolution_change_ = true;
}
pw_loop_signal_event(pw_thread_loop_get_loop(pw_main_loop_), renegotiate_);
}
}
RTC_NO_SANITIZE("cfi-icall")
void SharedScreenCastStreamPrivate::UpdateScreenCastStreamFrameRate(
uint32_t frame_rate) {
if (!pw_main_loop_) {
RTC_LOG(LS_WARNING) << "No main pipewire loop, ignoring frame rate change";
return;
}
if (!renegotiate_) {
RTC_LOG(LS_WARNING) << "Can not renegotiate stream params, ignoring "
<< "frame rate change";
return;
}
if (frame_rate_ != frame_rate) {
frame_rate_ = frame_rate;
pw_loop_signal_event(pw_thread_loop_get_loop(pw_main_loop_), renegotiate_);
}
}
@ -925,6 +948,11 @@ void SharedScreenCastStream::UpdateScreenCastStreamResolution(uint32_t width,
private_->UpdateScreenCastStreamResolution(width, height);
}
void SharedScreenCastStream::UpdateScreenCastStreamFrameRate(
uint32_t frame_rate) {
private_->UpdateScreenCastStreamFrameRate(frame_rate);
}
void SharedScreenCastStream::SetUseDamageRegion(bool use_damage_region) {
private_->SetUseDamageRegion(use_damage_region);
}

2
modules/desktop_capture/linux/wayland/shared_screencast_stream.h
View File

@ -35,6 +35,7 @@ class RTC_EXPORT SharedScreenCastStream
virtual void OnDesktopFrameChanged() = 0;
virtual void OnFailedToProcessBuffer() = 0;
virtual void OnStreamConfigured() = 0;
virtual void OnFrameRateChanged(uint32_t frame_rate) = 0;
protected:
Observer() = default;
@ -50,6 +51,7 @@ class RTC_EXPORT SharedScreenCastStream
uint32_t height = 0,
bool is_cursor_embedded = false);
void UpdateScreenCastStreamResolution(uint32_t width, uint32_t height);
void UpdateScreenCastStreamFrameRate(uint32_t frame_rate);
void SetUseDamageRegion(bool use_damage_region);
void SetObserver(SharedScreenCastStream::Observer* observer);
void StopScreenCastStream();

21
modules/desktop_capture/linux/wayland/shared_screencast_stream_unittest.cc
View File

@ -56,6 +56,7 @@ class PipeWireStreamTest : public ::testing::Test,
MOCK_METHOD(void, OnDesktopFrameChanged, (), (override));
MOCK_METHOD(void, OnFailedToProcessBuffer, (), (override));
MOCK_METHOD(void, OnStreamConfigured, (), (override));
MOCK_METHOD(void, OnFrameRateChanged, (uint32_t), (override));
void SetUp() override {
shared_screencast_stream_ = SharedScreenCastStream::CreateDefault();
@ -80,6 +81,8 @@ TEST_F(PipeWireStreamTest, TestPipeWire) {
// Set expectations for PipeWire to successfully connect both streams
rtc::Event waitConnectEvent;
rtc::Event waitStartStreamingEvent;
rtc::Event waitStreamParamChangedEvent1;
rtc::Event waitStreamParamChangedEvent2;
EXPECT_CALL(*this, OnStreamReady(_))
.WillOnce(Invoke(this, &PipeWireStreamTest::StartScreenCastStream));
@ -90,6 +93,7 @@ TEST_F(PipeWireStreamTest, TestPipeWire) {
EXPECT_CALL(*this, OnStartStreaming).WillOnce([&waitStartStreamingEvent] {
waitStartStreamingEvent.Set();
});
EXPECT_CALL(*this, OnFrameRateChanged(60)).Times(1); // Default frame rate.
// Give it some time to connect, the order between these shouldn't matter, but
// we need to be sure we are connected before we proceed to work with frames.
@ -152,6 +156,23 @@ TEST_F(PipeWireStreamTest, TestPipeWire) {
frameRetrievedEvent.Wait(kShortWait);
EXPECT_EQ(RgbaColor(frame->data()), blue_color);
// Update stream parameters.
EXPECT_CALL(*this, OnFrameRateChanged(0))
.Times(1)
.WillOnce([&waitStreamParamChangedEvent1] {
waitStreamParamChangedEvent1.Set();
});
shared_screencast_stream_->UpdateScreenCastStreamFrameRate(0);
waitStreamParamChangedEvent1.Wait(kShortWait);
EXPECT_CALL(*this, OnFrameRateChanged(22))
.Times(1)
.WillOnce([&waitStreamParamChangedEvent2] {
waitStreamParamChangedEvent2.Set();
});
shared_screencast_stream_->UpdateScreenCastStreamFrameRate(22);
waitStreamParamChangedEvent2.Wait(kShortWait);
// Test disconnection from stream
EXPECT_CALL(*this, OnStopStreaming);
shared_screencast_stream_->StopScreenCastStream();

4
modules/desktop_capture/linux/wayland/test/test_screencast_stream_provider.cc
View File

@ -90,8 +90,10 @@ TestScreenCastStreamProvider::TestScreenCastStreamProvider(Observer* observer,
spa_rectangle resolution =
SPA_RECTANGLE(uint32_t(width_), uint32_t(height_));
struct spa_fraction default_frame_rate = SPA_FRACTION(60, 1);
params.push_back(BuildFormat(&builder, SPA_VIDEO_FORMAT_BGRx,
/*modifiers=*/{}, &resolution));
/*modifiers=*/{}, &resolution,
&default_frame_rate));
auto flags =
pw_stream_flags(PW_STREAM_FLAG_DRIVER | PW_STREAM_FLAG_ALLOC_BUFFERS);