desktop_capturer: Support frame rate negotiation via pipewire
This change adds support for renegotiating the frame rate
via pipewire.
Bug: chromium:1291247
Change-Id: Iacd4a3c924900839a8db75a50b448df6c48c83ab
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/291460
Commit-Queue: Salman Malik <salmanmalik@chromium.org>
Reviewed-by: Alexander Cooper <alcooper@chromium.org>
Cr-Commit-Position: refs/heads/main@{#39216}
diff --git a/modules/desktop_capture/desktop_and_cursor_composer.cc b/modules/desktop_capture/desktop_and_cursor_composer.cc
index dd688ac..3bdc1ee 100644
--- a/modules/desktop_capture/desktop_and_cursor_composer.cc
+++ b/modules/desktop_capture/desktop_and_cursor_composer.cc
@@ -177,6 +177,10 @@
desktop_capturer_->Start(this);
}
+void DesktopAndCursorComposer::SetMaxFrameRate(uint32_t max_frame_rate) {
+ desktop_capturer_->SetMaxFrameRate(max_frame_rate);
+}
+
void DesktopAndCursorComposer::SetSharedMemoryFactory(
std::unique_ptr<SharedMemoryFactory> shared_memory_factory) {
desktop_capturer_->SetSharedMemoryFactory(std::move(shared_memory_factory));
diff --git a/modules/desktop_capture/desktop_and_cursor_composer.h b/modules/desktop_capture/desktop_and_cursor_composer.h
index a078b3e..d9208b0 100644
--- a/modules/desktop_capture/desktop_and_cursor_composer.h
+++ b/modules/desktop_capture/desktop_and_cursor_composer.h
@@ -61,6 +61,7 @@
bool SelectSource(SourceId id) override;
bool FocusOnSelectedSource() override;
bool IsOccluded(const DesktopVector& pos) override;
+ void SetMaxFrameRate(uint32_t max_frame_rate) override;
#if defined(WEBRTC_USE_GIO)
DesktopCaptureMetadata GetMetadata() override;
#endif // defined(WEBRTC_USE_GIO)
diff --git a/modules/desktop_capture/desktop_capturer.h b/modules/desktop_capture/desktop_capturer.h
index 3e8f0dc..9a054b6 100644
--- a/modules/desktop_capture/desktop_capturer.h
+++ b/modules/desktop_capture/desktop_capturer.h
@@ -100,6 +100,12 @@
// valid until capturer is destroyed.
virtual void Start(Callback* callback) = 0;
+ // Sets max frame rate for the capturer. This is best effort and may not be
+ // supported by all capturers. This will only affect the frequency at which
+ // new frames are available, not the frequency at which you are allowed to
+ // capture the frames.
+ virtual void SetMaxFrameRate(uint32_t max_frame_rate) {}
+
// Returns a valid pointer if the capturer requires the user to make a
// selection from a source list provided by the capturer.
// Returns nullptr if the capturer does not provide a UI for the user to make
diff --git a/modules/desktop_capture/linux/wayland/base_capturer_pipewire.cc b/modules/desktop_capture/linux/wayland/base_capturer_pipewire.cc
index cf4f7dc..66d0c30 100644
--- a/modules/desktop_capture/linux/wayland/base_capturer_pipewire.cc
+++ b/modules/desktop_capture/linux/wayland/base_capturer_pipewire.cc
@@ -118,6 +118,13 @@
}
}
+void BaseCapturerPipeWire::SetMaxFrameRate(uint32_t max_frame_rate) {
+ if (!capturer_failed_) {
+ options_.screencast_stream()->UpdateScreenCastStreamFrameRate(
+ max_frame_rate);
+ }
+}
+
void BaseCapturerPipeWire::Start(Callback* callback) {
RTC_DCHECK(!callback_);
RTC_DCHECK(callback);
diff --git a/modules/desktop_capture/linux/wayland/base_capturer_pipewire.h b/modules/desktop_capture/linux/wayland/base_capturer_pipewire.h
index 4b5cdc4..3b70807 100644
--- a/modules/desktop_capture/linux/wayland/base_capturer_pipewire.h
+++ b/modules/desktop_capture/linux/wayland/base_capturer_pipewire.h
@@ -49,6 +49,7 @@
bool GetSourceList(SourceList* sources) override;
bool SelectSource(SourceId id) override;
DelegatedSourceListController* GetDelegatedSourceListController() override;
+ void SetMaxFrameRate(uint32_t max_frame_rate) override;
// DelegatedSourceListController
void Observe(Observer* observer) override;
diff --git a/modules/desktop_capture/linux/wayland/screencast_stream_utils.cc b/modules/desktop_capture/linux/wayland/screencast_stream_utils.cc
index 0c4900d..8177cfd 100644
--- a/modules/desktop_capture/linux/wayland/screencast_stream_utils.cc
+++ b/modules/desktop_capture/linux/wayland/screencast_stream_utils.cc
@@ -67,11 +67,11 @@
spa_pod* BuildFormat(spa_pod_builder* builder,
uint32_t format,
const std::vector<uint64_t>& modifiers,
- const struct spa_rectangle* resolution) {
+ const struct spa_rectangle* resolution,
+ const struct spa_fraction* frame_rate) {
spa_pod_frame frames[2];
spa_rectangle pw_min_screen_bounds = spa_rectangle{1, 1};
spa_rectangle pw_max_screen_bounds = spa_rectangle{UINT32_MAX, UINT32_MAX};
-
spa_pod_builder_push_object(builder, &frames[0], SPA_TYPE_OBJECT_Format,
SPA_PARAM_EnumFormat);
spa_pod_builder_add(builder, SPA_FORMAT_mediaType,
@@ -116,7 +116,17 @@
&pw_max_screen_bounds),
0);
}
-
+ if (frame_rate) {
+ static const spa_fraction pw_min_frame_rate = spa_fraction{0, 1};
+ spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_framerate,
+ SPA_POD_CHOICE_RANGE_Fraction(
+ frame_rate, &pw_min_frame_rate, frame_rate),
+ 0);
+ spa_pod_builder_add(builder, SPA_FORMAT_VIDEO_maxFramerate,
+ SPA_POD_CHOICE_RANGE_Fraction(
+ frame_rate, &pw_min_frame_rate, frame_rate),
+ 0);
+ }
return static_cast<spa_pod*>(spa_pod_builder_pop(builder, &frames[0]));
}
diff --git a/modules/desktop_capture/linux/wayland/screencast_stream_utils.h b/modules/desktop_capture/linux/wayland/screencast_stream_utils.h
index e04d7db..2f44300 100644
--- a/modules/desktop_capture/linux/wayland/screencast_stream_utils.h
+++ b/modules/desktop_capture/linux/wayland/screencast_stream_utils.h
@@ -21,6 +21,7 @@
struct spa_pod;
struct spa_pod_builder;
struct spa_rectangle;
+struct spa_fraction;
namespace webrtc {
@@ -44,7 +45,8 @@
spa_pod* BuildFormat(spa_pod_builder* builder,
uint32_t format,
const std::vector<uint64_t>& modifiers,
- const struct spa_rectangle* resolution);
+ const struct spa_rectangle* resolution,
+ const struct spa_fraction* frame_rate);
} // namespace webrtc
diff --git a/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc b/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc
index 71bde9b..7c4a78d 100644
--- a/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc
+++ b/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc
@@ -80,6 +80,7 @@
uint32_t height = 0,
bool is_cursor_embedded = false);
void UpdateScreenCastStreamResolution(uint32_t width, uint32_t height);
+ void UpdateScreenCastStreamFrameRate(uint32_t frame_rate);
void SetUseDamageRegion(bool use_damage_region) {
use_damage_region_ = use_damage_region;
}
@@ -138,9 +139,8 @@
// Resolution parameters.
uint32_t width_ = 0;
uint32_t height_ = 0;
- webrtc::Mutex resolution_lock_;
- // Resolution changes are processed during buffer processing.
- bool pending_resolution_change_ RTC_GUARDED_BY(&resolution_lock_) = false;
+ // Frame rate.
+ uint32_t frame_rate_ = 60;
bool use_damage_region_ = true;
@@ -256,6 +256,12 @@
spa_format_video_raw_parse(format, &that->spa_video_format_);
+ if (that->observer_ && that->spa_video_format_.max_framerate.denom) {
+ that->observer_->OnFrameRateChanged(
+ that->spa_video_format_.max_framerate.num /
+ that->spa_video_format_.max_framerate.denom);
+ }
+
auto width = that->spa_video_format_.size.width;
auto height = that->spa_video_format_.size.height;
auto stride = SPA_ROUND_UP_N(width * kBytesPerPixel, 4);
@@ -355,22 +361,22 @@
std::vector<const spa_pod*> params;
struct spa_rectangle resolution =
SPA_RECTANGLE(that->width_, that->height_);
+ struct spa_fraction frame_rate = SPA_FRACTION(that->frame_rate_, 1);
- webrtc::MutexLock lock(&that->resolution_lock_);
for (uint32_t format : {SPA_VIDEO_FORMAT_BGRA, SPA_VIDEO_FORMAT_RGBA,
SPA_VIDEO_FORMAT_BGRx, SPA_VIDEO_FORMAT_RGBx}) {
if (!that->modifiers_.empty()) {
- params.push_back(BuildFormat(
- &builder, format, that->modifiers_,
- that->pending_resolution_change_ ? &resolution : nullptr));
+ params.push_back(
+ BuildFormat(&builder, format, that->modifiers_,
+ that->width_ && that->height_ ? &resolution : nullptr,
+ &frame_rate));
}
params.push_back(BuildFormat(
&builder, format, /*modifiers=*/{},
- that->pending_resolution_change_ ? &resolution : nullptr));
+ that->width_ && that->height_ ? &resolution : nullptr, &frame_rate));
}
pw_stream_update_params(that->pw_stream_, params.data(), params.size());
- that->pending_resolution_change_ = false;
}
}
@@ -479,6 +485,7 @@
resolution = SPA_RECTANGLE(width, height);
set_resolution = true;
}
+ struct spa_fraction default_frame_rate = SPA_FRACTION(frame_rate_, 1);
for (uint32_t format : {SPA_VIDEO_FORMAT_BGRA, SPA_VIDEO_FORMAT_RGBA,
SPA_VIDEO_FORMAT_BGRx, SPA_VIDEO_FORMAT_RGBx}) {
// Modifiers can be used with PipeWire >= 0.3.33
@@ -487,12 +494,14 @@
if (!modifiers_.empty()) {
params.push_back(BuildFormat(&builder, format, modifiers_,
- set_resolution ? &resolution : nullptr));
+ set_resolution ? &resolution : nullptr,
+ &default_frame_rate));
}
}
params.push_back(BuildFormat(&builder, format, /*modifiers=*/{},
- set_resolution ? &resolution : nullptr));
+ set_resolution ? &resolution : nullptr,
+ &default_frame_rate));
}
if (pw_stream_connect(pw_stream_, PW_DIRECTION_INPUT, pw_stream_node_id_,
@@ -528,10 +537,24 @@
if (width_ != width || height_ != height) {
width_ = width;
height_ = height;
- {
- webrtc::MutexLock lock(&resolution_lock_);
- pending_resolution_change_ = true;
- }
+ pw_loop_signal_event(pw_thread_loop_get_loop(pw_main_loop_), renegotiate_);
+ }
+}
+
+RTC_NO_SANITIZE("cfi-icall")
+void SharedScreenCastStreamPrivate::UpdateScreenCastStreamFrameRate(
+ uint32_t frame_rate) {
+ if (!pw_main_loop_) {
+ RTC_LOG(LS_WARNING) << "No main pipewire loop, ignoring frame rate change";
+ return;
+ }
+ if (!renegotiate_) {
+ RTC_LOG(LS_WARNING) << "Can not renegotiate stream params, ignoring "
+ << "frame rate change";
+ return;
+ }
+ if (frame_rate_ != frame_rate) {
+ frame_rate_ = frame_rate;
pw_loop_signal_event(pw_thread_loop_get_loop(pw_main_loop_), renegotiate_);
}
}
@@ -925,6 +948,11 @@
private_->UpdateScreenCastStreamResolution(width, height);
}
+void SharedScreenCastStream::UpdateScreenCastStreamFrameRate(
+ uint32_t frame_rate) {
+ private_->UpdateScreenCastStreamFrameRate(frame_rate);
+}
+
void SharedScreenCastStream::SetUseDamageRegion(bool use_damage_region) {
private_->SetUseDamageRegion(use_damage_region);
}
diff --git a/modules/desktop_capture/linux/wayland/shared_screencast_stream.h b/modules/desktop_capture/linux/wayland/shared_screencast_stream.h
index 9cdd3d8..a130e53 100644
--- a/modules/desktop_capture/linux/wayland/shared_screencast_stream.h
+++ b/modules/desktop_capture/linux/wayland/shared_screencast_stream.h
@@ -35,6 +35,7 @@
virtual void OnDesktopFrameChanged() = 0;
virtual void OnFailedToProcessBuffer() = 0;
virtual void OnStreamConfigured() = 0;
+ virtual void OnFrameRateChanged(uint32_t frame_rate) = 0;
protected:
Observer() = default;
@@ -50,6 +51,7 @@
uint32_t height = 0,
bool is_cursor_embedded = false);
void UpdateScreenCastStreamResolution(uint32_t width, uint32_t height);
+ void UpdateScreenCastStreamFrameRate(uint32_t frame_rate);
void SetUseDamageRegion(bool use_damage_region);
void SetObserver(SharedScreenCastStream::Observer* observer);
void StopScreenCastStream();
diff --git a/modules/desktop_capture/linux/wayland/shared_screencast_stream_unittest.cc b/modules/desktop_capture/linux/wayland/shared_screencast_stream_unittest.cc
index 1de5f19..6a72edd 100644
--- a/modules/desktop_capture/linux/wayland/shared_screencast_stream_unittest.cc
+++ b/modules/desktop_capture/linux/wayland/shared_screencast_stream_unittest.cc
@@ -56,6 +56,7 @@
MOCK_METHOD(void, OnDesktopFrameChanged, (), (override));
MOCK_METHOD(void, OnFailedToProcessBuffer, (), (override));
MOCK_METHOD(void, OnStreamConfigured, (), (override));
+ MOCK_METHOD(void, OnFrameRateChanged, (uint32_t), (override));
void SetUp() override {
shared_screencast_stream_ = SharedScreenCastStream::CreateDefault();
@@ -80,6 +81,8 @@
// Set expectations for PipeWire to successfully connect both streams
rtc::Event waitConnectEvent;
rtc::Event waitStartStreamingEvent;
+ rtc::Event waitStreamParamChangedEvent1;
+ rtc::Event waitStreamParamChangedEvent2;
EXPECT_CALL(*this, OnStreamReady(_))
.WillOnce(Invoke(this, &PipeWireStreamTest::StartScreenCastStream));
@@ -90,6 +93,7 @@
EXPECT_CALL(*this, OnStartStreaming).WillOnce([&waitStartStreamingEvent] {
waitStartStreamingEvent.Set();
});
+ EXPECT_CALL(*this, OnFrameRateChanged(60)).Times(1); // Default frame rate.
// Give it some time to connect, the order between these shouldn't matter, but
// we need to be sure we are connected before we proceed to work with frames.
@@ -152,6 +156,23 @@
frameRetrievedEvent.Wait(kShortWait);
EXPECT_EQ(RgbaColor(frame->data()), blue_color);
+ // Update stream parameters.
+ EXPECT_CALL(*this, OnFrameRateChanged(0))
+ .Times(1)
+ .WillOnce([&waitStreamParamChangedEvent1] {
+ waitStreamParamChangedEvent1.Set();
+ });
+ shared_screencast_stream_->UpdateScreenCastStreamFrameRate(0);
+ waitStreamParamChangedEvent1.Wait(kShortWait);
+
+ EXPECT_CALL(*this, OnFrameRateChanged(22))
+ .Times(1)
+ .WillOnce([&waitStreamParamChangedEvent2] {
+ waitStreamParamChangedEvent2.Set();
+ });
+ shared_screencast_stream_->UpdateScreenCastStreamFrameRate(22);
+ waitStreamParamChangedEvent2.Wait(kShortWait);
+
// Test disconnection from stream
EXPECT_CALL(*this, OnStopStreaming);
shared_screencast_stream_->StopScreenCastStream();
diff --git a/modules/desktop_capture/linux/wayland/test/test_screencast_stream_provider.cc b/modules/desktop_capture/linux/wayland/test/test_screencast_stream_provider.cc
index 3b82995..ffba137 100644
--- a/modules/desktop_capture/linux/wayland/test/test_screencast_stream_provider.cc
+++ b/modules/desktop_capture/linux/wayland/test/test_screencast_stream_provider.cc
@@ -90,8 +90,10 @@
spa_rectangle resolution =
SPA_RECTANGLE(uint32_t(width_), uint32_t(height_));
+ struct spa_fraction default_frame_rate = SPA_FRACTION(60, 1);
params.push_back(BuildFormat(&builder, SPA_VIDEO_FORMAT_BGRx,
- /*modifiers=*/{}, &resolution));
+ /*modifiers=*/{}, &resolution,
+ &default_frame_rate));
auto flags =
pw_stream_flags(PW_STREAM_FLAG_DRIVER | PW_STREAM_FLAG_ALLOC_BUFFERS);
