From 6eec049e481acb84549922e930341be4f7b62038 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20S=C3=A1lyi?= <salyigergo94@gmail.com>
Date: Thu, 24 Apr 2025 14:50:24 +0200
Subject: [PATCH] Flush Wayland events before having to send too many file
 descriptors at once

Works around a bug in the wayland-backend dependency failing to limit
the number of file descriptors sent in one message to MAX_FDS_OUT = 28
---
 src/main.rs    | 15 +++++++++------
 src/wayland.rs | 24 +++++++++++++++++++++---
 2 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index 5639f3d..b9cac6d 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -13,7 +13,8 @@ use std::{
     sync::{
         Arc,
         mpsc::{channel, Receiver},
-    }
+    },
+    rc::Rc,
 };
 
 use clap::Parser;
@@ -50,6 +51,7 @@ use crate::{
 };
 
 pub struct State {
+    pub connection: Rc<Connection>,
     pub compositor_state: CompositorState,
     pub registry_state: RegistryState,
     pub output_state: OutputState,
@@ -95,7 +97,7 @@ fn run() -> anyhow::Result<()> {
     //     Initialize wayland client
     // ********************************
 
-    let conn = Connection::connect_to_env().unwrap();
+    let conn = Rc::new(Connection::connect_to_env().unwrap());
     let (globals, mut event_queue) = registry_queue_init(&conn).unwrap();
     let qh = event_queue.handle();
 
@@ -150,6 +152,7 @@ fn run() -> anyhow::Result<()> {
         .unwrap_or(Compositor::Sway);
 
     let mut state = State {
+        connection: Rc::clone(&conn),
         compositor_state,
         registry_state,
         output_state: OutputState::new(&globals, &qh),
@@ -185,7 +188,7 @@ fn run() -> anyhow::Result<()> {
     let token_signal = signal_pipe.as_ref().map(|pipe| poll.add_readable(pipe));
 
     loop {
-        flush_blocking(&event_queue);
+        flush_blocking(&conn);
         let read_guard = ensure_prepare_read(&mut state, &mut event_queue);
         poll.poll().expect("Main event loop poll failed");
         if poll.ready(token_wayland) {
@@ -218,15 +221,15 @@ fn run() -> anyhow::Result<()> {
     }
 }
 
-fn flush_blocking(event_queue: &EventQueue<State>) {
+fn flush_blocking(connection: &Connection) {
     loop {
-        let result = event_queue.flush();
+        let result = connection.flush();
         if result.is_ok() { return }
         if let Err(WaylandError::Io(io_error)) = &result {
             if io_error.kind() == io::ErrorKind::WouldBlock {
                 warn!("Wayland flush needs to block");
                 let mut poll_fds = [PollFd::from_borrowed_fd(
-                    event_queue.as_fd(),
+                    connection.as_fd(),
                     PollFlags::OUT,
                 )];
                 retry_on_intr(|| poll(&mut poll_fds, -1)).unwrap();
diff --git a/src/wayland.rs b/src/wayland.rs
index 338e08b..8e05163 100644
--- a/src/wayland.rs
+++ b/src/wayland.rs
@@ -49,7 +49,7 @@ use smithay_client_toolkit::reexports::protocols::wp::{
 };
 
 use crate::{
-    State,
+    flush_blocking, State,
     gpu::{
         DRM_FORMAT_XRGB8888, fmt_modifier,
         GpuMemory, GpuUploader, GpuWallpaper,
@@ -57,6 +57,8 @@ use crate::{
     image::{load_wallpaper, output_wallpaper_files, WallpaperFile},
 };
 
+const MAX_FDS_OUT: usize = 28;
+
 impl CompositorHandler for State {
     fn scale_factor_changed(
         &mut self,
@@ -937,6 +939,8 @@ fn load_wallpapers(
     let mut reused_count = 0usize;
     let mut loaded_count = 0usize;
     let mut error_count = 0usize;
+    flush_blocking(&state.connection);
+    let mut fds_need_flush = 0usize;
     for wallpaper_file in wallpaper_files {
         if log::log_enabled!(log::Level::Debug) {
             if wallpaper_file.path == wallpaper_file.canon_path {
@@ -992,6 +996,12 @@ fn load_wallpapers(
             }
             match uploader.upload() {
                 Ok(gpu_wallpaper) => {
+                    let fds_count = gpu_wallpaper.memory_planes_len;
+                    if fds_need_flush + fds_count > MAX_FDS_OUT {
+                        flush_blocking(&state.connection);
+                        fds_need_flush = 0;
+                    }
+                    fds_need_flush += fds_count;
                     let wallpaper = wallpaper_dmabuf(
                         &state.dmabuf_state,
                         qh,
@@ -1015,6 +1025,11 @@ fn load_wallpapers(
                 }
             }
         }
+        if fds_need_flush + 1 > MAX_FDS_OUT {
+            flush_blocking(&state.connection);
+            fds_need_flush = 0;
+        }
+        fds_need_flush += 1;
         let mut shm_pool = match RawPool::new(shm_size, &state.shm) {
             Ok(shm_pool) => shm_pool,
             Err(e) => {
@@ -1058,8 +1073,11 @@ fn load_wallpapers(
         });
         loaded_count += 1;
     }
-    debug!("Wallpapers for new output: {} reused, {} loaded, {} errors",
-        reused_count, loaded_count, error_count);
+    if fds_need_flush > 0 {
+        flush_blocking(&state.connection);
+    }
+    debug!("Wallpapers for new output: {} loaded, {} reused, {} errors",
+        loaded_count, reused_count, error_count);
     debug!("Wallpapers are available for workspaces: {}",
         workspace_backgrounds.iter()
             .map(|bg| bg.workspace_name.as_str())