image-copy: Move duplicate code to a cursor_capture_constraints()

This ensures we avoid advertising a `0,0` buffer size (which will
protocol error to actually allocate), and avoids defining the supported
formats for cursor capture buffers in a bunch of places.

src/input/mod.rs

@@ -11,7 +11,7 @@ use crate::{
     },
     input::gestures::{GestureState, SwipeAction},
     shell::{
-        CursorGeometry, LastModifierChange, SeatExt, Trigger,
+        LastModifierChange, SeatExt, Trigger,
         focus::{
             Stage, render_input_order,
             target::{KeyboardFocusTarget, PointerFocusTarget},
@@ -25,7 +25,8 @@ use crate::{
     },
     utils::{prelude::*, quirks::workspace_overview_is_open},
     wayland::handlers::{
-        image_copy_capture::SessionHolder, xwayland_keyboard_grab::XWaylandGrabSeat,
+        image_copy_capture::{SessionHolder, cursor_capture_constraints},
+        xwayland_keyboard_grab::XWaylandGrabSeat,
     },
 };
 use calloop::{
@@ -60,15 +61,12 @@ use smithay::{
     output::Output,
     reexports::{
         input::Device as InputDevice,
-        wayland_server::{
+        wayland_server::{Resource as _, protocol::wl_surface::WlSurface},
-            Resource as _,
-            protocol::{wl_shm::Format as ShmFormat, wl_surface::WlSurface},
-        },
     },
-    utils::{Logical, Point, Rectangle, SERIAL_COUNTER, Serial, Size},
+    utils::{Logical, Point, Rectangle, SERIAL_COUNTER, Serial},
     wayland::{
         compositor::CompositorHandler,
-        image_copy_capture::{BufferConstraints, CursorSessionRef},
+        image_copy_capture::CursorSessionRef,
         keyboard_shortcuts_inhibit::KeyboardShortcutsInhibitorSeat,
         pointer_constraints::{PointerConstraint, with_pointer_constraint},
         seat::WaylandFocus,
@@ -651,7 +649,7 @@ impl State {
                     }
 
                     for session in cursor_sessions_for_output(&shell, &output) {
-                        if let Some(CursorGeometry { geometry, hotspot }) = seat.cursor_geometry(
+                        if let Some(cursor_geometry) = seat.cursor_geometry(
                             (position - output_geometry.loc.to_f64())
                                 .as_logical()
                                 .to_buffer(
@@ -661,24 +659,18 @@ impl State {
                                 ),
                             self.common.clock.now(),
                         ) {
+                            let constraints = cursor_capture_constraints(Some(cursor_geometry));
                             if session
                                 .current_constraints()
-                                .map(|constraint| constraint.size != geometry.size)
+                                .map(|current_constraints| {
+                                    current_constraints.size != constraints.size
+                                })
                                 .unwrap_or(true)
                             {
-                                let mut cursor_size = geometry.size;
+                                session.update_constraints(constraints);
-                                // Client shouldn't try to allocate 0x0 buffer
-                                if cursor_size == Size::new(0, 0) {
-                                    cursor_size = Size::new(1, 1);
-                                }
-                                session.update_constraints(BufferConstraints {
-                                    size: cursor_size,
-                                    shm: vec![ShmFormat::Argb8888],
-                                    dma: None,
-                                });
                             }
-                            session.set_cursor_hotspot(hotspot);
+                            session.set_cursor_hotspot(cursor_geometry.hotspot);
-                            session.set_cursor_pos(Some(geometry.loc));
+                            session.set_cursor_pos(Some(cursor_geometry.geometry.loc));
                         }
                     }
                 }
@@ -720,7 +712,7 @@ impl State {
 
                     let shell = self.common.shell.read();
                     for session in cursor_sessions_for_output(&shell, &output) {
-                        if let Some(CursorGeometry { geometry, hotspot }) = seat.cursor_geometry(
+                        if let Some(cursor_geometry) = seat.cursor_geometry(
                             (position - output_geometry.loc.to_f64())
                                 .as_logical()
                                 .to_buffer(
@@ -730,24 +722,18 @@ impl State {
                                 ),
                             self.common.clock.now(),
                         ) {
+                            let constraints = cursor_capture_constraints(Some(cursor_geometry));
                             if session
                                 .current_constraints()
-                                .map(|constraint| constraint.size != geometry.size)
+                                .map(|current_constraints| {
+                                    current_constraints.size != constraints.size
+                                })
                                 .unwrap_or(true)
                             {
-                                let mut cursor_size = geometry.size;
+                                session.update_constraints(constraints);
-                                // Client shouldn't try to allocate 0x0 buffer
-                                if cursor_size == Size::new(0, 0) {
-                                    cursor_size = Size::new(1, 1);
-                                }
-                                session.update_constraints(BufferConstraints {
-                                    size: cursor_size,
-                                    shm: vec![ShmFormat::Argb8888],
-                                    dma: None,
-                                });
                             }
-                            session.set_cursor_hotspot(hotspot);
+                            session.set_cursor_hotspot(cursor_geometry.hotspot);
-                            session.set_cursor_pos(Some(geometry.loc));
+                            session.set_cursor_pos(Some(cursor_geometry.geometry.loc));
                         }
                     }
                 }
@@ -2482,7 +2468,7 @@ impl State {
 
                 let output_geometry = output.geometry();
                 for session in cursor_sessions_for_output(&shell, &output) {
-                    if let Some(CursorGeometry { geometry, hotspot }) = seat.cursor_geometry(
+                    if let Some(cursor_geometry) = seat.cursor_geometry(
                         point.to_buffer(
                             output.current_scale().fractional_scale(),
                             output.current_transform(),
@@ -2490,19 +2476,16 @@ impl State {
                         ),
                         self.common.clock.now(),
                     ) {
+                        let constraints = cursor_capture_constraints(Some(cursor_geometry));
                         if session
                             .current_constraints()
-                            .map(|constraint| constraint.size != geometry.size)
+                            .map(|current_constraints| current_constraints.size != constraints.size)
                             .unwrap_or(true)
                         {
-                            session.update_constraints(BufferConstraints {
+                            session.update_constraints(constraints);
-                                size: geometry.size,
-                                shm: vec![ShmFormat::Argb8888],
-                                dma: None,
-                            });
                         }
-                        session.set_cursor_hotspot(hotspot);
+                        session.set_cursor_hotspot(cursor_geometry.hotspot);
-                        session.set_cursor_pos(Some(geometry.loc));
+                        session.set_cursor_pos(Some(cursor_geometry.geometry.loc));
                     }
                 }
             }

src/wayland/handlers/image_copy_capture/mod.rs

@@ -45,6 +45,10 @@ pub use self::render::*;
 use self::user_data::*;
 pub use self::user_data::{FrameHolder, ImageCopySessions, SessionData, SessionHolder};
 
+fn default_cursor_size() -> Size<i32, BufferCoords> {
+    Size::new(64, 64)
+}
+
 fn seat_for_wl_pointer<'a>(shell: &'a Shell, pointer: &WlPointer) -> Option<&'a Seat<State>> {
     let pointer_handle = PointerHandle::<State>::from_resource(pointer)?;
     shell
@@ -53,6 +57,24 @@ fn seat_for_wl_pointer<'a>(shell: &'a Shell, pointer: &WlPointer) -> Option<&'a
         .find(|seat| seat.get_pointer().is_some_and(|p| p == pointer_handle))
 }
 
+pub fn cursor_capture_constraints(cursor_geometry: Option<CursorGeometry>) -> BufferConstraints {
+    let size = if let Some(cursor_geometry) = cursor_geometry {
+        let mut size = cursor_geometry.geometry.size;
+        // Client shouldn't try to allocate 0x0 buffer
+        if size == Size::new(0, 0) {
+            size = Size::new(1, 1);
+        }
+        size
+    } else {
+        default_cursor_size()
+    };
+    BufferConstraints {
+        size,
+        shm: vec![ShmFormat::Argb8888],
+        dma: None,
+    }
+}
+
 impl ImageCopyCaptureHandler for State {
     fn image_copy_capture_state(&mut self) -> &mut ImageCopyCaptureState {
         &mut self.common.image_copy_capture_state
@@ -86,19 +108,8 @@ impl ImageCopyCaptureHandler for State {
     ) -> Option<BufferConstraints> {
         let shell = self.common.shell.read();
         let seat = seat_for_wl_pointer(&shell, pointer)?;
-        let size = if let Some(CursorGeometry { geometry, .. }) =
+        let cursor_geometry = seat.cursor_geometry((0.0, 0.0), self.common.clock.now());
-            seat.cursor_geometry((0.0, 0.0), self.common.clock.now())
+        Some(cursor_capture_constraints(cursor_geometry))
-        {
-            geometry.size
-        } else {
-            Size::from((64, 64))
-        };
-
-        Some(BufferConstraints {
-            size,
-            shm: vec![ShmFormat::Argb8888],
-            dma: None,
-        })
     }
 
     fn new_session(&mut self, session: Session) {
@@ -165,12 +176,12 @@ impl ImageCopyCaptureHandler for State {
                 {
                     (geometry.size, hotspot)
                 } else {
-                    (Size::from((64, 64)), Point::from((0, 0)))
+                    (default_cursor_size(), Point::from((0, 0)))
                 };
 
                 (pointer_loc, pointer_size, hotspot)
             } else {
-                (Point::new(0, 0), Size::from((64, 64)), Point::from((0, 0)))
+                (Point::new(0, 0), default_cursor_size(), Point::from((0, 0)))
             }
         };
 

src/wayland/handlers/image_copy_capture/render.rs

@@ -20,16 +20,14 @@ use smithay::{
     desktop::space::SpaceElement,
     input::Seat,
     output::{Output, OutputNoMode},
-    reexports::wayland_server::protocol::{wl_buffer::WlBuffer, wl_shm::Format as ShmFormat},
+    reexports::wayland_server::protocol::wl_buffer::WlBuffer,
     utils::{
         Buffer as BufferCoords, IsAlive, Logical, Physical, Point, Rectangle, Scale, Size,
         Transform,
     },
     wayland::{
         dmabuf::get_dmabuf,
-        image_copy_capture::{
+        image_copy_capture::{CaptureFailureReason, CursorSessionRef, Frame, SessionRef},
-            BufferConstraints, CaptureFailureReason, CursorSessionRef, Frame, SessionRef,
-        },
         seat::WaylandFocus,
         shm::{shm_format_to_fourcc, with_buffer_contents, with_buffer_contents_mut},
     },
@@ -45,7 +43,7 @@ use crate::{
         render_workspace,
         wayland::SurfaceRenderElement,
     },
-    shell::{CosmicMappedRenderElement, CosmicSurface, CursorGeometry, WorkspaceRenderElement},
+    shell::{CosmicMappedRenderElement, CosmicSurface, WorkspaceRenderElement},
     state::{Common, KmsNodes, State},
     utils::prelude::{PointExt, PointGlobalExt, RectExt, RectLocalExt, SeatExt},
     wayland::{
@@ -56,7 +54,9 @@ use crate::{
     },
 };
 
-use super::{super::data_device::get_dnd_icon, user_data::SessionHolder};
+use super::{
+    super::data_device::get_dnd_icon, cursor_capture_constraints, user_data::SessionHolder,
+};
 
 pub fn render_element_buffers<R, E>(
     renderer: &mut R,
@@ -802,25 +802,17 @@ pub fn render_cursor_to_buffer(
     seat: &Seat<State>,
 ) {
     let buffer = frame.buffer();
-    let mut cursor_size = seat
+    let cursor_geometry = seat.cursor_geometry((0.0, 0.0), state.common.clock.now());
-        .cursor_geometry((0.0, 0.0), state.common.clock.now())
+    let constraints = cursor_capture_constraints(cursor_geometry);
-        .map(|CursorGeometry { geometry, .. }| geometry.size)
-        .unwrap_or_else(|| Size::from((64, 64)));
     let buffer_size = buffer_dimensions(&buffer).unwrap();
-    // Client shouldn't try to allocate 0x0 buffer
+    if buffer_size != constraints.size {
-    if cursor_size == Size::new(0, 0) {
+        session.update_constraints(constraints.clone());
-        cursor_size = Size::new(1, 1);
-    }
-    if buffer_size != cursor_size {
-        let constraints = BufferConstraints {
-            size: cursor_size,
-            shm: vec![ShmFormat::Argb8888],
-            dma: None,
-        };
-        session.update_constraints(constraints);
         if let Some(data) = session.user_data().get::<SessionData>() {
             *data.lock().unwrap() = SessionUserData::new(OutputDamageTracker::new(
-                cursor_size.to_logical(1, Transform::Normal).to_physical(1),
+                constraints
+                    .size
+                    .to_logical(1, Transform::Normal)
+                    .to_physical(1),
                 1.0,
                 Transform::Normal,
             ));