kms: Allow diverging primary plane formats under certain conditions

src/backend/kms/device.rs

@@ -13,8 +13,9 @@ use libc::dev_t;
 use smithay::{
     backend::{
         allocator::{
+            format::FormatSet,
             gbm::{GbmAllocator, GbmDevice},
-            Fourcc,
+            Format, Fourcc,
         },
         drm::{
             compositor::FrameFlags, output::DrmOutputManager, DrmDevice, DrmDeviceFd, DrmEvent,
@@ -612,25 +613,31 @@ impl Device {
         }
     }
 
-    pub fn allow_overlay_scanout(
+    fn allow_frame_flags(
         &mut self,
         flag: bool,
+        flags: FrameFlags,
         renderer: &mut GlMultiRenderer,
         clock: &Clock<Monotonic>,
         shell: &Arc<RwLock<Shell>>,
+        startup_done: bool,
     ) -> Result<()> {
         for surface in self.surfaces.values_mut() {
-            surface.allow_overlay_scanout(flag);
+            surface.allow_frame_flags(flag, flags);
         }
 
         if !flag {
             let now = clock.now();
 
-            let output_map = self
+            let mut output_map = self
                 .surfaces
                 .iter()
+                .filter(|(_, s)| s.is_active())
                 .map(|(crtc, surface)| (*crtc, surface.output.clone()))
                 .collect::<HashMap<_, _>>();
+            if !startup_done {
+                output_map.clear();
+            }
 
             self.drm.with_compositors::<Result<()>>(|map| {
                 for (crtc, compositor) in map.iter() {
@@ -665,6 +672,65 @@ impl Device {
         Ok(())
     }
 
+    pub fn allow_overlay_scanout(
+        &mut self,
+        flag: bool,
+        renderer: &mut GlMultiRenderer,
+        clock: &Clock<Monotonic>,
+        shell: &Arc<RwLock<Shell>>,
+    ) -> Result<()> {
+        self.allow_frame_flags(
+            flag,
+            FrameFlags::ALLOW_OVERLAY_PLANE_SCANOUT,
+            renderer,
+            clock,
+            shell,
+            true,
+        )
+    }
+
+    pub fn allow_primary_scanout_any(
+        &mut self,
+        flag: bool,
+        renderer: &mut GlMultiRenderer,
+        clock: &Clock<Monotonic>,
+        shell: &Arc<RwLock<Shell>>,
+        startup_done: bool,
+    ) -> Result<()> {
+        self.allow_frame_flags(
+            flag,
+            FrameFlags::ALLOW_PRIMARY_PLANE_SCANOUT_ANY,
+            renderer,
+            clock,
+            shell,
+            startup_done,
+        )?;
+
+        self.drm.with_compositors(|comps| {
+            for (crtc, comp) in comps {
+                let Some(surface) = self.surfaces.get_mut(crtc) else {
+                    continue;
+                };
+                let comp = comp.lock().unwrap();
+                surface.primary_plane_formats = if flag {
+                    comp.surface().plane_info().formats.clone()
+                } else {
+                    // This certainly isn't perfect and might still miss the happy path,
+                    // but it is surprisingly difficult to hack an api into smithay,
+                    // to get the actual framebuffer format
+                    let code = comp.format();
+                    FormatSet::from_iter(comp.modifiers().iter().map(|mo| Format {
+                        code,
+                        modifier: *mo,
+                    }))
+                };
+                surface.feedback.clear();
+            }
+        });
+
+        Ok(())
+    }
+
     pub fn in_use(&self, primary: Option<&DrmNode>) -> bool {
         Some(&self.render_node) == primary
             || !self.surfaces.is_empty()

src/backend/kms/mod.rs

@@ -44,7 +44,10 @@ use std::{
     borrow::BorrowMut,
     collections::{HashMap, HashSet},
     path::Path,
-    sync::{atomic::AtomicBool, Arc, RwLock},
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc, RwLock,
+    },
 };
 
 mod device;
@@ -646,16 +649,44 @@ impl KmsState {
                 .context("Failed to enable devices")?;
         }
 
+        let startup_done = startup_done.load(Ordering::SeqCst);
         let mut all_outputs = Vec::new();
         for device in self.drm_devices.values_mut() {
-            // reconfigure existing
             let now = clock.now();
-            let output_map = device
+            let mut output_map = device
                 .surfaces
                 .iter()
+                .filter(|(_, s)| s.is_active())
                 .map(|(crtc, surface)| (*crtc, surface.output.clone()))
                 .collect::<HashMap<_, _>>();
+            if !startup_done {
+                output_map.clear();
+            }
 
+            // configure primary scanout allowance
+            if !device.surfaces.is_empty() {
+                let mut renderer = self
+                    .api
+                    .single_renderer(&device.render_node)
+                    .with_context(|| "Failed to create renderer")?;
+
+                device
+                    .allow_primary_scanout_any(
+                        device
+                            .surfaces
+                            .values()
+                            .filter(|s| s.output.is_enabled())
+                            .count()
+                            <= 1,
+                        &mut renderer,
+                        clock,
+                        &shell,
+                        startup_done,
+                    )
+                    .context("Failed to switch primary-plane scanout flags")?;
+            }
+
+            // reconfigure existing
             for (crtc, surface) in device.surfaces.iter_mut() {
                 let output_config = surface.output.config();
 

src/backend/kms/surface/mod.rs

@@ -108,8 +108,9 @@ pub struct Surface {
     known_nodes: HashSet<DrmNode>,
 
     active: Arc<AtomicBool>,
-    feedback: HashMap<DrmNode, SurfaceDmabufFeedback>,
+    pub(super) feedback: HashMap<DrmNode, SurfaceDmabufFeedback>,
-    plane_formats: FormatSet,
+    pub(super) primary_plane_formats: FormatSet,
+    overlay_plane_formats: FormatSet,
 
     loop_handle: LoopHandle<'static, State>,
     thread_command: Sender<ThreadCommand>,
@@ -239,7 +240,7 @@ pub enum ThreadCommand {
     ScheduleRender,
     AdaptiveSyncAvailable(SyncSender<Result<VrrSupport>>),
     UseAdaptiveSync(AdaptiveSync),
-    AllowOverlayScanout(bool, SyncSender<()>),
+    AllowFrameFlags(bool, FrameFlags, SyncSender<()>),
     End,
     DpmsOff,
 }
@@ -309,6 +310,7 @@ impl Surface {
                         .surfaces
                         .get_mut(&crtc)
                         .unwrap();
+
                     state
                         .common
                         .send_dmabuf_feedback(&output_clone, &states, |source_node| {
@@ -332,7 +334,8 @@ impl Surface {
                                             target_node,
                                             render_formats,
                                             target_formats,
-                                            surface.plane_formats.clone(),
+                                            surface.primary_plane_formats.clone(),
+                                            surface.overlay_plane_formats.clone(),
                                         )
                                     })
                                     .clone(),
@@ -350,7 +353,8 @@ impl Surface {
             known_nodes: HashSet::new(),
             active,
             feedback: HashMap::new(),
-            plane_formats: FormatSet::default(),
+            primary_plane_formats: FormatSet::default(),
+            overlay_plane_formats: FormatSet::default(),
             loop_handle: evlh.clone(),
             thread_command: tx,
             thread_token,
@@ -423,11 +427,11 @@ impl Surface {
         Ok(true)
     }
 
-    pub fn allow_overlay_scanout(&mut self, flag: bool) {
+    pub fn allow_frame_flags(&mut self, flag: bool, flags: FrameFlags) {
         let (tx, rx) = std::sync::mpsc::sync_channel(1);
         let _ = self
             .thread_command
-            .send(ThreadCommand::AllowOverlayScanout(flag, tx));
+            .send(ThreadCommand::AllowFrameFlags(flag, flags, tx));
         let _ = rx.recv();
     }
 
@@ -439,21 +443,18 @@ impl Surface {
 
     pub fn resume(&mut self, compositor: GbmDrmOutput) -> Result<()> {
         let (tx, rx) = std::sync::mpsc::sync_channel(1);
-        self.plane_formats = compositor.with_compositor(|c| {
+        (self.primary_plane_formats, self.overlay_plane_formats) =
-            c.surface()
+            compositor.with_compositor(|c| {
-                .plane_info()
+                (
-                .formats
+                    c.surface().plane_info().formats.clone(),
-                .iter()
-                .copied()
-                .chain(
                     c.surface()
                         .planes()
                         .overlay
                         .iter()
-                        .flat_map(|p| p.formats.iter().cloned()),
+                        .flat_map(|p| p.formats.iter().cloned())
+                        .collect::<FormatSet>(),
                 )
-                .collect::<FormatSet>()
+            });
-        });
 
         let _ = self.thread_command.send(ThreadCommand::Resume {
             compositor,
@@ -615,20 +616,18 @@ fn surface_thread(
                     };
                 }
             }
-            Event::Msg(ThreadCommand::AllowOverlayScanout(flag, tx)) => {
+            Event::Msg(ThreadCommand::AllowFrameFlags(flag, mut flags, tx)) => {
-                if !crate::utils::env::bool_var("COSMIC_DISABLE_DIRECT_SCANOUT").unwrap_or(false)
+                if crate::utils::env::bool_var("COSMIC_DISABLE_DIRECT_SCANOUT").unwrap_or(false) {
-                    && !crate::utils::env::bool_var("COSMIC_DISABLE_OVERLAY_SCANOUT")
+                    flags.remove(FrameFlags::ALLOW_SCANOUT);
-                        .unwrap_or(false)
+                }
-                {
+                if crate::utils::env::bool_var("COSMIC_DISABLE_OVERLAY_SCANOUT").unwrap_or(false) {
-                    if flag {
+                    flags.remove(FrameFlags::ALLOW_OVERLAY_PLANE_SCANOUT);
-                        state
+                }
-                            .frame_flags
+
-                            .insert(FrameFlags::ALLOW_OVERLAY_PLANE_SCANOUT);
+                if flag {
-                    } else {
+                    state.frame_flags.insert(flags);
-                        state
+                } else {
-                            .frame_flags
+                    state.frame_flags.remove(flags);
-                            .remove(FrameFlags::ALLOW_OVERLAY_PLANE_SCANOUT);
-                    }
                 }
                 let _ = tx.send(());
             }
@@ -1484,7 +1483,8 @@ fn get_surface_dmabuf_feedback(
     target_node: DrmNode,
     render_formats: FormatSet,
     target_formats: FormatSet,
-    plane_formats: FormatSet,
+    primary_plane_formats: FormatSet,
+    overlay_plane_formats: FormatSet,
 ) -> SurfaceDmabufFeedback {
     let combined_formats = render_formats
         .intersection(&target_formats)
@@ -1494,7 +1494,11 @@ fn get_surface_dmabuf_feedback(
     // We limit the scan-out trache to formats we can also render from
     // so that there is always a fallback render path available in case
     // the supplied buffer can not be scanned out directly
-    let planes_formats = plane_formats
+    let primary_plane_formats = primary_plane_formats
+        .intersection(&combined_formats)
+        .copied()
+        .collect::<FormatSet>();
+    let overlay_plane_formats = overlay_plane_formats
         .intersection(&combined_formats)
         .copied()
         .collect::<FormatSet>();
@@ -1514,12 +1518,29 @@ fn get_surface_dmabuf_feedback(
 
     let render_feedback = builder.clone().build().unwrap();
     // we would want to do this in other cases as well, but same thing as above applies
+    let primary_scanout_feedback = if target_node == render_node {
+        builder
+            .clone()
+            .add_preference_tranche(
+                target_node.dev_id(),
+                Some(zwp_linux_dmabuf_feedback_v1::TrancheFlags::Scanout),
+                primary_plane_formats.clone(),
+            )
+            .build()
+            .unwrap()
+    } else {
+        builder.clone().build().unwrap()
+    };
     let scanout_feedback = if target_node == render_node {
         builder
             .add_preference_tranche(
                 target_node.dev_id(),
                 Some(zwp_linux_dmabuf_feedback_v1::TrancheFlags::Scanout),
-                planes_formats,
+                FormatSet::from_iter(
+                    primary_plane_formats
+                        .into_iter()
+                        .chain(overlay_plane_formats.into_iter()),
+                ),
             )
             .build()
             .unwrap()
@@ -1530,5 +1551,6 @@ fn get_surface_dmabuf_feedback(
     SurfaceDmabufFeedback {
         render_feedback,
         scanout_feedback,
+        primary_scanout_feedback,
     }
 }

src/shell/element/surface.rs

@@ -552,13 +552,19 @@ impl CosmicSurface {
     ) where
         F1: FnMut(&WlSurface, &SurfaceData) -> Option<Output> + Copy,
     {
+        let is_fullscreen = self.is_fullscreen(false);
+
         self.0
             .send_dmabuf_feedback(output, primary_scan_out_output, |surface, _| {
                 select_dmabuf_feedback(
                     surface,
                     render_element_states,
                     &feedback.render_feedback,
-                    &feedback.scanout_feedback,
+                    if is_fullscreen {
+                        &feedback.primary_scanout_feedback
+                    } else {
+                        &feedback.scanout_feedback
+                    },
                 )
             })
     }

src/state.rs

@@ -253,6 +253,7 @@ pub enum BackendData {
 pub struct SurfaceDmabufFeedback {
     pub render_feedback: DmabufFeedback,
     pub scanout_feedback: DmabufFeedback,
+    pub primary_scanout_feedback: DmabufFeedback,
 }
 
 #[derive(Debug)]
@@ -800,7 +801,7 @@ impl Common {
                                 surface,
                                 render_element_states,
                                 &feedback.render_feedback,
-                                &feedback.scanout_feedback,
+                                &feedback.primary_scanout_feedback,
                             )
                         },
                     )