drm_lease: Disable overlay planes when active

src/backend/kms/device.rs

@@ -612,6 +612,59 @@ impl Device {
         }
     }
 
+    pub fn allow_overlay_scanout(
+        &mut self,
+        flag: bool,
+        renderer: &mut GlMultiRenderer,
+        clock: &Clock<Monotonic>,
+        shell: &Arc<RwLock<Shell>>,
+    ) -> Result<()> {
+        for surface in self.surfaces.values_mut() {
+            surface.allow_overlay_scanout(flag);
+        }
+
+        if !flag {
+            let now = clock.now();
+
+            let output_map = self
+                .surfaces
+                .iter()
+                .map(|(crtc, surface)| (*crtc, surface.output.clone()))
+                .collect::<HashMap<_, _>>();
+
+            self.drm.with_compositors::<Result<()>>(|map| {
+                for (crtc, compositor) in map.iter() {
+                    let elements = match output_map.get(crtc) {
+                        Some(output) => output_elements(
+                            Some(&self.render_node),
+                            renderer,
+                            shell,
+                            now,
+                            &output,
+                            CursorMode::All,
+                            None,
+                        )
+                        .with_context(|| "Failed to render outputs")?,
+                        None => Vec::new(),
+                    };
+
+                    let mut compositor = compositor.lock().unwrap();
+                    compositor.render_frame(
+                        renderer,
+                        &elements,
+                        CLEAR_COLOR,
+                        FrameFlags::empty(),
+                    )?;
+                    compositor.commit_frame()?;
+                }
+
+                Ok(())
+            })?;
+        }
+
+        Ok(())
+    }
+
     pub fn in_use(&self, primary: Option<&DrmNode>) -> bool {
         Some(&self.render_node) == primary
             || !self.surfaces.is_empty()

src/backend/kms/surface/mod.rs

@@ -423,6 +423,14 @@ impl Surface {
         Ok(true)
     }
 
+    pub fn allow_overlay_scanout(&mut self, flag: bool) {
+        let (tx, rx) = std::sync::mpsc::sync_channel(1);
+        let _ = self
+            .thread_command
+            .send(ThreadCommand::AllowOverlayScanout(flag, tx));
+        let _ = rx.recv();
+    }
+
     pub fn suspend(&mut self) {
         let (tx, rx) = std::sync::mpsc::sync_channel(1);
         let _ = self.thread_command.send(ThreadCommand::Suspend(tx));
@@ -607,6 +615,23 @@ fn surface_thread(
                     };
                 }
             }
+            Event::Msg(ThreadCommand::AllowOverlayScanout(flag, tx)) => {
+                if !crate::utils::env::bool_var("COSMIC_DISABLE_DIRECT_SCANOUT").unwrap_or(false)
+                    && !crate::utils::env::bool_var("COSMIC_DISABLE_OVERLAY_SCANOUT")
+                        .unwrap_or(false)
+                {
+                    if flag {
+                        state
+                            .frame_flags
+                            .insert(FrameFlags::ALLOW_OVERLAY_PLANE_SCANOUT);
+                    } else {
+                        state
+                            .frame_flags
+                            .remove(FrameFlags::ALLOW_OVERLAY_PLANE_SCANOUT);
+                    }
+                }
+                let _ = tx.send(());
+            }
             Event::Closed | Event::Msg(ThreadCommand::End) => {
                 signal.stop();
                 signal.wakeup();

src/wayland/handlers/drm_lease.rs

@@ -26,12 +26,31 @@ impl DrmLeaseHandler for State {
         node: DrmNode,
         request: DrmLeaseRequest,
     ) -> Result<DrmLeaseBuilder, LeaseRejected> {
-        let backend = self
+        let kms = self.backend.kms();
-            .backend
+        let backend = kms
-            .kms()
             .drm_devices
             .get_mut(&node)
             .ok_or(LeaseRejected::default())?;
+        let mut renderer = match kms.api.single_renderer(&backend.render_node) {
+            Ok(renderer) => renderer,
+            Err(err) => {
+                tracing::warn!(
+                    ?err,
+                    "Failed to create renderer to disable direct scanout, denying lease"
+                );
+                return Err(LeaseRejected::default());
+            }
+        };
+        if let Err(err) = backend.allow_overlay_scanout(
+            false,
+            &mut renderer,
+            &self.common.clock,
+            &self.common.shell,
+        ) {
+            tracing::warn!(?err, "Failed to disable direct scanout");
+            return Err(LeaseRejected::default());
+        }
+
         let mut builder = DrmLeaseBuilder::new(backend.drm.device());
         for conn in request.connectors {
             if let Some((_, crtc)) = backend
@@ -87,8 +106,27 @@ impl DrmLeaseHandler for State {
     }
 
     fn lease_destroyed(&mut self, node: DrmNode, lease: u32) {
-        if let Some(backend) = self.backend.kms().drm_devices.get_mut(&node) {
+        let kms = self.backend.kms();
+        if let Some(backend) = kms.drm_devices.get_mut(&node) {
             backend.active_leases.retain(|l| l.id() != lease);
+
+            if backend.active_leases.is_empty() {
+                let mut renderer = match kms.api.single_renderer(&backend.render_node) {
+                    Ok(renderer) => renderer,
+                    Err(err) => {
+                        tracing::warn!(?err, "Failed to create renderer to enable direct scanout.");
+                        return;
+                    }
+                };
+                if let Err(err) = backend.allow_overlay_scanout(
+                    true,
+                    &mut renderer,
+                    &self.common.clock,
+                    &self.common.shell,
+                ) {
+                    tracing::warn!(?err, "Failed to enable direct scanout");
+                }
+            }
         }
     }
 }