kms/surface: Simpify surface feedback creation

src/backend/kms/surface/mod.rs

@@ -120,7 +120,7 @@ pub struct Surface {
     active: Arc<AtomicBool>,
     pub(super) feedback: HashMap<DrmNode, SurfaceDmabufFeedback>,
     pub(super) primary_plane_formats: FormatSet,
-    overlay_plane_formats: FormatSet,
+    overlay_plane_formats: Option<FormatSet>,
 
     loop_handle: LoopHandle<'static, State>,
     thread_command: Sender<ThreadCommand>,
@@ -343,7 +343,7 @@ impl Surface {
             active,
             feedback: HashMap::new(),
             primary_plane_formats: FormatSet::default(),
-            overlay_plane_formats: FormatSet::default(),
+            overlay_plane_formats: None,
             loop_handle: evlh.clone(),
             thread_command: tx,
             thread_token,
@@ -436,7 +436,7 @@ impl Surface {
         &mut self,
         compositor: GbmDrmOutput,
         primary_plane_formats: FormatSet,
-        overlay_plane_formats: FormatSet,
+        overlay_plane_formats: Option<FormatSet>,
     ) {
         self.primary_plane_formats = primary_plane_formats;
         self.overlay_plane_formats = overlay_plane_formats;
@@ -1528,75 +1528,82 @@ fn get_surface_dmabuf_feedback(
     render_node: DrmNode,
     target_node: DrmNode,
     render_formats: FormatSet,
-    target_formats: FormatSet,
+    _target_formats: FormatSet,
     primary_plane_formats: FormatSet,
-    overlay_plane_formats: FormatSet,
+    overlay_plane_formats: Option<FormatSet>,
 ) -> SurfaceDmabufFeedback {
-    let combined_formats = render_formats
-        .intersection(&target_formats)
-        .copied()
-        .collect::<FormatSet>();
-
     // We limit the scan-out trache to formats we can also render from
     // so that there is always a fallback render path available in case
     // the supplied buffer can not be scanned out directly
-    let primary_plane_formats = primary_plane_formats
-        .intersection(&combined_formats)
-        .copied()
-        .collect::<FormatSet>();
-    let overlay_plane_formats = overlay_plane_formats
-        .intersection(&combined_formats)
-        .copied()
-        .collect::<FormatSet>();
 
+    let primary_plane_formats = primary_plane_formats
+        .intersection(&render_formats)
+        .cloned()
+        .collect::<FormatSet>();
+    let overlay_plane_formats = overlay_plane_formats.map(|formats| {
+        formats
+            .intersection(&render_formats)
+            .cloned()
+            .collect::<FormatSet>()
+    });
     let builder = DmabufFeedbackBuilder::new(render_node.dev_id(), render_formats);
+
     /*
-    // iris doesn't handle nvidia buffers very well (it hangs).
-    // so only do this in the future with v6 and clients telling us the gpu
+    // Sadly no implementation would pick this up as a preferred render tranche,
+    // where the combined formats would increase our chances of doing a dmabuf copy.
+    // .. So we should probably not advertise this on the off-chance it actually triggers bugs.
+    //
+
+    let combined_formats = render_formats.intersection(&target_formats).cloned().collect::<FormatSet>();
     if target_node != render_node.dev_id() && !combined_formats.is_empty() {
         builder = builder.add_preference_tranche(
-            target_node,
+            render_node.dev_id(),
+            None,
+            combined_formats,
+        );
+    };
+
+    // We also can't advertise scan out tranches for the actual display device,
+    // as e.g. the nvidia driver might then send us dmabufs, that makes e.g. the iris hangs on import...
+    if target_node != render_node.dev_id() && !combined_formats.is_empty() {
+        builder = builder.add_preference_tranche(
+            target_node.dev_id(),
             Some(zwp_linux_dmabuf_feedback_v1::TrancheFlags::Scanout),
             combined_formats,
         );
     };
+
+    // So no fun combinations, we gotta wait for dmabuf-v6
     */
 
     let render_feedback = builder.clone().build().unwrap();
-    // we would want to do this in other cases as well, but same thing as above applies
-    let primary_scanout_feedback = if target_node == render_node {
+    let primary_scanout_feedback = (target_node == render_node).then(|| {
         builder
             .clone()
             .add_preference_tranche(
-                target_node.dev_id(),
+                render_node.dev_id(),
                 Some(zwp_linux_dmabuf_feedback_v1::TrancheFlags::Scanout),
-                primary_plane_formats.clone(),
+                primary_plane_formats,
             )
             .build()
             .unwrap()
-    } else {
-        builder.clone().build().unwrap()
-    };
-    let scanout_feedback = if target_node == render_node {
-        builder
-            .add_preference_tranche(
-                target_node.dev_id(),
-                Some(zwp_linux_dmabuf_feedback_v1::TrancheFlags::Scanout),
-                FormatSet::from_iter(
-                    primary_plane_formats
-                        .into_iter()
-                        .chain(overlay_plane_formats),
-                ),
-            )
-            .build()
-            .unwrap()
-    } else {
-        builder.build().unwrap()
-    };
+    });
+    let overlay_scanout_feedback = overlay_plane_formats
+        .filter(|_| target_node == render_node)
+        .map(|formats| {
+            builder
+                .add_preference_tranche(
+                    render_node.dev_id(),
+                    Some(zwp_linux_dmabuf_feedback_v1::TrancheFlags::Scanout),
+                    formats,
+                )
+                .build()
+                .unwrap()
+        });
 
     SurfaceDmabufFeedback {
         render_feedback,
-        scanout_feedback,
+        overlay_scanout_feedback,
         primary_scanout_feedback,
     }
 }