shell: xdg-popup positioning logic

src/shell/layout/tiling/mod.rs

@@ -195,6 +195,8 @@ impl TilingLayout {
                 }
             }
         }
+
+        let mut changed = false;
         while let Some(dead_windows) = Some(TilingLayout::update_space_positions(
             &mut self.trees,
             space,
@@ -205,6 +207,12 @@ impl TilingLayout {
             for window in dead_windows {
                 self.unmap_window_internal(&window);
             }
+            changed = true;
+        }
+        if changed {
+            for window in &self.windows {
+                update_reactive_popups(space, window);
+            }
         }
     }
 

src/shell/mod.rs

@@ -568,6 +568,10 @@ impl Shell {
                 focus_stack.iter(),
             );
         }
+
+        for window in self.active_space(output).space.windows() {
+            self.update_reactive_popups(window);
+        }
     }
 
     pub fn map_layer(&mut self, layer_surface: &LayerSurface, dh: &DisplayHandle) {
@@ -644,6 +648,13 @@ impl Shell {
                 );
             }
         }
+
+        for window in self.active_space(output).space.windows() {
+            self.update_reactive_popups(window);
+        }
+        for window in self.spaces[idx].space.windows() {
+            self.update_reactive_popups(window);
+        }
     }
 }
 

src/utils/prelude.rs

@@ -6,6 +6,7 @@ use smithay::{
 use std::cell::RefCell;
 
 pub use crate::state::State;
+pub use crate::wayland::handlers::xdg_shell::popup::update_reactive_popups;
 
 pub trait OutputExt {
     fn geometry(&self) -> Rectangle<i32, Logical>;

src/wayland/handlers/compositor.rs

@@ -4,7 +4,7 @@ use crate::{state::BackendData, utils::prelude::*};
 use smithay::{
     backend::renderer::utils::on_commit_buffer_handler,
     delegate_compositor,
-    desktop::{Kind, LayerSurface, PopupKind, WindowSurfaceType, layer_map_for_output},
+    desktop::{layer_map_for_output, Kind, LayerSurface, PopupKind, WindowSurfaceType},
     reexports::wayland_server::{protocol::wl_surface::WlSurface, DisplayHandle},
     wayland::{
         compositor::{with_states, CompositorHandler, CompositorState},
@@ -77,7 +77,11 @@ impl State {
         }
     }
 
-    fn layer_surface_ensure_inital_configure(&mut self, surface: &LayerSurface, dh: &DisplayHandle) -> bool {
+    fn layer_surface_ensure_inital_configure(
+        &mut self,
+        surface: &LayerSurface,
+        dh: &DisplayHandle,
+    ) -> bool {
         // send the initial configure if relevant
         let initial_configure_sent = with_states(surface.wl_surface(), |states| {
             states
@@ -164,12 +168,16 @@ impl CompositorHandler for State {
                 );
             if let Some(location) = new_location {
                 space.map_window(&window, location, true);
+                for window in space.windows() {
+                    update_reactive_popups(space, window);
+                }
             }
         }
 
         if let Some(output) = self.common.shell.outputs().find(|o| {
             let map = layer_map_for_output(o);
-            map.layer_for_surface(surface, WindowSurfaceType::ALL).is_some()
+            map.layer_for_surface(surface, WindowSurfaceType::ALL)
+                .is_some()
         }) {
             layer_map_for_output(output).arrange(dh);
         }

src/wayland/handlers/xdg_shell/mod.rs

@@ -26,6 +26,8 @@ use smithay::{
 };
 use std::cell::Cell;
 
+pub mod popup;
+
 pub type PopupGrabData = Cell<Option<PopupGrab>>;
 
 impl XdgShellHandler for State {
@@ -50,15 +52,24 @@ impl XdgShellHandler for State {
         &mut self,
         _dh: &DisplayHandle,
         surface: PopupSurface,
-        _positioner: PositionerState,
+        positioner: PositionerState,
     ) {
         super::mark_dirty_on_drop(&self.common, surface.wl_surface());
 
-        self.common
+        surface.with_pending_state(|state| {
-            .shell
+            state.geometry = positioner.get_geometry();
-            .popups
+            state.positioner = positioner;
-            .track_popup(PopupKind::from(surface))
+        });
-            .unwrap();
+
+        self.common.shell.unconstrain_popup(&surface, &positioner);
+
+        if surface.send_configure().is_ok() {
+            self.common
+                .shell
+                .popups
+                .track_popup(PopupKind::from(surface))
+                .unwrap();
+        }
     }
 
     fn ack_configure(&mut self, _dh: &DisplayHandle, surface: WlSurface, configure: Configure) {
@@ -132,14 +143,13 @@ impl XdgShellHandler for State {
         token: u32,
     ) {
         surface.with_pending_state(|state| {
-            // TODO: This is a simplification, a proper compositor would
-            // calculate the geometry of the popup here.
-            // For now we just use the default implementation here that does not take the
-            // window position and output constraints into account.
             let geometry = positioner.get_geometry();
             state.geometry = geometry;
             state.positioner = positioner;
         });
+
+        self.common.shell.unconstrain_popup(&surface, &positioner);
+
         surface.send_repositioned(token);
     }
 

src/wayland/handlers/xdg_shell/popup.rs

@@ -0,0 +1,383 @@
+// SPDX-License-Identifier: GPL-3.0-only
+
+use crate::{shell::Shell, utils::prelude::*};
+use smithay::{
+    desktop::{PopupKind, PopupManager, Space, Window, WindowSurfaceType},
+    reexports::{
+        wayland_protocols::xdg::shell::server::xdg_positioner::{
+            Anchor, ConstraintAdjustment, Gravity,
+        },
+        wayland_server::protocol::wl_surface::WlSurface,
+    },
+    utils::{Logical, Point, Rectangle},
+    wayland::{
+        compositor::{get_role, with_states},
+        shell::xdg::{
+            PopupSurface, PositionerState, SurfaceCachedState, XdgPopupSurfaceRoleAttributes,
+            XDG_POPUP_ROLE,
+        },
+    },
+};
+use std::sync::Mutex;
+
+impl Shell {
+    pub fn unconstrain_popup(&self, surface: &PopupSurface, positioner: &PositionerState) {
+        if let Some(parent) = get_popup_toplevel(&surface) {
+            if let Some(workspace) = self.space_for_surface(&parent) {
+                let window = workspace
+                    .space
+                    .window_for_surface(&parent, WindowSurfaceType::TOPLEVEL)
+                    .unwrap();
+                unconstrain_popup(surface, positioner, &workspace.space, window);
+            }
+        }
+    }
+
+    pub fn update_reactive_popups(&self, window: &Window) {
+        if let Some(workspace) = self.space_for_surface(window.toplevel().wl_surface()) {
+            update_reactive_popups(&workspace.space, window);
+        }
+    }
+}
+
+pub fn update_reactive_popups(space: &Space, window: &Window) {
+    for (popup, _) in PopupManager::popups_for_surface(window.toplevel().wl_surface()) {
+        match popup {
+            PopupKind::Xdg(surface) => {
+                let positioner = with_states(&surface.wl_surface(), |states| {
+                    let attributes = states
+                        .data_map
+                        .get::<Mutex<XdgPopupSurfaceRoleAttributes>>()
+                        .unwrap()
+                        .lock()
+                        .unwrap();
+                    attributes.current.positioner.clone()
+                });
+                if positioner.reactive {
+                    unconstrain_popup(&surface, &positioner, space, window);
+                }
+            }
+        }
+    }
+}
+
+fn unconstrain_popup(
+    surface: &PopupSurface,
+    positioner: &PositionerState,
+    space: &Space,
+    window: &Window,
+) {
+    let anchor_point = get_anchor_point(&positioner) + space.window_location(&window).unwrap();
+    if let Some(output_rect) = space
+        .outputs_for_window(window)
+        .into_iter()
+        .find(|o| o.geometry().contains(anchor_point))
+        .map(|o| space.output_geometry(&o).unwrap())
+    {
+        let mut combined = output_rect
+            .intersection(space.window_bbox(&window).unwrap_or_default())
+            .unwrap_or_default();
+        combined.loc -= space.window_location(&window).unwrap();
+        let offset = check_constrained(&surface, positioner.get_geometry(), combined);
+
+        if offset.x != 0 || offset.y != 0 {
+            if !unconstrain_flip(&surface, &positioner, combined) {
+                if !unconstrain_slide(&surface, &positioner, combined) {
+                    unconstrain_resize(&surface, &positioner, combined);
+                }
+            }
+        }
+    }
+}
+
+fn unconstrain_flip(
+    popup: &PopupSurface,
+    positioner: &PositionerState,
+    toplevel_box: Rectangle<i32, Logical>,
+) -> bool {
+    let offset = check_constrained(popup, positioner.get_geometry(), toplevel_box);
+    if offset.x == 0 && offset.y == 0 {
+        return true;
+    }
+
+    let mut positioner = positioner.clone();
+
+    let flip_x = offset.x != 0
+        && positioner
+            .constraint_adjustment
+            .contains(ConstraintAdjustment::FlipX);
+    let flip_y = offset.y != 0
+        && positioner
+            .constraint_adjustment
+            .contains(ConstraintAdjustment::FlipY);
+
+    if flip_x {
+        positioner.anchor_edges = invert_anchor_x(positioner.anchor_edges);
+        positioner.gravity = invert_gravity_x(positioner.gravity);
+    }
+    if flip_y {
+        positioner.anchor_edges = invert_anchor_y(positioner.anchor_edges);
+        positioner.gravity = invert_gravity_y(positioner.gravity);
+    }
+
+    let offset = check_constrained(popup, positioner.get_geometry(), toplevel_box);
+    if offset.x == 0 && offset.y == 0 {
+        // no longer constrained
+        popup.with_pending_state(|state| {
+            state.geometry = positioner.get_geometry();
+            state.positioner = positioner;
+        });
+        true
+    } else {
+        false
+    }
+}
+
+fn unconstrain_slide(
+    popup: &PopupSurface,
+    positioner: &PositionerState,
+    toplevel_box: Rectangle<i32, Logical>,
+) -> bool {
+    let offset = check_constrained(popup, positioner.get_geometry(), toplevel_box);
+    if offset.x == 0 && offset.y == 0 {
+        return true;
+    }
+
+    let slide_x = offset.x != 0
+        && positioner
+            .constraint_adjustment
+            .contains(ConstraintAdjustment::SlideX);
+    let slide_y = offset.y != 0
+        && positioner
+            .constraint_adjustment
+            .contains(ConstraintAdjustment::SlideY);
+
+    let mut geometry = positioner.get_geometry();
+    if slide_x {
+        geometry.loc.x += offset.x;
+    }
+    if slide_y {
+        geometry.loc.y += offset.y;
+    }
+
+    let toplevel = get_popup_toplevel_coords(popup);
+    if slide_x && toplevel.x < toplevel_box.loc.x {
+        geometry.loc.x += toplevel_box.loc.x - toplevel.x;
+    }
+    if slide_y && toplevel.y < toplevel_box.loc.y {
+        geometry.loc.y += toplevel_box.loc.y - toplevel.y;
+    }
+
+    let offset = check_constrained(popup, geometry, toplevel_box);
+    if offset.x == 0 && offset.y == 0 {
+        // no longer constrained
+        popup.with_pending_state(|state| {
+            state.geometry = geometry;
+        });
+        true
+    } else {
+        false
+    }
+}
+
+fn unconstrain_resize(
+    popup: &PopupSurface,
+    positioner: &PositionerState,
+    toplevel_box: Rectangle<i32, Logical>,
+) -> bool {
+    let offset = check_constrained(popup, positioner.get_geometry(), toplevel_box);
+    if offset.x == 0 && offset.y == 0 {
+        return true;
+    }
+
+    let resize_x = offset.x != 0
+        && positioner
+            .constraint_adjustment
+            .contains(ConstraintAdjustment::ResizeX);
+    let resize_y = offset.y != 0
+        && positioner
+            .constraint_adjustment
+            .contains(ConstraintAdjustment::ResizeY);
+
+    let mut geometry = positioner.get_geometry();
+    if resize_x {
+        geometry.size.w -= offset.x;
+    }
+    if resize_y {
+        geometry.size.h -= offset.y;
+    }
+
+    let offset = check_constrained(popup, geometry, toplevel_box);
+    if offset.x == 0 && offset.y == 0 {
+        // no longer constrained
+        popup.with_pending_state(|state| {
+            state.geometry = geometry;
+        });
+        true
+    } else {
+        false
+    }
+}
+
+fn check_constrained(
+    popup: &PopupSurface,
+    geometry: Rectangle<i32, Logical>,
+    toplevel_box: Rectangle<i32, Logical>,
+) -> Point<i32, Logical> {
+    let relative_coords = Rectangle::from_loc_and_size(
+        geometry.loc + get_popup_toplevel_coords(popup),
+        geometry.size,
+    );
+
+    let mut offset = (0, 0).into();
+    if toplevel_box.contains_rect(relative_coords) {
+        return offset;
+    }
+
+    if relative_coords.loc.x < toplevel_box.loc.x {
+        offset.x = toplevel_box.loc.x - relative_coords.loc.x;
+    } else if relative_coords.loc.x + relative_coords.size.w
+        > toplevel_box.loc.x + toplevel_box.size.w
+    {
+        offset.x = toplevel_box.loc.x + toplevel_box.size.w
+            - (relative_coords.loc.x + relative_coords.size.w);
+    }
+
+    if relative_coords.loc.y < toplevel_box.loc.y {
+        offset.y = toplevel_box.loc.y - relative_coords.loc.y;
+    } else if relative_coords.loc.y + relative_coords.size.h
+        > toplevel_box.loc.y + toplevel_box.size.h
+    {
+        offset.y = toplevel_box.loc.y + toplevel_box.size.h
+            - (relative_coords.loc.y + relative_coords.size.h);
+    }
+
+    offset
+}
+
+fn get_anchor_point(positioner: &PositionerState) -> Point<i32, Logical> {
+    let rect = positioner.anchor_rect;
+    match positioner.anchor_edges {
+        Anchor::Top => (rect.loc.x + (rect.size.w / 2), rect.loc.y),
+        Anchor::Bottom => (rect.loc.x + (rect.size.w / 2), rect.loc.y + rect.size.h),
+        Anchor::Left => (rect.loc.x, rect.loc.y + (rect.size.h / 2)),
+        Anchor::Right => (rect.loc.x + rect.size.w, rect.loc.y + (rect.size.h / 2)),
+        Anchor::TopLeft => (rect.loc.x, rect.loc.y),
+        Anchor::TopRight => (rect.loc.x + rect.size.w, rect.loc.y),
+        Anchor::BottomLeft => (rect.loc.x, rect.loc.y + rect.size.h),
+        Anchor::BottomRight => (rect.loc.x + rect.size.w, rect.loc.y + rect.size.h),
+        Anchor::None | _ => (
+            rect.loc.x + (rect.size.w / 2),
+            rect.loc.y + (rect.size.h / 2),
+        ),
+    }
+    .into()
+}
+
+fn get_popup_toplevel(popup: &PopupSurface) -> Option<WlSurface> {
+    let mut parent = popup.get_parent_surface()?;
+    while get_role(&parent) == Some(XDG_POPUP_ROLE) {
+        parent = with_states(&parent, |states| {
+            states
+                .data_map
+                .get::<Mutex<XdgPopupSurfaceRoleAttributes>>()
+                .unwrap()
+                .lock()
+                .unwrap()
+                .parent
+                .as_ref()
+                .cloned()
+                .unwrap()
+        });
+    }
+    Some(parent)
+}
+
+fn get_popup_toplevel_coords(popup: &PopupSurface) -> Point<i32, Logical> {
+    let mut parent = match popup.get_parent_surface() {
+        Some(parent) => parent,
+        None => return (0, 0).into(),
+    };
+
+    let mut offset = (0, 0).into();
+    while get_role(&parent) == Some(XDG_POPUP_ROLE) {
+        offset += with_states(&parent, |states| {
+            states
+                .data_map
+                .get::<Mutex<XdgPopupSurfaceRoleAttributes>>()
+                .unwrap()
+                .lock()
+                .unwrap()
+                .current
+                .geometry
+                .loc
+        });
+        parent = with_states(&parent, |states| {
+            states
+                .data_map
+                .get::<Mutex<XdgPopupSurfaceRoleAttributes>>()
+                .unwrap()
+                .lock()
+                .unwrap()
+                .parent
+                .as_ref()
+                .cloned()
+                .unwrap()
+        });
+    }
+    offset += with_states(&parent, |states| {
+        states
+            .cached_state
+            .current::<SurfaceCachedState>()
+            .geometry
+            .map(|x| x.loc)
+            .unwrap_or_else(|| (0, 0).into())
+    });
+
+    offset
+}
+
+fn invert_anchor_x(anchor: Anchor) -> Anchor {
+    match anchor {
+        Anchor::Left => Anchor::Right,
+        Anchor::Right => Anchor::Left,
+        Anchor::TopLeft => Anchor::TopRight,
+        Anchor::TopRight => Anchor::TopLeft,
+        Anchor::BottomLeft => Anchor::BottomRight,
+        Anchor::BottomRight => Anchor::BottomLeft,
+        x => x,
+    }
+}
+fn invert_anchor_y(anchor: Anchor) -> Anchor {
+    match anchor {
+        Anchor::Top => Anchor::Bottom,
+        Anchor::Bottom => Anchor::Top,
+        Anchor::TopLeft => Anchor::BottomLeft,
+        Anchor::TopRight => Anchor::BottomRight,
+        Anchor::BottomLeft => Anchor::TopLeft,
+        Anchor::BottomRight => Anchor::TopRight,
+        x => x,
+    }
+}
+fn invert_gravity_x(gravity: Gravity) -> Gravity {
+    match gravity {
+        Gravity::Left => Gravity::Right,
+        Gravity::Right => Gravity::Left,
+        Gravity::TopLeft => Gravity::TopRight,
+        Gravity::TopRight => Gravity::TopLeft,
+        Gravity::BottomLeft => Gravity::BottomRight,
+        Gravity::BottomRight => Gravity::BottomLeft,
+        x => x,
+    }
+}
+fn invert_gravity_y(gravity: Gravity) -> Gravity {
+    match gravity {
+        Gravity::Top => Gravity::Bottom,
+        Gravity::Bottom => Gravity::Top,
+        Gravity::TopLeft => Gravity::BottomLeft,
+        Gravity::TopRight => Gravity::BottomRight,
+        Gravity::BottomLeft => Gravity::TopLeft,
+        Gravity::BottomRight => Gravity::TopRight,
+        x => x,
+    }
+}