diff --git a/src/shell/focus/mod.rs b/src/shell/focus/mod.rs
index 58301072..e109519e 100644
--- a/src/shell/focus/mod.rs
+++ b/src/shell/focus/mod.rs
@@ -1,6 +1,6 @@
 use crate::{
     shell::{CosmicSurface, MinimizedWindow, Shell, Trigger, element::CosmicMapped},
-    state::Common,
+    state::{Common, State},
     utils::prelude::*,
     wayland::handlers::{xdg_shell::PopupGrabData, xwayland_keyboard_grab::XWaylandGrabSeatData},
 };
@@ -12,6 +12,7 @@ use smithay::{
     reexports::wayland_server::{Resource, protocol::wl_surface::WlSurface},
     utils::{IsAlive, Point, SERIAL_COUNTER, Serial},
     wayland::{
+        pointer_constraints::with_pointer_constraint,
         seat::WaylandFocus,
         selection::{data_device::set_data_device_focus, primary_selection::set_primary_focus},
         shell::wlr_layer::{KeyboardInteractivity, Layer},
@@ -347,6 +348,20 @@ fn update_focus_state(
 ) {
     // update keyboard focus
     if let Some(keyboard) = seat.get_keyboard() {
+        // remove constraint when target changed
+        let old_focus = keyboard.current_focus();
+        if let Some(old_target) = old_focus
+            && target != Some(&old_target)
+            && let Some(surface) = old_target.wl_surface()
+            && let Some(pointer) = seat.get_pointer()
+        {
+            with_pointer_constraint(&surface, &pointer, |constraint| {
+                if let Some(constraint) = constraint {
+                    constraint.deactivate();
+                }
+            });
+        }
+
         if should_update_cursor
             && state.common.config.cosmic_conf.cursor_follows_focus
             && target.is_some()