diff --git a/CHANGELOG.md b/CHANGELOG.md
index f7365ad..8755ca1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- Fix primary selection storing, when releasing button outside of the surface
+
 ## 0.3.4 -- 2019-08-14
 
 - Add fallback to gtk primary selection, when zwp primary selection is not available
diff --git a/src/threaded.rs b/src/threaded.rs
index 5dc9641..6bf8662 100644
--- a/src/threaded.rs
+++ b/src/threaded.rs
@@ -665,6 +665,16 @@ fn implement_seat(
                 // Get serials from recieved events from the seat pointer
                 match evt {
                     PtrEvent::Enter { serial, .. } => {
+                        if let Some(seat) = seat_map
+                            .lock()
+                            .unwrap()
+                            .get_mut(&seat_name.lock().unwrap().clone())
+                        {
+                            // Update serial if "seat" is already presented
+                            seat.1 = serial;
+                            return;
+                        }
+
                         seat_map.lock().unwrap().insert(
                             seat_name.lock().unwrap().clone(),
                             (
@@ -678,6 +688,17 @@ fn implement_seat(
                         );
                     }
                     PtrEvent::Button { serial, .. } => {
+                        if let Some(seat) = seat_map
+                            .lock()
+                            .unwrap()
+                            .get_mut(&seat_name.lock().unwrap().clone())
+                        {
+                            // Update serial if seat is already presented
+                            seat.1 = serial;
+                            return;
+                        }
+
+                        // This is for consistency with `PtrEvent::Enter`
                         seat_map.lock().unwrap().insert(
                             seat_name.lock().unwrap().clone(),
                             (
@@ -690,9 +711,6 @@ fn implement_seat(
                             ),
                         );
                     }
-                    PtrEvent::Leave { .. } => {
-                        seat_map.lock().unwrap().remove(&*seat_name.lock().unwrap());
-                    }
                     _ => {}
                 }
             },