diff --git a/src/backend/kms/device.rs b/src/backend/kms/device.rs
index 5927ae60..806499c3 100644
--- a/src/backend/kms/device.rs
+++ b/src/backend/kms/device.rs
@@ -46,7 +46,7 @@ use std::{
     cell::RefCell,
     collections::{HashMap, HashSet},
     fmt,
-    path::{Path, PathBuf},
+    path::Path,
     sync::{atomic::AtomicBool, mpsc::Receiver, Arc, RwLock},
     time::Duration,
 };
@@ -144,7 +144,7 @@ pub fn init_egl(gbm: &GbmDevice<DrmDeviceFd>) -> Result<EGLInternals> {
 }
 
 impl State {
-    pub fn device_added(&mut self, dev: dev_t, path: PathBuf, dh: &DisplayHandle) -> Result<()> {
+    pub fn device_added(&mut self, dev: dev_t, path: &Path, dh: &DisplayHandle) -> Result<()> {
         if !self.backend.kms().session.is_active() {
             return Ok(());
         }
@@ -337,13 +337,7 @@ impl State {
             // TODO atomic commit all surfaces together and drop surfaces, if it fails due to bandwidth
 
             let kms = self.backend.kms();
-            let was_empty = kms.drm_devices.is_empty();
             kms.drm_devices.insert(drm_node, device);
-            if was_empty {
-                if let Err(err) = kms.select_primary_gpu(dh) {
-                    warn!("Failed to determine a new primary gpu: {}", err);
-                }
-            }
         }
 
         self.common
@@ -497,12 +491,11 @@ impl State {
                 dh.remove_global::<State>(socket.drm_global);
             }
             backend.api.as_mut().remove_node(&device.render_node);
-            let was_primary = *backend.primary_node.read().unwrap() == Some(device.render_node);
-            if was_primary {
-                if let Err(err) = backend.select_primary_gpu(dh) {
-                    warn!("Failed to determine a new primary gpu: {}", err);
-                }
-            }
+            backend
+                .primary_node
+                .write()
+                .unwrap()
+                .take_if(|node| node == &device.render_node);
         }
         self.common
             .output_configuration_state
diff --git a/src/backend/kms/mod.rs b/src/backend/kms/mod.rs
index 2725e3cd..8e0d39db 100644
--- a/src/backend/kms/mod.rs
+++ b/src/backend/kms/mod.rs
@@ -136,6 +136,10 @@ pub fn init_backend(
         }
     }
 
+    if let Err(err) = state.backend.kms().select_primary_gpu(dh) {
+        warn!("Failed to determine primary gpu: {}", err);
+    }
+
     // start x11
     let primary = state.backend.kms().primary_node.read().unwrap().clone();
     state.launch_xwayland(primary);
@@ -254,7 +258,10 @@ fn init_udev(
     let dispatcher = Dispatcher::new(udev_backend, move |event, _, state: &mut State| {
         let dh = state.common.display_handle.clone();
         match match event {
-            UdevEvent::Added { device_id, path } => state
+            UdevEvent::Added {
+                device_id,
+                ref path,
+            } => state
                 .device_added(device_id, path, &dh)
                 .with_context(|| format!("Failed to add drm device: {}", device_id)),
             UdevEvent::Changed { device_id } => state
@@ -265,7 +272,15 @@ fn init_udev(
                 .with_context(|| format!("Failed to remove drm device: {}", device_id)),
         } {
             Ok(()) => {
-                trace!("Successfully handled udev event.")
+                trace!("Successfully handled udev event.");
+                let backend = state.backend.kms();
+                if matches!(event, UdevEvent::Added { .. } | UdevEvent::Removed { .. })
+                    && backend.primary_node.read().unwrap().is_none()
+                {
+                    if let Err(err) = state.backend.kms().select_primary_gpu(&dh) {
+                        warn!("Failed to determine a new primary gpu: {}", err);
+                    }
+                }
             }
             Err(err) => {
                 error!(?err, "Error while handling udev event.")