diff --git a/tiny_skia/src/layer.rs b/tiny_skia/src/layer.rs
index b14458b8..4555cdfa 100644
--- a/tiny_skia/src/layer.rs
+++ b/tiny_skia/src/layer.rs
@@ -316,7 +316,11 @@ impl graphics::Layer for Layer {
             return 2;
         }
 
-        return 3;
+        if !self.text.is_empty() {
+            return 3;
+        }
+
+        0
     }
 
     fn end(&self) -> usize {
diff --git a/wgpu/src/layer.rs b/wgpu/src/layer.rs
index 93968886..a7d8b446 100644
--- a/wgpu/src/layer.rs
+++ b/wgpu/src/layer.rs
@@ -310,7 +310,11 @@ impl graphics::Layer for Layer {
             return 3;
         }
 
-        4
+        if !self.text.is_empty() {
+            return 4;
+        }
+
+        0
     }
 
     fn end(&self) -> usize {