diff --git a/src/font/font.rs b/src/font/font.rs
index bcdc359..c9a3adc 100644
--- a/src/font/font.rs
+++ b/src/font/font.rs
@@ -8,7 +8,7 @@ pub struct Font<'a> {
     pub index: u32,
     pub rustybuzz: rustybuzz::Face<'a>,
     #[cfg(feature = "swash")]
-    pub swash: swash::FontRef<'a>,
+    pub swash: (u32, swash::CacheKey),
 }
 
 impl<'a> Font<'a> {
@@ -28,7 +28,19 @@ impl<'a> Font<'a> {
             index: info.index,
             rustybuzz: rustybuzz::Face::from_slice(data, info.index)?,
             #[cfg(feature = "swash")]
-            swash: swash::FontRef::from_index(data, info.index as usize)?,
+            swash: {
+                let swash = swash::FontRef::from_index(data, info.index as usize)?;
+                (swash.offset, swash.key)
+            },
         })
     }
+
+    #[cfg(feature = "swash")]
+    pub fn as_swash(&self) -> swash::FontRef {
+        swash::FontRef {
+            data: self.data,
+            offset: self.swash.0,
+            key: self.swash.1,
+        }
+    }
 }
diff --git a/src/swash.rs b/src/swash.rs
index 09c5047..5afcb23 100644
--- a/src/swash.rs
+++ b/src/swash.rs
@@ -36,7 +36,7 @@ impl SwashCache {
 
             // Build the scaler
             let mut scaler = self.context
-                .builder(font.swash)
+                .builder(font.as_swash())
                 .size(cache_key.font_size as f32)
                 .hint(true)
                 .build();