diff --git a/examples/rich-text/src/main.rs b/examples/rich-text/src/main.rs
index 9d51bc0..0c0cc3e 100644
--- a/examples/rich-text/src/main.rs
+++ b/examples/rich-text/src/main.rs
@@ -32,7 +32,7 @@ fn set_buffer_text(buffer: &mut BorrowedWithFontSystem<'_, Buffer>) {
             attrs.clone().metrics(Metrics::relative(64.0, 1.2)),
         ),
         (
-            "Font size 8 ",
+            "\n\nFont size 8 \n\n",
             attrs.clone().metrics(Metrics::relative(8.0, 1.2)),
         ),
         (
diff --git a/src/buffer.rs b/src/buffer.rs
index ae15540..4e6b7d4 100644
--- a/src/buffer.rs
+++ b/src/buffer.rs
@@ -845,6 +845,11 @@ impl Buffer {
                 if *attrs != attrs_list.defaults() {
                     attrs_list.add_span(text_start..text_end, attrs);
                 }
+            } else if line_string.is_empty() && attrs.metrics_opt.is_some() {
+                // reset the attrs list with the span's attrs so the line height
+                // matches the span's font size rather than falling back to
+                // the buffer default
+                attrs_list = attrs_list.reset(attrs);
             }
 
             // we know that at the end of a line,
diff --git a/tests/richtext_layout.rs b/tests/richtext_layout.rs
new file mode 100644
index 0000000..e6280d3
--- /dev/null
+++ b/tests/richtext_layout.rs
@@ -0,0 +1,73 @@
+use cosmic_text::{Attrs, Buffer, FontSystem, Metrics, Shaping};
+
+// Test for https://github.com/pop-os/cosmic-text/issues/364
+//
+// Empty lines at the start/end of a span should use that span's line height,
+// not the buffer's default line height.
+#[test]
+fn empty_lines_use_span_metrics() {
+    let mut font_system = FontSystem::new();
+
+    let metrics = Metrics::new(32.0, 44.0);
+    let mut buffer = Buffer::new(&mut font_system, metrics);
+    let mut buffer = buffer.borrow_with(&mut font_system);
+
+    let attrs = Attrs::new();
+    let small_attrs = attrs.clone().metrics(Metrics::relative(8.0, 1.2));
+
+    // The empty lines from \n\n at start and end should use 8.0 * 1.2 = 9.6 line height.
+    // All newlines are inside the small_attrs span so the empty lines are clearly within it.
+    buffer.set_rich_text(
+        [
+            ("Before", attrs.clone()),
+            ("\n\n\nSmall\n\n", small_attrs),
+            ("After", attrs.clone()),
+        ],
+        &attrs,
+        Shaping::Advanced,
+        None,
+    );
+    buffer.set_size(Some(500.0), Some(500.0));
+    buffer.shape_until_scroll(false);
+
+    let line_heights: Vec<f32> = buffer.layout_runs().map(|run| run.line_height).collect();
+
+    // line_heights should be:
+    // [0] "Before"   -> 44.0 (buffer default)
+    // [1] "" (empty)  -> 9.6  (small span metrics: 8.0 * 1.2)
+    // [2] "" (empty)  -> 9.6  (small span metrics: 8.0 * 1.2)
+    // [3] "Small"     -> 9.6  (small span metrics from glyphs)
+    // [4] "" (empty)  -> 9.6  (small span metrics: 8.0 * 1.2)
+    // [5] "After"     -> 44.0 (buffer default)
+    assert_eq!(
+        line_heights.len(),
+        6,
+        "expected 6 layout runs, got {}",
+        line_heights.len()
+    );
+    assert!(
+        (line_heights[0] - 44.0).abs() < 0.1,
+        "line 0 should use buffer default: {}",
+        line_heights[0]
+    );
+    assert!(
+        (line_heights[1] - 9.6).abs() < 0.1,
+        "line 1 (empty) should use span metrics: {}",
+        line_heights[1]
+    );
+    assert!(
+        (line_heights[2] - 9.6).abs() < 0.1,
+        "line 2 (empty) should use span metrics: {}",
+        line_heights[2]
+    );
+    assert!(
+        (line_heights[4] - 9.6).abs() < 0.1,
+        "line 4 (empty) should use span metrics: {}",
+        line_heights[4]
+    );
+    assert!(
+        (line_heights[5] - 44.0).abs() < 0.1,
+        "line 5 should use buffer default: {}",
+        line_heights[5]
+    );
+}