Try harder to succeed at fall-backing to a Monospace font

A combination of some ideas:

 * Try all Monospace fonts before giving up.
 * Relax exact weight restriction on font matching when trying Monospace
   fall-back. Try smaller weights if needed.
 * Make the fall-back try order weight-offset aware, AND script-aware.
 * And finally, add the option to adjust the font size of glyphs using
   fall-back Monospace fonts, so the width of them matches the default
   font width.

   For my use-case, the current fall-back attempt always fails with
   Arabic script. And none of the Arabic-supporting Monospace fonts in
   my system also support medium weight. So, if my default font is set
   to medium weight, script-aware fall-back alone will still not work.

Signed-off-by: Mohammad AlSaleh <CE.Mohammad.AlSaleh@gmail.com>

Cargo.toml

@@ -24,6 +24,7 @@ self_cell = "1.0.1"
 swash = { version = "0.1.8", optional = true }
 syntect = { version = "5.1.0", optional = true }
 sys-locale = { version = "0.3.1", optional = true }
+ttf-parser = "0.20.0"
 unicode-linebreak = "0.1.5"
 unicode-script = "0.5.5"
 unicode-segmentation = "1.10.1"

src/attrs.rs

@@ -177,7 +177,8 @@ impl<'a> Attrs<'a> {
         //TODO: smarter way of including emoji
         face.post_script_name.contains("Emoji")
             || (face.style == self.style
-                && face.weight == self.weight
+                // Relax exact weight matching for the Monospace fallback use-case
+                && face.weight <= self.weight
                 && face.stretch == self.stretch)
     }
 

src/buffer.rs

@@ -232,6 +232,7 @@ pub struct Buffer {
     /// True if a redraw is requires. Set to false after processing
     redraw: bool,
     wrap: Wrap,
+    monospace_width: Option<f32>,
 
     /// Scratch buffer for shaping and laying out.
     scratch: ShapeBuffer,
@@ -247,6 +248,7 @@ impl Clone for Buffer {
             scroll: self.scroll,
             redraw: self.redraw,
             wrap: self.wrap,
+            monospace_width: self.monospace_width,
             scratch: ShapeBuffer::default(),
         }
     }
@@ -275,6 +277,7 @@ impl Buffer {
             redraw: false,
             wrap: Wrap::Word,
             scratch: ShapeBuffer::default(),
+            monospace_width: None,
         }
     }
 
@@ -313,6 +316,7 @@ impl Buffer {
                     self.metrics.font_size,
                     self.width,
                     self.wrap,
+                    self.monospace_width,
                 );
             }
         }
@@ -492,6 +496,7 @@ impl Buffer {
             self.metrics.font_size,
             self.width,
             self.wrap,
+            self.monospace_width,
         ))
     }
 
@@ -523,6 +528,20 @@ impl Buffer {
         }
     }
 
+    /// Get the current `monospace_width`
+    pub fn monospace_width(&self) -> Option<f32> {
+        self.monospace_width
+    }
+
+    /// Set monospace width monospace glyphs should be resized to match. `None` means don't resize
+    pub fn set_monospace_width(&mut self, font_system: &mut FontSystem, monospace_width: Option<f32>) {
+        if monospace_width != self.monospace_width {
+            self.monospace_width = monospace_width;
+            self.relayout(font_system);
+            self.shape_until_scroll(font_system, false);
+        }
+    }
+
     /// Get the current buffer dimensions (width, height)
     pub fn size(&self) -> (f32, f32) {
         (self.width, self.height)

src/buffer_line.rs

@@ -184,6 +184,7 @@ impl BufferLine {
         font_size: f32,
         width: f32,
         wrap: Wrap,
+        match_mono_width: Option<f32>,
     ) -> &[LayoutLine] {
         self.layout_in_buffer(
             &mut ShapeBuffer::default(),
@@ -191,6 +192,7 @@ impl BufferLine {
             font_size,
             width,
             wrap,
+            match_mono_width,
         )
     }
 
@@ -202,12 +204,13 @@ impl BufferLine {
         font_size: f32,
         width: f32,
         wrap: Wrap,
+        match_mono_width: Option<f32>,
     ) -> &[LayoutLine] {
         if self.layout_opt.is_none() {
             let align = self.align;
             let shape = self.shape_in_buffer(scratch, font_system);
             let mut layout = Vec::with_capacity(1);
-            shape.layout_to_buffer(scratch, font_size, width, wrap, align, &mut layout);
+            shape.layout_to_buffer(scratch, font_size, width, wrap, align, &mut layout, match_mono_width);
             self.layout_opt = Some(layout);
         }
         self.layout_opt.as_ref().expect("layout not found")

src/font/fallback/mod.rs

@@ -3,10 +3,11 @@
 use alloc::sync::Arc;
 #[cfg(not(feature = "std"))]
 use alloc::vec::Vec;
+use alloc::collections::BTreeSet;
 use fontdb::Family;
 use unicode_script::Script;
 
-use crate::{Font, FontSystem, ShapePlanCache};
+use crate::{Font, FontSystem, ShapePlanCache, FontMatchKey};
 
 use self::platform::*;
 
@@ -31,10 +32,21 @@ use log::debug as missing_warn;
 #[cfg(feature = "warn_on_missing_glyphs")]
 use log::warn as missing_warn;
 
+// Match on lowest weight_offset, then script_non_matches
+// Default font gets None for both `weight_offset` and `script_non_matches`, and thus, it is
+// always the first to be popped from the set.
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
+struct MonospaceFallbackInfo {
+    weight_offset: Option<u16>,
+    script_non_matches: Option<usize>,
+    id: fontdb::ID,
+}
+
 pub struct FontFallbackIter<'a> {
     font_system: &'a mut FontSystem,
-    font_ids: &'a [fontdb::ID],
+    font_match_keys: &'a [FontMatchKey],
     default_families: &'a [&'a Family<'a>],
+    monospace_fallbacks: BTreeSet<MonospaceFallbackInfo>,
     default_i: usize,
     scripts: &'a [Script],
     script_i: (usize, usize),
@@ -46,14 +58,15 @@ pub struct FontFallbackIter<'a> {
 impl<'a> FontFallbackIter<'a> {
     pub fn new(
         font_system: &'a mut FontSystem,
-        font_ids: &'a [fontdb::ID],
+        font_match_keys: &'a [FontMatchKey],
         default_families: &'a [&'a Family<'a>],
         scripts: &'a [Script],
     ) -> Self {
         Self {
             font_system,
-            font_ids,
+            font_match_keys,
             default_families,
+            monospace_fallbacks: BTreeSet::new(),
             default_i: 0,
             scripts,
             script_i: (0, 0),
@@ -76,7 +89,7 @@ impl<'a> FontFallbackIter<'a> {
                 "Failed to find preset fallback for {:?} locale '{}', used '{}': '{}'",
                 self.scripts,
                 self.font_system.locale(),
-                self.face_name(self.font_ids[self.other_i - 1]),
+                self.face_name(self.font_match_keys[self.other_i - 1].id),
                 word
             );
         } else if !self.scripts.is_empty() && self.common_i > 0 {
@@ -119,34 +132,71 @@ impl<'a> FontFallbackIter<'a> {
 impl<'a> Iterator for FontFallbackIter<'a> {
     type Item = Arc<Font>;
     fn next(&mut self) -> Option<Self::Item> {
+        if let Some(fallback_info) = self.monospace_fallbacks.pop_first() {
+            if let Some(font) = self.font_system.get_font(fallback_info.id) {
+                return Some(font);
+            }
+        }
+
+        let font_match_keys_iter = |is_mono| self.font_match_keys
+                .iter()
+                .filter(move |m_key| m_key.weight_offset == 0 || is_mono);
+
         while self.default_i < self.default_families.len() {
             self.default_i += 1;
-            let mut monospace_fallback = None;
+            let is_mono = self.default_families[self.default_i - 1] == &Family::Monospace;
-            for id in self.font_ids.iter() {
+
+            for m_key in font_match_keys_iter(is_mono) {
                 let default_family = self
                     .font_system
                     .db()
                     .family_name(self.default_families[self.default_i - 1]);
-                if self.face_contains_family(*id, default_family) {
+                if self.face_contains_family(m_key.id, default_family) {
-                    if let Some(font) = self.font_system.get_font(*id) {
+                    if let Some(font) = self.font_system.get_font(m_key.id) {
-                        return Some(font);
+                        if !is_mono {
+                            return Some(font);
+                        } else if m_key.weight_offset == 0 {
+                            // Default font
+                            let fallback_info = MonospaceFallbackInfo {
+                                weight_offset: None,
+                                script_non_matches: None,
+                                id: m_key.id
+                            };
+                            assert_eq!(self.monospace_fallbacks.insert(fallback_info), true);
+                        }
                     }
                 }
                 // Set a monospace fallback if Monospace family is not found
-                if self.default_families[self.default_i - 1] == &Family::Monospace
+                if is_mono {
-                    && monospace_fallback.is_none()
+                    let script_tags = self.scripts.iter()
-                {
+                        .filter_map(|script| {
-                    if let Some(face_info) = self.font_system.db().face(*id) {
+                            let script_as_lower = script.short_name().to_lowercase();
+                            <[u8; 4]>::try_from(script_as_lower.as_bytes()).ok()
+                        }).collect::<Vec<_>>();
+
+                    if let Some(face_info) = self.font_system.db().face(m_key.id) {
                         // Don't use emoji fonts as Monospace
                         if face_info.monospaced && !face_info.post_script_name.contains("Emoji") {
-                            monospace_fallback = Some(id);
+                            if let Some(font) = self.font_system.get_font(m_key.id) {
+                                let script_non_matches = self.scripts.len() - script_tags.iter()
+                                    .filter(|&&script_tag| {
+                                        font.scripts().iter().any(|&tag_bytes| tag_bytes == script_tag)
+                                    }).count();
+
+                                let fallback_info = MonospaceFallbackInfo {
+                                    weight_offset: Some(m_key.weight_offset),
+                                    script_non_matches: Some(script_non_matches),
+                                    id: m_key.id
+                                };
+                                assert_eq!(self.monospace_fallbacks.insert(fallback_info), true);
+                            }
                         }
                     }
                 }
             }
             // If default family is Monospace fallback to first monospaced font
-            if let Some(id) = monospace_fallback {
+            if let Some(fallback_info) = self.monospace_fallbacks.pop_first() {
-                if let Some(font) = self.font_system.get_font(*id) {
+                if let Some(font) = self.font_system.get_font(fallback_info.id) {
                     return Some(font);
                 }
             }
@@ -159,9 +209,9 @@ impl<'a> Iterator for FontFallbackIter<'a> {
             while self.script_i.1 < script_families.len() {
                 let script_family = script_families[self.script_i.1];
                 self.script_i.1 += 1;
-                for id in self.font_ids.iter() {
+                for m_key in font_match_keys_iter(false) {
-                    if self.face_contains_family(*id, script_family) {
+                    if self.face_contains_family(m_key.id, script_family) {
-                        if let Some(font) = self.font_system.get_font(*id) {
+                        if let Some(font) = self.font_system.get_font(m_key.id) {
                             return Some(font);
                         }
                     }
@@ -182,9 +232,9 @@ impl<'a> Iterator for FontFallbackIter<'a> {
         while self.common_i < common_families.len() {
             let common_family = common_families[self.common_i];
             self.common_i += 1;
-            for id in self.font_ids.iter() {
+            for m_key in font_match_keys_iter(false) {
-                if self.face_contains_family(*id, common_family) {
+                if self.face_contains_family(m_key.id, common_family) {
-                    if let Some(font) = self.font_system.get_font(*id) {
+                    if let Some(font) = self.font_system.get_font(m_key.id) {
                         return Some(font);
                     }
                 }
@@ -195,8 +245,8 @@ impl<'a> Iterator for FontFallbackIter<'a> {
         //TODO: do we need to do this?
         //TODO: do not evaluate fonts more than once!
         let forbidden_families = forbidden_fallback();
-        while self.other_i < self.font_ids.len() {
+        while self.other_i < self.font_match_keys.len() {
-            let id = self.font_ids[self.other_i];
+            let id = self.font_match_keys[self.other_i].id;
             self.other_i += 1;
             if forbidden_families
                 .iter()

src/font/mod.rs

@@ -27,6 +27,8 @@ pub struct Font {
     rustybuzz: OwnedFace,
     data: Arc<dyn AsRef<[u8]> + Send + Sync>,
     id: fontdb::ID,
+    monospace_em_width: Option<f32>,
+    scripts: Vec<[u8; 4]>,
 }
 
 impl fmt::Debug for Font {
@@ -42,6 +44,14 @@ impl Font {
         self.id
     }
 
+    pub fn monospace_em_width(&self) -> Option<f32> {
+        self.monospace_em_width
+    }
+
+    pub fn scripts(&self) -> &[[u8; 4]] {
+        &self.scripts
+    }
+
     pub fn data(&self) -> &[u8] {
         (*self.data).as_ref()
     }
@@ -62,7 +72,32 @@ impl Font {
 }
 
 impl Font {
-    pub fn new(info: &fontdb::FaceInfo) -> Option<Self> {
+    pub fn new(db: &fontdb::Database, id: fontdb::ID) -> Option<Self> {
+        let info = db.face(id)?;
+
+        let (monospace_em_width, scripts) = {
+            db.with_face_data(id, |font_data, face_index| {
+                let face = ttf_parser::Face::parse(font_data, face_index).ok()?;
+                let monospace_em_width = info.monospaced.then(|| {
+                    let hor_advance = face.glyph_hor_advance(face.glyph_index(' ')?)? as f32;
+                    let upem = face.units_per_em() as f32;
+                    Some(hor_advance/upem)
+                }).flatten();
+
+                if info.monospaced && monospace_em_width.is_none() {
+                    None?;
+                }
+
+                let scripts = face.tables().gpos.into_iter()
+                    .chain(face.tables().gsub)
+                    .map(|table| table.scripts)
+                    .flatten()
+                    .map(|script| script.tag.to_bytes())
+                    .collect();
+                Some((monospace_em_width, scripts))
+            })?
+        }?;
+
         let data = match &info.source {
             fontdb::Source::Binary(data) => Arc::clone(data),
             #[cfg(feature = "std")]
@@ -76,6 +111,8 @@ impl Font {
 
         Some(Self {
             id: info.id,
+            monospace_em_width,
+            scripts,
             #[cfg(feature = "swash")]
             swash: {
                 let swash = swash::FontRef::from_index((*data).as_ref(), info.index as usize)?;

src/font/system.rs

@@ -9,6 +9,12 @@ use core::ops::{Deref, DerefMut};
 pub use fontdb;
 pub use rustybuzz;
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct FontMatchKey {
+    pub(crate) weight_offset: u16,
+    pub(crate) id: fontdb::ID,
+}
+
 /// Access to the system fonts.
 pub struct FontSystem {
     /// The locale of the system.
@@ -21,7 +27,7 @@ pub struct FontSystem {
     font_cache: HashMap<fontdb::ID, Option<Arc<Font>>>,
 
     /// Cache for font matches.
-    font_matches_cache: HashMap<AttrsOwned, Arc<Vec<fontdb::ID>>>,
+    font_matches_cache: HashMap<AttrsOwned, Arc<Vec<FontMatchKey>>>,
 
     /// Cache for rustybuzz shape plans.
     shape_plan_cache: ShapePlanCache,
@@ -111,11 +117,10 @@ impl FontSystem {
                 unsafe {
                     self.db.make_shared_face_data(id);
                 }
-                let face = self.db.face(id)?;
+                match Font::new(&self.db, id) {
-                match Font::new(face) {
                     Some(font) => Some(Arc::new(font)),
                     None => {
-                        log::warn!("failed to load font '{}'", face.post_script_name);
+                        log::warn!("failed to load font '{}'", self.db.face(id)?.post_script_name);
                         None
                     }
                 }
@@ -123,7 +128,7 @@ impl FontSystem {
             .clone()
     }
 
-    pub fn get_font_matches(&mut self, attrs: Attrs<'_>) -> Arc<Vec<fontdb::ID>> {
+    pub fn get_font_matches(&mut self, attrs: Attrs<'_>) -> Arc<Vec<FontMatchKey>> {
         self.font_matches_cache
             //TODO: do not create AttrsOwned unless entry does not already exist
             .entry(AttrsOwned::new(attrs))
@@ -131,20 +136,23 @@ impl FontSystem {
                 #[cfg(all(feature = "std", not(target_arch = "wasm32")))]
                 let now = std::time::Instant::now();
 
-                let ids = self
+                let mut font_match_keys = self
                     .db
                     .faces()
                     .filter(|face| attrs.matches(face))
-                    .map(|face| face.id)
+                    .map(|face| FontMatchKey{ weight_offset: attrs.weight.0 - face.weight.0, id: face.id })
                     .collect::<Vec<_>>();
 
+                // Sort so we get the keys with weight_offset=0 first
+                font_match_keys.sort();
+
                 #[cfg(all(feature = "std", not(target_arch = "wasm32")))]
                 {
                     let elapsed = now.elapsed();
                     log::debug!("font matches for {:?} in {:?}", attrs, elapsed);
                 }
 
-                Arc::new(ids)
+                Arc::new(font_match_keys)
             })
             .clone()
     }

src/shape.rs

@@ -144,6 +144,7 @@ fn shape_fallback(
             y_offset,
             ascent,
             descent,
+            font_monospace_em_width: font.monospace_em_width(),
             font_id: font.id(),
             glyph_id: info.glyph_id.try_into().expect("failed to cast glyph ID"),
             //TODO: color should not be related to shaping
@@ -344,6 +345,7 @@ fn shape_skip(
 
     let font = font_iter.next().expect("no default font found");
     let font_id = font.id();
+    let font_monospace_em_width = font.monospace_em_width();
     let font = font.as_swash();
 
     let charmap = font.charmap();
@@ -371,6 +373,7 @@ fn shape_skip(
                     y_offset: 0.0,
                     ascent,
                     descent,
+                    font_monospace_em_width,
                     font_id,
                     glyph_id,
                     color_opt: attrs.color_opt,
@@ -392,6 +395,7 @@ pub struct ShapeGlyph {
     pub y_offset: f32,
     pub ascent: f32,
     pub descent: f32,
+    pub font_monospace_em_width: Option<f32>,
     pub font_id: fontdb::ID,
     pub glyph_id: u16,
     pub color_opt: Option<Color>,
@@ -875,6 +879,7 @@ impl ShapeLine {
         line_width: f32,
         wrap: Wrap,
         align: Option<Align>,
+        match_mono_width: Option<f32>,
     ) -> Vec<LayoutLine> {
         let mut lines = Vec::with_capacity(1);
         self.layout_to_buffer(
@@ -884,6 +889,7 @@ impl ShapeLine {
             wrap,
             align,
             &mut lines,
+            match_mono_width,
         );
         lines
     }
@@ -896,6 +902,7 @@ impl ShapeLine {
         wrap: Wrap,
         align: Option<Align>,
         layout_lines: &mut Vec<LayoutLine>,
+        match_mono_width: Option<f32>,
     ) {
         // For each visual line a list of  (span index,  and range of words in that span)
         // Note that a BiDi visual line could have multiple spans or parts of them
@@ -1236,8 +1243,20 @@ impl ShapeLine {
                             (false, true) => &word.glyphs[..ending_glyph],
                             (true, true) => &word.glyphs[starting_glyph..ending_glyph],
                         };
+
+                        let match_mono_em_width = match_mono_width.map(|w| w / font_size);
+
                         for glyph in included_glyphs {
-                            let x_advance = font_size * glyph.x_advance
+                            let glyph_font_size = match(match_mono_em_width, glyph.font_monospace_em_width) {
+                                (Some(match_em_width), Some(glyph_em_width)) if glyph_em_width != match_em_width => {
+                                    let glyph_font_size = font_size * (match_em_width / glyph_em_width);
+                                    log::trace!("Adjusted glyph font size ({font_size} => {glyph_font_size})");
+                                    glyph_font_size
+                                },
+                                _ => font_size,
+                            };
+
+                            let x_advance = glyph_font_size * glyph.x_advance
                                 + if word.blank {
                                     justification_expansion
                                 } else {
@@ -1246,8 +1265,8 @@ impl ShapeLine {
                             if self.rtl {
                                 x -= x_advance;
                             }
-                            let y_advance = font_size * glyph.y_advance;
+                            let y_advance = glyph_font_size * glyph.y_advance;
-                            glyphs.push(glyph.layout(font_size, x, y, x_advance, span.level));
+                            glyphs.push(glyph.layout(glyph_font_size, x, y, x_advance, span.level));
                             if !self.rtl {
                                 x += x_advance;
                             }