Implement fallback priorities and han unification on Unix

examples/text/Cargo.toml

@@ -7,6 +7,7 @@ publish = false
 
 [dependencies]
 ab_glyph = { version = "0.2", optional = true }
+env_logger = "0.9"
 fontdb = "0.9"
 log = "0.4"
 orbclient = "0.3"
@@ -14,8 +15,10 @@ memmap2 = "0.5"
 rusttype = { version = "0.9", optional = true }
 rustybuzz = "0.5"
 swash = { version = "0.1", optional = true }
+sys-locale = "0.2"
 unicode-bidi = "0.3"
 unicode-linebreak = "0.1"
+unicode-script = "0.5"
 
 [features]
 default = ["swash"]

examples/text/res/han.txt

@@ -0,0 +1,29 @@
+https://en.wikipedia.org/wiki/Han_unification#Examples_of_language-dependent_glyphs
+今
+令
+免
+入
+全
+关
+具
+刃
+化
+外
+情
+才
+抵
+次
+海
+画
+直
+真
+示
+神
+空
+者
+草
+蔥
+角
+道
+雇
+骨

examples/text/src/buffer.rs

@@ -67,7 +67,7 @@ impl<'a> TextBuffer<'a> {
 
         let duration = instant.elapsed();
         if reshaped > 0 {
-            eprintln!("shape_until {}: {:?}", reshaped, duration);
+            log::debug!("shape_until {}: {:?}", reshaped, duration);
         }
     }
 
@@ -84,7 +84,7 @@ impl<'a> TextBuffer<'a> {
         }
 
         let duration = instant.elapsed();
-        eprintln!("reshape line {}: {:?}", line_i.get(), duration);
+        log::debug!("reshape line {}: {:?}", line_i.get(), duration);
 
         self.relayout_line(line_i);
     }
@@ -106,7 +106,7 @@ impl<'a> TextBuffer<'a> {
         self.redraw = true;
 
         let duration = instant.elapsed();
-        eprintln!("relayout: {:?}", duration);
+        log::debug!("relayout: {:?}", duration);
     }
 
     pub fn relayout_line(&mut self, line_i: FontLineIndex) {
@@ -139,7 +139,7 @@ impl<'a> TextBuffer<'a> {
         self.redraw = true;
 
         let duration = instant.elapsed();
-        eprintln!("relayout line {}: {:?}", line_i.get(), duration);
+        log::debug!("relayout line {}: {:?}", line_i.get(), duration);
     }
 
     pub fn font_size(&self) -> i32 {

examples/text/src/font/fallback/macos.rs

@@ -0,0 +1,11 @@
+use unicode_script::Script;
+
+// Fallbacks to use after any script specific fallbacks
+pub fn common_fallback() -> &'static [&'static str] {
+    &[]
+}
+
+// Fallbacks to use per script
+pub fn script_fallback(script: &Script, locale: &str) -> &'static [&'static str] {
+    &[]
+}

examples/text/src/font/fallback/mod.rs

@@ -0,0 +1,86 @@
+use unicode_script::Script;
+
+use super::Font;
+
+#[cfg(not(any(macos, unix, windows)))]
+use self::other::*;
+#[cfg(not(any(macos, unix, windows)))]
+mod other;
+
+#[cfg(macos)]
+use self::macos::*;
+#[cfg(macos)]
+mod macos;
+
+#[cfg(unix)]
+use self::unix::*;
+#[cfg(unix)]
+mod unix;
+
+#[cfg(windows)]
+use self::windows::*;
+#[cfg(windows)]
+mod windows;
+
+pub struct FontFallbackIter<'a> {
+    fonts: &'a [Font<'a>],
+    locale: &'a str,
+    scripts: Vec<Script>,
+    script_i: (usize, usize),
+    common_i: usize,
+    other_i: usize,
+}
+
+impl<'a> FontFallbackIter<'a> {
+    pub fn new(fonts: &'a [Font<'a>], scripts: Vec<Script>, locale: &'a str) -> Self {
+        Self {
+            fonts,
+            locale,
+            scripts,
+            script_i: (0, 0),
+            common_i: 0,
+            other_i: 0,
+        }
+    }
+}
+
+impl<'a> Iterator for FontFallbackIter<'a> {
+    type Item = &'a Font<'a>;
+    fn next(&mut self) -> Option<Self::Item> {
+        while self.script_i.0 < self.scripts.len() {
+            let script_families = script_fallback(&self.scripts[self.script_i.0], self.locale);
+            while self.script_i.1 < script_families.len() {
+                let script_family = script_families[self.script_i.1];
+                self.script_i.1 += 1;
+                for font in self.fonts.iter() {
+                    if font.info.family == script_family {
+                        return Some(font);
+                    }
+                }
+            }
+            self.script_i.0 += 1;
+            self.script_i.1 = 0;
+        }
+
+        let common_families = common_fallback();
+        while self.common_i < common_families.len() {
+            let common_family = common_families[self.common_i];
+            self.common_i += 1;
+            for font in self.fonts.iter() {
+                if font.info.family == common_family {
+                    return Some(font);
+                }
+            }
+        }
+
+        //TODO: do we need to do this?
+        //TODO: do not evaluate fonts more than once!
+        while self.other_i < self.fonts.len() {
+            let font = &self.fonts[self.other_i];
+            self.other_i += 1;
+            return Some(font);
+        }
+
+        None
+    }
+}

examples/text/src/font/fallback/other.rs

@@ -0,0 +1,11 @@
+use unicode_script::Script;
+
+// Fallbacks to use after any script specific fallbacks
+pub fn common_fallback() -> &'static [&'static str] {
+    &[]
+}
+
+// Fallbacks to use per script
+pub fn script_fallback(script: &Script, locale: &str) -> &'static [&'static str] {
+    &[]
+}

examples/text/src/font/fallback/unix.rs

@@ -0,0 +1,34 @@
+use unicode_script::Script;
+
+// Fallbacks to use after any script specific fallbacks
+pub fn common_fallback() -> &'static [&'static str] {
+    //TODO: abstract style (sans/serif/monospaced)
+    &[
+        "Fira Sans",
+        "DejaVu Sans",
+        "DejaVu Serif",
+        "Noto Color Emoji",
+    ]
+}
+
+// Fallbacks to use per script
+pub fn script_fallback(script: &Script, locale: &str) -> &'static [&'static str] {
+    //TODO: abstract style (sans/serif/monospaced)
+    match script {
+        Script::Han => match locale {
+            // Japan
+            "ja" => &["Noto Sans CJK JA"],
+            // Korea
+            "ko" => &["Noto Sans CJK KR"],
+            // China
+            "zh-CN" => &["Noto Sans CJK SC"],
+            // Hong Kong
+            "zh-HK" => &["Noto Sans CJK HK"],
+            // Taiwan
+            "zh-TW" => &["Noto Sans CJK TC"],
+            // Simplified Chinese is the default
+            _ => &["Noto Sans CJK SC"],
+        }
+        _ => &[],
+    }
+}

examples/text/src/font/fallback/windows.rs

@@ -0,0 +1,11 @@
+use unicode_script::Script;
+
+// Fallbacks to use after any script specific fallbacks
+pub fn common_fallback() -> &'static [&'static str] {
+    &[]
+}
+
+// Fallbacks to use per script
+pub fn script_fallback(script: &Script, locale: &str) -> &'static [&'static str] {
+    &[]
+}

examples/text/src/font/layout.rs

@@ -117,7 +117,7 @@ impl<'a> FontLayoutLine<'a> {
                             }
                         }
                         Content::SubpixelMask => {
-                            println!("TODO: SubpixelMask");
+                            log::warn!("TODO: SubpixelMask");
                         }
                     }
                 }

examples/text/src/font/matches.rs

@@ -1,13 +1,17 @@
+use unicode_script::{Script, UnicodeScript};
+
 use super::{Font, FontLineIndex, FontShapeGlyph, FontShapeLine, FontShapeSpan, FontShapeWord};
+use super::fallback::{FontFallbackIter};
 
 pub struct FontMatches<'a> {
+    pub locale: &'a str,
     pub fonts: Vec<Font<'a>>,
 }
 
 impl<'a> FontMatches<'a> {
     fn shape_fallback(
         &self,
-        font_i: usize,
+        font: &'a Font<'a>,
         line: &str,
         start_word: usize,
         end_word: usize,
@@ -16,7 +20,7 @@ impl<'a> FontMatches<'a> {
     ) -> (Vec<FontShapeGlyph>, Vec<usize>) {
         let word = &line[start_word..end_word];
 
-        let font_scale = self.fonts[font_i].rustybuzz.units_per_em() as f32;
+        let font_scale = font.rustybuzz.units_per_em() as f32;
 
         let mut buffer = rustybuzz::UnicodeBuffer::new();
         buffer.set_direction(if span_rtl {
@@ -34,16 +38,7 @@ impl<'a> FontMatches<'a> {
         };
         assert_eq!(rtl, span_rtl);
 
-        if font_i == 0 {
-            log::debug!(
-                "    Word {}{}: '{}'",
-                if rtl { "RTL" } else { "LTR" },
-                if blank { " BLANK" } else { "" },
-                word
-            );
-        }
-
-        let glyph_buffer = rustybuzz::shape(&self.fonts[font_i].rustybuzz, &[], buffer);
+        let glyph_buffer = rustybuzz::shape(&font.rustybuzz, &[], buffer);
         let glyph_infos = glyph_buffer.glyph_infos();
         let glyph_positions = glyph_buffer.glyph_positions();
 
@@ -76,7 +71,7 @@ impl<'a> FontMatches<'a> {
                 y_advance,
                 x_offset,
                 y_offset,
-                font: &self.fonts[font_i],
+                font,
                 inner,
             });
         }
@@ -117,16 +112,62 @@ impl<'a> FontMatches<'a> {
         span_rtl: bool,
         blank: bool,
     ) -> FontShapeWord {
-        let mut font_i = 0;
-        let (mut glyphs, mut missing) =
-            self.shape_fallback(font_i, line, start_word, end_word, span_rtl, blank);
+        //TODO: use smallvec?
+        let mut scripts = Vec::new();
+        for c in line[start_word..end_word].chars() {
+            match c.script() {
+                Script::Common |
+                Script::Inherited |
+                Script::Latin |
+                Script::Unknown => (),
+                script => if ! scripts.contains(&script) {
+                    scripts.push(script);
+                },
+            }
+        }
+
+        if scripts.len() > 1 {
+            log::info!(
+                "    Word {:?}{}: '{}'",
+                scripts,
+                if blank { " BLANK" } else { "" },
+                &line[start_word..end_word],
+            );
+        } else {
+            log::debug!(
+                "    Word {:?}{}: '{}'",
+                scripts,
+                if blank { " BLANK" } else { "" },
+                &line[start_word..end_word],
+            );
+        }
+
+        let mut font_iter = FontFallbackIter::new(&self.fonts, scripts, &self.locale);
+
+        let (mut glyphs, mut missing) = self.shape_fallback(
+            font_iter.next().unwrap(),
+            line,
+            start_word,
+            end_word,
+            span_rtl,
+            blank
+        );
 
         //TODO: improve performance!
-        font_i += 1;
-        while !missing.is_empty() && font_i < self.fonts.len() {
-            // println!("Evaluating fallback with font {}", font_i);
-            let (mut fb_glyphs, fb_missing) =
-                self.shape_fallback(font_i, line, start_word, end_word, span_rtl, blank);
+        while let Some(font) = font_iter.next() {
+            if missing.is_empty() {
+                break;
+            }
+
+            log::trace!("Evaluating fallback with font '{}'", font.info.family);
+            let (mut fb_glyphs, fb_missing) = self.shape_fallback(
+                font,
+                line,
+                start_word,
+                end_word,
+                span_rtl,
+                blank
+            );
 
             // Insert all matching glyphs
             let mut fb_i = 0;
@@ -164,7 +205,7 @@ impl<'a> FontMatches<'a> {
                 while i < glyphs.len() {
                     if glyphs[i].start >= start && glyphs[i].end <= end {
                         let _glyph = glyphs.remove(i);
-                        // println!("Removed {},{} from {}", _glyph.start, _glyph.end, i);
+                        // log::trace!("Removed {},{} from {}", _glyph.start, _glyph.end, i);
                     } else {
                         break;
                     }
@@ -173,7 +214,7 @@ impl<'a> FontMatches<'a> {
                 while fb_i < fb_glyphs.len() {
                     if fb_glyphs[fb_i].start >= start && fb_glyphs[fb_i].end <= end {
                         let fb_glyph = fb_glyphs.remove(fb_i);
-                        // println!("Insert {},{} from font {} at {}", fb_glyph.start, fb_glyph.end, font_i, i);
+                        // log::trace!("Insert {},{} from font {} at {}", fb_glyph.start, fb_glyph.end, font_i, i);
                         glyphs.insert(i, fb_glyph);
                         i += 1;
                     } else {
@@ -181,13 +222,11 @@ impl<'a> FontMatches<'a> {
                     }
                 }
             }
-
-            font_i += 1;
         }
 
         /*
         for glyph in glyphs.iter() {
-            println!("'{}': {}, {}, {}, {}", &line[glyph.start..glyph.end], glyph.x_advance, glyph.y_advance, glyph.x_offset, glyph.y_offset);
+            log::trace!("'{}': {}, {}, {}, {}", &line[glyph.start..glyph.end], glyph.x_advance, glyph.y_advance, glyph.x_offset, glyph.y_offset);
         }
         */
 

examples/text/src/font/mod.rs

@@ -1,5 +1,7 @@
 use std::{collections::HashMap, sync::Mutex};
 
+pub mod fallback;
+
 pub use self::cache::*;
 mod cache;
 
@@ -34,7 +36,7 @@ impl FontLineIndex {
 }
 
 pub struct Font<'a> {
-    pub name: &'a str,
+    pub info: &'a fontdb::FaceInfo,
     pub data: &'a [u8],
     pub index: u32,
     pub rustybuzz: rustybuzz::Face<'a>,
@@ -50,9 +52,9 @@ pub struct Font<'a> {
 }
 
 impl<'a> Font<'a> {
-    pub fn new(name: &'a str, data: &'a [u8], index: u32) -> Option<Self> {
+    pub fn new(info: &'a fontdb::FaceInfo, data: &'a [u8], index: u32) -> Option<Self> {
         Some(Self {
-            name,
+            info,
             data,
             index,
             rustybuzz: rustybuzz::Face::from_slice(data, index)?,

examples/text/src/font/system.rs

@@ -3,11 +3,18 @@ use std::ops::Deref;
 use super::{Font, FontMatches};
 
 pub struct FontSystem {
+    pub locale: String,
     db: fontdb::Database,
 }
 
 impl FontSystem {
     pub fn new() -> Self {
+        let locale = sys_locale::get_locale().unwrap_or_else(|| {
+            log::warn!("failed to get system locale, falling back to en-US");
+            String::from("en-US")
+        });
+        log::info!("Locale: {}", locale);
+
         let mut db = fontdb::Database::new();
         let now = std::time::Instant::now();
         db.load_system_fonts();
@@ -15,7 +22,7 @@ impl FontSystem {
         db.set_monospace_family("Fira Mono");
         db.set_sans_serif_family("Fira Sans");
         db.set_serif_family("DejaVu Serif");
-        println!(
+        log::info!(
             "Loaded {} font faces in {}ms.",
             db.len(),
             now.elapsed().as_millis()
@@ -30,7 +37,7 @@ impl FontSystem {
             }
         }
 
-        Self { db }
+        Self { locale, db }
     }
 
     pub fn matches<'a, F: Fn(&fontdb::FaceInfo) -> bool>(
@@ -44,11 +51,11 @@ impl FontSystem {
             }
 
             let font_opt = Font::new(
-                &face.post_script_name,
+                face,
                 match &face.source {
                     fontdb::Source::Binary(data) => data.deref().as_ref(),
                     fontdb::Source::File(path) => {
-                        println!("Unsupported fontdb Source::File('{}')", path.display());
+                        log::warn!("Unsupported fontdb Source::File('{}')", path.display());
                         continue;
                     }
                     fontdb::Source::SharedFile(_path, data) => data.deref().as_ref(),
@@ -59,13 +66,16 @@ impl FontSystem {
             match font_opt {
                 Some(font) => fonts.push(font),
                 None => {
-                    eprintln!("failed to load font '{}'", face.post_script_name);
+                    log::warn!("failed to load font '{}'", face.post_script_name);
                 }
             }
         }
 
         if !fonts.is_empty() {
-            Some(FontMatches { fonts })
+            Some(FontMatches {
+                locale: &self.locale,
+                fonts
+            })
         } else {
             None
         }

examples/text/src/main.rs

@@ -3,29 +3,31 @@ use std::{cmp, env, fs, time::Instant};
 use text::{FontLineIndex, FontSystem, TextAction, TextBuffer, TextCursor};
 
 fn main() {
+    env_logger::init();
+
     let display_scale = match orbclient::get_display_size() {
         Ok((w, h)) => {
-            eprintln!("Display size: {}, {}", w, h);
+            log::info!("Display size: {}, {}", w, h);
             (h as i32 / 1600) + 1
         }
         Err(err) => {
-            eprintln!("Failed to get display size: {}", err);
+            log::warn!("Failed to get display size: {}", err);
             1
         }
     };
 
+    let font_system = FontSystem::new();
+
     let mut window = Window::new_flags(
         -1,
         -1,
         1024 * display_scale as u32,
         768 * display_scale as u32,
-        "COSMIC TEXT",
+        &format!("COSMIC TEXT - {}", font_system.locale),
         &[WindowFlag::Resizable],
     )
     .unwrap();
 
-    let font_system = FontSystem::new();
-
     let font_matches = font_system.matches(|info| -> bool {
         #[cfg(feature = "mono")]
         let monospaced = true;
@@ -66,7 +68,7 @@ fn main() {
         (28, 36), // Title 2
         (32, 44), // Title 1
     ];
-    let font_size_default = 2; // Title 4
+    let font_size_default = 1; // Body
     let mut font_size_i = font_size_default;
 
     let text = if let Some(arg) = env::args().nth(1) {
@@ -154,7 +156,7 @@ fn main() {
             rehit = false;
 
             let duration = instant.elapsed();
-            eprintln!("rehit: {:?}", duration);
+            log::debug!("rehit: {:?}", duration);
         }
 
         if buffer.redraw {
@@ -204,11 +206,12 @@ fn main() {
                         );
 
                         let text_line = &buffer.text_lines()[line.line_i.get()];
-                        eprintln!(
-                            "{}, {}: '{}': '{}'",
+                        log::info!(
+                            "{}, {}: '{}' ('{}'): '{}'",
                             glyph.start,
                             glyph.end,
-                            glyph.font.name,
+                            glyph.font.info.family,
+                            glyph.font.info.post_script_name,
                             &text_line[glyph.start..glyph.end],
                         );
                     }
@@ -243,7 +246,7 @@ fn main() {
             buffer.redraw = false;
 
             let duration = instant.elapsed();
-            eprintln!("redraw: {:?}", duration);
+            log::debug!("redraw: {:?}", duration);
         }
 
         for event in window.events() {