diff --git a/build.rs b/build.rs index ca4ba401..2ef0bdd2 100644 --- a/build.rs +++ b/build.rs @@ -34,28 +34,28 @@ enum Def<'a> { /// A symbol, either a leaf or with modifiers with optional deprecation. enum Symbol<'a> { - Single(char), - Multi(Vec<(ModifierSet<&'a str>, char, Option<&'a str>)>), + Single(String), + Multi(Vec<(ModifierSet<&'a str>, String, Option<&'a str>)>), } /// A single line during parsing. -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Clone)] enum Line<'a> { Blank, Deprecated(&'a str), ModuleStart(&'a str), ModuleEnd, - Symbol(&'a str, Option), - Variant(ModifierSet<&'a str>, char), + Symbol(&'a str, Option), + Variant(ModifierSet<&'a str>, String), Eof, } -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Clone)] enum Declaration<'a> { ModuleStart(&'a str, Option<&'a str>), ModuleEnd, - Symbol(&'a str, Option, Option<&'a str>), - Variant(ModifierSet<&'a str>, char, Option<&'a str>), + Symbol(&'a str, Option, Option<&'a str>), + Variant(ModifierSet<&'a str>, String, Option<&'a str>), } fn main() { @@ -103,11 +103,11 @@ fn process(buf: &mut String, file: &Path, name: &str, desc: &str) { Some(Ok(Declaration::ModuleEnd)) } } - Ok(Line::Symbol(name, c)) => { - Some(Ok(Declaration::Symbol(name, c, deprecation.take()))) + Ok(Line::Symbol(name, value)) => { + Some(Ok(Declaration::Symbol(name, value, deprecation.take()))) } - Ok(Line::Variant(modifiers, c)) => { - Some(Ok(Declaration::Variant(modifiers, c, deprecation.take()))) + Ok(Line::Variant(modifiers, value)) => { + Some(Ok(Declaration::Variant(modifiers, value, deprecation.take()))) } Ok(Line::Eof) => { deprecation.map(|_| Err(String::from("dangling `@deprecated:`"))) @@ -156,12 +156,12 @@ fn tokenize(line: &str) -> StrResult { for part in rest.split('.') { validate_ident(part)?; } - let c = decode_char(tail.ok_or("missing char")?)?; - Line::Variant(ModifierSet::from_raw_dotted(rest), c) + let value = decode_value(tail.ok_or("missing char")?)?; + Line::Variant(ModifierSet::from_raw_dotted(rest), value) } else { validate_ident(head)?; - let c = tail.map(decode_char).transpose()?; - Line::Symbol(head, c) + let value = tail.map(decode_value).transpose()?; + Line::Symbol(head, value) }) } @@ -174,20 +174,23 @@ fn validate_ident(string: &str) -> StrResult<()> { Err(format!("invalid identifier: {string:?}")) } -/// Extracts either a single char or parses a U+XXXX escape. -fn decode_char(text: &str) -> StrResult { - if let Some(hex) = text.strip_prefix("U+") { - u32::from_str_radix(hex, 16) - .ok() - .and_then(|n| char::try_from(n).ok()) - .ok_or_else(|| format!("invalid unicode escape {text:?}")) - } else { - let mut chars = text.chars(); - match (chars.next(), chars.next()) { - (Some(c), None) => Ok(c), - _ => Err(format!("expected exactly one char, found {text:?}")), - } +/// Extracts the value of a variant, parsing `\u{XXXX}` escapes +fn decode_value(text: &str) -> StrResult { + let mut iter = text.split("\\u{"); + let mut res = iter.next().unwrap().to_string(); + for other in iter { + let (hex, rest) = other.split_once("}").ok_or_else(|| { + format!("unclosed unicode escape \\u{{{}", other.escape_debug()) + })?; + res.push( + u32::from_str_radix(hex, 16) + .ok() + .and_then(|n| char::try_from(n).ok()) + .ok_or_else(|| format!("invalid unicode escape \\u{{{hex}}}"))?, + ); + res += rest; } + Ok(res) } /// Turns a stream of lines into a list of definitions. @@ -200,23 +203,23 @@ fn parse<'a>( None | Some(Declaration::ModuleEnd) => { break; } - Some(Declaration::Symbol(name, c, deprecation)) => { + Some(Declaration::Symbol(name, value, deprecation)) => { let mut variants = vec![]; - while let Some(Declaration::Variant(name, c, deprecation)) = + while let Some(Declaration::Variant(name, value, deprecation)) = p.peek().cloned().transpose()? { - variants.push((name, c, deprecation)); + variants.push((name, value, deprecation)); p.next(); } let symbol = if !variants.is_empty() { - if let Some(c) = c { - variants.insert(0, (ModifierSet::default(), c, None)); + if let Some(value) = value { + variants.insert(0, (ModifierSet::default(), value, None)); } Symbol::Multi(variants) } else { - let c = c.ok_or("symbol needs char or variants")?; - Symbol::Single(c) + let value = value.ok_or("symbol needs char or variants")?; + Symbol::Single(value) }; defs.push((name, Binding { def: Def::Symbol(symbol), deprecation })); @@ -251,7 +254,7 @@ fn encode(buf: &mut String, module: &Module) { Def::Symbol(symbol) => { buf.push_str("Def::Symbol(Symbol::"); match symbol { - Symbol::Single(c) => write!(buf, "Single({c:?})").unwrap(), + Symbol::Single(value) => write!(buf, "Single({value:?})").unwrap(), Symbol::Multi(list) => write!(buf, "Multi(&{list:?})").unwrap(), } buf.push(')'); diff --git a/src/lib.rs b/src/lib.rs index 8a7ad069..6fa777da 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,15 +59,15 @@ pub enum Def { #[derive(Debug, Copy, Clone)] pub enum Symbol { /// A symbol without modifiers. - Single(char), + Single(&'static str), /// A symbol with named modifiers. The symbol defaults to its first variant. - Multi(&'static [(ModifierSet<&'static str>, char, Option<&'static str>)]), + Multi(&'static [(ModifierSet<&'static str>, &'static str, Option<&'static str>)]), } impl Symbol { /// Get the symbol's character for a given set of modifiers, alongside an optional deprecation /// message. - pub fn get(&self, modifs: ModifierSet<&str>) -> Option<(char, Option<&str>)> { + pub fn get(&self, modifs: ModifierSet<&str>) -> Option<(&'static str, Option<&str>)> { match self { Self::Single(c) => modifs.is_empty().then_some((*c, None)), Self::Multi(list) => { @@ -81,13 +81,13 @@ impl Symbol { /// Each variant is represented by a tuple `(modifiers, character, deprecation)`. pub fn variants( &self, - ) -> impl Iterator, char, Option<&str>)> { + ) -> impl Iterator, &'static str, Option<&str>)> { enum Variants { - Single(std::iter::Once), + Single(std::iter::Once<&'static str>), Multi( std::slice::Iter< 'static, - (ModifierSet<&'static str>, char, Option<&'static str>), + (ModifierSet<&'static str>, &'static str, Option<&'static str>), >, ), } @@ -121,6 +121,7 @@ include!(concat!(env!("OUT_DIR"), "/out.rs")); #[cfg(test)] mod test { use super::*; + use std::collections::BTreeSet; #[test] fn all_modules_sorted() { @@ -136,4 +137,48 @@ mod test { assert_sorted_recursively(ROOT); } + + #[test] + fn unicode_escapes() { + let Def::Symbol(wj) = SYM.get("wj").unwrap().def else { panic!() }; + assert_eq!(wj.get(ModifierSet::default()).unwrap().0, "\u{2060}"); + let Def::Symbol(space) = SYM.get("space").unwrap().def else { panic!() }; + assert_eq!(space.get(ModifierSet::default()).unwrap().0, " "); + assert_eq!( + space.get(ModifierSet::from_raw_dotted("nobreak")).unwrap().0, + "\u{A0}" + ); + } + + #[test] + fn random_sample() { + for (key, control) in [ + ("backslash", [("", "\\"), ("circle", "⦸"), ("not", "⧷")].as_slice()), + ("chi", &[("", "χ")]), + ("forces", &[("", "⊩"), ("not", "⊮")]), + ("interleave", &[("", "⫴"), ("big", "⫼"), ("struck", "⫵")]), + ("uranus", &[("", "⛢"), ("alt", "♅")]), + ] { + let Def::Symbol(s) = SYM.get(key).unwrap().def else { + panic!("{key:?} is not a symbol") + }; + let variants = s + .variants() + .map(|(m, v, _)| (m.into_iter().collect::>(), v)) + .collect::>(); + let control = control + .iter() + .map(|&(m, v)| { + ( + ModifierSet::from_raw_dotted(m) + .into_iter() + .collect::>(), + v, + ) + }) + .collect::>(); + + assert_eq!(variants, control); + } + } } diff --git a/src/modules/sym.txt b/src/modules/sym.txt index bd2a6d33..824aa6f9 100644 --- a/src/modules/sym.txt +++ b/src/modules/sym.txt @@ -1,25 +1,25 @@ // Control. -wj U+2060 -zwj U+200D -zwnj U+200C -zws U+200B -lrm U+200E -rlm U+200F +wj \u{2060} +zwj \u{200D} +zwnj \u{200C} +zws \u{200B} +lrm \u{200E} +rlm \u{200F} // Spaces. -space U+20 - .nobreak U+A0 - .nobreak.narrow U+202F - .en U+2002 - .quad U+2003 - .third U+2004 - .quarter U+2005 - .sixth U+2006 - .med U+205F - .fig U+2007 - .punct U+2008 - .thin U+2009 - .hair U+200A +space \u{20} + .nobreak \u{A0} + .nobreak.narrow \u{202F} + .en \u{2002} + .quad \u{2003} + .third \u{2004} + .quarter \u{2005} + .sixth \u{2006} + .med \u{205F} + .fig \u{2007} + .punct \u{2008} + .thin \u{2009} + .hair \u{200A} // Delimiters. paren @@ -30,9 +30,9 @@ paren .t ⏜ .b ⏝ brace - .l U+7B + .l \u{7B} .l.double ⦃ - .r U+7D + .r \u{7D} .r.double ⦄ .t ⏞ .b ⏟ @@ -141,14 +141,14 @@ dash .wave.double 〰 dot .op ⋅ - .basic U+2E + .basic \u{2E} .c · .circle ⊙ .circle.big ⨀ .square ⊡ .double ¨ - .triple U+20DB - .quad U+20DC + .triple \u{20DB} + .quad \u{20DC} excl ! .double ‼ .inv ¡ @@ -161,10 +161,10 @@ interrobang ‽ .inv ⸘ hash # hyph ‐ - .minus U+2D - .nobreak U+2011 + .minus \u{2D} + .nobreak \u{2011} .point ‧ - .soft U+AD + .soft \u{AD} numero № percent % permille ‰