From b65bbe6f42b7cd4598335d0ea2079e4d7e1e86be Mon Sep 17 00:00:00 2001 From: Robert Mongold Date: Tue, 27 Oct 2026 10:25:52 -0400 Subject: [PATCH 1/4] Tokenize Linux text with Hunspell WORDCHARS and ICONV Parse the dictionary .aff once at open, keep curly apostrophes in words, trim edge quotes, and skip URL and email spans before spelling checks. --- src/lib.rs | 5 +- src/unix.rs | 229 +++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 204 insertions(+), 30 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 53a7c39..eb6d5ec 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -211,7 +211,10 @@ impl Checker { /// Ranges are UTF-8 byte offsets. Linux tokenizes words itself (alphanumeric /// and `'`). macOS and Windows use the OS spell-checking APIs, so word breaks /// may differ. - pub fn check<'a>(&self, text: &'a str) -> impl Iterator + 'a + use<'a> { + pub fn check<'a>( + &'a self, + text: &'a str, + ) -> impl Iterator + 'a + use<'a> { self.0.check(text).map(SpellingError) } diff --git a/src/unix.rs b/src/unix.rs index 62e0ecb..f1c6f6f 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -1,5 +1,6 @@ use crate::Error; +use std::collections::HashSet; use std::ffi::{CStr, CString}; use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; @@ -47,7 +48,85 @@ fn find_dictionary(dirs: &[PathBuf], locales: &[&str]) -> Option<(PathBuf, PathB None } -fn open_dictionary(locales: &[&str]) -> Result<(*mut Hunhandle, String), Error> { +#[derive(Clone, Debug)] +struct WordChars { + /// Extra chars beyond alphanumeric and ASCII `'`. + extra: HashSet, +} + +impl WordChars { + fn contains(&self, c: char) -> bool { + c.is_alphanumeric() || c == '\'' || self.extra.contains(&c) + } + + fn from_aff(aff: &Path) -> Self { + let mut extra = HashSet::new(); + let Ok(contents) = std::fs::read_to_string(aff) else { + return Self { extra }; + }; + + let mut lines = contents.lines(); + while let Some(line) = lines.next() { + let line = line.trim(); + if line.is_empty() || line.starts_with("#") { + continue; + } + + if let Some(rest) = line.strip_prefix("WORDCHARS") { + for c in rest.trim().chars() { + if !c.is_whitespace() { + extra.insert(c); + } + } + continue; + } + + // `ICONV N` then N mapping lines: `from to` or `ICONV from to` + let mut parts = line.split_whitespace(); + if parts.next() != Some("ICONV") { + continue; + } + let Some(count_str) = parts.next() else { + continue; + }; + let Ok(count) = count_str.parse::() else { + continue; + }; + if parts.next().is_some() { + // Same line is a mapping (`ICONV ' '`), not a count. + continue; + } + + for _ in 0..count { + let Some(map_line) = lines.next() else { + break; + }; + let map_line = map_line.trim(); + let map_line = map_line.strip_prefix("ICONV").unwrap_or(map_line).trim(); + let mut map = map_line.split_whitespace(); + let Some(from) = map.next() else { + continue; + }; + let Some(to) = map.next() else { + continue; + }; + // Add ICONV sources whose dest is `'` or already a word char. + let to_is_word = to + .chars() + .all(|c| c.is_alphanumeric() || c == '\'' || extra.contains(&c)); + if to == "'" || to_is_word { + for c in from.chars() { + extra.insert(c); + } + } + } + } + + Self { extra } + } +} + +fn open_dictionary(locales: &[&str]) -> Result<(*mut Hunhandle, String, WordChars), Error> { let dirs = dict_search_dirs(); let Some((aff, dic, locale)) = find_dictionary(&dirs, locales) else { return Err(Error::DictionaryNotFound { @@ -55,6 +134,7 @@ fn open_dictionary(locales: &[&str]) -> Result<(*mut Hunhandle, String), Error> searched: dirs, }); }; + let word_chars = WordChars::from_aff(&aff); let label = Some(locale.clone()); let aff_c = CString::new(aff.as_os_str().as_bytes()).map_err(|_| Error::InitializationFailed { @@ -73,43 +153,50 @@ fn open_dictionary(locales: &[&str]) -> Result<(*mut Hunhandle, String), Error> message: "Hunspell_create returned null".into(), }); } - Ok((hunspell, locale)) + Ok((hunspell, locale, word_chars)) } #[derive(Debug)] pub struct Checker { hunspell: *mut Hunhandle, locale: String, + word_chars: WordChars, } impl Checker { pub fn new() -> Result { if let Some(raw) = crate::locale_from_vars(|key| std::env::var(key).ok()) { - let (hunspell, _) = crate::normalize_locale(&raw); - let mut candidates = crate::lookup_candidates(&raw, &hunspell); - if let Some((lang, _)) = hunspell.split_once('_') { + let (hunspell_tag, _) = crate::normalize_locale(&raw); + let mut candidates = crate::lookup_candidates(&raw, &hunspell_tag); + if let Some((lang, _)) = hunspell_tag.split_once('_') { if !candidates.contains(&lang) { candidates.push(lang); } } - if let Ok((handle, locale)) = open_dictionary(&candidates) { + if let Ok((handle, locale, word_chars)) = open_dictionary(&candidates) { return Ok(Checker { hunspell: handle, locale, + word_chars, }); } } - let (handle, locale) = open_dictionary(DEFAULT_LOCALES)?; + let (handle, locale, word_chars) = open_dictionary(DEFAULT_LOCALES)?; Ok(Checker { hunspell: handle, locale, + word_chars, }) } pub fn with_locale(raw: &str, hunspell_locale: &str, _bcp47: &str) -> Result { let candidates = crate::lookup_candidates(raw, hunspell_locale); - let (hunspell, locale) = open_dictionary(&candidates)?; - Ok(Checker { hunspell, locale }) + let (hunspell, locale, word_chars) = open_dictionary(&candidates)?; + Ok(Checker { + hunspell, + locale, + word_chars, + }) } pub fn locale(&self) -> &str { @@ -171,10 +258,10 @@ impl Checker { } } - pub fn check<'a>(&self, text: &'a str) -> impl Iterator + 'a + use<'a> { + pub fn check<'a>(&'a self, text: &'a str) -> impl Iterator + 'a { let hunspell = self.hunspell; - - words(text).filter_map(move |(start, end, word)| { + let word_chars = &self.word_chars; + words(text, word_chars).filter_map(move |(start, end, word)| { let cstr = CString::new(word).ok()?; let ok = unsafe { Hunspell_spell(hunspell, cstr.as_bytes_with_nul().as_ptr() as *const i8) } @@ -231,28 +318,112 @@ impl SpellingError { } } -fn is_word_char(c: char) -> bool { - c.is_alphanumeric() || c == '\'' +fn trim_word_edges<'a>( + text: &'a str, + start: usize, + end: usize, + wc: &WordChars, +) -> Option<(usize, usize, &'a str)> { + let mut s = start; + let mut e = end; + while s < e { + let c = text[s..e].chars().next()?; + if c.is_alphanumeric() { + break; + } + if !wc.contains(c) { + break; + } + s += c.len_utf8(); + } + while e > s { + let c = text[s..e].chars().next_back()?; + if c.is_alphanumeric() { + break; + } + if !wc.contains(c) { + break; + } + e -= c.len_utf8(); + } + if s >= e { + return None; + } + Some((s, e, &text[s..e])) } -fn words(text: &str) -> impl Iterator { - let mut words = Vec::new(); - let mut chars = text.char_indices().peekable(); - - while let Some((start, c)) = chars.next() { - if !is_word_char(c) { - continue; +/// If `text[from..]` starts a URL or email token, return the byte index after it. +fn skip_url_or_email(text: &str, from: usize) -> Option { + let rest = &text[from..]; + let token_end = rest + .find(|c: char| c.is_whitespace()) + .map(|o| from + o) + .unwrap_or(text.len()); + let token = &text[from..token_end]; + if token.contains("://") { + return Some(token_end); + } + if let Some(at) = token.find('@') { + let domain = &token[at + 1..]; + if domain.contains('.') { + return Some(token_end); } + } + None +} + +struct Words<'a> { + text: &'a str, + word_chars: &'a WordChars, + pos: usize, +} - let mut end = start + c.len_utf8(); - while let Some(&(i, next)) = chars.peek() { - if !is_word_char(next) { - break; +impl<'a> Iterator for Words<'a> { + type Item = (usize, usize, &'a str); + + fn next(&mut self) -> Option { + let bytes = self.text.as_bytes(); + while self.pos < bytes.len() { + // Skip whitespace. + let c = self.text[self.pos..].chars().next()?; + if c.is_whitespace() { + self.pos += c.len_utf8(); + continue; + } + + if let Some(after) = skip_url_or_email(self.text, self.pos) { + self.pos = after; + continue; + } + + if !self.word_chars.contains(c) { + self.pos += c.len_utf8(); + continue; + } + + let start = self.pos; + self.pos += c.len_utf8(); + while self.pos < bytes.len() { + let next = self.text[self.pos..].chars().next()?; + if !self.word_chars.contains(next) { + break; + } + self.pos += next.len_utf8(); + } + let end = self.pos; + + if let Some(item) = trim_word_edges(self.text, start, end, self.word_chars) { + return Some(item); } - end = i + next.len_utf8(); - chars.next(); } - words.push((start, end, &text[start..end])); + None + } +} + +fn words<'a>(text: &'a str, word_chars: &'a WordChars) -> Words<'a> { + Words { + text, + word_chars, + pos: 0, } - words.into_iter() } From c54b23f5bb3ea4ddfae34d385cb16c324b832021 Mon Sep 17 00:00:00 2001 From: Robert Mongold Date: Tue, 27 Oct 2026 11:01:32 -0400 Subject: [PATCH 2/4] Add tests --- src/lib.rs | 28 ++++++++++++++++++++++++++++ src/unix.rs | 15 +++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index eb6d5ec..bb7c2ce 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -552,4 +552,32 @@ mod tests { fn new_succeeds() { assert!(Checker::with_locale("en_US").is_ok()); } + + #[test] + #[cfg(all(unix, not(target_os = "macos")))] + fn curly_apostrophe_is_one_word() { + let checker = Checker::with_locale("en_US").unwrap(); + // U+2019 RIGHT SINGLE QUOTATION MARK between n and t + assert_eq!(checker.check("don\u{2019}t").count(), 0); + assert_eq!(checker.check("don't").count(), 0); + } + + #[test] + #[cfg(all(unix, not(target_os = "macos")))] + fn quoted_word_strips_quotes() { + let checker = Checker::with_locale("en_US").unwrap(); + assert_eq!(checker.check("'hello'").count(), 0); + } + + #[test] + #[cfg(all(unix, not(target_os = "macos")))] + fn skips_url_and_email() { + let checker = Checker::with_locale("en_US").unwrap(); + assert_eq!( + checker + .check("see https://example.com/foo and a@b.co please") + .count(), + 0 + ); + } } diff --git a/src/unix.rs b/src/unix.rs index f1c6f6f..f9930d6 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -427,3 +427,18 @@ fn words<'a>(text: &'a str, word_chars: &'a WordChars) -> Words<'a> { pos: 0, } } + +#[cfg(test)] +mod word_tests { + use super::*; + #[test] + fn trim_keeps_interior_apostrophe() { + let wc = WordChars { + extra: HashSet::new(), + }; + let text = "'don't'"; + let (s, e, w) = trim_word_edges(text, 0, text.len(), &wc).unwrap(); + assert_eq!(w, "don't"); + assert_eq!(&text[s..e], "don't"); + } +} From cdd1fcf837e53d17a0be4796d7533ea9bdeb9182 Mon Sep 17 00:00:00 2001 From: Robert Mongold Date: Tue, 27 Oct 2026 11:03:50 -0400 Subject: [PATCH 3/4] Add documentation --- README.md | 2 +- src/lib.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bee0834..250014b 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ fn main() -> Result<(), spellkit::Error> { Linux also honors `DICPATH` (colon-separated directories) before `/usr/share/hunspell` and the other built-in paths. -Word breaks are **not** identical: Linux tokenizes alphanumeric / `'` runs; macOS and Windows use the OS checker. +Word breaks are **not** identical: Linux uses the dictionary `.aff` (`WORDCHARS` / `ICONV`), skips URLs and emails, and trims edge quotes; macOS and Windows use the OS checker. ## How it works diff --git a/src/lib.rs b/src/lib.rs index bb7c2ce..b79d27c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -208,9 +208,9 @@ impl Checker { /// Check `text` for spelling errors. /// - /// Ranges are UTF-8 byte offsets. Linux tokenizes words itself (alphanumeric - /// and `'`). macOS and Windows use the OS spell-checking APIs, so word breaks - /// may differ. + /// Ranges are UTF-8 byte offsets. Linux tokenizes with Hunspell `WORDCHARS` / + /// `ICONV` from the dictionary `.aff`, skips URLs and emails, and trims + /// leading/trailing quote-like word characters. macOS and Windows use the OS APIs. pub fn check<'a>( &'a self, text: &'a str, From c3ea3a86c32947010c66390df0d68e2f52552ace Mon Sep 17 00:00:00 2001 From: Robert Mongold Date: Tue, 27 Oct 2026 11:03:59 -0400 Subject: [PATCH 4/4] Add changelog entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f36142d..1894b78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,16 @@ This project follows [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Changed +- Linux `Checker::check` iterators now borrow the checker (needed for affix-based word breaks) + ### Fixed - Locale tags keep script and region case (`zh-Hans-CN`), and a POSIX encoding or `@modifier` is stripped before lookup - Linux opens the dictionary stem it was given (`en_US-large`) before the normalized name - Linux `Checker::new()` reads `LC_CTYPE` instead of `LC_MESSAGES`, and falls back to a bare `en` dictionary - `with_locale("en")` tries `en`, then `en_US`, then `en_GB` - macOS `with_locale` stores the installed language tag that matched, and accepts a bare language when only a regional tag is installed +- Linux tokenization uses Hunspell `WORDCHARS`/`ICONV`, keeps curly apostrophes in words, trims edge quotes, and skips URLs and emails ## [0.4.0] - 2026-08-25