diff --git a/CHANGELOG.md b/CHANGELOG.md index f36142d..1894b78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,16 @@ This project follows [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Changed +- Linux `Checker::check` iterators now borrow the checker (needed for affix-based word breaks) + ### Fixed - Locale tags keep script and region case (`zh-Hans-CN`), and a POSIX encoding or `@modifier` is stripped before lookup - Linux opens the dictionary stem it was given (`en_US-large`) before the normalized name - Linux `Checker::new()` reads `LC_CTYPE` instead of `LC_MESSAGES`, and falls back to a bare `en` dictionary - `with_locale("en")` tries `en`, then `en_US`, then `en_GB` - macOS `with_locale` stores the installed language tag that matched, and accepts a bare language when only a regional tag is installed +- Linux tokenization uses Hunspell `WORDCHARS`/`ICONV`, keeps curly apostrophes in words, trims edge quotes, and skips URLs and emails ## [0.4.0] - 2026-08-25 diff --git a/README.md b/README.md index bee0834..250014b 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ fn main() -> Result<(), spellkit::Error> { Linux also honors `DICPATH` (colon-separated directories) before `/usr/share/hunspell` and the other built-in paths. -Word breaks are **not** identical: Linux tokenizes alphanumeric / `'` runs; macOS and Windows use the OS checker. +Word breaks are **not** identical: Linux uses the dictionary `.aff` (`WORDCHARS` / `ICONV`), skips URLs and emails, and trims edge quotes; macOS and Windows use the OS checker. ## How it works diff --git a/src/lib.rs b/src/lib.rs index 53a7c39..b79d27c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -208,10 +208,13 @@ impl Checker { /// Check `text` for spelling errors. /// - /// Ranges are UTF-8 byte offsets. Linux tokenizes words itself (alphanumeric - /// and `'`). macOS and Windows use the OS spell-checking APIs, so word breaks - /// may differ. - pub fn check<'a>(&self, text: &'a str) -> impl Iterator + 'a + use<'a> { + /// Ranges are UTF-8 byte offsets. Linux tokenizes with Hunspell `WORDCHARS` / + /// `ICONV` from the dictionary `.aff`, skips URLs and emails, and trims + /// leading/trailing quote-like word characters. macOS and Windows use the OS APIs. + pub fn check<'a>( + &'a self, + text: &'a str, + ) -> impl Iterator + 'a + use<'a> { self.0.check(text).map(SpellingError) } @@ -549,4 +552,32 @@ mod tests { fn new_succeeds() { assert!(Checker::with_locale("en_US").is_ok()); } + + #[test] + #[cfg(all(unix, not(target_os = "macos")))] + fn curly_apostrophe_is_one_word() { + let checker = Checker::with_locale("en_US").unwrap(); + // U+2019 RIGHT SINGLE QUOTATION MARK between n and t + assert_eq!(checker.check("don\u{2019}t").count(), 0); + assert_eq!(checker.check("don't").count(), 0); + } + + #[test] + #[cfg(all(unix, not(target_os = "macos")))] + fn quoted_word_strips_quotes() { + let checker = Checker::with_locale("en_US").unwrap(); + assert_eq!(checker.check("'hello'").count(), 0); + } + + #[test] + #[cfg(all(unix, not(target_os = "macos")))] + fn skips_url_and_email() { + let checker = Checker::with_locale("en_US").unwrap(); + assert_eq!( + checker + .check("see https://example.com/foo and a@b.co please") + .count(), + 0 + ); + } } diff --git a/src/unix.rs b/src/unix.rs index 62e0ecb..f9930d6 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -1,5 +1,6 @@ use crate::Error; +use std::collections::HashSet; use std::ffi::{CStr, CString}; use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; @@ -47,7 +48,85 @@ fn find_dictionary(dirs: &[PathBuf], locales: &[&str]) -> Option<(PathBuf, PathB None } -fn open_dictionary(locales: &[&str]) -> Result<(*mut Hunhandle, String), Error> { +#[derive(Clone, Debug)] +struct WordChars { + /// Extra chars beyond alphanumeric and ASCII `'`. + extra: HashSet, +} + +impl WordChars { + fn contains(&self, c: char) -> bool { + c.is_alphanumeric() || c == '\'' || self.extra.contains(&c) + } + + fn from_aff(aff: &Path) -> Self { + let mut extra = HashSet::new(); + let Ok(contents) = std::fs::read_to_string(aff) else { + return Self { extra }; + }; + + let mut lines = contents.lines(); + while let Some(line) = lines.next() { + let line = line.trim(); + if line.is_empty() || line.starts_with("#") { + continue; + } + + if let Some(rest) = line.strip_prefix("WORDCHARS") { + for c in rest.trim().chars() { + if !c.is_whitespace() { + extra.insert(c); + } + } + continue; + } + + // `ICONV N` then N mapping lines: `from to` or `ICONV from to` + let mut parts = line.split_whitespace(); + if parts.next() != Some("ICONV") { + continue; + } + let Some(count_str) = parts.next() else { + continue; + }; + let Ok(count) = count_str.parse::() else { + continue; + }; + if parts.next().is_some() { + // Same line is a mapping (`ICONV ' '`), not a count. + continue; + } + + for _ in 0..count { + let Some(map_line) = lines.next() else { + break; + }; + let map_line = map_line.trim(); + let map_line = map_line.strip_prefix("ICONV").unwrap_or(map_line).trim(); + let mut map = map_line.split_whitespace(); + let Some(from) = map.next() else { + continue; + }; + let Some(to) = map.next() else { + continue; + }; + // Add ICONV sources whose dest is `'` or already a word char. + let to_is_word = to + .chars() + .all(|c| c.is_alphanumeric() || c == '\'' || extra.contains(&c)); + if to == "'" || to_is_word { + for c in from.chars() { + extra.insert(c); + } + } + } + } + + Self { extra } + } +} + +fn open_dictionary(locales: &[&str]) -> Result<(*mut Hunhandle, String, WordChars), Error> { let dirs = dict_search_dirs(); let Some((aff, dic, locale)) = find_dictionary(&dirs, locales) else { return Err(Error::DictionaryNotFound { @@ -55,6 +134,7 @@ fn open_dictionary(locales: &[&str]) -> Result<(*mut Hunhandle, String), Error> searched: dirs, }); }; + let word_chars = WordChars::from_aff(&aff); let label = Some(locale.clone()); let aff_c = CString::new(aff.as_os_str().as_bytes()).map_err(|_| Error::InitializationFailed { @@ -73,43 +153,50 @@ fn open_dictionary(locales: &[&str]) -> Result<(*mut Hunhandle, String), Error> message: "Hunspell_create returned null".into(), }); } - Ok((hunspell, locale)) + Ok((hunspell, locale, word_chars)) } #[derive(Debug)] pub struct Checker { hunspell: *mut Hunhandle, locale: String, + word_chars: WordChars, } impl Checker { pub fn new() -> Result { if let Some(raw) = crate::locale_from_vars(|key| std::env::var(key).ok()) { - let (hunspell, _) = crate::normalize_locale(&raw); - let mut candidates = crate::lookup_candidates(&raw, &hunspell); - if let Some((lang, _)) = hunspell.split_once('_') { + let (hunspell_tag, _) = crate::normalize_locale(&raw); + let mut candidates = crate::lookup_candidates(&raw, &hunspell_tag); + if let Some((lang, _)) = hunspell_tag.split_once('_') { if !candidates.contains(&lang) { candidates.push(lang); } } - if let Ok((handle, locale)) = open_dictionary(&candidates) { + if let Ok((handle, locale, word_chars)) = open_dictionary(&candidates) { return Ok(Checker { hunspell: handle, locale, + word_chars, }); } } - let (handle, locale) = open_dictionary(DEFAULT_LOCALES)?; + let (handle, locale, word_chars) = open_dictionary(DEFAULT_LOCALES)?; Ok(Checker { hunspell: handle, locale, + word_chars, }) } pub fn with_locale(raw: &str, hunspell_locale: &str, _bcp47: &str) -> Result { let candidates = crate::lookup_candidates(raw, hunspell_locale); - let (hunspell, locale) = open_dictionary(&candidates)?; - Ok(Checker { hunspell, locale }) + let (hunspell, locale, word_chars) = open_dictionary(&candidates)?; + Ok(Checker { + hunspell, + locale, + word_chars, + }) } pub fn locale(&self) -> &str { @@ -171,10 +258,10 @@ impl Checker { } } - pub fn check<'a>(&self, text: &'a str) -> impl Iterator + 'a + use<'a> { + pub fn check<'a>(&'a self, text: &'a str) -> impl Iterator + 'a { let hunspell = self.hunspell; - - words(text).filter_map(move |(start, end, word)| { + let word_chars = &self.word_chars; + words(text, word_chars).filter_map(move |(start, end, word)| { let cstr = CString::new(word).ok()?; let ok = unsafe { Hunspell_spell(hunspell, cstr.as_bytes_with_nul().as_ptr() as *const i8) } @@ -231,28 +318,127 @@ impl SpellingError { } } -fn is_word_char(c: char) -> bool { - c.is_alphanumeric() || c == '\'' +fn trim_word_edges<'a>( + text: &'a str, + start: usize, + end: usize, + wc: &WordChars, +) -> Option<(usize, usize, &'a str)> { + let mut s = start; + let mut e = end; + while s < e { + let c = text[s..e].chars().next()?; + if c.is_alphanumeric() { + break; + } + if !wc.contains(c) { + break; + } + s += c.len_utf8(); + } + while e > s { + let c = text[s..e].chars().next_back()?; + if c.is_alphanumeric() { + break; + } + if !wc.contains(c) { + break; + } + e -= c.len_utf8(); + } + if s >= e { + return None; + } + Some((s, e, &text[s..e])) } -fn words(text: &str) -> impl Iterator { - let mut words = Vec::new(); - let mut chars = text.char_indices().peekable(); - - while let Some((start, c)) = chars.next() { - if !is_word_char(c) { - continue; +/// If `text[from..]` starts a URL or email token, return the byte index after it. +fn skip_url_or_email(text: &str, from: usize) -> Option { + let rest = &text[from..]; + let token_end = rest + .find(|c: char| c.is_whitespace()) + .map(|o| from + o) + .unwrap_or(text.len()); + let token = &text[from..token_end]; + if token.contains("://") { + return Some(token_end); + } + if let Some(at) = token.find('@') { + let domain = &token[at + 1..]; + if domain.contains('.') { + return Some(token_end); } + } + None +} - let mut end = start + c.len_utf8(); - while let Some(&(i, next)) = chars.peek() { - if !is_word_char(next) { - break; +struct Words<'a> { + text: &'a str, + word_chars: &'a WordChars, + pos: usize, +} + +impl<'a> Iterator for Words<'a> { + type Item = (usize, usize, &'a str); + + fn next(&mut self) -> Option { + let bytes = self.text.as_bytes(); + while self.pos < bytes.len() { + // Skip whitespace. + let c = self.text[self.pos..].chars().next()?; + if c.is_whitespace() { + self.pos += c.len_utf8(); + continue; + } + + if let Some(after) = skip_url_or_email(self.text, self.pos) { + self.pos = after; + continue; + } + + if !self.word_chars.contains(c) { + self.pos += c.len_utf8(); + continue; + } + + let start = self.pos; + self.pos += c.len_utf8(); + while self.pos < bytes.len() { + let next = self.text[self.pos..].chars().next()?; + if !self.word_chars.contains(next) { + break; + } + self.pos += next.len_utf8(); + } + let end = self.pos; + + if let Some(item) = trim_word_edges(self.text, start, end, self.word_chars) { + return Some(item); } - end = i + next.len_utf8(); - chars.next(); } - words.push((start, end, &text[start..end])); + None + } +} + +fn words<'a>(text: &'a str, word_chars: &'a WordChars) -> Words<'a> { + Words { + text, + word_chars, + pos: 0, + } +} + +#[cfg(test)] +mod word_tests { + use super::*; + #[test] + fn trim_keeps_interior_apostrophe() { + let wc = WordChars { + extra: HashSet::new(), + }; + let text = "'don't'"; + let (s, e, w) = trim_word_edges(text, 0, text.len(), &wc).unwrap(); + assert_eq!(w, "don't"); + assert_eq!(&text[s..e], "don't"); } - words.into_iter() }