Spellcheck

use spellcheck.qweruse spell_load, spell_add from spellcheck      // Quillo 0.6: only these

strsim = "0.11"unicode-segmentation = "1"serde_json = { version = "1", features = ["preserve_order"] }

// A support form that checks spelling before sending, and a search box that asks "did you mean".// spellcheck_sample.txt is a small English list; load a full one (a word and its count per line) for real use.////   quillo run spellcheck.qwe      then open http://localhost:8080/checkuse "spellcheck.qwer"let WORDS = spell_load("en", "spellcheck_sample.txt")?let OURS = spell_add("en", ["Quillo", "espresso", "barista"])route GET /check {    let message = "Hello, I recieved my ordr yesterday but the pakage was opend. Can I get a refnd? Order 1042, me@example.com"    {        words: WORDS,        problems: spell_check("en", message)?,        corrected: spell_correct("en", message)?,        suggest: spell_suggest("en", "beleive", 3)?,        upper: spell_suggest("en", "Quilo", 1)?,        known: [spell_known("en", "barista")?, spell_known("en", "baristta")?],    }}// /search?q=cofee+delivry: "did you mean coffee delivery?"route GET /search ? { q: str } {    let fixed = spell_correct("en", q)?    { query: q, did_you_mean: fixed == q ? null : fixed }}

//! Spelling checked against a word list you load (any language: a word per line, with how common it is if you know): unknown words found with their places, suggestions ranked by closeness and frequency (SymSpell's fast deletion method), text corrected, and your own words (product names, jargon) added.//! version: 1.0.0//!//! [dependencies]//! strsim = "0.11"//! unicode-segmentation = "1"//! serde_json = { version = "1", features = ["preserve_order"] }use serde_json::{json, Value};use std::collections::HashMap;use std::sync::{Arc, Mutex, OnceLock, RwLock};use unicode_segmentation::UnicodeSegmentation;const MAX_EDIT: usize = 2;const PREFIX: usize = 7;#[derive(Default)]struct Dict { counts: HashMap<String, u64>, deletes: HashMap<String, Vec<u32>>, words: Vec<String> }impl Dict {    fn add(&mut self, word: &str, count: u64) {        let w = word.to_lowercase();        if w.is_empty() { return }        if let Some(c) = self.counts.get_mut(&w) { *c = (*c).max(count); return }        self.counts.insert(w.clone(), count);        let id = self.words.len() as u32;        let key: String = w.chars().take(PREFIX).collect();        for d in deletes(&key) { self.deletes.entry(d).or_default().push(id) }        self.words.push(w);    }    fn suggest(&self, word: &str, max: usize) -> Vec<(String, usize, u64)> {        let w = word.to_lowercase();        if let Some(c) = self.counts.get(&w) { return vec![(w, 0, *c)] }        let key: String = w.chars().take(PREFIX).collect();        let mut seen: HashMap<u32, ()> = HashMap::new();        let mut out: Vec<(String, usize, u64)> = Vec::new();        for d in deletes(&key) {            if let Some(ids) = self.deletes.get(&d) {                for &id in ids {                    if seen.insert(id, ()).is_some() { continue }                    let cand = &self.words[id as usize];                    let dist = strsim::osa_distance(&w, cand);                    if dist <= MAX_EDIT { out.push((cand.clone(), dist, self.counts[cand])) }                }            }        }        out.sort_by(|a, b| a.1.cmp(&b.1).then(b.2.cmp(&a.2)).then(a.0.cmp(&b.0)));        out.truncate(max);        out    }}// Every string made by deleting up to MAX_EDIT characters, the word itself included.fn deletes(w: &str) -> Vec<String> {    let mut all = vec![w.to_string()];    let mut frontier = vec![w.to_string()];    for _ in 0..MAX_EDIT {        let mut next = Vec::new();        for s in &frontier {            let c: Vec<char> = s.chars().collect();            if c.len() <= 1 { continue }            for i in 0..c.len() {                let d: String = c[..i].iter().chain(c[i + 1..].iter()).collect();                if !all.contains(&d) { all.push(d.clone()); next.push(d) }            }        }        frontier = next;    }    all}fn store() -> &'static Mutex<HashMap<String, Arc<RwLock<Dict>>>> {    static S: OnceLock<Mutex<HashMap<String, Arc<RwLock<Dict>>>>> = OnceLock::new();    S.get_or_init(|| Mutex::new(HashMap::new()))}fn dict(name: &str) -> Result<Arc<RwLock<Dict>>, String> {    store().lock().map_err(|_| "the dictionaries are unavailable".to_string())?.get(name).cloned().ok_or_else(|| format!("no dictionary `{}`: load one with spell_load", name))}// The words of a text to check, with their character positions; numbers, words with digits, e-mail addresses and links skipped.fn words(text: &str) -> Vec<(String, usize, usize)> {    let mut out = Vec::new();    let mut char_at = 0;    let mut last_byte = 0;    for (b, w) in text.split_word_bound_indices() {        char_at += text[last_byte..b].chars().count();        last_byte = b;        let n = w.chars().count();        let skip = !w.chars().any(|c| c.is_alphabetic()) || w.chars().any(|c| c.is_numeric()) || w.contains('@') || w.contains('.') || text[..b].ends_with('@')            || text[b + w.len()..].starts_with("://") || text[..b].ends_with("://") || text[..b].ends_with('.') && !text[..b].ends_with(". ");        if !skip { out.push((w.to_string(), char_at, char_at + n)) }    }    out}fn with_case(original: &str, word: &str) -> String {    if original.chars().all(|c| !c.is_lowercase()) && original.chars().count() > 1 { return word.to_uppercase() }    if original.chars().next().map(|c| c.is_uppercase()).unwrap_or(false) {        let mut c = word.chars();        return c.next().map(|f| f.to_uppercase().collect::<String>() + c.as_str()).unwrap_or_default();    }    word.to_string()}/// Loads a word list from `path` as the dictionary `name` (loading again replaces it): a word per line, optionally followed by a space or tab and how common it is (higher first in suggestions); lines starting with `#` are comments. The number of words.pub fn spell_load(name: &str, path: &str) -> Result<i64, String> {    let text = std::fs::read_to_string(path).map_err(|e| format!("{}: {}", path, e))?;    let mut d = Dict::default();    for line in text.lines() {        let l = line.trim();        if l.is_empty() || l.starts_with('#') { continue }        let mut parts = l.split_whitespace();        let w = parts.next().unwrap_or("");        let c = parts.next().and_then(|c| c.parse::<u64>().ok()).unwrap_or(1);        d.add(w, c);    }    let n = d.words.len() as i64;    store().lock().map_err(|_| "the dictionaries are unavailable".to_string())?.insert(name.to_string(), Arc::new(RwLock::new(d)));    Ok(n)}/// Adds words to the dictionary `name` (made if it does not exist): your brand, product names, jargon. They count as common words. The dictionary's size after.pub fn spell_add(name: &str, words: Vec<String>) -> Result<i64, String> {    let d = { let mut s = store().lock().map_err(|_| "the dictionaries are unavailable".to_string())?; s.entry(name.to_string()).or_default().clone() };    let mut d = d.write().map_err(|_| "the dictionary is unavailable".to_string())?;    for w in words { d.add(w.trim(), 1_000_000) }    Ok(d.words.len() as i64)}/// Whether `word` is in the dictionary (any case).pub fn spell_known(name: &str, word: &str) -> Result<bool, String> {    Ok(dict(name)?.read().map_err(|_| "the dictionary is unavailable".to_string())?.counts.contains_key(&word.to_lowercase()))}/// Up to `max` suggestions for `word`, closest first, then most common: `[{ word, distance }]` (edits: a letter added, removed, changed, or two swapped; at most 2). A known word is its own only suggestion, at distance 0.pub fn spell_suggest(name: &str, word: &str, max: i64) -> Result<Vec<Value>, String> {    let d = dict(name)?;    let d = d.read().map_err(|_| "the dictionary is unavailable".to_string())?;    Ok(d.suggest(word.trim(), max.max(1) as usize).into_iter().map(|(w, dist, _)| json!({ "word": with_case(word.trim(), &w), "distance": dist })).collect())}/// The unknown words of `text` with where they are and what they might be: `[{ word, start, end, suggestions }]`, positions in characters as `slice` counts, up to 3 suggestions each (in the word's case). Numbers, words with digits, e-mail addresses and links are not checked.pub fn spell_check(name: &str, text: &str) -> Result<Vec<Value>, String> {    let d = dict(name)?;    let d = d.read().map_err(|_| "the dictionary is unavailable".to_string())?;    Ok(words(text).into_iter().filter(|(w, _, _)| !d.counts.contains_key(&w.to_lowercase())).map(|(w, s, e)| {        let sug: Vec<String> = d.suggest(&w, 3).into_iter().map(|x| with_case(&w, &x.0)).collect();        json!({ "word": w, "start": s, "end": e, "suggestions": sug })    }).collect())}/// `text` with each unknown word replaced by its best suggestion, its case kept; a word with no suggestion is left as it is. For search queries ("did you mean") more than for others' writing.pub fn spell_correct(name: &str, text: &str) -> Result<String, String> {    let d = dict(name)?;    let d = d.read().map_err(|_| "the dictionary is unavailable".to_string())?;    let chars: Vec<char> = text.chars().collect();    let mut out = String::new();    let mut pos = 0;    for (w, s, e) in words(text) {        if d.counts.contains_key(&w.to_lowercase()) { continue }        if let Some(best) = d.suggest(&w, 1).into_iter().next() {            out.extend(&chars[pos..s]);            out.push_str(&with_case(&w, &best.0));            pos = e;        }    }    out.extend(&chars[pos..]);    Ok(out)}