Sentences

use sentences.qweruse sentences_split, sentences_spans from sentences      // Quillo 0.6: only these

unicode-segmentation = "1"serde_json = { version = "1", features = ["preserve_order"] }

// A news site: articles split into sentences for a reading aid, and meta descriptions of whole sentences.////   quillo run sentences.qwe      then open http://localhost:8080/articleuse "sentences.qwer"let ARTICLE = "Dr. Smith arrived at 3 p.m. on Jan. 5. She met J. R. R. Tolkien's biographer at St. Mary's, e.g. near the U.S. embassy. The bill came to $4.50, i.e. less than expected! \"Was it worth it?\" she asked. Nobody knew.\n\nNext steps\nThe board meets again in March."route GET /article {    let parts = sentences_split(ARTICLE, "en")    {        count: sentences_count(ARTICLE, "en"),        sentences: parts,        meta: sentences_preview(ARTICLE, 160, "en"),        tiny: sentences_preview(ARTICLE, 20, "en"),        german: sentences_split("Wir treffen uns z.B. bei Hrn. Müller in der Goethestr. 5. Er kommt ca. um 8 Uhr.", "de"),        french: sentences_split("M. Dupont habite av. Foch. Mme Martin aussi.", "fr"),        where: first(sentences_spans(ARTICLE, "en")),    }}

//! Text split into sentences the way a reader would: "Dr. Smith arrived at 3 p.m. on Jan. 5." stays one sentence, as do initials, decimals, "e.g." and quotes, with abbreviations for English, French, German, Spanish, Italian, Dutch and Portuguese; and previews of whole sentences that fit a length.//! version: 1.0.0//!//! [dependencies]//! unicode-segmentation = "1"//! serde_json = { version = "1", features = ["preserve_order"] }use serde_json::{json, Value};use unicode_segmentation::UnicodeSegmentation;const COMMON: &[&str] = &["e.g", "i.e", "etc", "vs", "approx", "ca", "cf", "no", "nos", "nr", "fig", "figs", "p", "pp", "vol", "vols", "ch", "ed", "eds", "al", "dr", "prof", "st", "mt", "inc", "ltd", "co", "corp", "jr", "sr", "a.m", "p.m", "u.s", "u.k", "u.n", "e.u", "ph.d", "m.d", "b.a", "m.a"];const EN: &[&str] = &["mr", "mrs", "ms", "messrs", "rev", "hon", "gen", "col", "capt", "sgt", "lt", "maj", "gov", "sen", "rep", "pres", "dept", "est", "jan", "feb", "mar", "apr", "jun", "jul", "aug", "sep", "sept", "oct", "nov", "dec", "mon", "tue", "wed", "thu", "fri", "sat", "sun", "ave", "blvd", "rd", "ft", "oz", "lb", "lbs", "min", "max", "misc", "dist", "univ", "assn", "bros"];const FR: &[&str] = &["m", "mm", "mme", "mmes", "mlle", "mlles", "me", "mgr", "av", "bd", "boul", "env", "art", "chap", "éd", "janv", "févr", "avr", "juil", "sept", "oct", "nov", "déc", "c.-à-d", "p. ex", "cie", "ste", "st"];const DE: &[&str] = &["z.b", "bzw", "usw", "u.a", "d.h", "u.ä", "o.ä", "s.o", "s.u", "vgl", "ggf", "inkl", "exkl", "evtl", "hr", "hrn", "fr", "frl", "str", "tel", "abs", "abt", "jh", "jhd", "mio", "mrd", "geb", "gest", "nr", "bspw", "allg", "sog", "zzt", "z.zt", "u.v.m", "max", "min"];const ES: &[&str] = &["sr", "sra", "srta", "dra", "ud", "uds", "vd", "vds", "pág", "págs", "art", "avda", "av", "c/", "dña", "d", "lic", "ing", "arq", "núm", "tel", "aprox", "ej", "p.ej", "ee.uu"];const IT: &[&str] = &["sig", "sigg", "sig.ra", "sig.na", "dott", "dott.ssa", "ing", "avv", "arch", "geom", "rag", "pag", "pagg", "tel", "ecc", "es", "ca", "c.a", "n", "nn", "v", "ss"];const NL: &[&str] = &["dhr", "mevr", "mw", "drs", "ir", "ing", "mr", "bijv", "bv", "o.a", "m.a.w", "d.w.z", "z.g.a.n", "enz", "ca", "nl", "resp", "tel", "jl", "blz", "evt", "i.p.v", "t.a.v", "m.b.t"];const PT: &[&str] = &["sr", "sra", "srta", "dr", "dra", "exmo", "exma", "v.exa", "pág", "págs", "av", "r", "tel", "aprox", "ex", "obs", "nº", "ltda"];fn abbreviations(lang: &str) -> Vec<&'static str> {    let mut v: Vec<&'static str> = COMMON.to_vec();    let extra: &[&str] = match lang.trim().to_lowercase().get(..2).unwrap_or("en") {        "fr" => FR, "de" => DE, "es" => ES, "it" => IT, "nl" => NL, "pt" => PT, _ => EN,    };    v.extend_from_slice(extra);    if !lang.to_lowercase().starts_with("en") { v.extend_from_slice(EN) }    v}// The word just before a final period, lower-cased, inner periods kept ("e.g", "u.s").fn last_token(s: &str) -> Option<String> {    let t = s.trim_end();    let t = t.trim_end_matches(|c: char| c == '"' || c == '\'' || c == ')' || c == '’' || c == '”');    let body = t.strip_suffix('.')?;    let word: String = body.chars().rev().take_while(|c| !c.is_whitespace() && *c != '(' && *c != '"' && *c != '“').collect::<Vec<_>>().into_iter().rev().collect();    Some(word.to_lowercase())}fn ends_in_abbreviation(seg: &str, abbr: &[&str], de: bool) -> bool {    let Some(w) = last_token(seg) else { return false };    if w.is_empty() { return false }    let chars: Vec<char> = w.chars().collect();    // An initial ("J."), or initials ("J.R.R").    if chars.len() == 1 && chars[0].is_alphabetic() { return true }    if w.split('.').all(|p| p.chars().count() == 1 && p.chars().all(|c| c.is_alphabetic())) { return true }    abbr.contains(&w.as_str()) || (de && w.ends_with("str") && w.chars().count() > 4)}fn spans(text: &str, lang: &str) -> Vec<(usize, usize)> {    let abbr = abbreviations(lang);    // German compounds end in abbreviations too: "Goethestr.".    let de = lang.trim().to_lowercase().starts_with("de");    let mut out: Vec<(usize, usize)> = Vec::new();    let mut pending: Option<(usize, usize)> = None;    for (start, seg) in text.split_sentence_bound_indices() {        let end = start + seg.len();        let span = match pending.take() { Some((s, _)) => (s, end), None => (start, end) };        let piece = &text[span.0..span.1];        let breaks_paragraph = seg.trim_end_matches([' ', '\t']).ends_with('\n');        // A piece that goes on in lower case ("\"Was it?\" she asked.") belongs to the sentence before it.        let continues = piece.trim_start().chars().find(|c| c.is_alphabetic()).map(|c| c.is_lowercase()).unwrap_or(false)            && piece.trim_start().chars().next().map(|c| c.is_alphabetic()).unwrap_or(false);        let span = match (continues, out.last()) {            (true, Some(&(ps, pe))) if !text[ps..pe].trim_end_matches([' ', '\t']).ends_with('\n') => { out.pop(); (ps, span.1) }            _ => span,        };        let piece = &text[span.0..span.1];        if !breaks_paragraph && ends_in_abbreviation(piece, &abbr, de) {            pending = Some(span);        } else {            out.push(span);        }    }    if let Some(p) = pending { out.push(p) }    // Trim each span of its surrounding whitespace, and drop the empty ones.    out.into_iter().filter_map(|(s, e)| {        let piece = &text[s..e];        let lead = piece.len() - piece.trim_start().len();        let trail = piece.len() - piece.trim_end().len();        if piece.trim().is_empty() { None } else { Some((s + lead, e - trail)) }    }).collect()}/// The sentences of `text`, trimmed, by the abbreviations of `lang` (`"en"`, `"fr"`, `"de"`, `"es"`, `"it"`, `"nl"`, `"pt"`; others use the English list): a line break between paragraphs always ends one.pub fn sentences_split(text: &str, lang: &str) -> Vec<String> {    spans(text, lang).into_iter().map(|(s, e)| text[s..e].to_string()).collect()}/// The sentences with where each is in `text`: `[{ text, start, end }]`, positions counting characters as Quillo's `slice` does, for highlighting.pub fn sentences_spans(text: &str, lang: &str) -> Vec<Value> {    spans(text, lang).into_iter().map(|(s, e)| {        let cs = text[..s].chars().count();        json!({ "text": &text[s..e], "start": cs, "end": cs + text[s..e].chars().count() })    }).collect()}/// How many sentences `text` has.pub fn sentences_count(text: &str, lang: &str) -> i64 {    spans(text, lang).len() as i64}/// As many whole sentences from the start of `text` as fit in `max_chars`, for meta descriptions and previews: never a sentence cut in the middle, except a first sentence longer than `max_chars`, cut at a word with "…".pub fn sentences_preview(text: &str, max_chars: i64, lang: &str) -> String {    let max = max_chars.max(1) as usize;    let mut out = String::new();    for (s, e) in spans(text, lang) {        let sentence = text[s..e].split_whitespace().collect::<Vec<_>>().join(" ");        let next = if out.is_empty() { sentence.clone() } else { format!("{} {}", out, sentence) };        if next.chars().count() <= max { out = next; continue }        if out.is_empty() {            let mut cut = String::new();            for w in sentence.split_whitespace() {                if cut.chars().count() + w.chars().count() + 2 > max { break }                if !cut.is_empty() { cut.push(' ') }                cut.push_str(w);            }            if cut.is_empty() { cut = sentence.chars().take(max.saturating_sub(1)).collect() }            out = format!("{}…", cut.trim_end_matches([',', ';', ':']));        }        break;    }    out}