Summaries

use summarize.qweruse summarize_text, summarize_ratio from summarize      // Quillo 0.6: only these

unicode-segmentation = "1"serde_json = { version = "1", features = ["preserve_order"] }

// A news digest: each article's two key sentences for the e-mail, and the key sentences highlighted on the page.////   quillo run summarize.qwe      then open http://localhost:8080/digestuse "summarize.qwer"let ARTICLE = "The city council approved a new plan for electric buses on Tuesday. The plan replaces the diesel buses on twelve routes with electric buses by 2028. Council members said the electric buses will cut noise and air pollution in the city centre. The weather on Tuesday was mild, and the meeting started late. Charging stations for the buses will be built at the two main depots. The plan costs 40 million euros, half of it paid by a national grant for clean transport. Some residents asked whether the charging stations at the depots would be ready in time. The mayor thanked the staff for their work. Bus drivers will be trained to drive the electric buses next year."route GET /digest {    let ranked = summarize_ranked(ARTICLE, "en")    {        summary: summarize_text(ARTICLE, 2, "en"),        third: summarize_ratio(ARTICLE, 0.34, "en")?,        top: first(ranked),        least: last(ranked).text,        french: summarize_text("Le conseil a voté un nouveau budget pour les écoles. Le budget des écoles augmente de dix pour cent. Il faisait beau ce jour-là. Les écoles recevront le budget en janvier.", 2, "fr"),    }}

//! Extractive summaries: the sentences that matter most in an article, found by TextRank (sentences ranked by how much they share with the others), given back in their order, as a count or a share of the text; for previews, digests and "TL;DR" boxes. English, French, German, Spanish, Italian, Dutch and Portuguese stop words built in.//! version: 1.0.0//!//! [dependencies]//! unicode-segmentation = "1"//! serde_json = { version = "1", features = ["preserve_order"] }use serde_json::{json, Value};use std::collections::HashSet;use unicode_segmentation::UnicodeSegmentation;const COMMON: &[&str] = &["e.g", "i.e", "etc", "vs", "approx", "ca", "cf", "no", "nos", "nr", "fig", "figs", "p", "pp", "vol", "vols", "ch", "ed", "eds", "al", "dr", "prof", "st", "mt", "inc", "ltd", "co", "corp", "jr", "sr", "a.m", "p.m", "u.s", "u.k", "u.n", "e.u", "ph.d", "m.d", "b.a", "m.a"];const EN: &[&str] = &["mr", "mrs", "ms", "messrs", "rev", "hon", "gen", "col", "capt", "sgt", "lt", "maj", "gov", "sen", "rep", "pres", "dept", "est", "jan", "feb", "mar", "apr", "jun", "jul", "aug", "sep", "sept", "oct", "nov", "dec", "mon", "tue", "wed", "thu", "fri", "sat", "sun", "ave", "blvd", "rd", "ft", "oz", "lb", "lbs", "min", "max", "misc", "dist", "univ", "assn", "bros"];const FR: &[&str] = &["m", "mm", "mme", "mmes", "mlle", "mlles", "me", "mgr", "av", "bd", "boul", "env", "art", "chap", "éd", "janv", "févr", "avr", "juil", "sept", "oct", "nov", "déc", "c.-à-d", "p. ex", "cie", "ste", "st"];const DE: &[&str] = &["z.b", "bzw", "usw", "u.a", "d.h", "u.ä", "o.ä", "s.o", "s.u", "vgl", "ggf", "inkl", "exkl", "evtl", "hr", "hrn", "fr", "frl", "str", "tel", "abs", "abt", "jh", "jhd", "mio", "mrd", "geb", "gest", "nr", "bspw", "allg", "sog", "zzt", "z.zt", "u.v.m", "max", "min"];const ES: &[&str] = &["sr", "sra", "srta", "dra", "ud", "uds", "vd", "vds", "pág", "págs", "art", "avda", "av", "c/", "dña", "d", "lic", "ing", "arq", "núm", "tel", "aprox", "ej", "p.ej", "ee.uu"];const IT: &[&str] = &["sig", "sigg", "sig.ra", "sig.na", "dott", "dott.ssa", "ing", "avv", "arch", "geom", "rag", "pag", "pagg", "tel", "ecc", "es", "ca", "c.a", "n", "nn", "v", "ss"];const NL: &[&str] = &["dhr", "mevr", "mw", "drs", "ir", "ing", "mr", "bijv", "bv", "o.a", "m.a.w", "d.w.z", "z.g.a.n", "enz", "ca", "nl", "resp", "tel", "jl", "blz", "evt", "i.p.v", "t.a.v", "m.b.t"];const PT: &[&str] = &["sr", "sra", "srta", "dr", "dra", "exmo", "exma", "v.exa", "pág", "págs", "av", "r", "tel", "aprox", "ex", "obs", "nº", "ltda"];fn abbreviations(lang: &str) -> Vec<&'static str> {    let mut v: Vec<&'static str> = COMMON.to_vec();    let extra: &[&str] = match lang.trim().to_lowercase().get(..2).unwrap_or("en") {        "fr" => FR, "de" => DE, "es" => ES, "it" => IT, "nl" => NL, "pt" => PT, _ => EN,    };    v.extend_from_slice(extra);    if !lang.to_lowercase().starts_with("en") { v.extend_from_slice(EN) }    v}// The word just before a final period, lower-cased, inner periods kept ("e.g", "u.s").fn last_token(s: &str) -> Option<String> {    let t = s.trim_end();    let t = t.trim_end_matches(|c: char| c == '"' || c == '\'' || c == ')' || c == '’' || c == '”');    let body = t.strip_suffix('.')?;    let word: String = body.chars().rev().take_while(|c| !c.is_whitespace() && *c != '(' && *c != '"' && *c != '“').collect::<Vec<_>>().into_iter().rev().collect();    Some(word.to_lowercase())}fn ends_in_abbreviation(seg: &str, abbr: &[&str], de: bool) -> bool {    let Some(w) = last_token(seg) else { return false };    if w.is_empty() { return false }    let chars: Vec<char> = w.chars().collect();    // An initial ("J."), or initials ("J.R.R").    if chars.len() == 1 && chars[0].is_alphabetic() { return true }    if w.split('.').all(|p| p.chars().count() == 1 && p.chars().all(|c| c.is_alphabetic())) { return true }    abbr.contains(&w.as_str()) || (de && w.ends_with("str") && w.chars().count() > 4)}fn spans(text: &str, lang: &str) -> Vec<(usize, usize)> {    let abbr = abbreviations(lang);    // German compounds end in abbreviations too: "Goethestr.".    let de = lang.trim().to_lowercase().starts_with("de");    let mut out: Vec<(usize, usize)> = Vec::new();    let mut pending: Option<(usize, usize)> = None;    for (start, seg) in text.split_sentence_bound_indices() {        let end = start + seg.len();        let span = match pending.take() { Some((s, _)) => (s, end), None => (start, end) };        let piece = &text[span.0..span.1];        let breaks_paragraph = seg.trim_end_matches([' ', '\t']).ends_with('\n');        // A piece that goes on in lower case ("\"Was it?\" she asked.") belongs to the sentence before it.        let continues = piece.trim_start().chars().find(|c| c.is_alphabetic()).map(|c| c.is_lowercase()).unwrap_or(false)            && piece.trim_start().chars().next().map(|c| c.is_alphabetic()).unwrap_or(false);        let span = match (continues, out.last()) {            (true, Some(&(ps, pe))) if !text[ps..pe].trim_end_matches([' ', '\t']).ends_with('\n') => { out.pop(); (ps, span.1) }            _ => span,        };        let piece = &text[span.0..span.1];        if !breaks_paragraph && ends_in_abbreviation(piece, &abbr, de) {            pending = Some(span);        } else {            out.push(span);        }    }    if let Some(p) = pending { out.push(p) }    // Trim each span of its surrounding whitespace, and drop the empty ones.    out.into_iter().filter_map(|(s, e)| {        let piece = &text[s..e];        let lead = piece.len() - piece.trim_start().len();        let trail = piece.len() - piece.trim_end().len();        if piece.trim().is_empty() { None } else { Some((s + lead, e - trail)) }    }).collect()}const STOP_EN: &str = "a about above after again against all am an and any are as at be because been before being below between both but by can could did do does doing down during each few for from further had has have having he her here hers herself him himself his how i if in into is it its itself just me more most my myself no nor not now of off on once only or other our ours ourselves out over own same she should so some such than that the their theirs them themselves then there these they this those through to too under until up very was we were what when where which while who whom why will with would you your yours yourself yourselves also its it's said says one two new";const STOP_FR: &str = "au aux avec ce ces dans de des du elle en et eux il ils je la le les leur lui ma mais me même mes moi mon ne nos notre nous on ou par pas pour qu que qui sa se ses son sur ta te tes toi ton tu un une vos votre vous c d j l à m n s t y été est sont ont a plus cette comme fait";const STOP_DE: &str = "aber als am an auch auf aus bei bin bis bist da dadurch daher darum das dass dein deine dem den der des dessen deshalb die dies dieser dieses doch dort du durch ein eine einem einen einer eines er es euer eure für hatte hatten hattest hattet hier hinter ich ihr ihre im in ist ja jede jedem jeden jeder jedes jener jenes jetzt kann kannst können könnt machen mein meine mit muß mußt musst müssen müßt nach nachdem nein nicht nun oder seid sein seine sich sie sind soll sollen sollst sollt sonst soweit sowie und unser unsere unter vom von vor wann warum was weiter weitere wenn wer werde werden werdet weshalb wie wieder wieso wir wird wirst wo woher wohin zu zum zur über wurde wurden";const STOP_ES: &str = "a al algo algunos ante antes como con contra cual cuando de del desde donde durante e el ella ellas ellos en entre era es esa esas ese eso esos esta estas este esto estos fue fueron ha han hasta hay la las le les lo los mas me mi mis mucho muy más ni no nos o otra otros para pero poco por porque que quien se ser si sido sin sobre son su sus también tanto te tiene todo todos tu tus un una uno unos y ya";const STOP_IT: &str = "a ad al alla alle anche avere che chi con cui da dal dalla dei del della delle di dove e ed era essere gli ha hanno i il in io la le lei lo loro lui ma mi ne nei nel nella non o per perché più quale quando questa questo se si sia siamo sono su sua sue suo suoi tra tu un una uno è stato";const STOP_NL: &str = "aan al alles als altijd andere ben bij daar dan dat de der deze die dit doch doen door dus een eens en er ge geen geweest haar had heb hebben heeft hem het hier hij hoe hun iemand iets ik in is ja je kan kon kunnen maar me meer men met mij mijn moet na naar niet niets nog nu of om omdat onder ons ook op over reeds te tegen toch toen tot u uit uw van veel voor want waren was wat werd wezen wie wil worden wordt zal ze zelf zich zij zijn zo zonder zou";const STOP_PT: &str = "a ao aos as até com como da das de dela dele deles do dos e ela elas ele eles em entre era essa esse esta este eu foi foram há isso isto já lhe mais mas me mesmo meu minha muito na nas nem no nos nossa nosso num numa não o os ou para pela pelas pelo pelos por qual quando que quem se sem ser seu seus sua suas são só também te tem um uma umas uns você é";fn stop_words(lang: &str) -> HashSet<String> {    let list = match lang.trim().to_lowercase().get(..2).unwrap_or("en") {        "fr" => STOP_FR, "de" => STOP_DE, "es" => STOP_ES, "it" => STOP_IT, "nl" => STOP_NL, "pt" => STOP_PT, _ => STOP_EN,    };    list.split_whitespace().map(|w| w.to_string()).collect()}fn words(s: &str, stop: &HashSet<String>) -> HashSet<String> {    s.unicode_words()        .map(|w| w.to_lowercase())        .filter(|w| w.chars().count() > 1 && !stop.contains(w) && !w.chars().all(|c| c.is_numeric()))        // A light stem: plurals and verb endings, so "prices" meets "price".        .map(|w| {            for suf in ["ing", "ed", "es", "s"] {                if w.len() > suf.len() + 3 && w.ends_with(suf) { return w[..w.len() - suf.len()].to_string() }            }            w        })        .collect()}struct Ranked { text: String, position: usize, score: f64 }fn rank(text: &str, lang: &str) -> Vec<Ranked> {    let sp = spans(text, lang);    let sents: Vec<String> = sp.iter().map(|(s, e)| text[*s..*e].split_whitespace().collect::<Vec<_>>().join(" ")).collect();    let stop = stop_words(lang);    let bags: Vec<HashSet<String>> = sents.iter().map(|s| words(s, &stop)).collect();    let n = sents.len();    let mut w = vec![vec![0.0f64; n]; n];    for i in 0..n {        for j in (i + 1)..n {            let (a, b) = (&bags[i], &bags[j]);            if a.len() < 2 || b.len() < 2 { continue }            let common = a.intersection(b).count() as f64;            if common == 0.0 { continue }            let sim = common / ((a.len() as f64).ln() + (b.len() as f64).ln());            w[i][j] = sim;            w[j][i] = sim;        }    }    let out: Vec<f64> = w.iter().map(|r| r.iter().sum()).collect();    let mut score = vec![1.0f64; n];    for _ in 0..60 {        let mut next = vec![0.15f64; n];        for i in 0..n {            for j in 0..n {                if w[j][i] > 0.0 && out[j] > 0.0 { next[i] += 0.85 * w[j][i] / out[j] * score[j] }            }        }        let delta: f64 = next.iter().zip(&score).map(|(a, b)| (a - b).abs()).sum();        score = next;        if delta < 1e-6 { break }    }    sents.into_iter().enumerate().map(|(i, t)| Ranked { text: t, position: i, score: score[i] }).collect()}fn top(text: &str, count: usize, lang: &str) -> Vec<Ranked> {    let mut r = rank(text, lang);    r.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal).then(a.position.cmp(&b.position)));    r.truncate(count);    r.sort_by_key(|x| x.position);    r}/// A summary of `text` in its `count` most central sentences, in the order they come, joined by spaces. Text with no more sentences than that comes back whole. `lang` (`"en"`, `"fr"`, `"de"`...) picks the stop words and abbreviations.pub fn summarize_text(text: &str, count: i64, lang: &str) -> String {    top(text, count.max(1) as usize, lang).into_iter().map(|r| r.text).collect::<Vec<_>>().join(" ")}/// A summary of about `ratio` of the sentences (0.3 keeps 3 in 10, at least one), in their order.pub fn summarize_ratio(text: &str, ratio: f64, lang: &str) -> Result<String, String> {    if !(0.0..=1.0).contains(&ratio) { return Err(format!("the ratio is 0 to 1, not {}", ratio)) }    let n = spans(text, lang).len();    Ok(summarize_text(text, ((n as f64 * ratio).round() as i64).max(1), lang))}/// Every sentence with its TextRank score, most central first: `[{ text, position, score }]`, `position` counted from 0, `score` rounded to 4 places. For highlighting key sentences in place.pub fn summarize_ranked(text: &str, lang: &str) -> Vec<Value> {    let mut r = rank(text, lang);    r.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal).then(a.position.cmp(&b.position)));    r.into_iter().map(|x| json!({ "text": x.text, "position": x.position, "score": (x.score * 10000.0).round() / 10000.0 })).collect()}