Keywords

use keywords.qweruse keywords_rake, keywords_textrank from keywords      // Quillo 0.6: only these

keyword_extraction = { version = "1.5", default-features = false, features = ["tf_idf", "rake", "text_rank"] }stop-words = "0.10"

// Tags suggested for a blog post, and the words that set a shop's products apart.////   quillo run keywords.qwe      then open http://localhost:8080/tagsuse "keywords.qwer"let post = "Solar panels on the roof of a house turn sunlight into electricity. A solar panel works best facing south, without shade. Battery storage keeps the electricity for the evening, so a house with solar panels and battery storage buys less electricity from the grid. The price of solar panels has fallen every year for a decade."route GET /tags = { rake: keywords_rake(post, 5, "en")?, textrank: keywords_textrank(post, 5, "en")? }route GET /products = keywords_tfidf([    "Leather wallet with coin pocket, brown leather, hand stitched",    "Canvas backpack with laptop pocket, waterproof canvas",    "Leather belt, black leather, brass buckle",    "Waterproof jacket with hood, breathable",], 5, "en")?

//! The keywords and key phrases of a text, for tags, SEO and "related posts": RAKE, TextRank and TF-IDF, with stop words of 58 languages left out.//! version: 1.0.0//!//! [dependencies]//! keyword_extraction = { version = "1.5", default-features = false, features = ["tf_idf", "rake", "text_rank"] }//! stop-words = "0.10"use keyword_extraction::rake::{Rake, RakeParams};use keyword_extraction::text_rank::{TextRank, TextRankParams};use keyword_extraction::tf_idf::{TfIdf, TfIdfParams};use serde_json::{json, Value};const MAX: usize = 1_000_000;fn stops(language: &str) -> Result<Vec<String>, String> {    let l = language.trim().to_lowercase();    let l = if l.is_empty() { "en".to_string() } else { l };    let mut w: Vec<String> = stop_words::lookup(&l).map(|w| w.iter().map(|s| s.to_string()).collect()).ok_or_else(|| format!("no stop words for {:?}", language))?;    w.push(BREAK.to_string());    Ok(w)}/// The crate skips punctuation instead of ending a phrase there ("shade. Battery" would be one/// phrase), so each mark that ends a phrase becomes a stop word of our own.const BREAK: &str = "qqbreakqq";fn breaks(text: &str) -> String {    let mut out = String::with_capacity(text.len() + 64);    for c in text.chars() {        if ".,;:!?()[]{}\"“”«»—–|/\n".contains(c) {            out.push(' ');            out.push_str(BREAK);            out.push(' ');        } else {            out.push(c);        }    }    out}fn scored(v: Vec<(String, f32)>, key: &str) -> Vec<Value> {    v.into_iter().map(|(w, s)| json!({ key: w, "score": ((s as f64) * 1000.0).round() / 1000.0 })).collect()}fn check(text: &str) -> Result<(), String> {    if text.len() > MAX { Err("text up to 1 MB".into()) } else { Ok(()) }}/// The `n` best key phrases of `text` by RAKE, as `[{ phrase, score }]`, best first: runs of words between stop words and punctuation, scored by how their words are used together. Quick, good for one article. `language` is a two-letter code for the stop words.pub fn keywords_rake(text: &str, n: i64, language: &str) -> Result<Vec<Value>, String> {    check(text)?;    let sw = stops(language)?;    let r = Rake::new(RakeParams::WithDefaultsAndPhraseLength(&breaks(text), &sw, Some(4)));    Ok(scored(r.get_ranked_phrases_scores(n.clamp(1, 100) as usize), "phrase"))}/// The `n` best key phrases of `text` by TextRank, as `[{ phrase, score }]`: words ranked by the words around them, as PageRank ranks pages. Slower than RAKE, often better on long texts.pub fn keywords_textrank(text: &str, n: i64, language: &str) -> Result<Vec<Value>, String> {    check(text)?;    let sw = stops(language)?;    let r = TextRank::new(TextRankParams::WithDefaultsAndPhraseLength(&breaks(text), &sw, Some(3)));    Ok(scored(r.get_ranked_phrase_scores(n.clamp(1, 100) as usize), "phrase"))}/// The `n` words that best set `documents` apart from one another, by TF-IDF, as `[{ word, score }]`: common in some documents, rare in the rest. Give it a set of posts, products or tickets. Up to 10,000 documents.pub fn keywords_tfidf(documents: Vec<String>, n: i64, language: &str) -> Result<Vec<Value>, String> {    if documents.len() > 10_000 || documents.iter().map(|d| d.len()).sum::<usize>() > 20 * MAX {        return Err("up to 10,000 documents and 20 MB".into());    }    let sw = stops(language)?;    let t = TfIdf::new(TfIdfParams::UnprocessedDocuments(&documents, &sw, None));    Ok(scored(t.get_ranked_word_scores(n.clamp(1, 1000) as usize), "word"))}