Full-text search

use search.qweruse search_add, search_query from search      // Quillo 0.6: only these

tantivy = "0.27"

// Search over a help center's articles, with highlighted snippets.////   quillo run search.qwe      then open http://localhost:8080/search?q=refunduse "search.qwer"let INDEX = "data/search-index"let articles = [    { id: "returns", title: "Returns and refunds", body: "You can return any item within 30 days. Refunds go back to the card you paid with, in 5 to 10 days." },    { id: "shipping", title: "Shipping times", body: "Orders ship in 2 days. Express shipping arrives the next day in most cities." },    { id: "account", title: "Changing your password", body: "Open Settings, then Security, and choose a new password. A refund of a gift card is not possible." },]route GET /reindex = { added: search_add(INDEX, articles)?, count: search_count(INDEX)? }route GET /search ? { q: str } {    if search_count(INDEX)? == 0 { search_add(INDEX, articles)? }    search_query(INDEX, q, 10)?}route GET /delete ? { id: str } {    search_delete(INDEX, id)?    { count: search_count(INDEX)? }}

//! Full-text search for a site or an app: documents indexed in a folder, found by words (English stems, so "refunds" finds "refund") with ranking and highlighted snippets.//! version: 1.0.0//!//! [dependencies]//! tantivy = "0.27"use serde_json::{json, Value};use std::collections::HashMap;use std::sync::{Arc, Mutex};use tantivy::collector::TopDocs;use tantivy::query::QueryParser;use tantivy::schema::{Field, IndexRecordOption, Schema, TextFieldIndexing, TextOptions, Value as _, STORED, STRING};use tantivy::snippet::SnippetGenerator;use tantivy::{doc, Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term};struct Store {    index: Index,    writer: Mutex<IndexWriter>,    reader: IndexReader,    id: Field,    title: Field,    body: Field,}static STORES: Mutex<Option<HashMap<String, Arc<Store>>>> = Mutex::new(None);fn open(dir: &str) -> Result<Arc<Store>, String> {    let mut all = STORES.lock().unwrap_or_else(|e| e.into_inner());    let all = all.get_or_insert_with(HashMap::new);    if let Some(s) = all.get(dir) {        return Ok(s.clone());    }    std::fs::create_dir_all(dir).map_err(|e| format!("{}: {}", dir, e))?;    let index = match Index::open_in_dir(dir) {        Ok(i) => i,        Err(_) => {            let mut sb = Schema::builder();            sb.add_text_field("id", STRING | STORED);            // English stemming: "refunds" finds "refund".            let text = TextOptions::default().set_stored().set_indexing_options(                TextFieldIndexing::default().set_tokenizer("en_stem").set_index_option(IndexRecordOption::WithFreqsAndPositions));            sb.add_text_field("title", text.clone());            sb.add_text_field("body", text);            Index::create_in_dir(dir, sb.build()).map_err(|e| e.to_string())?        }    };    let schema = index.schema();    let field = |n: &str| schema.get_field(n).map_err(|_| format!("{} is not an index made by this plugin", dir));    let (id, title, body) = (field("id")?, field("title")?, field("body")?);    let writer = index.writer_with_num_threads(1, 20_000_000).map_err(|e| e.to_string())?;    let reader = index.reader_builder().reload_policy(ReloadPolicy::Manual).try_into().map_err(|e: tantivy::TantivyError| e.to_string())?;    let s = Arc::new(Store { index, writer: Mutex::new(writer), reader, id, title, body });    all.insert(dir.to_string(), s.clone());    Ok(s)}fn text(v: &Value, k: &str) -> String {    match v.get(k) {        Some(Value::String(s)) => s.clone(),        Some(Value::Null) | None => String::new(),        Some(o) => o.to_string(),    }}/// Adds documents to the index in folder `dir` (made if missing): each a map `{ id, title, body }`. A document with an id already there replaces it. Up to 10,000 per call.pub fn search_add(dir: &str, docs: Vec<serde_json::Value>) -> Result<i64, String> {    if docs.len() > 10_000 {        return Err("at most 10,000 documents per call".into());    }    let s = open(dir)?;    let mut w = s.writer.lock().unwrap_or_else(|e| e.into_inner());    for d in &docs {        let id = text(d, "id");        if id.is_empty() {            return Err("each document needs an id".into());        }        w.delete_term(Term::from_field_text(s.id, &id));        w.add_document(doc!(s.id => id, s.title => text(d, "title"), s.body => text(d, "body"))).map_err(|e| e.to_string())?;    }    w.commit().map_err(|e| e.to_string())?;    s.reader.reload().map_err(|e| e.to_string())?;    Ok(docs.len() as i64)}/// The documents matching `query`, best first, as `[{ id, title, score, snippet }]`: words in the title count double; `snippet` is HTML with the matched words in `<b>`. The query takes `"exact phrases"`, `-word` to leave out, `title:word` and `word*` prefixes. At most `limit` (up to 100).pub fn search_query(dir: &str, query: &str, limit: i64) -> Result<Vec<Value>, String> {    let s = open(dir)?;    let searcher = s.reader.searcher();    let mut qp = QueryParser::for_index(&s.index, vec![s.title, s.body]);    qp.set_field_boost(s.title, 2.0);    let (q, _) = qp.parse_query_lenient(query);    let top = searcher.search(&q, &TopDocs::with_limit(limit.clamp(1, 100) as usize).order_by_score()).map_err(|e| e.to_string())?;    let snip = SnippetGenerator::create(&searcher, &*q, s.body).map_err(|e| e.to_string())?;    let mut out = Vec::new();    for (score, addr) in top {        let d: TantivyDocument = searcher.doc(addr).map_err(|e| e.to_string())?;        let get = |f: Field| d.get_first(f).and_then(|v| v.as_str()).unwrap_or("").to_string();        let mut snippet = snip.snippet_from_doc(&d).to_html();        if snippet.is_empty() {            snippet = get(s.body).chars().take(150).collect();        }        out.push(json!({ "id": get(s.id), "title": get(s.title), "score": (score as f64 * 1000.0).round() / 1000.0, "snippet": snippet }));    }    Ok(out)}/// Removes the document with `id` from the index; nothing happens when there is none.pub fn search_delete(dir: &str, id: &str) -> Result<(), String> {    let s = open(dir)?;    let mut w = s.writer.lock().unwrap_or_else(|e| e.into_inner());    w.delete_term(Term::from_field_text(s.id, id));    w.commit().map_err(|e| e.to_string())?;    s.reader.reload().map_err(|e| e.to_string())}/// How many documents the index holds.pub fn search_count(dir: &str) -> Result<i64, String> {    Ok(open(dir)?.reader.searcher().num_docs() as i64)}