Readability

use readability.qweruse readability_extract, readability_markdown from readability      // Quillo 0.6: only these

dom_smoothie = "0.19"

// A read-later view: a saved page's article, without its menus, ads and footer.////   quillo run readability.qwe      then open http://localhost:8080/articleuse "readability.qwer"let page = read_file("readability_sample.html")?route GET /article {    let a = readability_extract(page, "https://blog.example.com/posts/faster-builds")?    { title: a.title, by: a.byline, site: a.site_name, published: a.published_time, image: a.image, words: len(split(a.text, " ")), text: a.text }}route GET /read {    let a = readability_extract(page, "https://blog.example.com/posts/faster-builds")?    <html>        <body>            <h1>{a.title}</h1>            <p><i>{a.byline}</i></p>            {raw(a.html)}        </body>    </html>}route GET /markdown = respond(200, readability_markdown(page, "https://blog.example.com/posts/faster-builds")?, { "content-type": "text/markdown; charset=utf-8" })route GET /probable = { article: readability_probable(page), menu: readability_probable("<nav><a href='/'>Home</a></nav>") }

//! A web page's article without its menus, ads and footers: title, author, date, main text and clean HTML, for read-later apps, previews and summaries.//! version: 1.0.0//!//! [dependencies]//! dom_smoothie = "0.19"use dom_smoothie::{Config, Readability, TextMode};use serde_json::{json, Value};const MAX: usize = 5_000_000;fn parse(html: &str, url: &str, mode: TextMode) -> Result<dom_smoothie::Article, String> {    if html.len() > MAX {        return Err("pages up to 5 MB".into());    }    let cfg = Config { text_mode: mode, max_elements_to_parse: 100_000, ..Default::default() };    let url = if url.trim().is_empty() { None } else { Some(url.trim()) };    let mut r = Readability::new(html, url, Some(cfg)).map_err(|e| e.to_string())?;    r.parse().map_err(|e| e.to_string())}/// The article in a page's `html`: `{ title, byline, excerpt, site_name, lang, published_time, image, html, text, length }`. `url` is the page's address (or ""), so relative links in the article become whole ones. `text` keeps paragraphs apart; `length` counts its characters. An error when no article is found.pub fn readability_extract(html: &str, url: &str) -> Result<Value, String> {    let a = parse(html, url, TextMode::Formatted)?;    Ok(json!({        "title": a.title, "byline": a.byline, "excerpt": a.excerpt, "site_name": a.site_name, "lang": a.lang,        "published_time": a.published_time, "image": a.image,        "html": a.content.to_string(), "text": a.text_content.trim().to_string(), "length": a.length,    }))}/// The article's main text as Markdown: headings, lists, links and emphasis kept, for an LLM or a plain-text view.pub fn readability_markdown(html: &str, url: &str) -> Result<String, String> {    Ok(parse(html, url, TextMode::Markdown)?.text_content.trim().to_string())}/// Whether a page looks like it holds an article worth extracting (enough paragraphs of text), cheaply.pub fn readability_probable(html: &str) -> bool {    html.len() <= MAX && Readability::new(html, None, None).map(|r| r.is_probably_readable()).unwrap_or(false)}