Scrape pages

use scrape.qweruse select_text, select_attr from scrape      // Quillo 0.6: only these

scraper = "0.20"

// Link previews, and the headlines of a page.////   quillo run scrape.qwe      then open http://localhost:8080/preview//   (http.get(url)?.body gives a live page's HTML to the same functions)use "scrape.qwer"let page = `<html><head><title>Quillo</title><meta property="og:title" content="Quillo: the web, as a language"><meta name="description" content="A small web programming language."><meta property="og:image" content="https://quillolang.org/banner.png"></head><body><h2 class="t">Routes</h2><h2 class="t">Components</h2><a href="/docs/">Docs</a><script>var x = 1</script></body></html>`route GET /preview = page_preview(page)route GET /headlines = { titles: select_text(page, "h2.t")?, links: select_attr(page, "a", "href")?, text: page_text(page) }

//! Read web pages: CSS selectors over HTML, a page's title, description and preview image, and its text.//! version: 1.0.0//!//! [dependencies]//! scraper = "0.20"use scraper::{Html, Selector};use std::collections::HashMap;fn sel(s: &str) -> Result<Selector, String> {    Selector::parse(s).map_err(|e| format!("`{}` is not a selector: {:?}", s, e))}fn squash(s: String) -> String {    s.split_whitespace().collect::<Vec<_>>().join(" ")}/// The text of each element `selector` matches, spaces squashed: `select_text(page, "h2.title")`.pub fn select_text(html: &str, selector: &str) -> Result<Vec<String>, String> {    let s = sel(selector)?;    Ok(Html::parse_document(html).select(&s).map(|e| squash(e.text().collect())).collect())}/// An attribute of each element `selector` matches that has it: `select_attr(page, "a", "href")`.pub fn select_attr(html: &str, selector: &str, attr: &str) -> Result<Vec<String>, String> {    let s = sel(selector)?;    Ok(Html::parse_document(html).select(&s).filter_map(|e| e.value().attr(attr).map(str::to_string)).collect())}/// The inner HTML of each element `selector` matches.pub fn select_html(html: &str, selector: &str) -> Result<Vec<String>, String> {    let s = sel(selector)?;    Ok(Html::parse_document(html).select(&s).map(|e| e.inner_html()).collect())}/// What a link preview shows: `title`, `description`, `image`, `site_name` and `url`, from Open Graph tags, else the page's own.pub fn page_preview(html: &str) -> HashMap<String, String> {    let doc = Html::parse_document(html);    let meta = |key: &str| -> Option<String> {        let s = Selector::parse(&format!("meta[property=\"og:{0}\"], meta[name=\"og:{0}\"], meta[name=\"twitter:{0}\"]", key)).ok()?;        doc.select(&s).find_map(|e| e.value().attr("content").map(|c| c.trim().to_string())).filter(|c| !c.is_empty())    };    let one = |css: &str, attr: Option<&str>| -> Option<String> {        let s = Selector::parse(css).ok()?;        doc.select(&s).next().and_then(|e| match attr { Some(a) => e.value().attr(a).map(str::to_string), None => Some(squash(e.text().collect())) })    };    let mut out = HashMap::new();    for (k, v) in [        ("title", meta("title").or_else(|| one("title", None))),        ("description", meta("description").or_else(|| one("meta[name=description]", Some("content")))),        ("image", meta("image")),        ("site_name", meta("site_name")),        ("url", meta("url").or_else(|| one("link[rel=canonical]", Some("href")))),    ] {        if let Some(v) = v {            out.insert(k.to_string(), v);        }    }    out}/// All the text of a page, without scripts and styles, spaces squashed.pub fn page_text(html: &str) -> String {    let doc = Html::parse_document(html);    let body = Selector::parse("body").unwrap();    let skip = ["script", "style", "noscript", "template"];    let root = doc.select(&body).next().unwrap_or(doc.root_element());    let mut out = String::new();    for node in root.descendants() {        if let Some(t) = node.value().as_text() {            let hidden = node.ancestors().any(|a| a.value().as_element().is_some_and(|e| skip.contains(&e.name())));            if !hidden {                out.push_str(t);                out.push(' ');            }        }    }    squash(out)}