Sitemaps

use sitemap_read.qweruse sitemap_read, sitemap_urls from sitemap_read      // Quillo 0.6: only these

roxmltree = "0.20"flate2 = "1"

// A crawler deciding what to fetch again from a site's sitemap, and the site's own sitemap written.////   quillo run sitemap_read.qwe      then open http://localhost:8080/changed?since=2026-10-01use "sitemap_read.qwer"route GET /changed ? { since: str = "" } = sitemap_urls("sitemap_read_sample.xml", since)?route GET /pages {    let s = sitemap_read("sitemap_read_sample.xml")?    { kind: s.kind, count: len(s.entries), images: flatten(s.entries | map((e) => e.images)), daily: s.entries | filter((e) => e.changefreq == "daily") | map((e) => e.loc) }}route GET /index = sitemap_read(`<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">  <sitemap><loc>https://shop.example.com/sitemap-products.xml.gz</loc><lastmod>2026-10-20</lastmod></sitemap>  <sitemap><loc>https://shop.example.com/sitemap-blog.xml</loc></sitemap></sitemapindex>`)?route GET /sitemap.xml = respond(200, sitemap_write([    { loc: "https://quillo.example/", changefreq: "weekly", priority: 1 },    { loc: "https://quillo.example/docs?page=1&lang=en", lastmod: "2026-10-28" },])?, { "content-type": "application/xml" })

//! Sitemaps read: the URLs of a sitemap or the sitemaps of a sitemap index, with lastmod, changefreq, priority and images, from XML text, a file, a .xml.gz, or a plain text list; and a sitemap written for your own pages.//! version: 1.0.0//!//! [dependencies]//! roxmltree = "0.20"//! flate2 = "1"use serde_json::{json, Value};use std::io::Read;const MAX_SITEMAP: u64 = 100 << 20;const MAX_URLS: usize = 200_000;/// The sitemap's text: given as XML or text, or read from a path (gunzipped when it is gzip), at most 100 MB unpacked as the protocol says.fn text(source: &str) -> Result<String, String> {    let s = source.trim_start();    if s.starts_with('<') || s.starts_with("http://") || s.starts_with("https://") {        return Ok(source.to_string());    }    let f = std::fs::File::open(source.trim()).map_err(|e| format!("{}: {}", source, e))?;    let mut raw = Vec::new();    f.take(MAX_SITEMAP + 1).read_to_end(&mut raw).map_err(|e| e.to_string())?;    if raw.starts_with(&[0x1f, 0x8b]) {        let mut out = Vec::new();        flate2::read::GzDecoder::new(&raw[..]).take(MAX_SITEMAP + 1).read_to_end(&mut out).map_err(|e| format!("{}: {}", source, e))?;        raw = out;    }    if raw.len() as u64 > MAX_SITEMAP {        return Err("a sitemap is at most 100 MB".into());    }    String::from_utf8(raw).map_err(|_| format!("{} is not UTF-8 text", source))}fn child<'a>(n: roxmltree::Node<'a, 'a>, name: &str) -> Option<String> {    n.children().find(|c| c.is_element() && c.tag_name().name() == name).and_then(|c| c.text()).map(|t| t.trim().to_string())}/// A sitemap's entries: `{ kind, entries }`, `kind` "urlset" (pages: `{ loc, lastmod, changefreq, priority, images }`) or "sitemapindex" (more sitemaps: `{ loc, lastmod }`, to read each in turn),/// or "text" for a list of URLs one per line. `source` is the XML (or text), or a file's path (.xml or .xml.gz). At most 200,000 entries.pub fn sitemap_read(source: &str) -> Result<Value, String> {    let t = text(source)?;    if !t.trim_start().starts_with('<') {        let entries: Vec<Value> = t.lines().map(str::trim).filter(|l| l.starts_with("http://") || l.starts_with("https://")).take(MAX_URLS).map(|l| json!({ "loc": l })).collect();        return Ok(json!({ "kind": "text", "entries": entries }));    }    let opts = roxmltree::ParsingOptions { allow_dtd: false, nodes_limit: 20_000_000 };    let doc = roxmltree::Document::parse_with_options(&t, opts).map_err(|e| format!("not XML: {}", e))?;    let root = doc.root_element();    let kind = root.tag_name().name().to_string();    let item = match kind.as_str() {        "urlset" => "url",        "sitemapindex" => "sitemap",        other => return Err(format!("<{}> is not a sitemap (urlset or sitemapindex)", other)),    };    let mut entries = Vec::new();    for n in root.children().filter(|c| c.is_element() && c.tag_name().name() == item) {        let Some(loc) = child(n, "loc") else { continue };        let mut e = json!({ "loc": loc, "lastmod": child(n, "lastmod") });        if item == "url" {            e["changefreq"] = json!(child(n, "changefreq"));            e["priority"] = json!(child(n, "priority").and_then(|p| p.parse::<f64>().ok()));            let images: Vec<String> = n.children().filter(|c| c.is_element() && c.tag_name().name() == "image").filter_map(|c| child(c, "loc")).collect();            e["images"] = json!(images);        }        entries.push(e);        if entries.len() >= MAX_URLS {            break;        }    }    Ok(json!({ "kind": kind, "entries": entries }))}/// Only the page URLs, those changed on or after `since` (`2026-10-01`; "" for all), in the sitemap's order: what a crawler should fetch again./// Entries without a lastmod are kept. For a sitemap index these are the sitemaps' URLs.pub fn sitemap_urls(source: &str, since: &str) -> Result<Vec<String>, String> {    let s = sitemap_read(source)?;    let since = since.trim();    Ok(s["entries"]        .as_array()        .map(|es| {            es.iter()                .filter(|e| since.is_empty() || e["lastmod"].as_str().map(|m| m[..m.len().min(since.len())] >= *since).unwrap_or(true))                .filter_map(|e| e["loc"].as_str().map(str::to_string))                .collect()        })        .unwrap_or_default())}fn esc(s: &str) -> String {    s.replace('&', "&amp;").replace('<', "&lt;").replace('>', "&gt;").replace('"', "&quot;").replace('\'', "&apos;")}/// A sitemap for your pages: `pages` are `[{ loc, lastmod, changefreq, priority }]` (only `loc` needed, a full URL). At most 50,000, as the protocol allows in one file.pub fn sitemap_write(pages: Vec<serde_json::Value>) -> Result<String, String> {    if pages.len() > 50_000 {        return Err("a sitemap holds at most 50,000 URLs: split it, with a sitemap index".into());    }    let mut out = String::from("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n");    for p in &pages {        let loc = p["loc"].as_str().ok_or("each page has a `loc`")?;        out.push_str(&format!("  <url><loc>{}</loc>", esc(loc)));        if let Some(m) = p["lastmod"].as_str() {            out.push_str(&format!("<lastmod>{}</lastmod>", esc(m)));        }        if let Some(c) = p["changefreq"].as_str() {            out.push_str(&format!("<changefreq>{}</changefreq>", esc(c)));        }        if let Some(pr) = p["priority"].as_f64() {            out.push_str(&format!("<priority>{:.1}</priority>", pr.clamp(0.0, 1.0)));        }        out.push_str("</url>\n");    }    out.push_str("</urlset>\n");    Ok(out)}