Structured data

use structured_data.qweruse structured_extract, structured_jsonld from structured_data      // Quillo 0.6: only these

scraper = "0.20"url = "2"serde_json = { version = "1", features = ["preserve_order"] }

// A shop's product pages with schema.org JSON-LD in their head, and a checker for any page's// structured data (here the sample page beside this file).////   quillo run structured_data.qwe      then open http://localhost:8080/extract and http://localhost:8080/p/kettleuse "structured_data.qwer"route GET /extract {    let found = structured_extract(read_file("structured_data_sample.html")?, "https://home.example.com/p/kettle-17")    { ...found, checks: found.json_ld + found.microdata | map(structured_check) }}let KETTLE = { name: "Stovetop Kettle 1.7 L", sku: "KT-17", price: 39.9, currency: "EUR", image: "https://home.example.com/img/kettle.jpg" }route GET /p/kettle {    let product = structured_jsonld("Product", KETTLE)?    let crumbs = structured_jsonld("BreadcrumbList", { items: [{ name: "Kitchen", url: "https://home.example.com/kitchen" }, { name: "Kettles" }] })?    let faq = structured_jsonld("FAQPage", { questions: [{ question: "Does it work on induction?", answer: "Yes, the base is magnetic steel." }] })?    respond(200, `<!doctype html><html><head><title>${KETTLE.name}</title>${structured_script([product, crumbs, faq])}</head><body><h1>${KETTLE.name}</h1></body></html>`, { "content-type": "text/html; charset=utf-8" })}route GET /check = {    ours: structured_check(structured_jsonld("Product", KETTLE)?),    bare: structured_check({ "@type": "Event", name: "Launch party" }),}

//! Structured data in web pages: JSON-LD (graphs too), microdata and Open Graph and Twitter tags read out of HTML; schema.org JSON-LD written for products, articles, breadcrumbs and FAQs; and checked for the properties Google's rich results need.//! version: 1.0.0//!//! [dependencies]//! scraper = "0.20"//! url = "2"//! serde_json = { version = "1", features = ["preserve_order"] }use scraper::{ElementRef, Html, Selector};use serde_json::{json, Map, Value};fn sel(s: &str) -> Selector {    Selector::parse(s).expect("a valid selector")}fn abs(base: &Option<url::Url>, u: &str) -> String {    base.as_ref().and_then(|b| b.join(u.trim()).ok()).map(|u| u.to_string()).unwrap_or_else(|| u.trim().to_string())}fn text(e: &ElementRef) -> String {    e.text().collect::<Vec<_>>().join(" ").split_whitespace().collect::<Vec<_>>().join(" ")}fn prop_value(e: &ElementRef, base: &Option<url::Url>) -> Value {    let v = e.value();    if v.attr("itemscope").is_some() {        return micro_item(e, base);    }    let a = |n: &str| v.attr(n).map(str::to_string);    let s = match v.name() {        "meta" => a("content"),        "a" | "link" | "area" => a("href").map(|h| abs(base, &h)),        "img" | "audio" | "video" | "source" | "iframe" | "embed" | "track" => a("src").map(|h| abs(base, &h)),        "object" => a("data").map(|h| abs(base, &h)),        "time" => a("datetime"),        "data" | "meter" => a("value"),        _ => None,    };    json!(s.unwrap_or_else(|| text(e)))}fn owner_is(e: &ElementRef, item: &ElementRef) -> bool {    // The nearest ancestor with itemscope must be `item`.    let mut p = e.parent();    while let Some(n) = p {        if let Some(el) = ElementRef::wrap(n) {            if el.value().attr("itemscope").is_some() {                return el.id() == item.id();            }        }        p = n.parent();    }    false}fn micro_item(item: &ElementRef, base: &Option<url::Url>) -> Value {    let mut props: Map<String, Value> = Map::new();    for d in item.select(&sel("[itemprop]")) {        if !owner_is(&d, item) {            continue;        }        let val = prop_value(&d, base);        for name in d.value().attr("itemprop").unwrap_or("").split_whitespace() {            props.entry(name.to_string()).or_insert_with(|| json!([])).as_array_mut().unwrap().push(val.clone());        }    }    // One value is written as itself, several as a list, like JSON-LD.    let props: Map<String, Value> = props.into_iter().map(|(k, v)| if v.as_array().is_some_and(|a| a.len() == 1) { (k, v[0].clone()) } else { (k, v) }).collect();    let types: Vec<String> = item.value().attr("itemtype").unwrap_or("").split_whitespace().map(|t| t.rsplit('/').next().unwrap_or(t).to_string()).collect();    let mut m = Map::new();    m.insert("@type".into(), if types.len() == 1 { json!(types[0]) } else { json!(types) });    if let Some(id) = item.value().attr("itemid") {        m.insert("@id".into(), json!(id));    }    m.extend(props);    Value::Object(m)}/// Everything structured in a page: `{ json_ld, microdata, open_graph, twitter, meta, errors }`. `json_ld` lists the items of every/// `<script type="application/ld+json">`, `@graph`s flattened; `microdata` the top-level `itemscope` items, nested ones inside/// as JSON-LD writes them; `open_graph` the `og:` and other `property` metas, `twitter` the `twitter:` ones; `meta` the title,/// description, canonical URL and language. URLs are made absolute against `page_url`. `errors` says which JSON-LD blocks did not parse.pub fn structured_extract(html: &str, page_url: &str) -> Value {    let base = url::Url::parse(page_url).ok();    let doc = Html::parse_document(html);    let mut json_ld = vec![];    let mut errors = vec![];    for (i, s) in doc.select(&sel("script[type]")).filter(|s| s.value().attr("type").is_some_and(|t| t.trim().eq_ignore_ascii_case("application/ld+json"))).enumerate() {        let raw: String = s.text().collect();        match serde_json::from_str::<Value>(raw.trim()) {            Ok(v) => {                let items = match &v {                    Value::Array(xs) => xs.clone(),                    Value::Object(o) if o.contains_key("@graph") => {                        let ctx = o.get("@context").cloned();                        o["@graph"].as_array().cloned().unwrap_or_default().into_iter().map(|mut it| {                            if let (Some(c), Some(obj)) = (&ctx, it.as_object_mut()) {                                obj.entry("@context").or_insert(c.clone());                            }                            it                        }).collect()                    }                    other => vec![other.clone()],                };                json_ld.extend(items);            }            Err(e) => errors.push(json!({ "block": i + 1, "error": e.to_string() })),        }    }    let microdata: Vec<Value> = doc.select(&sel("[itemscope]")).filter(|e| e.value().attr("itemprop").is_none()).map(|e| micro_item(&e, &base)).collect();    let mut og = Map::new();    let mut tw = Map::new();    for m in doc.select(&sel("meta[content]")) {        let c = m.value().attr("content").unwrap_or("").trim().to_string();        if let Some(p) = m.value().attr("property") {            og.entry(p.to_string()).or_insert(json!(c));        } else if let Some(n) = m.value().attr("name").filter(|n| n.starts_with("twitter:")) {            tw.entry(n.to_string()).or_insert(json!(c));        }    }    let first = |s: &str, attr: Option<&str>| doc.select(&sel(s)).next().map(|e| attr.and_then(|a| e.value().attr(a)).map(str::to_string).unwrap_or_else(|| text(&e)));    json!({        "json_ld": json_ld,        "microdata": microdata,        "open_graph": og,        "twitter": tw,        "meta": {            "title": first("title", None),            "description": first("meta[name=description]", Some("content")),            "canonical": first("link[rel=canonical]", Some("href")).map(|h| abs(&base, &h)),            "lang": first("html", Some("lang")),        },        "errors": errors,    })}/// A schema.org item as JSON-LD: `@context`, `@type` and the fields given. Two shortcuts: for "BreadcrumbList", `items` is a list of/// `{ name, url }`; for "FAQPage", `questions` is a list of `{ question, answer }`. A Product's `price` and `currency` become its Offer.pub fn structured_jsonld(schema_type: &str, fields: serde_json::Value) -> Result<Value, String> {    let f = fields.as_object().cloned().unwrap_or_default();    let mut m = Map::new();    m.insert("@context".into(), json!("https://schema.org"));    m.insert("@type".into(), json!(schema_type));    match schema_type {        "BreadcrumbList" => {            let items = f.get("items").and_then(|v| v.as_array()).ok_or("a BreadcrumbList takes items: [{ name, url }]")?;            let list: Vec<Value> = items.iter().enumerate().map(|(i, it)| {                let mut li = json!({ "@type": "ListItem", "position": i + 1, "name": it["name"] });                if let Some(u) = it["url"].as_str() {                    li["item"] = json!(u);                }                li            }).collect();            m.insert("itemListElement".into(), json!(list));        }        "FAQPage" => {            let qs = f.get("questions").and_then(|v| v.as_array()).ok_or("an FAQPage takes questions: [{ question, answer }]")?;            let list: Vec<Value> = qs.iter().map(|q| json!({ "@type": "Question", "name": q["question"], "acceptedAnswer": { "@type": "Answer", "text": q["answer"] } })).collect();            m.insert("mainEntity".into(), json!(list));        }        _ => {            let mut f = f;            if schema_type == "Product" && f.contains_key("price") {                let price = f.remove("price").unwrap();                let currency = f.remove("currency").unwrap_or(json!("USD"));                let availability = f.remove("availability").and_then(|a| a.as_str().map(|s| if s.starts_with("http") { s.to_string() } else { format!("https://schema.org/{}", s) })).unwrap_or_else(|| "https://schema.org/InStock".into());                let price = match price { Value::Number(n) => json!(format!("{:.2}", n.as_f64().unwrap_or(0.0))), other => other };                f.insert("offers".into(), json!({ "@type": "Offer", "price": price, "priceCurrency": currency, "availability": availability }));            }            m.extend(f);        }    }    Ok(Value::Object(m))}/// A JSON-LD item (or a list of them) as the `<script type="application/ld+json">` tag for a page's head, safe to put in HTML/// with `raw(...)`: `<`, `>` and `&` in it are escaped so no text inside can close the tag.pub fn structured_script(item: serde_json::Value) -> String {    let s = serde_json::to_string(&item).unwrap_or_default().replace('<', "\\u003c").replace('>', "\\u003e").replace('&', "\\u0026");    format!("<script type=\"application/ld+json\">{}</script>", s)}const RULES: &[(&str, &[&str], &[&str])] = &[    ("Product", &["name"], &["image", "description", "offers", "sku", "brand", "aggregateRating", "review"]),    ("Offer", &["price", "priceCurrency"], &["availability", "url", "priceValidUntil"]),    ("Article", &["headline"], &["image", "datePublished", "dateModified", "author"]),    ("NewsArticle", &["headline"], &["image", "datePublished", "dateModified", "author"]),    ("BlogPosting", &["headline"], &["image", "datePublished", "dateModified", "author"]),    ("BreadcrumbList", &["itemListElement"], &[]),    ("FAQPage", &["mainEntity"], &[]),    ("Recipe", &["name", "image"], &["recipeIngredient", "recipeInstructions", "totalTime", "recipeYield", "author", "nutrition"]),    ("Event", &["name", "startDate", "location"], &["endDate", "eventStatus", "eventAttendanceMode", "image", "description", "offers", "organizer"]),    ("Organization", &[], &["name", "url", "logo", "sameAs", "contactPoint"]),    ("LocalBusiness", &["name", "address"], &["telephone", "openingHoursSpecification", "geo", "url", "priceRange", "image"]),    ("JobPosting", &["title", "description", "datePosted", "hiringOrganization", "jobLocation"], &["validThrough", "employmentType", "baseSalary"]),    ("Review", &["author", "itemReviewed", "reviewRating"], &["datePublished", "reviewBody"]),    ("Person", &["name"], &["url", "image", "sameAs"]),    ("VideoObject", &["name", "thumbnailUrl", "uploadDate"], &["description", "duration", "contentUrl", "embedUrl"]),];/// What a JSON-LD or microdata item lacks for Google's rich results: `{ type, missing_required, missing_recommended, ok }`, nested/// Offers checked too. Types without rules give empty lists.pub fn structured_check(item: serde_json::Value) -> Value {    let ty = match &item["@type"] {        Value::String(s) => s.clone(),        Value::Array(a) => a.first().and_then(|v| v.as_str()).unwrap_or("").to_string(),        _ => String::new(),    };    let present = |k: &str| match item.get(k) {        None | Some(Value::Null) => false,        Some(Value::String(s)) => !s.trim().is_empty(),        Some(Value::Array(a)) => !a.is_empty(),        _ => true,    };    let (mut req, mut rec) = (vec![], vec![]);    if let Some((_, r, o)) = RULES.iter().find(|(t, _, _)| *t == ty) {        req = r.iter().filter(|k| !present(k)).map(|k| k.to_string()).collect();        rec = o.iter().filter(|k| !present(k)).map(|k| k.to_string()).collect();    }    if ty == "Product" && !present("offers") && !present("review") && !present("aggregateRating") {        req.push("offers, review or aggregateRating".into());    }    if let Some(o) = item.get("offers").filter(|o| o.is_object()) {        let mut o = o.clone();        o.as_object_mut().unwrap().entry("@type").or_insert(json!("Offer"));        let sub = structured_check(o);        req.extend(sub["missing_required"].as_array().into_iter().flatten().filter_map(|v| v.as_str()).map(|s| format!("offers.{}", s)));        rec.extend(sub["missing_recommended"].as_array().into_iter().flatten().filter_map(|v| v.as_str()).map(|s| format!("offers.{}", s)));    }    json!({ "type": ty, "ok": req.is_empty(), "missing_required": req, "missing_recommended": rec })}