oEmbed

use oembed.qweruse oembed_discover, oembed_endpoint from oembed      // Quillo 0.6: only these

scraper = "0.20"url = "2"ureq = "2"

// Links pasted into posts shown as players: a known provider's endpoint is asked, or the one the// page names; the answer's HTML is cut down to one sandboxed iframe from a known player.////   quillo run oembed.qwe      then open http://localhost:8080/offline//   http://localhost:8080/embed?url=https://www.youtube.com/watch%3Fv%3DdQw4w9WgXcQ goes to the network.use "oembed.qwer"route GET /offline = {    discovered: oembed_discover(read_file("oembed_sample.html")?, "https://kitchen.example.com/sourdough")?,    youtube: oembed_endpoint("https://www.youtube.com/watch?v=dQw4w9WgXcQ", 720),    unknown: oembed_endpoint("https://blog.example.com/post/1", 0),    answer: oembed_read(read_file("oembed_sample.json")?, [])?,    player: oembed_player("https://youtu.be/dQw4w9WgXcQ?t=42", 42),}route GET /embed ? { url: str } {    let e = oembed_fetch(url, 720, 5000)    if is_err(e) { reply 502 e.error }    <figure>        {raw(e.iframe ?? "")}        <figcaption>{e.title} · {e.provider_name}</figcaption>    </figure>}

//! Rich embeds from links people paste: the oEmbed endpoint found in a page or from a list of known providers (YouTube, Vimeo, Spotify, SoundCloud, X, TikTok and more), the answer read and its HTML cut down to one iframe from an allowed host, and privacy-friendly players made without a request.//! version: 1.0.0//!//! [dependencies]//! scraper = "0.20"//! url = "2"//! ureq = "2"use scraper::{Html, Selector};use serde_json::{json, Value};/// (name, endpoint, URL patterns with `*` for any characters but `/`... `**` for anything)const PROVIDERS: &[(&str, &str, &[&str])] = &[    ("YouTube", "https://www.youtube.com/oembed", &["https://*.youtube.com/watch**", "https://*.youtube.com/shorts/**", "https://*.youtube.com/live/**", "https://youtu.be/**", "https://*.youtube.com/playlist**"]),    ("Vimeo", "https://vimeo.com/api/oembed.json", &["https://vimeo.com/**", "https://player.vimeo.com/video/**"]),    ("Spotify", "https://open.spotify.com/oembed", &["https://open.spotify.com/**"]),    ("SoundCloud", "https://soundcloud.com/oembed?format=json", &["https://soundcloud.com/**", "https://on.soundcloud.com/**"]),    ("X", "https://publish.twitter.com/oembed", &["https://twitter.com/*/status/**", "https://x.com/*/status/**"]),    ("TikTok", "https://www.tiktok.com/oembed", &["https://www.tiktok.com/@*/video/**"]),    ("Dailymotion", "https://www.dailymotion.com/services/oembed", &["https://www.dailymotion.com/video/**", "https://dai.ly/**"]),    ("Flickr", "https://www.flickr.com/services/oembed/?format=json", &["https://*.flickr.com/photos/**", "https://flic.kr/**"]),    ("Reddit", "https://www.reddit.com/oembed", &["https://www.reddit.com/r/*/comments/**", "https://reddit.com/r/*/comments/**"]),    ("CodePen", "https://codepen.io/api/oembed?format=json", &["https://codepen.io/*/pen/**"]),    ("Giphy", "https://giphy.com/services/oembed", &["https://giphy.com/gifs/**", "https://gph.is/**"]),    ("Loom", "https://www.loom.com/v1/oembed", &["https://www.loom.com/share/**"]),    ("Figma", "https://www.figma.com/api/oembed", &["https://www.figma.com/file/**", "https://www.figma.com/design/**", "https://www.figma.com/proto/**"]),    ("Mixcloud", "https://app.mixcloud.com/oembed/?format=json", &["https://www.mixcloud.com/**"]),    ("SlideShare", "https://www.slideshare.net/api/oembed/2?format=json", &["https://www.slideshare.net/**"]),    ("Bluesky", "https://embed.bsky.app/oembed", &["https://bsky.app/profile/*/post/**"]),];/// The hosts whose iframes `oembed_read` keeps when no list is given.const PLAYER_HOSTS: &[&str] = &["www.youtube.com", "www.youtube-nocookie.com", "player.vimeo.com", "open.spotify.com", "w.soundcloud.com", "www.tiktok.com", "www.dailymotion.com", "geo.dailymotion.com", "embed.reddit.com", "codepen.io", "giphy.com", "www.loom.com", "www.figma.com", "embed.figma.com", "player-widget.mixcloud.com", "www.slideshare.net", "embed.bsky.app"];fn glob(pattern: &str, s: &str) -> bool {    fn go(p: &[u8], s: &[u8]) -> bool {        match p.first() {            None => s.is_empty(),            Some(b'*') if p.get(1) == Some(&b'*') => (0..=s.len()).any(|i| go(&p[2..], &s[i..])),            Some(b'*') => (0..=s.len()).take_while(|&i| i == 0 || s[i - 1] != b'/').any(|i| go(&p[1..], &s[i..])),            Some(c) => s.first() == Some(c) && go(&p[1..], &s[1..]),        }    }    go(pattern.as_bytes(), s.as_bytes())}fn with_query(endpoint: &str, pairs: &[(&str, String)]) -> String {    let mut u = url::Url::parse(endpoint).expect("endpoints are URLs");    {        let mut q = u.query_pairs_mut();        for (k, v) in pairs {            q.append_pair(k, v);        }    }    u.to_string()}/// The oEmbed endpoints a page names in its `<link rel="alternate" type="application/json+oembed">` tags (and the XML ones):/// `[{ format, url, title }]`, each URL made absolute against `page_url`. JSON ones come first.pub fn oembed_discover(html: &str, page_url: &str) -> Result<Vec<Value>, String> {    let base = url::Url::parse(page_url).map_err(|e| format!("page_url: {}", e))?;    let doc = Html::parse_document(html);    let sel = Selector::parse("link[rel~=alternate][href]").expect("a valid selector");    let mut out: Vec<Value> = doc        .select(&sel)        .filter_map(|l| {            let t = l.value().attr("type")?.to_lowercase();            let format = if t.contains("json+oembed") { "json" } else if t.contains("xml+oembed") { "xml" } else { return None };            let href = base.join(l.value().attr("href")?).ok()?;            Some(json!({ "format": format, "url": href.to_string(), "title": l.value().attr("title") }))        })        .collect();    out.sort_by_key(|v| v["format"] != "json");    Ok(out)}/// The known provider of `url` and the oEmbed request to make for it: `{ provider, request }`, or null when no provider in the/// list matches. `max_width` (0 for none) asks for a player at most that wide.pub fn oembed_endpoint(url: &str, max_width: i64) -> Option<Value> {    let u = url.trim();    let (name, endpoint, _) = PROVIDERS.iter().find(|(_, _, pats)| pats.iter().any(|p| glob(p, u) || glob(p, &u.replacen("http://", "https://", 1))))?;    let mut pairs = vec![("url", u.to_string())];    if max_width > 0 {        pairs.push(("maxwidth", max_width.to_string()));    }    if !endpoint.contains("format=") {        pairs.push(("format", "json".into()));    }    Some(json!({ "provider": name, "request": with_query(endpoint, &pairs) }))}fn safe_iframe(html: &str, hosts: &[String]) -> Option<String> {    let frag = Html::parse_fragment(html);    let sel = Selector::parse("iframe[src]").expect("a valid selector");    let f = frag.select(&sel).next()?;    let src = url::Url::parse(f.value().attr("src")?).ok()?;    if src.scheme() != "https" || !hosts.iter().any(|h| src.host_str() == Some(h.as_str())) {        return None;    }    let num = |a: &str| f.value().attr(a).and_then(|v| v.trim_end_matches("px").parse::<u32>().ok());    let esc = |s: &str| s.replace('&', "&amp;").replace('"', "&quot;").replace('<', "&lt;");    let title = f.value().attr("title").unwrap_or("Embedded content");    let allow: Vec<&str> = f.value().attr("allow").unwrap_or("").split(';').map(str::trim).filter(|p| ["autoplay", "fullscreen", "picture-in-picture", "encrypted-media", "clipboard-write"].contains(&p.split_whitespace().next().unwrap_or(""))).collect();    Some(format!(        "<iframe src=\"{}\" width=\"{}\" height=\"{}\" title=\"{}\" allow=\"{}\" allowfullscreen loading=\"lazy\" referrerpolicy=\"strict-origin-when-cross-origin\" sandbox=\"allow-scripts allow-same-origin allow-popups allow-presentation\" style=\"border:0\"></iframe>",        esc(src.as_str()),        num("width").unwrap_or(640),        num("height").unwrap_or(360),        esc(title),        esc(&allow.join("; "))    ))}/// An oEmbed answer (its JSON text) read into `{ type, title, author_name, author_url, provider_name, thumbnail_url, width,/// height, url, iframe }`. `iframe` is the answer's HTML cut down to its one `<iframe>`, rebuilt with only safe attributes,/// sandboxed and lazy, when its https host is in `iframe_hosts` (empty: the known players); otherwise null, as for a `link`.pub fn oembed_read(response_json: &str, iframe_hosts: Vec<String>) -> Result<Value, String> {    let v: Value = serde_json::from_str(response_json).map_err(|e| format!("not an oEmbed JSON answer: {}", e))?;    let t = v["type"].as_str().ok_or("an oEmbed answer has a type (photo, video, link or rich)")?;    let hosts: Vec<String> = if iframe_hosts.is_empty() { PLAYER_HOSTS.iter().map(|s| s.to_string()).collect() } else { iframe_hosts };    let iframe = v["html"].as_str().and_then(|h| safe_iframe(h, &hosts));    let text = |k: &str| v[k].as_str().map(str::to_string);    let https = |k: &str| text(k).filter(|u| u.starts_with("https://") || u.starts_with("http://"));    Ok(json!({        "type": t,        "title": text("title"),        "author_name": text("author_name"),        "author_url": https("author_url"),        "provider_name": text("provider_name"),        "thumbnail_url": https("thumbnail_url"),        "width": v["width"].as_i64().or_else(|| v["width"].as_str().and_then(|s| s.parse().ok())),        "height": v["height"].as_i64().or_else(|| v["height"].as_str().and_then(|s| s.parse().ok())),        "url": if t == "photo" { https("url") } else { None },        "iframe": iframe,    }))}/// A player for a YouTube or Vimeo link made without asking anyone: `{ provider, id, iframe, thumbnail_url }` (YouTube's from/// youtube-nocookie.com, Vimeo's with `dnt=1`), or null for other links. `start` seconds where to begin (0 for the start).pub fn oembed_player(url: &str, start: i64) -> Option<Value> {    let u = url::Url::parse(url.trim()).ok()?;    let host = u.host_str()?.trim_start_matches("www.").trim_start_matches("m.");    let seg: Vec<&str> = u.path_segments().map(|s| s.filter(|x| !x.is_empty()).collect()).unwrap_or_default();    let ok_id = |s: &str| !s.is_empty() && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_');    let (provider, id) = match host {        "youtube.com" | "music.youtube.com" => {            let id = match seg.first() {                Some(&"watch") => u.query_pairs().find(|(k, _)| k == "v").map(|(_, v)| v.to_string())?,                Some(&("shorts" | "embed" | "live")) => seg.get(1)?.to_string(),                _ => return None,            };            ("YouTube", id)        }        "youtu.be" => ("YouTube", seg.first()?.to_string()),        "vimeo.com" => ("Vimeo", seg.iter().find(|s| s.chars().all(|c| c.is_ascii_digit()))?.to_string()),        "player.vimeo.com" => ("Vimeo", seg.get(1)?.to_string()),        _ => return None,    };    if !ok_id(&id) {        return None;    }    let (src, thumb) = if provider == "YouTube" {        let t = if start > 0 { format!("?start={}", start) } else { String::new() };        (format!("https://www.youtube-nocookie.com/embed/{}{}", id, t), Some(format!("https://i.ytimg.com/vi/{}/hqdefault.jpg", id)))    } else {        let t = if start > 0 { format!("#t={}s", start) } else { String::new() };        (format!("https://player.vimeo.com/video/{}?dnt=1{}", id, t), None)    };    let iframe = format!("<iframe src=\"{}\" width=\"640\" height=\"360\" title=\"{} video\" allow=\"autoplay; fullscreen; picture-in-picture; encrypted-media\" allowfullscreen loading=\"lazy\" referrerpolicy=\"strict-origin-when-cross-origin\" style=\"border:0\"></iframe>", src, provider);    Some(json!({ "provider": provider, "id": id, "iframe": iframe, "thumbnail_url": thumb }))}/// Fetches the embed for `url`: from a known provider's endpoint, else from the endpoint the page itself names, and reads it as/// `oembed_read` does with the known players. `timeout_ms` limits each request.pub fn oembed_fetch(url: &str, max_width: i64, timeout_ms: i64) -> Result<Value, String> {    let agent = ureq::AgentBuilder::new().timeout(std::time::Duration::from_millis(timeout_ms.clamp(100, 30_000) as u64)).redirects(3).build();    let get = |u: &str| -> Result<String, String> {        let r = agent.get(u).set("user-agent", "Mozilla/5.0 (compatible; QuilloEmbed/1.0)").call().map_err(|e| e.to_string())?;        let mut s = String::new();        std::io::Read::read_to_string(&mut std::io::Read::take(r.into_reader(), 2_000_000), &mut s).map_err(|e| e.to_string())?;        Ok(s)    };    let request = match oembed_endpoint(url, max_width) {        Some(e) => e["request"].as_str().unwrap_or_default().to_string(),        None => {            let page = get(url)?;            let found = oembed_discover(&page, url)?;            let first = found.iter().find(|f| f["format"] == "json").ok_or("the page names no JSON oEmbed endpoint")?;            first["url"].as_str().unwrap_or_default().to_string()        }    };    oembed_read(&get(&request)?, vec![])}