Bots and crawlers

use bots.qweruse bot_detect, bot_verify from bots      // Quillo 0.6: only these

dns-lookup = "2"

// Who is visiting: AI crawlers kept out by robots.txt and refused if they come anyway, link// previews let in, and a visitor that claims to be Googlebot checked by DNS.////   quillo run bots.qwe      then open http://localhost:8080/robots.txt and http://localhost:8080/whouse "bots.qwer"route GET /robots.txt = respond(200, bot_robots(["ai_training", "seo"], ["Applebot-Extended"], "https://shop.example.com/sitemap.xml")?, { "content-type": "text/plain" })route GET /who = {    gptbot: bot_detect("Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)"),    google: bot_detect("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"),    slack: bot_detect("Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)").category,    unknown: bot_detect("Mozilla/5.0 (compatible; ShinyNewCrawler/0.3)"),    firefox: bot_detect("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0"),    // A request from this machine that claims to be Googlebot.    fake_google: bot_verify("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "127.0.0.1")?,    ai_training_bots: len(bot_list("ai_training")),}// Pages refuse AI training crawlers that ignore robots.txt.route GET /articles/:id {    let who = bot_detect(request.headers["user-agent"] ?? "")    if who.bot && who.category == "ai_training" { reply 403 "AI training crawlers are not allowed here" }    `Article ${id}`}

//! Crawlers and bots told apart by their User-Agent: search engines, AI training and AI answer bots, link previews, SEO tools, monitors, feed readers, scanners and HTTP libraries, with who runs each; a claimed Googlebot or GPTBot verified by reverse and forward DNS; and robots.txt rules that block the categories you choose.//! version: 1.0.0//!//! [dependencies]//! dns-lookup = "2"use serde_json::{json, Value};struct Bot {    /// Matched case-insensitively in the User-Agent; empty for robots.txt tokens no User-Agent carries.    pattern: &'static str,    name: &'static str,    category: &'static str,    owner: &'static str,    /// The product token robots.txt rules name.    robots: &'static str,    /// Host names a verified visit's IP reverse-resolves to.    rdns: &'static [&'static str],    /// Where the operator publishes its IP ranges.    ranges: &'static str,}const fn b(pattern: &'static str, name: &'static str, category: &'static str, owner: &'static str, robots: &'static str, rdns: &'static [&'static str], ranges: &'static str) -> Bot {    Bot { pattern, name, category, owner, robots, rdns, ranges }}const GOOGLE: &[&str] = &["googlebot.com", "google.com", "googleusercontent.com"];const BOTS: &[Bot] = &[    // AI crawlers first: some of their User-Agents also say "compatible; ...bot".    b("GPTBot", "GPTBot", "ai_training", "OpenAI", "GPTBot", &[], "https://openai.com/gptbot.json"),    b("OAI-SearchBot", "OAI-SearchBot", "ai_search", "OpenAI", "OAI-SearchBot", &[], "https://openai.com/searchbot.json"),    b("ChatGPT-User", "ChatGPT-User", "ai_assistant", "OpenAI", "ChatGPT-User", &[], "https://openai.com/chatgpt-user.json"),    b("ClaudeBot", "ClaudeBot", "ai_training", "Anthropic", "ClaudeBot", &[], ""),    b("Claude-SearchBot", "Claude-SearchBot", "ai_search", "Anthropic", "Claude-SearchBot", &[], ""),    b("Claude-User", "Claude-User", "ai_assistant", "Anthropic", "Claude-User", &[], ""),    b("anthropic-ai", "anthropic-ai", "ai_training", "Anthropic", "anthropic-ai", &[], ""),    b("PerplexityBot", "PerplexityBot", "ai_search", "Perplexity", "PerplexityBot", &[], "https://www.perplexity.com/perplexitybot.json"),    b("Perplexity-User", "Perplexity-User", "ai_assistant", "Perplexity", "Perplexity-User", &[], "https://www.perplexity.com/perplexity-user.json"),    b("CCBot", "CCBot", "ai_training", "Common Crawl", "CCBot", &[], ""),    b("Bytespider", "Bytespider", "ai_training", "ByteDance", "Bytespider", &[], ""),    b("meta-externalagent", "Meta-ExternalAgent", "ai_training", "Meta", "meta-externalagent", &[], ""),    b("meta-externalfetcher", "Meta-ExternalFetcher", "ai_assistant", "Meta", "meta-externalfetcher", &[], ""),    b("FacebookBot", "FacebookBot", "ai_training", "Meta", "FacebookBot", &[], ""),    b("Amazonbot", "Amazonbot", "ai_search", "Amazon", "Amazonbot", &["crawl.amazonbot.amazon"], ""),    b("cohere-ai", "cohere-ai", "ai_training", "Cohere", "cohere-ai", &[], ""),    b("Diffbot", "Diffbot", "ai_training", "Diffbot", "Diffbot", &[], ""),    b("YouBot", "YouBot", "ai_search", "You.com", "YouBot", &[], ""),    b("MistralAI-User", "MistralAI-User", "ai_assistant", "Mistral AI", "MistralAI-User", &[], ""),    b("", "Google-Extended", "ai_training", "Google", "Google-Extended", &[], ""),    b("", "Applebot-Extended", "ai_training", "Apple", "Applebot-Extended", &[], ""),    // Search engines.    b("Googlebot", "Googlebot", "search", "Google", "Googlebot", GOOGLE, "https://developers.google.com/static/search/apis/ipranges/googlebot.json"),    b("Storebot-Google", "Storebot-Google", "search", "Google", "Storebot-Google", GOOGLE, "https://developers.google.com/static/search/apis/ipranges/googlebot.json"),    b("Google-InspectionTool", "Google-InspectionTool", "search", "Google", "Google-InspectionTool", GOOGLE, "https://developers.google.com/static/search/apis/ipranges/googlebot.json"),    b("GoogleOther", "GoogleOther", "search", "Google", "GoogleOther", GOOGLE, "https://developers.google.com/static/search/apis/ipranges/googlebot.json"),    b("AdsBot-Google", "AdsBot-Google", "ads", "Google", "AdsBot-Google", GOOGLE, "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json"),    b("Mediapartners-Google", "Mediapartners-Google", "ads", "Google", "Mediapartners-Google", GOOGLE, "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json"),    b("FeedFetcher-Google", "FeedFetcher-Google", "feed_reader", "Google", "FeedFetcher-Google", GOOGLE, "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json"),    b("bingbot", "Bingbot", "search", "Microsoft", "bingbot", &["search.msn.com"], "https://www.bing.com/toolbox/bingbot.json"),    b("BingPreview", "BingPreview", "social_preview", "Microsoft", "bingbot", &["search.msn.com"], ""),    b("AdIdxBot", "AdIdxBot", "ads", "Microsoft", "adidxbot", &["search.msn.com"], ""),    b("Applebot", "Applebot", "search", "Apple", "Applebot", &["applebot.apple.com"], "https://search.developer.apple.com/applebot.json"),    b("DuckDuckBot", "DuckDuckBot", "search", "DuckDuckGo", "DuckDuckBot", &[], "https://duckduckgo.com/duckduckbot.json"),    b("YandexBot", "YandexBot", "search", "Yandex", "Yandex", &["yandex.ru", "yandex.net", "yandex.com"], ""),    b("YandexImages", "YandexImages", "search", "Yandex", "Yandex", &["yandex.ru", "yandex.net", "yandex.com"], ""),    b("Baiduspider", "Baiduspider", "search", "Baidu", "Baiduspider", &["baidu.com", "baidu.jp"], ""),    b("PetalBot", "PetalBot", "search", "Huawei", "PetalBot", &["petalsearch.com"], ""),    b("SeznamBot", "SeznamBot", "search", "Seznam", "SeznamBot", &["seznam.cz"], ""),    b("Qwantbot", "Qwantbot", "search", "Qwant", "Qwantbot", &[], ""),    b("MojeekBot", "MojeekBot", "search", "Mojeek", "MojeekBot", &[], ""),    b("Sogou", "Sogou spider", "search", "Sogou", "Sogou", &[], ""),    b("Yeti", "Yeti", "search", "Naver", "Yeti", &["naver.com"], ""),    b("Pinterestbot", "Pinterestbot", "social_preview", "Pinterest", "Pinterestbot", &["pinterest.com"], ""),    // Link previews.    b("facebookexternalhit", "facebookexternalhit", "social_preview", "Meta", "facebookexternalhit", &[], ""),    b("Twitterbot", "Twitterbot", "social_preview", "X", "Twitterbot", &[], ""),    b("LinkedInBot", "LinkedInBot", "social_preview", "LinkedIn", "LinkedInBot", &[], ""),    b("Slackbot", "Slackbot", "social_preview", "Slack", "Slackbot", &[], ""),    b("Slack-ImgProxy", "Slack-ImgProxy", "social_preview", "Slack", "Slackbot", &[], ""),    b("Discordbot", "Discordbot", "social_preview", "Discord", "Discordbot", &[], ""),    b("TelegramBot", "TelegramBot", "social_preview", "Telegram", "TelegramBot", &[], ""),    b("WhatsApp", "WhatsApp", "social_preview", "Meta", "WhatsApp", &[], ""),    b("SkypeUriPreview", "Skype", "social_preview", "Microsoft", "SkypeUriPreview", &[], ""),    b("redditbot", "redditbot", "social_preview", "Reddit", "redditbot", &[], ""),    b("Mastodon", "Mastodon", "social_preview", "Mastodon servers", "Mastodon", &[], ""),    b("Embedly", "Embedly", "social_preview", "Embedly", "Embedly", &[], ""),    b("Iframely", "Iframely", "social_preview", "Iframely", "Iframely", &[], ""),    // SEO tools.    b("AhrefsBot", "AhrefsBot", "seo", "Ahrefs", "AhrefsBot", &["ahrefs.com", "ahrefs.net"], ""),    b("SemrushBot", "SemrushBot", "seo", "Semrush", "SemrushBot", &["semrush.com"], ""),    b("MJ12bot", "MJ12bot", "seo", "Majestic", "MJ12bot", &[], ""),    b("DotBot", "DotBot", "seo", "Moz", "dotbot", &[], ""),    b("rogerbot", "rogerbot", "seo", "Moz", "rogerbot", &[], ""),    b("Screaming Frog", "Screaming Frog SEO Spider", "seo", "Screaming Frog", "Screaming Frog SEO Spider", &[], ""),    b("BLEXBot", "BLEXBot", "seo", "WebMeUp", "BLEXBot", &[], ""),    b("DataForSeoBot", "DataForSeoBot", "seo", "DataForSEO", "DataForSeoBot", &[], ""),    b("serpstatbot", "serpstatbot", "seo", "Serpstat", "serpstatbot", &[], ""),    // Monitors.    b("UptimeRobot", "UptimeRobot", "monitoring", "UptimeRobot", "UptimeRobot", &[], ""),    b("Pingdom", "Pingdom", "monitoring", "SolarWinds", "Pingdom", &[], ""),    b("StatusCake", "StatusCake", "monitoring", "StatusCake", "StatusCake", &[], ""),    b("Better Uptime", "Better Stack", "monitoring", "Better Stack", "BetterUptimeBot", &[], ""),    b("Site24x7", "Site24x7", "monitoring", "Zoho", "Site24x7", &[], ""),    b("Datadog", "Datadog Synthetics", "monitoring", "Datadog", "Datadog", &[], ""),    b("Chrome-Lighthouse", "Lighthouse", "monitoring", "Google", "Chrome-Lighthouse", &[], ""),    // Feed readers.    b("Feedly", "Feedly", "feed_reader", "Feedly", "Feedly", &[], ""),    b("Inoreader", "Inoreader", "feed_reader", "Inoreader", "Inoreader", &[], ""),    b("NewsBlur", "NewsBlur", "feed_reader", "NewsBlur", "NewsBlur", &[], ""),    b("Feedbin", "Feedbin", "feed_reader", "Feedbin", "Feedbin", &[], ""),    b("FreshRSS", "FreshRSS", "feed_reader", "FreshRSS", "FreshRSS", &[], ""),    b("Miniflux", "Miniflux", "feed_reader", "Miniflux", "Miniflux", &[], ""),    // Archivers.    b("archive.org_bot", "Internet Archive", "archiver", "Internet Archive", "archive.org_bot", &["archive.org"], ""),    b("ia_archiver", "ia_archiver", "archiver", "Internet Archive", "ia_archiver", &[], ""),    // Scanners.    b("CensysInspect", "Censys", "scanner", "Censys", "CensysInspect", &[], ""),    b("Expanse", "Expanse", "scanner", "Palo Alto Networks", "Expanse", &[], ""),    b("zgrab", "zgrab", "scanner", "", "zgrab", &[], ""),    b("Nuclei", "Nuclei", "scanner", "", "Nuclei", &[], ""),    b("Nmap Scripting Engine", "Nmap", "scanner", "", "Nmap", &[], ""),    b("masscan", "masscan", "scanner", "", "masscan", &[], ""),    b("sqlmap", "sqlmap", "scanner", "", "sqlmap", &[], ""),    b("Nikto", "Nikto", "scanner", "", "Nikto", &[], ""),    b("WPScan", "WPScan", "scanner", "", "WPScan", &[], ""),    // Automation and libraries.    b("HeadlessChrome", "Headless Chrome", "headless", "", "HeadlessChrome", &[], ""),    b("PhantomJS", "PhantomJS", "headless", "", "PhantomJS", &[], ""),    b("Scrapy", "Scrapy", "library", "", "Scrapy", &[], ""),    b("curl/", "curl", "library", "", "curl", &[], ""),    b("Wget", "Wget", "library", "", "Wget", &[], ""),    b("python-requests", "Python requests", "library", "", "python-requests", &[], ""),    b("python-urllib", "Python urllib", "library", "", "python-urllib", &[], ""),    b("python-httpx", "Python httpx", "library", "", "python-httpx", &[], ""),    b("aiohttp", "aiohttp", "library", "", "aiohttp", &[], ""),    b("Go-http-client", "Go net/http", "library", "", "Go-http-client", &[], ""),    b("okhttp", "OkHttp", "library", "", "okhttp", &[], ""),    b("Apache-HttpClient", "Apache HttpClient", "library", "", "Apache-HttpClient", &[], ""),    b("Java/", "Java", "library", "", "Java", &[], ""),    b("node-fetch", "node-fetch", "library", "", "node-fetch", &[], ""),    b("axios/", "axios", "library", "", "axios", &[], ""),    b("undici", "undici", "library", "", "undici", &[], ""),    b("libwww-perl", "libwww-perl", "library", "", "libwww-perl", &[], ""),    b("PostmanRuntime", "Postman", "library", "", "PostmanRuntime", &[], ""),    b("HTTPie", "HTTPie", "library", "", "HTTPie", &[], ""),    b("colly", "Colly", "library", "", "colly", &[], ""),];const CATEGORIES: &[&str] = &["search", "ads", "ai_training", "ai_search", "ai_assistant", "social_preview", "seo", "monitoring", "feed_reader", "archiver", "scanner", "headless", "library", "other"];fn find(ua: &str) -> Option<&'static Bot> {    let l = ua.to_lowercase();    BOTS.iter().find(|b| !b.pattern.is_empty() && l.contains(&b.pattern.to_lowercase()))}fn describe(b: &Bot) -> Value {    json!({        "bot": true,        "name": b.name,        "category": b.category,        "owner": if b.owner.is_empty() { Value::Null } else { json!(b.owner) },        "robots_token": b.robots,        "verify": if !b.rdns.is_empty() { "reverse_dns" } else if !b.ranges.is_empty() { "ip_ranges" } else { "none" },        "rdns_domains": b.rdns,        "ip_ranges_url": if b.ranges.is_empty() { Value::Null } else { json!(b.ranges) },    })}/// What a User-Agent says about who is visiting: `{ bot, name, category, owner, robots_token, verify, rdns_domains, ip_ranges_url }`./// `category` is one of search, ads, ai_training, ai_search, ai_assistant (fetches for a person's question), social_preview, seo,/// monitoring, feed_reader, archiver, scanner, headless, library or other (an unknown "bot", "crawler" or "spider"). `verify` says/// how its operator lets you check a visit is real. A browser gives `{ bot: false }`; an empty User-Agent counts as a bot.pub fn bot_detect(user_agent: &str) -> Value {    let ua = user_agent.trim();    if ua.is_empty() {        return json!({ "bot": true, "name": null, "category": "other", "owner": null, "robots_token": null, "verify": "none", "rdns_domains": [], "ip_ranges_url": null });    }    if let Some(b) = find(ua) {        return describe(b);    }    let l = ua.to_lowercase();    if ["bot", "crawler", "spider", "crawl", "fetcher", "scraper", "slurp"].iter().any(|w| l.contains(w)) {        let name = ua.split(|c: char| c == ';' || c == '(' || c == ')' || c == ' ').find(|w| { let w = w.to_lowercase(); w.contains("bot") || w.contains("crawl") || w.contains("spider") }).map(|w| w.split('/').next().unwrap_or(w).to_string());        return json!({ "bot": true, "name": name, "category": "other", "owner": null, "robots_token": name, "verify": "none", "rdns_domains": [], "ip_ranges_url": null });    }    json!({ "bot": false })}/// Whether a visit that says it is `user_agent` comes from its operator: `ip`'s reverse DNS name must end in one of the operator's/// domains, and that name must resolve back to `ip` (Google's, Bing's and Apple's documented check). Gives `{ claimed, verified,/// hostname, reason }`; bots without published domains are never verified this way (compare with their IP ranges instead).pub fn bot_verify(user_agent: &str, ip: &str) -> Result<Value, String> {    let addr: std::net::IpAddr = ip.trim().parse().map_err(|_| format!("`{}` is not an IP address", ip))?;    let Some(b) = find(user_agent) else {        return Ok(json!({ "claimed": null, "verified": false, "hostname": null, "reason": "the User-Agent names no known bot" }));    };    if b.rdns.is_empty() {        let how = if b.ranges.is_empty() { "its operator publishes no way to check".to_string() } else { format!("check the IP against {}", b.ranges) };        return Ok(json!({ "claimed": b.name, "verified": false, "hostname": null, "reason": format!("{} cannot be checked by DNS: {}", b.name, how) }));    }    let host = match dns_lookup::lookup_addr(&addr) {        Ok(h) => h.trim_end_matches('.').to_lowercase(),        Err(_) => return Ok(json!({ "claimed": b.name, "verified": false, "hostname": null, "reason": format!("{} has no reverse DNS name", ip) })),    };    if !b.rdns.iter().any(|d| host == *d || host.ends_with(&format!(".{}", d))) {        return Ok(json!({ "claimed": b.name, "verified": false, "hostname": host, "reason": format!("{} is not under {}: an impostor", host, b.rdns.join(" or ")) }));    }    let back = dns_lookup::lookup_host(&host).unwrap_or_default();    if !back.contains(&addr) {        return Ok(json!({ "claimed": b.name, "verified": false, "hostname": host, "reason": format!("{} does not resolve back to {}", host, ip) }));    }    Ok(json!({ "claimed": b.name, "verified": true, "hostname": host, "reason": "reverse and forward DNS agree" }))}/// Every known bot of `category` (empty for all): `[{ name, category, owner, robots_token, ... }]`.pub fn bot_list(category: &str) -> Vec<Value> {    BOTS.iter().filter(|b| category.is_empty() || b.category == category).map(describe).collect()}/// robots.txt rules that keep out every bot of the `block` categories (say `["ai_training"]`) except those named in `allow`,/// then let everyone else in. Each token is written once; `sitemap` (a URL, or empty) is added at the end.pub fn bot_robots(block: Vec<String>, allow: Vec<String>, sitemap: &str) -> Result<String, String> {    for c in &block {        if !CATEGORIES.contains(&c.as_str()) {            return Err(format!("unknown category `{}`: {}", c, CATEGORIES.join(", ")));        }    }    let allowed = |b: &Bot| allow.iter().any(|a| a.eq_ignore_ascii_case(b.name) || a.eq_ignore_ascii_case(b.robots));    let mut tokens: Vec<&str> = vec![];    for b in BOTS.iter().filter(|b| block.iter().any(|c| c == b.category) && !allowed(b)) {        if !tokens.contains(&b.robots) {            tokens.push(b.robots);        }    }    let mut out = String::new();    if !tokens.is_empty() {        for t in &tokens {            out.push_str(&format!("User-agent: {}\n", t));        }        out.push_str("Disallow: /\n\n");    }    out.push_str("User-agent: *\nAllow: /\n");    if !sitemap.trim().is_empty() {        out.push_str(&format!("\nSitemap: {}\n", sitemap.trim()));    }    Ok(out)}