Secrets scanner

use secrets_scan.qweruse secrets_find, secrets_any from secrets_scan      // Quillo 0.6: only these

regex = "1"

// Secrets kept out of support tickets: each message is scanned before it is saved, and what is// saved has the keys masked.////   quillo run secrets_scan.qwe      then open http://localhost:8080/scan//// The fake tokens below are put together from pieces, so this file itself holds none.use "secrets_scan.qwer"let TICKET = join([    "Hi, our deploy fails since this morning. Config:",    "  AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE",    "  DATABASE_URL=postgres://shop:" + "s3cr3t-Pa55" + "@db.internal:5432/shop",    "  STRIPE_KEY=" + "sk_" + "live_" + "4eC39HqLyjWDarjtT1zdp7dc",    "  api_secret = \"" + "q8Zr2Lw0" + "Vx9kTn4M" + "pC7s\"",    "  slack: " + "xox" + "b-1234567890-" + "abcdefABCDEF1234",    "The token in the header was " + "eyJhbGciOiJIUzI1NiJ9" + ".eyJzdWIiOiI0MiIsIm5hbWUiOiJBZGEifQ" + ".c2lnbmF0dXJlLW5vdC1yZWFs",    "Thanks, Ada",], "\n")route GET /scan = {    found: secrets_find(TICKET),    any_real: secrets_any(TICKET),    saved: secrets_redact(TICKET),}route GET /clean = {    any: secrets_any("Order 1042 shipped, tracking 1Z999AA10123456784."),    placeholder_only: secrets_any("AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE"),    rules: len(secrets_rules()),}route POST /tickets { json: { text: str } } {    if secrets_any(text) {        reply 422 { error: "this message holds a key or a password: it was not saved", found: secrets_find(text) }    }    { saved: true }}

//! Secrets found in text before it is saved, logged or published: cloud keys, API tokens of two dozen services, private keys, JWTs, passwords in URLs and high-entropy values in assignments, by gitleaks-style rules; and the text given back with them masked.//! version: 1.0.0//!//! [dependencies]//! regex = "1"use regex::Regex;use serde_json::{json, Value};use std::sync::OnceLock;struct Rule {    id: &'static str,    title: &'static str,    pattern: &'static str,    /// The capture group holding the secret (0: the whole match).    group: usize,    /// Minimum Shannon entropy (bits per character) of the secret, for the generic rules.    entropy: f64,}const RULES: &[Rule] = &[    Rule { id: "private-key", title: "Private key", pattern: r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP |ENCRYPTED )?PRIVATE KEY(?: BLOCK)?-----", group: 0, entropy: 0.0 },    Rule { id: "aws-access-key-id", title: "AWS access key ID", pattern: r"\b((?:AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16})\b", group: 1, entropy: 0.0 },    Rule { id: "aws-secret-access-key", title: "AWS secret access key", pattern: r#"(?i)aws.{0,20}?(?:secret|key).{0,20}?[=:]\s*["']?([A-Za-z0-9/+]{40})\b"#, group: 1, entropy: 3.0 },    Rule { id: "github-pat", title: "GitHub personal access token", pattern: r"\b(gh[pousr]_[A-Za-z0-9]{36,76})\b", group: 1, entropy: 0.0 },    Rule { id: "github-fine-grained-pat", title: "GitHub fine-grained token", pattern: r"\b(github_pat_[A-Za-z0-9_]{82})\b", group: 1, entropy: 0.0 },    Rule { id: "gitlab-pat", title: "GitLab personal access token", pattern: r"\b(glpat-[A-Za-z0-9_-]{20,})\b", group: 1, entropy: 0.0 },    Rule { id: "slack-token", title: "Slack token", pattern: r"\b(xox[baprse]-[0-9A-Za-z-]{10,})\b", group: 1, entropy: 0.0 },    Rule { id: "slack-webhook", title: "Slack incoming webhook", pattern: r"(https://hooks\.slack\.com/(?:services|workflows|triggers)/[A-Za-z0-9+/]{30,})", group: 1, entropy: 0.0 },    Rule { id: "discord-webhook", title: "Discord webhook", pattern: r"(https://(?:canary\.|ptb\.)?discord(?:app)?\.com/api/webhooks/\d+/[A-Za-z0-9_-]{30,})", group: 1, entropy: 0.0 },    Rule { id: "stripe-key", title: "Stripe secret or restricted key", pattern: r"\b((?:sk|rk)_(?:live|test)_[A-Za-z0-9]{20,})\b", group: 1, entropy: 0.0 },    Rule { id: "google-api-key", title: "Google API key", pattern: r"\b(AIza[0-9A-Za-z_-]{35})\b", group: 1, entropy: 0.0 },    Rule { id: "google-oauth-secret", title: "Google OAuth client secret", pattern: r"\b(GOCSPX-[A-Za-z0-9_-]{28})\b", group: 1, entropy: 0.0 },    Rule { id: "anthropic-key", title: "Anthropic API key", pattern: r"\b(sk-ant-(?:api|admin)\d\d-[A-Za-z0-9_-]{20,})", group: 1, entropy: 0.0 },    Rule { id: "openai-key", title: "OpenAI API key", pattern: r"\b(sk-(?:proj-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}T3BlbkFJ[A-Za-z0-9_-]{20,})", group: 1, entropy: 0.0 },    Rule { id: "openai-project-key", title: "OpenAI project key", pattern: r"\b(sk-(?:proj|svcacct|admin)-[A-Za-z0-9_-]{40,})", group: 1, entropy: 0.0 },    Rule { id: "huggingface-token", title: "Hugging Face token", pattern: r"\b(hf_[A-Za-z]{34})\b", group: 1, entropy: 0.0 },    Rule { id: "twilio-api-key", title: "Twilio API key", pattern: r"\b(SK[0-9a-fA-F]{32})\b", group: 1, entropy: 0.0 },    Rule { id: "sendgrid-key", title: "SendGrid API key", pattern: r"\b(SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43})\b", group: 1, entropy: 0.0 },    Rule { id: "mailgun-key", title: "Mailgun API key", pattern: r"\b(key-[0-9a-f]{32})\b", group: 1, entropy: 0.0 },    Rule { id: "postmark-token", title: "Postmark server token", pattern: r#"(?i)postmark.{0,30}?[=:]\s*["']?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"#, group: 1, entropy: 0.0 },    Rule { id: "npm-token", title: "npm access token", pattern: r"\b(npm_[A-Za-z0-9]{36})\b", group: 1, entropy: 0.0 },    Rule { id: "pypi-token", title: "PyPI upload token", pattern: r"\b(pypi-AgEIcHlwaS5vcmc[A-Za-z0-9_-]{50,})", group: 1, entropy: 0.0 },    Rule { id: "shopify-token", title: "Shopify access token", pattern: r"\b(shp(?:at|ca|pa|ss)_[a-fA-F0-9]{32})\b", group: 1, entropy: 0.0 },    Rule { id: "digitalocean-token", title: "DigitalOcean token", pattern: r"\b(do[opr]_v1_[a-f0-9]{64})\b", group: 1, entropy: 0.0 },    Rule { id: "telegram-bot-token", title: "Telegram bot token", pattern: r"\b(\d{8,10}:AA[A-Za-z0-9_-]{33})\b", group: 1, entropy: 0.0 },    Rule { id: "square-token", title: "Square access token", pattern: r"\b(EAAA[A-Za-z0-9_-]{60})\b", group: 1, entropy: 0.0 },    Rule { id: "azure-storage-key", title: "Azure storage account key", pattern: r"AccountKey=([A-Za-z0-9+/]{86}==)", group: 1, entropy: 0.0 },    Rule { id: "jwt", title: "JSON Web Token", pattern: r"\b(eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})", group: 1, entropy: 0.0 },    Rule { id: "password-in-url", title: "Password in a URL", pattern: r"\b[a-zA-Z][a-zA-Z0-9+.-]*://[^/\s:@]+:([^/\s:@]{3,})@[^/\s]+", group: 1, entropy: 0.0 },    Rule {        id: "generic-secret",        title: "Secret in an assignment",        pattern: r#"(?i)\b[\w.-]*(?:api[_-]?key|secret|token|passw(?:or)?d|pwd|credential|auth)[\w.-]*["']?\s*(?:=|:|:=|=>)\s*["']([^"'\s]{12,})["']"#,        group: 1,        entropy: 3.5,    },];fn compiled() -> &'static Vec<Regex> {    static C: OnceLock<Vec<Regex>> = OnceLock::new();    C.get_or_init(|| RULES.iter().map(|r| Regex::new(r.pattern).expect("rule patterns are valid")).collect())}fn entropy(s: &str) -> f64 {    let n = s.chars().count() as f64;    if n == 0.0 {        return 0.0;    }    let mut counts = std::collections::HashMap::new();    for c in s.chars() {        *counts.entry(c).or_insert(0usize) += 1;    }    counts.values().map(|&c| { let p = c as f64 / n; -p * p.log2() }).sum()}fn mask(s: &str) -> String {    let cs: Vec<char> = s.chars().collect();    if cs.len() <= 8 {        return "*".repeat(cs.len());    }    format!("{}{}{}", cs[..4].iter().collect::<String>(), "*".repeat(cs.len() - 6), cs[cs.len() - 2..].iter().collect::<String>())}fn placeholder(s: &str) -> bool {    let l = s.to_lowercase();    l.contains("example") || l.contains("xxxx") || l.contains("****") || l.contains("${") || l.contains("{{") || l.starts_with('<') || l.contains("changeme") || l.contains("your_") || l.contains("dummy")}struct Hit {    rule: usize,    start: usize,    end: usize,    secret: String,}fn hits(text: &str) -> Vec<Hit> {    let mut out: Vec<Hit> = vec![];    for (i, re) in compiled().iter().enumerate() {        let rule = &RULES[i];        for caps in re.captures_iter(text) {            let Some(m) = caps.get(rule.group) else { continue };            if rule.entropy > 0.0 && entropy(m.as_str()) < rule.entropy {                continue;            }            // A more specific rule found the same place first.            if out.iter().any(|h| m.start() < h.end && h.start < m.end()) {                continue;            }            out.push(Hit { rule: i, start: m.start(), end: m.end(), secret: m.as_str().to_string() });        }    }    out.sort_by_key(|h| h.start);    out}/// Every secret in `text`: `[{ rule, title, line, column, masked, entropy, placeholder }]`, lines and columns from 1./// `placeholder` is true for values that look like documentation (AWS's `...EXAMPLE` keys, `xxxx`, `${VAR}`)./// The secret itself is never given back, only masked.pub fn secrets_find(text: &str) -> Vec<Value> {    hits(text)        .into_iter()        .map(|h| {            let before = &text[..h.start];            let line = before.matches('\n').count() + 1;            let column = before.rsplit('\n').next().map_or(0, |l| l.chars().count()) + 1;            json!({                "rule": RULES[h.rule].id,                "title": RULES[h.rule].title,                "line": line,                "column": column,                "masked": mask(&h.secret),                "entropy": (entropy(&h.secret) * 100.0).round() / 100.0,                "placeholder": placeholder(&h.secret),            })        })        .collect()}/// Whether `text` holds any secret that is not a placeholder: a quick gate before saving or logging it.pub fn secrets_any(text: &str) -> bool {    hits(text).iter().any(|h| !placeholder(&h.secret))}/// `text` with every secret replaced by `[REDACTED:<rule>]`, the rest kept as it was.pub fn secrets_redact(text: &str) -> String {    let mut out = String::with_capacity(text.len());    let mut at = 0;    for h in hits(text) {        out.push_str(&text[at..h.start]);        out.push_str(&format!("[REDACTED:{}]", RULES[h.rule].id));        at = h.end;    }    out.push_str(&text[at..]);    out}/// The secrets in a file (a config, a log, a commit's diff), as `secrets_find` gives them, each with the file's path.pub fn secrets_find_file(path: &str) -> Result<Vec<Value>, String> {    let bytes = std::fs::read(path).map_err(|e| format!("{}: {}", path, e))?;    let text = String::from_utf8_lossy(&bytes);    Ok(secrets_find(&text).into_iter().map(|mut v| { v["file"] = json!(path); v }).collect())}/// The rules: `[{ id, title }]`.pub fn secrets_rules() -> Vec<Value> {    RULES.iter().map(|r| json!({ "id": r.id, "title": r.title })).collect()}