HTML to Markdown

use html_markdown.qweruse html_to_markdown, html_to_markdown_with from html_markdown      // Quillo 0.6: only these

htmd = "=0.5.6"

// A blog moving to Markdown files: old HTML posts imported, the menus and footers dropped, and rich-text editor output saved as Markdown.////   quillo run html_markdown.qwe      then open http://localhost:8080/post.mduse "html_markdown.qwer"route GET /post.md = respond(200, html_file_to_markdown("html_markdown_sample.html", { skip: ["nav", "footer"] })?, { "content-type": "text/markdown; charset=utf-8" })route GET /snippets {    {        editor: html_to_markdown("<p>Hello <b>world</b>!</p><ul><li>one</li><li>two</li></ul>")?,        stars: html_to_markdown_with("<ul><li>one</li></ul><hr>", { bullet: "*", hr: "***" })?,        references: html_to_markdown_with("<p>See <a href=\"https://example.com/a\">this</a> and <a href=\"https://example.com/b\">that</a>.</p>", { links: "referenced" })?,        no_script: html_to_markdown("<p>Text</p><script>alert(1)</script>")?,    }}// POST HTML from a rich-text editor, get Markdown to store.route POST /convert = { markdown: html_to_markdown(request.body)? }

//! HTML turned into clean Markdown, as Turndown does: headings, lists, links, images, code blocks, quotes and tables kept, scripts, styles and the parts you name dropped; for importing old blog posts, rich-text editor output, and pages fed to an LLM.//! version: 1.0.0//!//! [dependencies]//! htmd = "=0.5.6"use htmd::options::{BulletListMarker, CodeBlockStyle, HeadingStyle, HrStyle, LinkStyle, Options};use htmd::HtmlToMarkdown;const ALWAYS_SKIP: [&str; 7] = ["script", "style", "noscript", "template", "iframe", "object", "head"];const MAX_FILE: u64 = 50 * 1024 * 1024;fn convert(html: &str, options: &serde_json::Value) -> Result<String, String> {    let s = |k: &str| options.get(k).and_then(|v| v.as_str()).unwrap_or("").to_lowercase();    let mut o = Options::default();    o.bullet_list_marker = if s("bullet") == "*" { BulletListMarker::Asterisk } else { BulletListMarker::Dash };    o.ul_bullet_spacing = 1;    o.ol_number_spacing = 1;    o.hr_style = match s("hr").as_str() { "***" => HrStyle::Asterisks, "___" => HrStyle::Underscores, _ => HrStyle::Dashes };    o.code_block_style = if s("code") == "indented" { CodeBlockStyle::Indented } else { CodeBlockStyle::Fenced };    o.heading_style = if s("headings") == "setext" { HeadingStyle::Setex } else { HeadingStyle::Atx };    o.link_style = match s("links").as_str() { "referenced" => LinkStyle::Referenced, "" | "inline" => LinkStyle::Inlined, l => return Err(format!("links is inline or referenced, not `{}`", l)) };    let mut skip: Vec<String> = ALWAYS_SKIP.iter().map(|x| x.to_string()).collect();    if let Some(extra) = options.get("skip").and_then(|v| v.as_array()) {        skip.extend(extra.iter().filter_map(|x| x.as_str()).map(|x| x.to_lowercase()));    }    let skip_refs: Vec<&str> = skip.iter().map(|x| x.as_str()).collect();    let conv = HtmlToMarkdown::builder().options(o).skip_tags(skip_refs).build();    let md = conv.convert(html).map_err(|e| format!("the HTML could not be read: {}", e))?;    // At most one blank line in a row, none at the ends.    let mut out = String::new();    let mut blank = 0;    for line in md.lines() {        let l = line.trim_end();        if l.is_empty() { blank += 1; if blank > 1 { continue } } else { blank = 0 }        out.push_str(l);        out.push('\n');    }    Ok(out.trim().to_string())}/// `html` as Markdown: `#` headings, `-` bullets, fenced code blocks, inline links; scripts, styles and other non-content left out.pub fn html_to_markdown(html: &str) -> Result<String, String> {    convert(html, &serde_json::Value::Null)}/// `html` as Markdown with `options`: `skip` (tags to drop with their content, such as `["nav", "footer", "aside"]`), `bullet` (`"-"` or `"*"`), `links` (`"inline"` or `"referenced"`, numbered at the end), `code` (`"fenced"` or `"indented"`), `headings` (`"atx"` or `"setext"`), `hr` (`"---"`, `"***"`).pub fn html_to_markdown_with(html: &str, options: serde_json::Value) -> Result<String, String> {    convert(html, &options)}/// An HTML file (an export, a saved page) as Markdown, with the same `options`.pub fn html_file_to_markdown(path: &str, options: serde_json::Value) -> Result<String, String> {    let f = std::fs::File::open(path).map_err(|e| format!("{}: {}", path, e))?;    let mut bytes = Vec::new();    std::io::Read::read_to_end(&mut std::io::Read::take(f, MAX_FILE), &mut bytes).map_err(|e| format!("{}: {}", path, e))?;    convert(&String::from_utf8_lossy(&bytes), &options)}