Text encodings

use text_encoding.qweruse charset_detect, charset_read from text_encoding      // Quillo 0.6: only these

encoding_rs = "0.8"chardetng = "0.1"

// Imports of files saved by old systems: a French menu in Windows-1252 and a Japanese price list in Shift_JIS.////   quillo run text_encoding.qwe      then open http://localhost:8080/menuuse "text_encoding.qwer"route GET /menu = { detected: charset_detect("text_encoding_sample.txt")?, lines: split(trim(charset_read("text_encoding_sample.txt", "auto")?), "\n") }route GET /prices {    let text = charset_read("text_encoding_sample.csv", "shift_jis")?    let rows = split(trim(text), "\n") | map((l) => split(l, ","))    drop(rows, 1) | map((r) => { item: r[0], yen: int(r[1]), note: r[2] })}// A UTF-8 copy for the rest of the app, and an export back to Windows-1252 for an old till.route GET /convert = {    copy: charset_to_utf8("text_encoding_sample.txt", "data/menu.utf8.txt", "auto")?,    till: charset_write("data/till.txt", "Crème brûlée 4,50 € ✓", "windows-1252")?,    back: charset_read("data/till.txt", "windows-1252")?,}

//! Text files in old encodings (Windows-1252, Latin-1, Shift_JIS, GBK, Big5, EUC-KR...) detected and read as UTF-8, and written back in them for systems that need it.//! version: 1.0.0//!//! [dependencies]//! encoding_rs = "0.8"//! chardetng = "0.1"use encoding_rs::Encoding;use serde_json::{json, Value};const MAX_BYTES: u64 = 256 * 1024 * 1024;fn read(path: &str) -> Result<Vec<u8>, String> {    let size = std::fs::metadata(path).map_err(|e| format!("{}: {}", path, e))?.len();    if size > MAX_BYTES {        return Err(format!("{}: larger than 256 MB", path));    }    std::fs::read(path).map_err(|e| format!("{}: {}", path, e))}// The encoding the bytes start with a mark for, else UTF-8 when they are valid UTF-8, else chardetng's guess.fn detect(bytes: &[u8]) -> (&'static Encoding, bool, bool) {    if let Some((enc, _)) = Encoding::for_bom(bytes) {        return (enc, true, true);    }    if std::str::from_utf8(bytes).is_ok() {        return (encoding_rs::UTF_8, true, false);    }    let mut d = chardetng::EncodingDetector::new();    d.feed(&bytes[..bytes.len().min(1 << 20)], true);    let (enc, sure) = d.guess_assess(None, true);    (enc, sure, false)}fn named(label: &str) -> Result<&'static Encoding, String> {    let l = label.trim().to_lowercase();    let l = match l.as_str() {        "latin1" | "latin-1" => "iso-8859-1",        "sjis" | "shift-jis" => "shift_jis",        "cp1252" | "ansi" => "windows-1252",        _ => l.as_str(),    };    Encoding::for_label(l.as_bytes()).ok_or_else(|| format!("`{}` is not an encoding it knows (try windows-1252, iso-8859-1, shift_jis, gbk, big5, euc-kr, utf-16le)", label))}fn decode(bytes: &[u8], encoding: &str) -> Result<(String, &'static Encoding, bool), String> {    let enc = if encoding.is_empty() || encoding.eq_ignore_ascii_case("auto") { detect(bytes).0 } else { named(encoding)? };    let (text, used, errors) = enc.decode(bytes);    Ok((text.into_owned(), used, errors))}/// The encoding of a text file, guessed from its bytes: `{ encoding, confident, bom }`, `encoding` a name like "windows-1252", "Shift_JIS", "GBK" or "UTF-8"./// Plain ASCII and valid UTF-8 are "UTF-8"; `confident` is false when the guess is a close call (short files, few accented letters).pub fn charset_detect(path: &str) -> Result<Value, String> {    let bytes = read(path)?;    let (enc, sure, bom) = detect(&bytes);    Ok(json!({ "encoding": enc.name(), "confident": sure, "bom": bom }))}/// The file's text, decoded from `encoding` ("" or "auto" to detect it): an import of a CSV saved by Excel, or a file from an old system./// A byte the encoding has no character for becomes "�"; a byte-order mark is dropped.pub fn charset_read(path: &str, encoding: &str) -> Result<String, String> {    Ok(decode(&read(path)?, encoding)?.0)}/// Rewrites the file `from` as UTF-8 at `to` (folders made), from `encoding` ("" or "auto" to detect it): `{ encoding, chars, replaced }`,/// `replaced` true when some bytes could not be read and became "�".pub fn charset_to_utf8(from: &str, to: &str, encoding: &str) -> Result<Value, String> {    let (text, enc, errors) = decode(&read(from)?, encoding)?;    if let Some(dir) = std::path::Path::new(to).parent().filter(|d| !d.as_os_str().is_empty()) {        std::fs::create_dir_all(dir).map_err(|e| format!("{}: {}", dir.display(), e))?;    }    std::fs::write(to, text.as_bytes()).map_err(|e| format!("{}: {}", to, e))?;    Ok(json!({ "encoding": enc.name(), "chars": text.chars().count(), "replaced": errors }))}/// Writes `text` to `path` in `encoding` (folders made), for a system that cannot read UTF-8: `{ bytes, unmappable }`./// Characters the encoding lacks are written as `&#NNNN;`, and `unmappable` is true then. UTF-16 is not written this way: use "utf-8" or a legacy encoding.pub fn charset_write(path: &str, text: &str, encoding: &str) -> Result<Value, String> {    let enc = named(encoding)?;    let (bytes, _, unmappable) = enc.encode(text);    if let Some(dir) = std::path::Path::new(path).parent().filter(|d| !d.as_os_str().is_empty()) {        std::fs::create_dir_all(dir).map_err(|e| format!("{}: {}", dir.display(), e))?;    }    std::fs::write(path, &bytes).map_err(|e| format!("{}: {}", path, e))?;    Ok(json!({ "bytes": bytes.len(), "unmappable": unmappable }))}