bzip2

use bzip2.qweruse bzip2_file, bunzip2_file from bzip2      // Quillo 0.6: only these

bzip2 = "0.6"base64 = "0.22"

// A nightly export of orders packed with bzip2, a look at its first lines without unpacking it, and text both ways.////   quillo run bzip2.qwe      then open http://localhost:8080/packuse "bzip2.qwer"route GET /pack {    let rows = for i in range(3000) => `${1000 + i},customer ${i % 50},${i % 90}.50,${["paid", "open", "refunded"][i % 3]}`    write_file("data/orders.csv", "id,customer,total,status\n" + join(rows, "\n"))    let size = bzip2_file("data/orders.csv", "data/orders.csv.bz2", 9)?    let packed = bzip2_text("hello, hello, hello", 9)?    {        csv_bytes: len(read_file("data/orders.csv")),        bz2_bytes: size,        unpacked_bytes: bunzip2_file("data/orders.csv.bz2", "data/orders.copy.csv")?,        first: bzip2_head("data/orders.csv.bz2", 3)?,        back: bunzip2_text(packed)?,    }}

//! bzip2 compression of files and text in pure Rust: smaller than gzip for text, and the .bz2 dumps and archives of other systems unpacked.//! version: 1.0.0//!//! [dependencies]//! bzip2 = "0.6"//! base64 = "0.22"use base64::engine::general_purpose::STANDARD;use base64::Engine;use bzip2::read::MultiBzDecoder;use bzip2::write::BzEncoder;use bzip2::Compression;use std::io::{Read, Write};const MAX_FILE: u64 = 16 * 1024 * 1024 * 1024;const MAX_TEXT: u64 = 64 * 1024 * 1024;fn clamp_level(l: i64) -> Compression {    Compression::new(l.clamp(1, 9) as u32)}fn create(to: &str) -> Result<std::fs::File, String> {    if let Some(dir) = std::path::Path::new(to).parent().filter(|d| !d.as_os_str().is_empty()) {        std::fs::create_dir_all(dir).map_err(|e| format!("{}: {}", dir.display(), e))?;    }    std::fs::File::create(to).map_err(|e| format!("{}: {}", to, e))}/// Compresses the file at `from` into `to` (often `from` + ".bz2"), at `level` 1 (fastest) to 9 (smallest, bzip2's usual): the size written, in bytes.pub fn bzip2_file(from: &str, to: &str, level: i64) -> Result<i64, String> {    let mut input = std::fs::File::open(from).map_err(|e| format!("{}: {}", from, e))?;    let mut enc = BzEncoder::new(std::io::BufWriter::new(create(to)?), clamp_level(level));    std::io::copy(&mut input, &mut enc).map_err(|e| e.to_string())?;    enc.finish().map_err(|e| e.to_string())?.flush().map_err(|e| e.to_string())?;    Ok(std::fs::metadata(to).map_err(|e| e.to_string())?.len() as i64)}/// Unpacks the .bz2 file at `from` into `to` (several streams one after another too, as `pbzip2` writes them): the size written, in bytes. Stops with an error past 16 GB.pub fn bunzip2_file(from: &str, to: &str) -> Result<i64, String> {    let input = std::fs::File::open(from).map_err(|e| format!("{}: {}", from, e))?;    let mut out = std::io::BufWriter::new(create(to)?);    let n = std::io::copy(&mut MultiBzDecoder::new(std::io::BufReader::new(input)).take(MAX_FILE + 1), &mut out).map_err(|e| format!("{}: {}", from, e))?;    if n > MAX_FILE {        return Err(format!("{}: more than 16 GB unpacked", from));    }    out.flush().map_err(|e| e.to_string())?;    Ok(n as i64)}/// `text` compressed with bzip2 at `level` 1 to 9, as base64 (a value cannot hold bytes).pub fn bzip2_text(text: &str, level: i64) -> Result<String, String> {    let mut enc = BzEncoder::new(Vec::new(), clamp_level(level));    enc.write_all(text.as_bytes()).map_err(|e| e.to_string())?;    Ok(STANDARD.encode(enc.finish().map_err(|e| e.to_string())?))}/// Base64 of bzip2 data, back to the text; up to 64 MB of it.pub fn bunzip2_text(b64: &str) -> Result<String, String> {    let bytes = STANDARD.decode(b64.trim()).map_err(|e| e.to_string())?;    let mut out = Vec::new();    MultiBzDecoder::new(&bytes[..]).take(MAX_TEXT + 1).read_to_end(&mut out).map_err(|e| e.to_string())?;    if out.len() as u64 > MAX_TEXT {        return Err("more than 64 MB of text".into());    }    String::from_utf8(out).map_err(|_| "the data is not UTF-8 text".to_string())}/// The first lines of a .bz2 text file, unpacked only as far as needed (at most 10,000 lines): a look into a large dump or log without unpacking it all.pub fn bzip2_head(path: &str, lines: i64) -> Result<Vec<String>, String> {    use std::io::BufRead;    let input = std::fs::File::open(path).map_err(|e| format!("{}: {}", path, e))?;    let r = std::io::BufReader::new(MultiBzDecoder::new(std::io::BufReader::new(input)));    r.lines().take(lines.clamp(0, 10_000) as usize).map(|l| l.map_err(|e| format!("{}: {}", path, e))).collect()}