Roaring sets

use roaring.qweruse roaring_from, roaring_change from roaring      // Quillo 0.6: only these

roaring = "0.12"base64 = "0.22"

// Audience segments for a newsletter: customers who bought, who opened, and who did not unsubscribe, combined as sets.////   quillo run roaring.qwe      then open http://localhost:8080/segmentuse "roaring.qwer"route GET /segment {    let bought = roaring_from(range(1, 100001))?                          // customers 1 to 100,000    let opened = roaring_from(range(1, 200001) | filter((i) => i % 3 == 0))?    let unsubscribed = roaring_from([3, 6, 9, 12, 99999])?    let target = roaring_op("difference", [roaring_op("intersect", [bought, opened])?, unsubscribed])?    {        target: roaring_count(target)?,        first: roaring_ids(target, 0, 5)?,        has_15: roaring_contains(target, 15)?,        has_12: roaring_contains(target, 12)?,        stored_chars: len(bought),    }}route GET /likes {    let likes = roaring_change("", [7, 42, 1001], [])?    let after = roaring_change(likes, [5], [42])?    { likes: roaring_ids(after, 0, 10)?, count: roaring_count(after)?, both: roaring_count(roaring_op("union", [likes, after])?)? }}

//! Compressed sets of ids (Roaring bitmaps): who liked a post, who is in a segment, which products match a filter; added to, combined (union, intersection, difference) and counted fast, and kept as compact base64 text in a column or a cache.//! version: 1.0.0//!//! [dependencies]//! roaring = "0.12"//! base64 = "0.22"use base64::Engine;use roaring::RoaringBitmap;const B64: base64::engine::GeneralPurpose = base64::engine::general_purpose::STANDARD;fn load(set: &str) -> Result<RoaringBitmap, String> {    if set.is_empty() {        return Ok(RoaringBitmap::new());    }    if set.len() > 100_000_000 {        return Err("sets up to 100 MB of text".into());    }    let bytes = B64.decode(set.trim()).map_err(|_| "a set is the base64 text these functions give".to_string())?;    RoaringBitmap::deserialize_from(&bytes[..]).map_err(|e| format!("not a Roaring set: {}", e))}fn save(mut b: RoaringBitmap) -> Result<String, String> {    // Runs of ids in a row are kept as ranges: a million in a row is a few bytes.    b.optimize();    let mut out = Vec::with_capacity(b.serialized_size());    b.serialize_into(&mut out).map_err(|e| e.to_string())?;    Ok(B64.encode(out))}fn ids(v: &[i64]) -> Result<Vec<u32>, String> {    if v.len() > 10_000_000 {        return Err("up to 10 million ids a call".into());    }    v.iter().map(|&i| u32::try_from(i).map_err(|_| format!("ids go from 0 to 4294967295, not {}", i))).collect()}/// A set of `ids` (whole numbers from 0 to 4,294,967,295; repeats count once) as base64 text: ids in a row take a few bytes a run, a million scattered ones up to 2.7 MB.pub fn roaring_from(ids: Vec<i64>) -> Result<String, String> {    let mut b = RoaringBitmap::new();    b.extend(self::ids(&ids)?);    save(b)}/// `set` (base64 text from these functions, "" for an empty set) with `add` put in and `remove` taken out.pub fn roaring_change(set: &str, add: Vec<i64>, remove: Vec<i64>) -> Result<String, String> {    let mut b = load(set)?;    b.extend(ids(&add)?);    for i in ids(&remove)? {        b.remove(i);    }    save(b)}/// `sets` combined by `op`: "union" (in any), "intersect" (in all), "difference" (in the first and none of the others) or "xor" (in an odd number of them). Liked by followers of both: `roaring_op("intersect", [likes, followers])`.pub fn roaring_op(op: &str, sets: Vec<String>) -> Result<String, String> {    if sets.len() > 10_000 {        return Err("up to 10,000 sets".into());    }    let mut it = sets.iter();    let Some(first) = it.next() else { return save(RoaringBitmap::new()) };    let mut acc = load(first)?;    for s in it {        let b = load(s)?;        match op {            "union" => acc |= b,            "intersect" => acc &= b,            "difference" => acc -= b,            "xor" => acc ^= b,            o => return Err(format!("op is union, intersect, difference or xor, not {:?}", o)),        }    }    save(acc)}/// How many ids `set` holds.pub fn roaring_count(set: &str) -> Result<i64, String> {    Ok(load(set)?.len() as i64)}/// Whether `set` holds `id`.pub fn roaring_contains(set: &str, id: i64) -> Result<bool, String> {    Ok(u32::try_from(id).map_or(false, |i| load(set).map_or(false, |b| b.contains(i))))}/// The ids of `set` in order, `limit` of them (up to 100,000) after skipping `offset`: for a page of results.pub fn roaring_ids(set: &str, offset: i64, limit: i64) -> Result<Vec<i64>, String> {    let b = load(set)?;    Ok(b.iter().skip(offset.max(0) as usize).take(limit.clamp(0, 100_000) as usize).map(|i| i as i64).collect())}