Chunker

use chunker.qweruse chunk_text, chunk_markdown from chunker      // Quillo 0.6: only these

text-splitter = { version = "0.33", features = ["markdown", "tiktoken-rs"] }tiktoken-rs = "0.12"

// A help article split into chunks for an embeddings index: by headings, with a little overlap.////   quillo run chunker.qwe      then open http://localhost:8080/chunksuse "chunker.qwer"let guide = read_file("chunker_sample.md")?route GET /chunks ? { size: int = 300, overlap: int = 0 } = chunk_markdown(guide, size, overlap)? | map((c) => { chars: len(c), text: c })route GET /tokens = chunk_tokens(guide, 80, 10, true)? | map((c) => { start: slice(c, 0, 40) })route GET /text = chunk_text("One sentence here. Another sentence there. And a third one to finish.", 40, 0)?

//! Text and Markdown split into chunks of a given size, with overlap, for embeddings, vector search and LLM prompts: splits fall between sections, paragraphs and sentences before words.//! version: 1.0.0//!//! [dependencies]//! text-splitter = { version = "0.33", features = ["markdown", "tiktoken-rs"] }//! tiktoken-rs = "0.12"use text_splitter::{ChunkConfig, MarkdownSplitter, TextSplitter};const MAX: usize = 20_000_000;fn sizes(text: &str, size: i64, overlap: i64) -> Result<(usize, usize), String> {    if text.len() > MAX {        return Err("text up to 20 MB".into());    }    if size < 1 {        return Err("size is at least 1".into());    }    if overlap < 0 || overlap >= size {        return Err("overlap is from 0 to less than the size".into());    }    Ok((size as usize, overlap as usize))}/// `text` in chunks of at most `max_chars` characters, each starting with the last `overlap` characters or so of the one before (0 for none), cut at the largest boundary that fits: blank lines, then line breaks, sentences, words. Spaces at the ends are trimmed.pub fn chunk_text(text: &str, max_chars: i64, overlap: i64) -> Result<Vec<String>, String> {    let (s, o) = sizes(text, max_chars, overlap)?;    let cfg = ChunkConfig::new(s).with_overlap(o).map_err(|e| e.to_string())?;    Ok(TextSplitter::new(cfg).chunks(text).map(String::from).collect())}/// Markdown in chunks of at most `max_chars` characters, cut at headings first, then blocks (lists, code, paragraphs), sentences and words, so a chunk is about one thing.pub fn chunk_markdown(text: &str, max_chars: i64, overlap: i64) -> Result<Vec<String>, String> {    let (s, o) = sizes(text, max_chars, overlap)?;    let cfg = ChunkConfig::new(s).with_overlap(o).map_err(|e| e.to_string())?;    Ok(MarkdownSplitter::new(cfg).chunks(text).map(String::from).collect())}/// As `chunk_markdown` (or `chunk_text` when `markdown` is false), sized in OpenAI tokens (cl100k, as text-embedding-3 counts them) rather than characters: `max_tokens` per chunk, `overlap` tokens shared.pub fn chunk_tokens(text: &str, max_tokens: i64, overlap: i64, markdown: bool) -> Result<Vec<String>, String> {    let (s, o) = sizes(text, max_tokens, overlap)?;    let bpe = tiktoken_rs::cl100k_base_singleton();    let cfg = ChunkConfig::new(s).with_sizer(bpe).with_overlap(o).map_err(|e| e.to_string())?;    Ok(if markdown {        MarkdownSplitter::new(cfg).chunks(text).map(String::from).collect()    } else {        TextSplitter::new(cfg).chunks(text).map(String::from).collect()    })}