Tokens

use tokens.qweruse tokens_count, tokens_cut from tokens      // Quillo 0.6: only these

tiktoken-rs = "0.12"

// A prompt kept inside a budget of tokens before it is sent to an OpenAI model.////   quillo run tokens.qwe      then open http://localhost:8080/count?text=Hello%20worlduse "tokens.qwer"let article = repeat("Quillo is a small language for web apps. Routes, pages and components live in one file, and the server is built in. ", 20)route GET /count ? { text: str, model: str = "gpt-4o" } = { tokens: tokens_count(text, model)?, window: tokens_window(model) }route GET /budget {    let prompt = tokens_cut(article, "gpt-4o", 50)?    { full: tokens_count(article, "gpt-4o")?, kept: tokens_count(prompt, "gpt-4o")?, prompt, pieces: len(tokens_split(article, "gpt-4o", 200)?) }}

//! Text counted and cut in OpenAI model tokens (GPT-4o, GPT-4, GPT-3.5, o1...), to keep prompts inside a model's window and estimate their cost; Claude and other models count their own way.//! version: 1.0.0//!//! [dependencies]//! tiktoken-rs = "0.12"use tiktoken_rs::{get_bpe_from_model, CoreBPE};const MAX: usize = 10_000_000;fn bpe(model: &str) -> Result<&'static CoreBPE, String> {    let m = if model.trim().is_empty() { "gpt-4o" } else { model.trim() };    get_bpe_from_model(m).map_err(|_| format!("no tokenizer known for model {:?}; try \"gpt-4o\", \"gpt-4\" or \"gpt-3.5-turbo\"", model))}fn check(text: &str) -> Result<(), String> {    if text.len() > MAX { Err("text up to 10 MB".into()) } else { Ok(()) }}/// Decodes tokens to text, dropping tokens at the end that would split a character.fn decode(b: &CoreBPE, toks: &[u32]) -> String {    let mut n = toks.len();    while n > 0 {        if let Ok(s) = b.decode(&toks[..n]) {            return s;        }        n -= 1;    }    String::new()}/// How many tokens `text` is for `model` ("gpt-4o", "gpt-4", "gpt-3.5-turbo", "o1"...; "" is gpt-4o). Special tokens such as `<|endoftext|>` count as plain text. Text up to 10 MB.pub fn tokens_count(text: &str, model: &str) -> Result<i64, String> {    check(text)?;    Ok(bpe(model)?.encode_ordinary(text).len() as i64)}/// The start of `text` that fits in `max_tokens` tokens for `model`, cut at a token (never inside a character); all of it when it fits.pub fn tokens_cut(text: &str, model: &str, max_tokens: i64) -> Result<String, String> {    check(text)?;    let b = bpe(model)?;    let toks = b.encode_ordinary(text);    let n = max_tokens.max(0) as usize;    if toks.len() <= n {        return Ok(text.to_string());    }    Ok(decode(b, &toks[..n]))}/// `text` in pieces of at most `max_tokens` tokens each (at least 10), in order, for models with a small window.pub fn tokens_split(text: &str, model: &str, max_tokens: i64) -> Result<Vec<String>, String> {    check(text)?;    let b = bpe(model)?;    let n = max_tokens.max(10) as usize;    let toks = b.encode_ordinary(text);    let mut out = Vec::new();    let mut i = 0;    while i < toks.len() {        let mut end = (i + n).min(toks.len());        // Back off so a piece never ends inside a character.        while end > i + 1 && b.decode(&toks[i..end]).is_err() {            end -= 1;        }        out.push(decode(b, &toks[i..end]));        i = end;    }    Ok(out)}/// The size of `model`'s context window in tokens (128000 for gpt-4o), or null when it is not known.pub fn tokens_window(model: &str) -> Option<i64> {    tiktoken_rs::model::get_context_size(model.trim()).map(|n| n as i64)}