//! Token-balance gate for master/spec.md form-only blocks. //! //! The mini-spec is the experimental treatment; if the JSON-cohort's //! form-only scaffolding is meaningfully longer than the AIL-cohort's //! (or vice versa), the experiment is biased before the model ever //! sees the prompts. This test catches gross imbalance. use std::path::PathBuf; use xmodel_render::splitter::{self, Form, Segment}; fn master_dir() -> PathBuf { let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")); manifest.parent().unwrap().join("master") } #[test] fn form_only_block_token_totals_within_five_percent() { let spec_text = std::fs::read_to_string(master_dir().join("spec.md")) .expect("master/spec.md must exist"); let segments = splitter::split(&spec_text); let mut json_buf = String::new(); let mut ail_buf = String::new(); for seg in &segments { if let Segment::FormOnly { form, body } = seg { match form { Form::Json => json_buf.push_str(body), Form::Ail => ail_buf.push_str(body), } } } let bpe = tiktoken_rs::cl100k_base().expect("cl100k_base tokenizer should load"); let json_count = bpe.encode_with_special_tokens(&json_buf).len(); let ail_count = bpe.encode_with_special_tokens(&ail_buf).len(); assert!(json_count > 0, "JSON-form blocks produced zero tokens"); assert!(ail_count > 0, "AIL-form blocks produced zero tokens"); let max = json_count.max(ail_count) as f64; let diff = (json_count as i64 - ail_count as i64).unsigned_abs() as f64; let ratio = diff / max; assert!( ratio <= 0.05, "form-only block token imbalance: json={json_count}, ail={ail_count}, ratio={ratio:.4} > 0.05", ); }