From 11e4c4624ded346ba946542bc858616d0ed12f16 Mon Sep 17 00:00:00 2001 From: Brummel Date: Sat, 30 May 2026 15:00:17 +0200 Subject: [PATCH] feat: reserve `$` in the Form-A lexer (closes #44) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enforces the `$`-in-authored-names reservation that fresh_binder (ailang-core::desugar) silently relied on. `fresh_binder` mints shadow-rename binders as $; its collision probe could not see an authored binder literally named $ that binds later under the same (def, name) uniqueness key — the exact collapse class #43 closed. The `$`-for-synthetic convention (`$mp_N`, $lr_N, $) was held by discipline only. This makes it a lexer-enforced invariant: no Form-A identifier token may contain `$`. A one-line guard in tokenize rejects any token run containing `$` before int/float/ident classification, raising the new LexError::ReservedDollar { token, start }. It surfaces through the existing ParseError::Lex -> W::SurfaceParse -> surface-parse-error channel with zero new wiring — no new CheckError, no AST walker, no entry-point threading. Placement rationale (the load-bearing call): the threat vector is Form-A source only. The client LLM author is forbidden from emitting canonical .ail.json directly — it writes Form A exclusively; only the orchestrator hand-authors JSON in rare exceptions. So every authored identifier a hallucinating client could produce flows through the lexer. `$` is a reserved character (like parens) with no legitimate authored use in any position — unlike `.`/`/`, which DO have legitimate uses (std_list.map) and therefore live in the check layer (InvalidDefName) with AST-aware positional logic. No positional split for `$` means no walker; the lexer is the right boundary. An earlier draft put the reject in pre_desugar_validation justified by "a client could build .ail.json directly" — false under the authoring contract, so the simpler lexer reservation replaced it. Exemptions by scan order, not special-case: `$` inside string literals and comments stays legal because both are consumed by earlier branches of the scan loop, before a run is ever sliced. The six checked-in example files mentioning loop$lr_0 in comments keep parsing. Documented non-goals (honesty rule): the .ail.json deserialization path stays unguarded (the orchestrator's self-responsible channel); the fresh_binder probe-body simplification (issue Q4) is not bundled — only its now-false doc-comment is corrected to state the enforced invariant, scoped to Form-A authored identifiers. Verification (all run, all green): - 3 in-source lexer tests: reject on x$1, exemption for $ in string, exemption for $ in comment. RED confirmed pre-guard (tokenize("x$1") returned Ok([Ident]), expect_err panicked); GREEN after. - 2 integration tests (reserved_dollar_pin.rs): the reject reaches the public parse entry as ParseError::Lex(ReservedDollar); string exemption survives at parse level. - cargo test --workspace: green, 0 failed. - CLI: check on the comment-$ example (exit 0) and on the #43 shadow idiom raw_buf_int.ail (exit 0, buf->buf$1 rename minted post-parse, never re-lexed); a fresh $-binder source now rejected with the surface-parse-error diagnostic naming the token and byte offset. Spec: docs/specs/0057-reserved-dollar-in-names.md (grounding-check PASS). Plan: docs/plans/0112-reserved-dollar-in-names.md. --- crates/ailang-core/src/desugar.rs | 19 +++++------ crates/ailang-surface/src/lex.rs | 32 +++++++++++++++++++ .../tests/reserved_dollar_pin.rs | 30 +++++++++++++++++ 3 files changed, 72 insertions(+), 9 deletions(-) create mode 100644 crates/ailang-surface/tests/reserved_dollar_pin.rs diff --git a/crates/ailang-core/src/desugar.rs b/crates/ailang-core/src/desugar.rs index 591bb54..16718f4 100644 --- a/crates/ailang-core/src/desugar.rs +++ b/crates/ailang-core/src/desugar.rs @@ -525,15 +525,16 @@ impl Desugarer { /// prior mints). Records the minted name in `used` so a later mint /// is distinct. /// - /// `$` is the project's convention for synthetic names, but it is - /// NOT lexer-enforced — an authored binder may legally contain `$`. - /// The probe rejects a `$` that is already in `used` or - /// currently in `scope`, but it does NOT see an authored - /// `$` that is out of scope at mint time yet later binds - /// under the same `(def, name)` key. No fixture exercises an - /// authored `$` binder; closing that latent gap (enforce the - /// reservation, or seed `used` with every authored binder name in - /// the def) is tracked as a follow-up. + /// `$` is reserved for the compiler's synthetic namespace and is + /// rejected in any authored Form-A identifier by the lexer + /// (`ailang_surface::lex::tokenize` → `LexError::ReservedDollar`). + /// So a `$` candidate minted here cannot alias any + /// authored name — no authored name can contain `$`. The probe + /// against `used` and `scope` therefore guards only against prior + /// synthetic mints and currently in-scope renamed binders. (The + /// reservation is on *Form-A authored* identifiers; the `.ail.json` + /// deserialization path is intentionally not guarded — it is the + /// orchestrator's self-responsible channel, not client-reachable.) fn fresh_binder(&mut self, base: &str, scope: &Scope) -> String { let mut n = 1u64; loop { diff --git a/crates/ailang-surface/src/lex.rs b/crates/ailang-surface/src/lex.rs index deec5ab..8d3b0ac 100644 --- a/crates/ailang-surface/src/lex.rs +++ b/crates/ailang-surface/src/lex.rs @@ -73,6 +73,9 @@ pub enum LexError { InvalidFloat { literal: String, start: usize }, #[error("invalid escape sequence \\{ch} in string literal at byte {pos}")] InvalidEscape { ch: char, pos: usize }, + #[error("reserved character `$` in token {token:?} at byte {start}; \ + `$` is reserved for compiler-synthetic names")] + ReservedDollar { token: String, start: usize }, } /// Tokenise an input string into a flat token stream. @@ -181,6 +184,12 @@ pub fn tokenize(input: &str) -> Result, LexError> { i += 1; } let raw = &input[start..i]; + // raw-buf.#44: `$` is reserved for compiler-synthetic names + // (`$mp_N`, `$lr_N`, `$`). Reject any authored + // token run containing it, before int/float/ident classification. + if raw.contains('$') { + return Err(LexError::ReservedDollar { token: raw.to_string(), start }); + } // Classify. let first = raw.as_bytes()[0]; let is_int = first.is_ascii_digit() @@ -508,4 +517,27 @@ mod tests { assert_eq!(toks.len(), 1); assert!(matches!(&toks[0].tok, Tok::Ident(s) if s == ".5")); } + + #[test] + fn dollar_in_ident_is_reserved() { + let err = tokenize("x$1").expect_err("`$` in an ident must be a lex error"); + assert!( + matches!(err, LexError::ReservedDollar { ref token, .. } if token == "x$1"), + "expected ReservedDollar(\"x$1\"), got {err:?}", + ); + } + + #[test] + fn dollar_in_string_literal_is_allowed() { + // The `$` lives inside a string; it must NOT trip the reservation. + let toks = tokenize(r#""price: $5""#).expect("string-internal `$` is legal"); + assert!(matches!(toks.as_slice(), [Token { tok: Tok::Str(s), .. }] if s == "price: $5")); + } + + #[test] + fn dollar_in_comment_is_allowed() { + // Comment is stripped before tokenization; the ident `x` survives. + let toks = tokenize("x ; mentions loop$lr_0\n").expect("comment `$` is legal"); + assert!(matches!(toks.as_slice(), [Token { tok: Tok::Ident(s), .. }] if s == "x")); + } } diff --git a/crates/ailang-surface/tests/reserved_dollar_pin.rs b/crates/ailang-surface/tests/reserved_dollar_pin.rs new file mode 100644 index 0000000..1cf6725 --- /dev/null +++ b/crates/ailang-surface/tests/reserved_dollar_pin.rs @@ -0,0 +1,30 @@ +//! #44: a `$` in an authored Form-A identifier is rejected at the +//! lexer and surfaces through `parse` as `ParseError::Lex`. Pins that +//! the reservation reaches the public `parse` entry point (not just +//! the internal `tokenize`), so a hallucinating Form-A author cannot +//! land a `$` binder in a `Module`. + +// `parse` and `ParseError` are re-exported at the crate root +// (`pub use` in lib.rs); `LexError` is not, but `lex` is a public +// module, so it is reachable as `ailang_surface::lex::LexError`. +use ailang_surface::lex::LexError; +use ailang_surface::{parse, ParseError}; + +#[test] +fn dollar_binder_in_module_is_rejected_at_parse() { + // (module bad (fn main (type (fn-type (params) (ret (con Int)))) (params) (body (let x$1 7 x$1)))) + let src = "(module bad\n (fn main\n (type (fn-type (params) (ret (con Int))))\n (params)\n (body\n (let x$1 7 x$1))))\n"; + let err = parse(src).expect_err("authored `$` binder must be rejected at parse"); + assert!( + matches!(err, ParseError::Lex(LexError::ReservedDollar { ref token, .. }) if token == "x$1"), + "expected ParseError::Lex(ReservedDollar(\"x$1\")), got {err:?}", + ); +} + +#[test] +fn dollar_in_string_in_module_parses_clean() { + // The exemption survives at the `parse` level too: `$` inside a + // string literal in a real module is fine. + let src = "(module dollar_ok\n (fn main\n (type (fn-type (params) (ret (con Str))))\n (params)\n (body \"price: $5\")))\n"; + parse(src).expect("string-internal `$` must parse clean"); +}