From 706f90bacd11766e31d36d4d96f52cb5b223db3f Mon Sep 17 00:00:00 2001 From: Brummel Date: Thu, 7 May 2026 16:22:14 +0200 Subject: [PATCH] =?UTF-8?q?Iter=2014c:=20ailang-surface=20ships=20?= =?UTF-8?q?=E2=80=94=20form=20(A)=20parser=20+=20pretty-printer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Strictly additive new crate per Decision 6 architectural pin. JSON-AST stays the source of truth; ailang-surface is one producer/consumer of ailang_core::ast::Module values. ailang-check and ailang-codegen unchanged. Crate contents: - src/lex.rs (~264 LOC): 3-rule lexical core. Whitespace and parens delimit tokens; semicolon to EOL is comment; first-character classifier (digit -> int, " -> string, else -> ident). - src/parse.rs (~1041 LOC): hand-written recursive descent. One Rust fn per EBNF production. No parser-combinator dep. - src/print.rs (~371 LOC): deterministic pretty-printer. Round-trip contract with parse() is the surface's correctness gate. - tests/round_trip.rs (~128 LOC): integration test runs every examples/*.ail.json fixture through print -> parse -> canonical JSON, asserts canonical-byte equality with the original. Two AST-driven form widenings beyond the 14b sketch (both folded into DESIGN.md Decision 6): - lam-term carries (typed name type) params, ret type, and optional effects (Term::Lam has parallel param_tys/ret_ty/ effects fields). - import-clause admits (import name (as alias)?) (Import.alias is Option). Production count ~28, under 30-rule budget. Constraint 1 (formalisable for foreign LLM) intact. Verification: - cargo build --workspace green. - cargo test --workspace: 76 tests green (was 64; +9 surface unit, +2 round-trip integration). All 17 fixtures round-trip byte-identical at canonical level. 3 hand-written .ailx exhibits parse to canonical JSON identical to their .ail.json siblings. - cargo doc --no-deps zero warnings (DESIGN.md item 6 invariant). Manual smoke test (ail parse → ail run): hello, box, list_map_poly all produce expected output through the form-(A) authoring lane end-to-end. CLI: ail parse [-o ]. .ail.json remains a first-class input to every existing subcommand. Plan 14d: stdlib (std_list.ailx with length/filter/fold/concat/ reverse/head/tail), authored in form (A) from day one. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 10 + Cargo.toml | 2 + crates/ail/Cargo.toml | 1 + crates/ail/src/main.rs | 36 + crates/ailang-surface/Cargo.toml | 10 + crates/ailang-surface/src/lex.rs | 264 ++++++ crates/ailang-surface/src/lib.rs | 39 + crates/ailang-surface/src/parse.rs | 1041 +++++++++++++++++++++ crates/ailang-surface/src/print.rs | 371 ++++++++ crates/ailang-surface/tests/round_trip.rs | 128 +++ docs/DESIGN.md | 38 + docs/JOURNAL.md | 118 +++ examples/list_map_poly.ailx | 11 +- 13 files changed, 2065 insertions(+), 4 deletions(-) create mode 100644 crates/ailang-surface/Cargo.toml create mode 100644 crates/ailang-surface/src/lex.rs create mode 100644 crates/ailang-surface/src/lib.rs create mode 100644 crates/ailang-surface/src/parse.rs create mode 100644 crates/ailang-surface/src/print.rs create mode 100644 crates/ailang-surface/tests/round_trip.rs diff --git a/Cargo.lock b/Cargo.lock index 03d1bff..9bd6d2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9,6 +9,7 @@ dependencies = [ "ailang-check", "ailang-codegen", "ailang-core", + "ailang-surface", "anyhow", "clap", "serde_json", @@ -46,6 +47,15 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ailang-surface" +version = "0.0.1" +dependencies = [ + "ailang-core", + "serde_json", + "thiserror", +] + [[package]] name = "anstream" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index 403f0bc..c4de061 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "crates/ailang-core", "crates/ailang-check", "crates/ailang-codegen", + "crates/ailang-surface", "crates/ail", ] @@ -26,6 +27,7 @@ indexmap = { version = "2", features = ["serde"] } ailang-core = { path = "crates/ailang-core" } ailang-check = { path = "crates/ailang-check" } ailang-codegen = { path = "crates/ailang-codegen" } +ailang-surface = { path = "crates/ailang-surface" } [profile.release] lto = "thin" diff --git a/crates/ail/Cargo.toml b/crates/ail/Cargo.toml index f0667b3..5832605 100644 --- a/crates/ail/Cargo.toml +++ b/crates/ail/Cargo.toml @@ -12,6 +12,7 @@ path = "src/main.rs" ailang-core.workspace = true ailang-check.workspace = true ailang-codegen.workspace = true +ailang-surface.workspace = true serde_json.workspace = true clap.workspace = true anyhow.workspace = true diff --git a/crates/ail/src/main.rs b/crates/ail/src/main.rs index d96e874..6e1b859 100644 --- a/crates/ail/src/main.rs +++ b/crates/ail/src/main.rs @@ -171,6 +171,17 @@ enum Cmd { #[arg(long)] json: bool, }, + /// Parses a `.ailx` source file (form (A)) into canonical + /// `.ail.json`. Iter 14c addition; symmetric to `render`. + /// + /// The form-(A) projection is one of potentially many producers of + /// `Module` values. The JSON-AST remains the source of truth and + /// is what every other subcommand consumes. + Parse { + path: PathBuf, + #[arg(short, long)] + output: Option, + }, } fn main() -> Result<()> { @@ -531,6 +542,31 @@ fn main() -> Result<()> { } } } + Cmd::Parse { path, output } => { + // Read .ailx, parse via the surface crate, emit canonical + // JSON. Symmetric to `render` (which goes the other way). + let src = std::fs::read_to_string(&path) + .with_context(|| format!("reading {}", path.display()))?; + let module = match ailang_surface::parse(&src) { + Ok(m) => m, + Err(e) => { + eprintln!("parse error: {e}"); + std::process::exit(1); + } + }; + let bytes = ailang_core::canonical::to_bytes(&module); + match output { + Some(p) => { + std::fs::write(&p, &bytes)?; + eprintln!("wrote {}", p.display()); + } + None => { + use std::io::Write; + std::io::stdout().write_all(&bytes)?; + println!(); + } + } + } Cmd::Deps { path, of, json, workspace } => { if workspace { let ws = ailang_core::load_workspace(&path)?; diff --git a/crates/ailang-surface/Cargo.toml b/crates/ailang-surface/Cargo.toml new file mode 100644 index 0000000..38a802d --- /dev/null +++ b/crates/ailang-surface/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "ailang-surface" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +ailang-core.workspace = true +serde_json.workspace = true +thiserror.workspace = true diff --git a/crates/ailang-surface/src/lex.rs b/crates/ailang-surface/src/lex.rs new file mode 100644 index 0000000..fa68b83 --- /dev/null +++ b/crates/ailang-surface/src/lex.rs @@ -0,0 +1,264 @@ +//! Lexer for form (A). +//! +//! The lexical layer has three rules: +//! +//! 1. Whitespace (` `, `\t`, `\n`, `\r`) and parens (`(`, `)`) are the +//! only delimiters. Any maximal non-whitespace, non-paren run is a +//! single token. +//! 2. `;` introduces a comment to end-of-line. Comments produce no +//! tokens. +//! 3. After tokenisation, classify by first character: +//! - `"` ⇒ string literal (consume until closing `"`; `\"` and +//! `\n` escapes supported). +//! - digit, or `-` followed by digit ⇒ integer literal. +//! - otherwise ⇒ ident. +//! +//! Operators (`+`, `==`, `<=`, `**`), qualified names like +//! `io/print_int`, and dotted cross-module references like +//! `std_list.map` are all single-token idents — there is no special +//! lexical rule for any of them. `(`, `)`, and whitespace are the only +//! reserved tokens. Bool literals (`true`, `false`) and unit +//! (`(lit-unit)`) are disambiguated by the parser via context. + +use thiserror::Error; + +/// Source byte offset (start, end] into the original input. `end` is +/// one past the last byte. Used for error reporting only. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Span { + pub start: usize, + pub end: usize, +} + +/// One lexical token. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Tok { + /// `(` + LParen, + /// `)` + RParen, + /// Integer literal. Original textual form preserved for error messages; + /// parsed `i64` value carried for direct use. + Int(i64), + /// String literal contents (after escape processing). + Str(String), + /// Identifier. Anything that isn't a paren, integer, or string. + Ident(String), +} + +/// A token plus its span. +#[derive(Debug, Clone)] +pub struct Token { + pub tok: Tok, + pub span: Span, +} + +/// Lexer error. +#[derive(Debug, Error)] +pub enum LexError { + #[error("unterminated string literal at byte {start}")] + UnterminatedString { start: usize }, + #[error("invalid integer literal {literal:?} at byte {start}")] + InvalidInteger { literal: String, start: usize }, + #[error("invalid escape sequence \\{ch} in string literal at byte {pos}")] + InvalidEscape { ch: char, pos: usize }, +} + +/// Tokenise an input string into a flat token stream. +/// +/// Whitespace and comments are dropped. Returns a [`LexError`] on +/// unterminated string literals or numeric tokens that fail to parse. +pub fn tokenize(input: &str) -> Result, LexError> { + let bytes = input.as_bytes(); + let mut out = Vec::new(); + let mut i = 0usize; + while i < bytes.len() { + let b = bytes[i]; + // Whitespace. + if b == b' ' || b == b'\t' || b == b'\n' || b == b'\r' { + i += 1; + continue; + } + // Comment to end-of-line. + if b == b';' { + while i < bytes.len() && bytes[i] != b'\n' { + i += 1; + } + continue; + } + // Parens. + if b == b'(' { + out.push(Token { tok: Tok::LParen, span: Span { start: i, end: i + 1 } }); + i += 1; + continue; + } + if b == b')' { + out.push(Token { tok: Tok::RParen, span: Span { start: i, end: i + 1 } }); + i += 1; + continue; + } + // String literal. Iterate over UTF-8 chars (not raw bytes) so + // multi-byte glyphs in the source survive round-trip. + if b == b'"' { + let start = i; + i += 1; + let mut value = String::new(); + // Walk via char_indices on the unread tail to keep i a byte offset. + loop { + if i >= bytes.len() { + return Err(LexError::UnterminatedString { start }); + } + // Quick byte-level checks for ASCII control bytes that + // can never start a multi-byte UTF-8 sequence. + let c0 = bytes[i]; + if c0 == b'"' { + i += 1; + break; + } + if c0 == b'\\' { + if i + 1 >= bytes.len() { + return Err(LexError::UnterminatedString { start }); + } + let esc = bytes[i + 1]; + match esc { + b'"' => value.push('"'), + b'\\' => value.push('\\'), + b'n' => value.push('\n'), + b't' => value.push('\t'), + b'r' => value.push('\r'), + other => { + return Err(LexError::InvalidEscape { + ch: other as char, + pos: i, + }); + } + } + i += 2; + continue; + } + // Decode one full UTF-8 character starting at i. The + // input is guaranteed to be valid UTF-8 by Rust's + // `&str` invariant, so a single `chars()` step on the + // tail gives us the next scalar. + if let Some(ch) = input[i..].chars().next() { + value.push(ch); + i += ch.len_utf8(); + } else { + return Err(LexError::UnterminatedString { start }); + } + } + out.push(Token { + tok: Tok::Str(value), + span: Span { start, end: i }, + }); + continue; + } + // Otherwise: maximal non-paren / non-whitespace / non-comment run. + let start = i; + while i < bytes.len() { + let c = bytes[i]; + if c == b' ' + || c == b'\t' + || c == b'\n' + || c == b'\r' + || c == b'(' + || c == b')' + || c == b';' + { + break; + } + i += 1; + } + let raw = &input[start..i]; + // Classify. + let first = raw.as_bytes()[0]; + let is_int = first.is_ascii_digit() + || (first == b'-' && raw.len() > 1 && raw.as_bytes()[1].is_ascii_digit()); + if is_int { + // Numeric parse must succeed; else parse error. + match raw.parse::() { + Ok(v) => out.push(Token { + tok: Tok::Int(v), + span: Span { start, end: i }, + }), + Err(_) => { + return Err(LexError::InvalidInteger { + literal: raw.to_string(), + start, + }); + } + } + } else { + out.push(Token { + tok: Tok::Ident(raw.to_string()), + span: Span { start, end: i }, + }); + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parens_and_idents() { + let toks = tokenize("(foo bar)").unwrap(); + assert_eq!(toks.len(), 4); + assert!(matches!(toks[0].tok, Tok::LParen)); + assert!(matches!(&toks[1].tok, Tok::Ident(s) if s == "foo")); + assert!(matches!(&toks[2].tok, Tok::Ident(s) if s == "bar")); + assert!(matches!(toks[3].tok, Tok::RParen)); + } + + #[test] + fn comments_are_dropped() { + let toks = tokenize("(a ; this is a comment\n b)").unwrap(); + assert_eq!(toks.len(), 4); + } + + #[test] + fn integers() { + let toks = tokenize("42 -7 0").unwrap(); + assert_eq!(toks.len(), 3); + assert!(matches!(toks[0].tok, Tok::Int(42))); + assert!(matches!(toks[1].tok, Tok::Int(-7))); + assert!(matches!(toks[2].tok, Tok::Int(0))); + } + + #[test] + fn operators_are_idents() { + let toks = tokenize("+ == <= io/print_int std_list.map").unwrap(); + let names: Vec<_> = toks + .iter() + .map(|t| match &t.tok { + Tok::Ident(s) => s.clone(), + _ => panic!("not ident"), + }) + .collect(); + assert_eq!(names, vec!["+", "==", "<=", "io/print_int", "std_list.map"]); + } + + #[test] + fn string_with_escapes() { + let toks = tokenize(r#""hello\nworld""#).unwrap(); + assert_eq!(toks.len(), 1); + assert!(matches!(&toks[0].tok, Tok::Str(s) if s == "hello\nworld")); + } + + #[test] + fn negative_lone_minus_is_ident() { + // `-` alone (no digit follows) is an ident, not a (failed) integer. + let toks = tokenize("-").unwrap(); + assert_eq!(toks.len(), 1); + assert!(matches!(&toks[0].tok, Tok::Ident(s) if s == "-")); + } + + #[test] + fn string_preserves_multibyte_utf8() { + let toks = tokenize(r#""em — dash and naïve""#).unwrap(); + assert_eq!(toks.len(), 1); + assert!(matches!(&toks[0].tok, Tok::Str(s) if s == "em — dash and naïve")); + } +} diff --git a/crates/ailang-surface/src/lib.rs b/crates/ailang-surface/src/lib.rs new file mode 100644 index 0000000..f1d3a80 --- /dev/null +++ b/crates/ailang-surface/src/lib.rs @@ -0,0 +1,39 @@ +//! AILang authoring surface — form (A) S-expression projection. +//! +//! This crate is **one of potentially many** producers / consumers of +//! [`ailang_core::ast::Module`] values. The JSON-AST in `ailang-core` +//! remains the canonical, hashable, content-addressed source of truth; +//! form (A) is the AI-authoring projection optimised for token-efficient +//! production by LLMs (see `docs/DESIGN.md` Decision 6). +//! +//! ## Modules +//! +//! - [`mod@lex`] — hand-written lexer for the 3-rule lexical core +//! (whitespace / parens / comments delimit tokens; first character +//! classifies into integer / string / ident). +//! - [`mod@parse`] — hand-written recursive-descent parser. One Rust +//! function per EBNF production, line-by-line auditable. No +//! parser-combinator dependency. +//! - [`mod@print`] — deterministic pretty-printer that emits form (A). +//! Round-trip with the parser is the contract that gates the +//! surface (see the integration test `tests/round_trip.rs`). +//! +//! ## Round-trip contract +//! +//! For every well-formed [`ailang_core::ast::Module`] value `m`, +//! [`parse()`] applied to [`print()`]`(m)` produces a module whose +//! canonical JSON bytes equal those of `m`. This is enforced by the +//! integration test against every `examples/*.ail.json` fixture. +//! +//! ## Architectural pin +//! +//! No new AST nodes, no schema changes, no new hashable form. +//! `ailang-check` and `ailang-codegen` are projection-agnostic — they +//! consume `Module` values regardless of which front-end produced them. + +pub mod lex; +pub mod parse; +pub mod print; + +pub use parse::{parse, ParseError}; +pub use print::print; diff --git a/crates/ailang-surface/src/parse.rs b/crates/ailang-surface/src/parse.rs new file mode 100644 index 0000000..676ad8b --- /dev/null +++ b/crates/ailang-surface/src/parse.rs @@ -0,0 +1,1041 @@ +//! Recursive-descent parser for form (A). +//! +//! The grammar (EBNF) lives next to its implementation. Each production +//! is one Rust function in this file. No look-ahead beyond a single +//! token is required; the parser is line-by-line auditable. +//! +//! ```text +//! module ::= "(" "module" ident def* ")" +//! def ::= data-def | fn-def | const-def | import-clause +//! data-def ::= "(" "data" ident vars-clause? data-attr* ")" +//! vars-clause ::= "(" "vars" ident+ ")" +//! data-attr ::= doc-attr | ctor-decl +//! ctor-decl ::= "(" "ctor" ident type* ")" +//! doc-attr ::= "(" "doc" string ")" +//! +//! fn-def ::= "(" "fn" ident fn-attr* ")" +//! fn-attr ::= doc-attr | type-attr | params-attr | body-attr +//! type-attr ::= "(" "type" type ")" +//! params-attr ::= "(" "params" ident* ")" +//! body-attr ::= "(" "body" term ")" +//! +//! const-def ::= "(" "const" ident const-attr+ ")" +//! const-attr ::= type-attr | body-attr | doc-attr +//! +//! import-clause ::= "(" "import" ident ("as" ident)? ")" +//! +//! type ::= type-var | type-con | fn-type | forall-type +//! type-var ::= ident +//! type-con ::= "(" "con" ident type* ")" +//! fn-type ::= "(" "fn-type" "(" "params" type* ")" +//! "(" "ret" type ")" +//! effects-clause? ")" +//! forall-type ::= "(" "forall" "(" "vars" ident+ ")" type ")" +//! effects-clause::= "(" "effects" ident+ ")" +//! +//! term ::= var-ref | int-lit | str-lit | bool-lit | unit-lit +//! | app-term | match-term | ctor-term | do-term | seq-term +//! | lam-term | if-term | let-term +//! var-ref ::= ident ; reserved: true/false → bool-lit +//! int-lit ::= integer ; numeric atom +//! str-lit ::= string ; string atom +//! bool-lit ::= "true" | "false" +//! unit-lit ::= "(" "lit-unit" ")" +//! app-term ::= "(" "app" term term+ ")" +//! ctor-term ::= "(" "term-ctor" ident ident term* ")" +//! match-term ::= "(" "match" term case-arm+ ")" +//! case-arm ::= "(" "case" pattern term ")" +//! do-term ::= "(" "do" ident term* ")" +//! seq-term ::= "(" "seq" term term ")" +//! lam-term ::= "(" "lam" "(" "params" typed-param* ")" +//! "(" "ret" type ")" +//! effects-clause? body-attr ")" +//! typed-param ::= "(" "typed" ident type ")" +//! if-term ::= "(" "if" term term term ")" +//! let-term ::= "(" "let" ident term term ")" +//! +//! pattern ::= pat-var | pat-ctor | pat-lit | pat-wild +//! pat-var ::= ident +//! pat-ctor ::= "(" "pat-ctor" ident pattern* ")" +//! pat-lit ::= "(" "pat-lit" lit-form ")" +//! pat-wild ::= "_" +//! lit-form ::= integer | "true" | "false" | string +//! ``` +//! +//! Notes on the form (deviations from the spec in DESIGN.md Decision 6): +//! +//! - The `lam` form carries `paramTypes`, a `ret` type, and an +//! optional `effects` clause. The original DESIGN.md sketch left +//! these out; the AST stores them per-lambda and so the form must +//! round-trip them. +//! - The `import` form admits an optional `as` alias to round-trip +//! [`ailang_core::ast::Import::alias`]. + +use ailang_core::ast::{ + Arm, ConstDef, Ctor, Def, FnDef, Import, Literal, Module, Pattern, Term, Type, TypeDef, +}; +use ailang_core::SCHEMA; +use thiserror::Error; + +use crate::lex::{tokenize, LexError, Tok, Token}; + +/// Errors raised during parsing. +#[derive(Debug, Error)] +pub enum ParseError { + #[error("lex error: {0}")] + Lex(#[from] LexError), + #[error("parse error: expected {expected}, got {got} at byte {pos}")] + Unexpected { + expected: String, + got: String, + pos: usize, + }, + #[error("parse error: unexpected end of input, expected {expected}")] + UnexpectedEof { expected: String }, + #[error("parse error in {production}: {message} at byte {pos}")] + Production { + production: &'static str, + message: String, + pos: usize, + }, +} + +/// Parse a form-(A) source string into an [`ailang_core::ast::Module`]. +/// +/// The schema field is set to [`ailang_core::SCHEMA`]; the form does +/// not carry it explicitly because the form itself implies the +/// version. +pub fn parse(input: &str) -> Result { + let toks = tokenize(input)?; + let mut p = Parser::new(&toks); + let m = p.parse_module()?; + if p.cur < p.toks.len() { + return Err(ParseError::Unexpected { + expected: "end of input".into(), + got: tok_label(&p.toks[p.cur].tok), + pos: p.toks[p.cur].span.start, + }); + } + Ok(m) +} + +fn tok_label(t: &Tok) -> String { + match t { + Tok::LParen => "`(`".into(), + Tok::RParen => "`)`".into(), + Tok::Int(v) => format!("integer `{v}`"), + Tok::Str(s) => format!("string {s:?}"), + Tok::Ident(s) => format!("ident `{s}`"), + } +} + +struct Parser<'a> { + toks: &'a [Token], + cur: usize, +} + +impl<'a> Parser<'a> { + fn new(toks: &'a [Token]) -> Self { + Self { toks, cur: 0 } + } + + fn peek(&self) -> Option<&Token> { + self.toks.get(self.cur) + } + + fn expect_lparen(&mut self, ctx: &'static str) -> Result<(), ParseError> { + match self.peek() { + Some(Token { tok: Tok::LParen, .. }) => { + self.cur += 1; + Ok(()) + } + Some(t) => Err(ParseError::Unexpected { + expected: format!("`(` (start of {ctx})"), + got: tok_label(&t.tok), + pos: t.span.start, + }), + None => Err(ParseError::UnexpectedEof { + expected: format!("`(` (start of {ctx})"), + }), + } + } + + fn expect_rparen(&mut self, ctx: &'static str) -> Result<(), ParseError> { + match self.peek() { + Some(Token { tok: Tok::RParen, .. }) => { + self.cur += 1; + Ok(()) + } + Some(t) => Err(ParseError::Unexpected { + expected: format!("`)` (end of {ctx})"), + got: tok_label(&t.tok), + pos: t.span.start, + }), + None => Err(ParseError::UnexpectedEof { + expected: format!("`)` (end of {ctx})"), + }), + } + } + + /// Consume an ident atom matching `expected`. Used for keyword tags + /// like `module`, `data`, `con`, etc. + fn expect_keyword(&mut self, kw: &'static str) -> Result<(), ParseError> { + match self.peek() { + Some(Token { tok: Tok::Ident(s), span }) if s == kw => { + let _ = span; + self.cur += 1; + Ok(()) + } + Some(t) => Err(ParseError::Unexpected { + expected: format!("`{kw}`"), + got: tok_label(&t.tok), + pos: t.span.start, + }), + None => Err(ParseError::UnexpectedEof { + expected: format!("`{kw}`"), + }), + } + } + + /// Consume any ident atom and return its text. + fn expect_ident(&mut self, ctx: &'static str) -> Result { + match self.peek().cloned() { + Some(Token { tok: Tok::Ident(s), .. }) => { + self.cur += 1; + Ok(s) + } + Some(t) => Err(ParseError::Unexpected { + expected: format!("ident ({ctx})"), + got: tok_label(&t.tok), + pos: t.span.start, + }), + None => Err(ParseError::UnexpectedEof { + expected: format!("ident ({ctx})"), + }), + } + } + + /// Try to consume an ident matching `kw`. On success advance and + /// return true; otherwise leave position unchanged and return + /// false. + fn try_keyword(&mut self, kw: &str) -> bool { + match self.peek() { + Some(Token { tok: Tok::Ident(s), .. }) if s == kw => { + self.cur += 1; + true + } + _ => false, + } + } + + /// Look at the head ident of a parenthesised form without + /// consuming. Used to dispatch on the head keyword. + fn peek_head_ident(&self) -> Option<&str> { + if let Some(Token { tok: Tok::LParen, .. }) = self.toks.get(self.cur) { + if let Some(Token { tok: Tok::Ident(s), .. }) = self.toks.get(self.cur + 1) { + return Some(s.as_str()); + } + } + None + } + + // ---- module --------------------------------------------------------- + + fn parse_module(&mut self) -> Result { + self.expect_lparen("module")?; + self.expect_keyword("module")?; + let name = self.expect_ident("module name")?; + let mut imports: Vec = Vec::new(); + let mut defs: Vec = Vec::new(); + loop { + match self.peek() { + Some(Token { tok: Tok::RParen, .. }) => break, + None => { + return Err(ParseError::UnexpectedEof { + expected: "`)` to end module or another def".into(), + }); + } + _ => {} + } + let head = self.peek_head_ident().ok_or_else(|| { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + ParseError::Production { + production: "module", + message: "expected `(` followed by a def-head keyword (\ + `data`, `fn`, `const`, `import`)" + .into(), + pos, + } + })?; + match head { + "import" => imports.push(self.parse_import()?), + "data" => defs.push(Def::Type(self.parse_data()?)), + "fn" => defs.push(Def::Fn(self.parse_fn()?)), + "const" => defs.push(Def::Const(self.parse_const()?)), + other => { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + return Err(ParseError::Production { + production: "module", + message: format!( + "unknown def head `{other}`; expected `data`, `fn`, `const`, or `import`" + ), + pos, + }); + } + } + } + self.expect_rparen("module")?; + Ok(Module { + schema: SCHEMA.to_string(), + name, + imports, + defs, + }) + } + + // ---- imports -------------------------------------------------------- + + fn parse_import(&mut self) -> Result { + self.expect_lparen("import-clause")?; + self.expect_keyword("import")?; + let module = self.expect_ident("import module name")?; + let alias = if self.try_keyword("as") { + Some(self.expect_ident("import alias")?) + } else { + None + }; + self.expect_rparen("import-clause")?; + Ok(Import { module, alias }) + } + + // ---- data def ------------------------------------------------------- + + fn parse_data(&mut self) -> Result { + self.expect_lparen("data-def")?; + self.expect_keyword("data")?; + let name = self.expect_ident("data name")?; + // Optional vars clause: `(vars a b ...)` + let mut vars: Vec = Vec::new(); + if let Some("vars") = self.peek_head_ident() { + self.expect_lparen("vars-clause")?; + self.expect_keyword("vars")?; + // vars+ : at least one + let first = self.expect_ident("type variable")?; + vars.push(first); + while !matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + vars.push(self.expect_ident("type variable")?); + } + self.expect_rparen("vars-clause")?; + } + let mut doc: Option = None; + let mut ctors: Vec = Vec::new(); + loop { + match self.peek_head_ident() { + Some("doc") => { + let s = self.parse_doc()?; + doc = Some(s); + } + Some("ctor") => { + ctors.push(self.parse_ctor()?); + } + Some(other) => { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + return Err(ParseError::Production { + production: "data-def", + message: format!( + "unknown data attribute `{other}`; expected `doc` or `ctor`" + ), + pos, + }); + } + None => break, + } + } + self.expect_rparen("data-def")?; + Ok(TypeDef { + name, + vars, + ctors, + doc, + }) + } + + fn parse_doc(&mut self) -> Result { + self.expect_lparen("doc-attr")?; + self.expect_keyword("doc")?; + let s = match self.peek().cloned() { + Some(Token { tok: Tok::Str(s), .. }) => { + self.cur += 1; + s + } + Some(t) => { + return Err(ParseError::Unexpected { + expected: "string literal (doc body)".into(), + got: tok_label(&t.tok), + pos: t.span.start, + }); + } + None => { + return Err(ParseError::UnexpectedEof { + expected: "string literal (doc body)".into(), + }); + } + }; + self.expect_rparen("doc-attr")?; + Ok(s) + } + + fn parse_ctor(&mut self) -> Result { + self.expect_lparen("ctor-decl")?; + self.expect_keyword("ctor")?; + let name = self.expect_ident("ctor name")?; + let mut fields: Vec = Vec::new(); + while !matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + fields.push(self.parse_type()?); + } + self.expect_rparen("ctor-decl")?; + Ok(Ctor { name, fields }) + } + + // ---- fn def --------------------------------------------------------- + + fn parse_fn(&mut self) -> Result { + self.expect_lparen("fn-def")?; + self.expect_keyword("fn")?; + let name = self.expect_ident("fn name")?; + let mut doc: Option = None; + let mut ty: Option = None; + let mut params: Option> = None; + let mut body: Option = None; + loop { + match self.peek_head_ident() { + Some("doc") => doc = Some(self.parse_doc()?), + Some("type") => { + let t = self.parse_type_attr()?; + ty = Some(t); + } + Some("params") => { + let p = self.parse_params_attr()?; + params = Some(p); + } + Some("body") => { + let b = self.parse_body_attr()?; + body = Some(b); + } + Some(other) => { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + return Err(ParseError::Production { + production: "fn-def", + message: format!( + "unknown fn attribute `{other}`; expected `doc`, `type`, `params`, or `body`" + ), + pos, + }); + } + None => break, + } + } + self.expect_rparen("fn-def")?; + let ty = ty.ok_or_else(|| ParseError::Production { + production: "fn-def", + message: format!("fn `{name}` is missing required `(type ...)` attribute"), + pos: 0, + })?; + let params = params.ok_or_else(|| ParseError::Production { + production: "fn-def", + message: format!("fn `{name}` is missing required `(params ...)` attribute"), + pos: 0, + })?; + let body = body.ok_or_else(|| ParseError::Production { + production: "fn-def", + message: format!("fn `{name}` is missing required `(body ...)` attribute"), + pos: 0, + })?; + Ok(FnDef { + name, + ty, + params, + body, + doc, + }) + } + + fn parse_type_attr(&mut self) -> Result { + self.expect_lparen("type-attr")?; + self.expect_keyword("type")?; + let t = self.parse_type()?; + self.expect_rparen("type-attr")?; + Ok(t) + } + + fn parse_params_attr(&mut self) -> Result, ParseError> { + self.expect_lparen("params-attr")?; + self.expect_keyword("params")?; + let mut out = Vec::new(); + while !matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + out.push(self.expect_ident("param name")?); + } + self.expect_rparen("params-attr")?; + Ok(out) + } + + fn parse_body_attr(&mut self) -> Result { + self.expect_lparen("body-attr")?; + self.expect_keyword("body")?; + let t = self.parse_term()?; + self.expect_rparen("body-attr")?; + Ok(t) + } + + // ---- const def ------------------------------------------------------ + + fn parse_const(&mut self) -> Result { + self.expect_lparen("const-def")?; + self.expect_keyword("const")?; + let name = self.expect_ident("const name")?; + let mut doc: Option = None; + let mut ty: Option = None; + let mut value: Option = None; + loop { + match self.peek_head_ident() { + Some("doc") => doc = Some(self.parse_doc()?), + Some("type") => ty = Some(self.parse_type_attr()?), + Some("body") => value = Some(self.parse_body_attr()?), + Some(other) => { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + return Err(ParseError::Production { + production: "const-def", + message: format!( + "unknown const attribute `{other}`; expected `doc`, `type`, or `body`" + ), + pos, + }); + } + None => break, + } + } + self.expect_rparen("const-def")?; + let ty = ty.ok_or_else(|| ParseError::Production { + production: "const-def", + message: format!("const `{name}` is missing required `(type ...)`"), + pos: 0, + })?; + let value = value.ok_or_else(|| ParseError::Production { + production: "const-def", + message: format!("const `{name}` is missing required `(body ...)`"), + pos: 0, + })?; + Ok(ConstDef { + name, + ty, + value, + doc, + }) + } + + // ---- types ---------------------------------------------------------- + + fn parse_type(&mut self) -> Result { + match self.peek() { + Some(Token { tok: Tok::LParen, .. }) => { + let head = self.peek_head_ident().ok_or_else(|| { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + ParseError::Production { + production: "type", + message: "expected `(con ...)`, `(fn-type ...)`, or `(forall ...)`" + .into(), + pos, + } + })?; + match head { + "con" => self.parse_type_con(), + "fn-type" => self.parse_fn_type(), + "forall" => self.parse_forall_type(), + other => { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + Err(ParseError::Production { + production: "type", + message: format!( + "unknown type head `{other}`; expected `con`, `fn-type`, or `forall`" + ), + pos, + }) + } + } + } + Some(Token { tok: Tok::Ident(s), .. }) => { + let s = s.clone(); + self.cur += 1; + Ok(Type::Var { name: s }) + } + Some(t) => Err(ParseError::Unexpected { + expected: "type expression".into(), + got: tok_label(&t.tok), + pos: t.span.start, + }), + None => Err(ParseError::UnexpectedEof { + expected: "type expression".into(), + }), + } + } + + fn parse_type_con(&mut self) -> Result { + self.expect_lparen("type-con")?; + self.expect_keyword("con")?; + let name = self.expect_ident("type constructor name")?; + let mut args: Vec = Vec::new(); + while !matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + args.push(self.parse_type()?); + } + self.expect_rparen("type-con")?; + Ok(Type::Con { name, args }) + } + + fn parse_fn_type(&mut self) -> Result { + self.expect_lparen("fn-type")?; + self.expect_keyword("fn-type")?; + // (params type*) + self.expect_lparen("fn-type params")?; + self.expect_keyword("params")?; + let mut params: Vec = Vec::new(); + while !matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + params.push(self.parse_type()?); + } + self.expect_rparen("fn-type params")?; + // (ret type) + self.expect_lparen("fn-type ret")?; + self.expect_keyword("ret")?; + let ret = self.parse_type()?; + self.expect_rparen("fn-type ret")?; + // optional (effects ident+) + let mut effects: Vec = Vec::new(); + if let Some("effects") = self.peek_head_ident() { + effects = self.parse_effects_clause()?; + } + self.expect_rparen("fn-type")?; + Ok(Type::Fn { + params, + ret: Box::new(ret), + effects, + }) + } + + fn parse_effects_clause(&mut self) -> Result, ParseError> { + self.expect_lparen("effects-clause")?; + self.expect_keyword("effects")?; + // 1+ idents per grammar; round-trip allows empty too (to support + // any future use), but the printer never emits an empty clause. + let mut out = Vec::new(); + while !matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + out.push(self.expect_ident("effect name")?); + } + if out.is_empty() { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + return Err(ParseError::Production { + production: "effects-clause", + message: "expected at least one effect name".into(), + pos, + }); + } + self.expect_rparen("effects-clause")?; + Ok(out) + } + + fn parse_forall_type(&mut self) -> Result { + self.expect_lparen("forall-type")?; + self.expect_keyword("forall")?; + // (vars ident+) + self.expect_lparen("forall vars")?; + self.expect_keyword("vars")?; + let mut vars = Vec::new(); + let first = self.expect_ident("type variable")?; + vars.push(first); + while !matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + vars.push(self.expect_ident("type variable")?); + } + self.expect_rparen("forall vars")?; + let body = self.parse_type()?; + self.expect_rparen("forall-type")?; + Ok(Type::Forall { + vars, + body: Box::new(body), + }) + } + + // ---- terms ---------------------------------------------------------- + + fn parse_term(&mut self) -> Result { + match self.peek().cloned() { + Some(Token { tok: Tok::LParen, .. }) => { + let head = self.peek_head_ident().ok_or_else(|| { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + ParseError::Production { + production: "term", + message: "expected a term-head keyword after `(`".into(), + pos, + } + })?; + match head { + "lit-unit" => self.parse_lit_unit(), + "app" => self.parse_app(), + "term-ctor" => self.parse_term_ctor(), + "match" => self.parse_match(), + "do" => self.parse_do(), + "seq" => self.parse_seq(), + "lam" => self.parse_lam(), + "if" => self.parse_if(), + "let" => self.parse_let(), + other => { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + Err(ParseError::Production { + production: "term", + message: format!( + "unknown term head `{other}`; expected one of \ + `app`, `lam`, `let`, `if`, `match`, `do`, `seq`, \ + `term-ctor`, `lit-unit`" + ), + pos, + }) + } + } + } + Some(Token { tok: Tok::Ident(s), .. }) => { + self.cur += 1; + if s == "true" { + Ok(Term::Lit { lit: Literal::Bool { value: true } }) + } else if s == "false" { + Ok(Term::Lit { lit: Literal::Bool { value: false } }) + } else { + Ok(Term::Var { name: s }) + } + } + Some(Token { tok: Tok::Int(v), .. }) => { + self.cur += 1; + Ok(Term::Lit { lit: Literal::Int { value: v } }) + } + Some(Token { tok: Tok::Str(s), .. }) => { + self.cur += 1; + Ok(Term::Lit { lit: Literal::Str { value: s } }) + } + Some(t) => Err(ParseError::Unexpected { + expected: "term".into(), + got: tok_label(&t.tok), + pos: t.span.start, + }), + None => Err(ParseError::UnexpectedEof { + expected: "term".into(), + }), + } + } + + fn parse_lit_unit(&mut self) -> Result { + self.expect_lparen("unit-lit")?; + self.expect_keyword("lit-unit")?; + self.expect_rparen("unit-lit")?; + Ok(Term::Lit { lit: Literal::Unit }) + } + + fn parse_app(&mut self) -> Result { + self.expect_lparen("app-term")?; + self.expect_keyword("app")?; + let callee = self.parse_term()?; + // 1+ args + if matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + return Err(ParseError::Production { + production: "app-term", + message: "expected at least one argument".into(), + pos, + }); + } + let mut args = Vec::new(); + while !matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + args.push(self.parse_term()?); + } + self.expect_rparen("app-term")?; + Ok(Term::App { + callee: Box::new(callee), + args, + }) + } + + fn parse_term_ctor(&mut self) -> Result { + self.expect_lparen("ctor-term")?; + self.expect_keyword("term-ctor")?; + let type_name = self.expect_ident("ADT type name")?; + let ctor = self.expect_ident("ctor name")?; + let mut args = Vec::new(); + while !matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + args.push(self.parse_term()?); + } + self.expect_rparen("ctor-term")?; + Ok(Term::Ctor { + type_name, + ctor, + args, + }) + } + + fn parse_match(&mut self) -> Result { + self.expect_lparen("match-term")?; + self.expect_keyword("match")?; + let scrutinee = self.parse_term()?; + let mut arms = Vec::new(); + while !matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + arms.push(self.parse_case_arm()?); + } + if arms.is_empty() { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + return Err(ParseError::Production { + production: "match-term", + message: "expected at least one `(case ...)` arm".into(), + pos, + }); + } + self.expect_rparen("match-term")?; + Ok(Term::Match { + scrutinee: Box::new(scrutinee), + arms, + }) + } + + fn parse_case_arm(&mut self) -> Result { + self.expect_lparen("case-arm")?; + self.expect_keyword("case")?; + let pat = self.parse_pattern()?; + let body = self.parse_term()?; + self.expect_rparen("case-arm")?; + Ok(Arm { pat, body }) + } + + fn parse_do(&mut self) -> Result { + self.expect_lparen("do-term")?; + self.expect_keyword("do")?; + let op = self.expect_ident("effect op (e.g. `io/print_int`)")?; + let mut args = Vec::new(); + while !matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + args.push(self.parse_term()?); + } + self.expect_rparen("do-term")?; + Ok(Term::Do { op, args }) + } + + fn parse_seq(&mut self) -> Result { + self.expect_lparen("seq-term")?; + self.expect_keyword("seq")?; + let lhs = self.parse_term()?; + let rhs = self.parse_term()?; + self.expect_rparen("seq-term")?; + Ok(Term::Seq { + lhs: Box::new(lhs), + rhs: Box::new(rhs), + }) + } + + fn parse_lam(&mut self) -> Result { + self.expect_lparen("lam-term")?; + self.expect_keyword("lam")?; + // (params (typed name type)*) + self.expect_lparen("lam params")?; + self.expect_keyword("params")?; + let mut params: Vec = Vec::new(); + let mut param_tys: Vec = Vec::new(); + while !matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + self.expect_lparen("typed-param")?; + self.expect_keyword("typed")?; + let pname = self.expect_ident("lambda param name")?; + let pty = self.parse_type()?; + self.expect_rparen("typed-param")?; + params.push(pname); + param_tys.push(pty); + } + self.expect_rparen("lam params")?; + // (ret type) + self.expect_lparen("lam ret")?; + self.expect_keyword("ret")?; + let ret_ty = self.parse_type()?; + self.expect_rparen("lam ret")?; + // optional effects clause + let mut effects: Vec = Vec::new(); + if let Some("effects") = self.peek_head_ident() { + effects = self.parse_effects_clause()?; + } + // (body term) + let body = self.parse_body_attr()?; + self.expect_rparen("lam-term")?; + Ok(Term::Lam { + params, + param_tys, + ret_ty: Box::new(ret_ty), + effects, + body: Box::new(body), + }) + } + + fn parse_if(&mut self) -> Result { + self.expect_lparen("if-term")?; + self.expect_keyword("if")?; + let cond = self.parse_term()?; + let then = self.parse_term()?; + let else_ = self.parse_term()?; + self.expect_rparen("if-term")?; + Ok(Term::If { + cond: Box::new(cond), + then: Box::new(then), + else_: Box::new(else_), + }) + } + + fn parse_let(&mut self) -> Result { + self.expect_lparen("let-term")?; + self.expect_keyword("let")?; + let name = self.expect_ident("let-bound name")?; + let value = self.parse_term()?; + let body = self.parse_term()?; + self.expect_rparen("let-term")?; + Ok(Term::Let { + name, + value: Box::new(value), + body: Box::new(body), + }) + } + + // ---- patterns ------------------------------------------------------- + + fn parse_pattern(&mut self) -> Result { + match self.peek().cloned() { + Some(Token { tok: Tok::LParen, .. }) => { + let head = self.peek_head_ident().ok_or_else(|| { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + ParseError::Production { + production: "pattern", + message: "expected `pat-ctor` or `pat-lit` after `(`".into(), + pos, + } + })?; + match head { + "pat-ctor" => self.parse_pat_ctor(), + "pat-lit" => self.parse_pat_lit(), + other => { + let pos = self.peek().map(|t| t.span.start).unwrap_or(0); + Err(ParseError::Production { + production: "pattern", + message: format!( + "unknown pattern head `{other}`; expected `pat-ctor` or `pat-lit`" + ), + pos, + }) + } + } + } + Some(Token { tok: Tok::Ident(s), .. }) => { + self.cur += 1; + if s == "_" { + Ok(Pattern::Wild) + } else { + Ok(Pattern::Var { name: s }) + } + } + Some(t) => Err(ParseError::Unexpected { + expected: "pattern".into(), + got: tok_label(&t.tok), + pos: t.span.start, + }), + None => Err(ParseError::UnexpectedEof { + expected: "pattern".into(), + }), + } + } + + fn parse_pat_ctor(&mut self) -> Result { + self.expect_lparen("pat-ctor")?; + self.expect_keyword("pat-ctor")?; + let ctor = self.expect_ident("ctor name")?; + let mut fields = Vec::new(); + while !matches!(self.peek(), Some(Token { tok: Tok::RParen, .. })) { + fields.push(self.parse_pattern()?); + } + self.expect_rparen("pat-ctor")?; + Ok(Pattern::Ctor { ctor, fields }) + } + + fn parse_pat_lit(&mut self) -> Result { + self.expect_lparen("pat-lit")?; + self.expect_keyword("pat-lit")?; + let lit = match self.peek().cloned() { + Some(Token { tok: Tok::Int(v), .. }) => { + self.cur += 1; + Literal::Int { value: v } + } + Some(Token { tok: Tok::Str(s), .. }) => { + self.cur += 1; + Literal::Str { value: s } + } + Some(Token { tok: Tok::Ident(s), span }) => { + if s == "true" { + self.cur += 1; + Literal::Bool { value: true } + } else if s == "false" { + self.cur += 1; + Literal::Bool { value: false } + } else { + return Err(ParseError::Unexpected { + expected: "literal form (integer, string, `true`, or `false`)".into(), + got: tok_label(&Tok::Ident(s)), + pos: span.start, + }); + } + } + Some(t) => { + return Err(ParseError::Unexpected { + expected: "literal form (integer, string, `true`, or `false`)".into(), + got: tok_label(&t.tok), + pos: t.span.start, + }); + } + None => { + return Err(ParseError::UnexpectedEof { + expected: "literal form".into(), + }); + } + }; + self.expect_rparen("pat-lit")?; + Ok(Pattern::Lit { lit }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_minimal_module() { + let m = parse( + r#" + (module hello + (fn main + (type (fn-type (params) (ret (con Unit)) (effects IO))) + (params) + (body (do io/print_str "Hello, AILang.")))) + "#, + ) + .unwrap(); + assert_eq!(m.name, "hello"); + assert_eq!(m.defs.len(), 1); + } + + #[test] + fn parses_var_and_int() { + let m = parse( + r#" + (module m + (fn id + (type (fn-type (params (con Int)) (ret (con Int)))) + (params x) + (body x))) + "#, + ) + .unwrap(); + assert!(matches!(m.defs.len(), 1)); + } +} diff --git a/crates/ailang-surface/src/print.rs b/crates/ailang-surface/src/print.rs new file mode 100644 index 0000000..e297427 --- /dev/null +++ b/crates/ailang-surface/src/print.rs @@ -0,0 +1,371 @@ +//! Pretty-printer for form (A). +//! +//! Mirrors the parser in [`mod@crate::parse`]. Output is deterministic +//! and parseable by the parser — round-trip is the gating contract +//! (see `tests/round_trip.rs`). +//! +//! Indentation is informational only (the lexer ignores it). Two-space +//! per level. Comments are NOT emitted. + +use ailang_core::ast::{ + Arm, ConstDef, Ctor, Def, FnDef, Import, Literal, Module, Pattern, Term, Type, TypeDef, +}; + +/// Print a module in form (A). +pub fn print(module: &Module) -> String { + let mut out = String::new(); + out.push('('); + out.push_str("module "); + out.push_str(&module.name); + for imp in &module.imports { + out.push('\n'); + write_import(&mut out, imp, 1); + } + for def in &module.defs { + out.push('\n'); + write_def(&mut out, def, 1); + } + out.push(')'); + out.push('\n'); + out +} + +// ---- helpers -------------------------------------------------------------- + +fn indent(out: &mut String, level: usize) { + for _ in 0..level { + out.push_str(" "); + } +} + +fn write_string_lit(out: &mut String, s: &str) { + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\t' => out.push_str("\\t"), + '\r' => out.push_str("\\r"), + other => out.push(other), + } + } + out.push('"'); +} + +// ---- imports & defs ------------------------------------------------------- + +fn write_import(out: &mut String, imp: &Import, level: usize) { + indent(out, level); + out.push_str("(import "); + out.push_str(&imp.module); + if let Some(alias) = &imp.alias { + out.push_str(" as "); + out.push_str(alias); + } + out.push(')'); +} + +fn write_def(out: &mut String, def: &Def, level: usize) { + match def { + Def::Type(td) => write_type_def(out, td, level), + Def::Fn(fd) => write_fn_def(out, fd, level), + Def::Const(cd) => write_const_def(out, cd, level), + } +} + +fn write_type_def(out: &mut String, td: &TypeDef, level: usize) { + indent(out, level); + out.push_str("(data "); + out.push_str(&td.name); + if !td.vars.is_empty() { + out.push_str(" (vars"); + for v in &td.vars { + out.push(' '); + out.push_str(v); + } + out.push(')'); + } + if let Some(doc) = &td.doc { + out.push('\n'); + indent(out, level + 1); + out.push_str("(doc "); + write_string_lit(out, doc); + out.push(')'); + } + for c in &td.ctors { + out.push('\n'); + write_ctor(out, c, level + 1); + } + out.push(')'); +} + +fn write_ctor(out: &mut String, c: &Ctor, level: usize) { + indent(out, level); + out.push_str("(ctor "); + out.push_str(&c.name); + for f in &c.fields { + out.push(' '); + write_type(out, f); + } + out.push(')'); +} + +fn write_fn_def(out: &mut String, fd: &FnDef, level: usize) { + indent(out, level); + out.push_str("(fn "); + out.push_str(&fd.name); + if let Some(doc) = &fd.doc { + out.push('\n'); + indent(out, level + 1); + out.push_str("(doc "); + write_string_lit(out, doc); + out.push(')'); + } + out.push('\n'); + indent(out, level + 1); + out.push_str("(type "); + write_type(out, &fd.ty); + out.push(')'); + out.push('\n'); + indent(out, level + 1); + out.push_str("(params"); + for p in &fd.params { + out.push(' '); + out.push_str(p); + } + out.push(')'); + out.push('\n'); + indent(out, level + 1); + out.push_str("(body "); + write_term(out, &fd.body, level + 2); + out.push(')'); + out.push(')'); +} + +fn write_const_def(out: &mut String, cd: &ConstDef, level: usize) { + indent(out, level); + out.push_str("(const "); + out.push_str(&cd.name); + if let Some(doc) = &cd.doc { + out.push('\n'); + indent(out, level + 1); + out.push_str("(doc "); + write_string_lit(out, doc); + out.push(')'); + } + out.push('\n'); + indent(out, level + 1); + out.push_str("(type "); + write_type(out, &cd.ty); + out.push(')'); + out.push('\n'); + indent(out, level + 1); + out.push_str("(body "); + write_term(out, &cd.value, level + 2); + out.push(')'); + out.push(')'); +} + +// ---- types ---------------------------------------------------------------- + +fn write_type(out: &mut String, t: &Type) { + match t { + Type::Var { name } => out.push_str(name), + Type::Con { name, args } => { + out.push_str("(con "); + out.push_str(name); + for a in args { + out.push(' '); + write_type(out, a); + } + out.push(')'); + } + Type::Fn { + params, + ret, + effects, + } => { + out.push_str("(fn-type (params"); + for p in params { + out.push(' '); + write_type(out, p); + } + out.push_str(") (ret "); + write_type(out, ret); + out.push(')'); + if !effects.is_empty() { + out.push_str(" (effects"); + for e in effects { + out.push(' '); + out.push_str(e); + } + out.push(')'); + } + out.push(')'); + } + Type::Forall { vars, body } => { + out.push_str("(forall (vars"); + for v in vars { + out.push(' '); + out.push_str(v); + } + out.push_str(") "); + write_type(out, body); + out.push(')'); + } + } +} + +// ---- terms ---------------------------------------------------------------- + +fn write_term(out: &mut String, t: &Term, level: usize) { + match t { + Term::Lit { lit } => write_lit(out, lit), + Term::Var { name } => out.push_str(name), + Term::App { callee, args } => { + out.push_str("(app "); + write_term(out, callee, level); + for a in args { + out.push(' '); + write_term(out, a, level); + } + out.push(')'); + } + Term::Let { name, value, body } => { + out.push_str("(let "); + out.push_str(name); + out.push(' '); + write_term(out, value, level); + out.push(' '); + write_term(out, body, level); + out.push(')'); + } + Term::If { cond, then, else_ } => { + out.push_str("(if "); + write_term(out, cond, level); + out.push(' '); + write_term(out, then, level); + out.push(' '); + write_term(out, else_, level); + out.push(')'); + } + Term::Do { op, args } => { + out.push_str("(do "); + out.push_str(op); + for a in args { + out.push(' '); + write_term(out, a, level); + } + out.push(')'); + } + Term::Ctor { type_name, ctor, args } => { + out.push_str("(term-ctor "); + out.push_str(type_name); + out.push(' '); + out.push_str(ctor); + for a in args { + out.push(' '); + write_term(out, a, level); + } + out.push(')'); + } + Term::Match { scrutinee, arms } => { + out.push_str("(match "); + write_term(out, scrutinee, level); + for arm in arms { + out.push('\n'); + indent(out, level); + write_arm(out, arm, level); + } + out.push(')'); + } + Term::Lam { + params, + param_tys, + ret_ty, + effects, + body, + } => { + out.push_str("(lam (params"); + for (name, ty) in params.iter().zip(param_tys.iter()) { + out.push_str(" (typed "); + out.push_str(name); + out.push(' '); + write_type(out, ty); + out.push(')'); + } + out.push_str(") (ret "); + write_type(out, ret_ty); + out.push(')'); + if !effects.is_empty() { + out.push_str(" (effects"); + for e in effects { + out.push(' '); + out.push_str(e); + } + out.push(')'); + } + out.push_str(" (body "); + write_term(out, body, level); + out.push_str("))"); + } + Term::Seq { lhs, rhs } => { + out.push_str("(seq "); + write_term(out, lhs, level); + out.push(' '); + write_term(out, rhs, level); + out.push(')'); + } + } +} + +fn write_arm(out: &mut String, arm: &Arm, level: usize) { + out.push_str("(case "); + write_pattern(out, &arm.pat); + out.push(' '); + write_term(out, &arm.body, level + 1); + out.push(')'); +} + +fn write_lit(out: &mut String, lit: &Literal) { + match lit { + Literal::Int { value } => out.push_str(&value.to_string()), + Literal::Bool { value } => out.push_str(if *value { "true" } else { "false" }), + Literal::Str { value } => write_string_lit(out, value), + Literal::Unit => out.push_str("(lit-unit)"), + } +} + +fn write_pattern(out: &mut String, p: &Pattern) { + match p { + Pattern::Wild => out.push('_'), + Pattern::Var { name } => out.push_str(name), + Pattern::Lit { lit } => { + out.push_str("(pat-lit "); + // Inside pat-lit, write the bare literal form (no `(lit-unit)` + // — Unit is not allowed in pat-lit per the grammar). + match lit { + Literal::Int { value } => out.push_str(&value.to_string()), + Literal::Bool { value } => out.push_str(if *value { "true" } else { "false" }), + Literal::Str { value } => write_string_lit(out, value), + Literal::Unit => { + // Unit is not a valid pat-lit; fall back to a bare unit-lit + // form. This is unreachable for any well-formed AST that + // came through the typechecker. + out.push_str("(lit-unit)"); + } + } + out.push(')'); + } + Pattern::Ctor { ctor, fields } => { + out.push_str("(pat-ctor "); + out.push_str(ctor); + for f in fields { + out.push(' '); + write_pattern(out, f); + } + out.push(')'); + } + } +} diff --git a/crates/ailang-surface/tests/round_trip.rs b/crates/ailang-surface/tests/round_trip.rs new file mode 100644 index 0000000..4d6d181 --- /dev/null +++ b/crates/ailang-surface/tests/round_trip.rs @@ -0,0 +1,128 @@ +//! Round-trip gate for the form-(A) projection. +//! +//! For every `examples/*.ail.json` fixture, this test: +//! +//! 1. Loads the original module via `ailang_core::load_module`. +//! 2. Prints it with `ailang_surface::print`. +//! 3. Re-parses the printed text with `ailang_surface::parse`. +//! 4. Canonicalises both modules and asserts byte-equal canonical JSON. +//! +//! In addition, the three hand-written exhibits (`hello.ailx`, +//! `box.ailx`, `list_map_poly.ailx`) are parsed and asserted to produce +//! canonical JSON identical to their corresponding `.ail.json` +//! fixtures. This is the human/AI authoring ground-truth check that +//! the form is writeable without going through the printer. + +use std::path::{Path, PathBuf}; + +fn examples_dir() -> PathBuf { + // `CARGO_MANIFEST_DIR` is the surface crate root; examples live two + // levels up. + let crate_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + crate_dir.parent().unwrap().parent().unwrap().join("examples") +} + +fn list_json_fixtures() -> Vec { + let dir = examples_dir(); + let mut paths: Vec = std::fs::read_dir(&dir) + .unwrap_or_else(|e| panic!("read_dir({}): {e}", dir.display())) + .filter_map(|entry| entry.ok()) + .map(|e| e.path()) + .filter(|p| { + p.file_name() + .and_then(|n| n.to_str()) + .map(|n| n.ends_with(".ail.json")) + .unwrap_or(false) + }) + .collect(); + paths.sort(); + paths +} + +#[test] +fn print_then_parse_round_trips_every_fixture() { + let fixtures = list_json_fixtures(); + assert!( + !fixtures.is_empty(), + "no .ail.json fixtures found under {}", + examples_dir().display() + ); + + let mut failures = Vec::::new(); + let mut passed = 0usize; + for path in &fixtures { + match round_trip_one(path) { + Ok(()) => passed += 1, + Err(msg) => failures.push(format!("{}: {msg}", path.display())), + } + } + if !failures.is_empty() { + panic!( + "round-trip failed for {} of {} fixtures (passed: {}):\n{}", + failures.len(), + fixtures.len(), + passed, + failures.join("\n") + ); + } + eprintln!("round-trip ok for {passed} fixtures"); +} + +fn round_trip_one(path: &Path) -> Result<(), String> { + let original = ailang_core::load_module(path).map_err(|e| format!("load: {e}"))?; + let text = ailang_surface::print(&original); + let parsed = ailang_surface::parse(&text) + .map_err(|e| format!("re-parse failed: {e}\n--- printed text ---\n{text}"))?; + let bytes_orig = ailang_core::canonical::to_bytes(&original); + let bytes_round = ailang_core::canonical::to_bytes(&parsed); + if bytes_orig != bytes_round { + let s_orig = String::from_utf8_lossy(&bytes_orig).into_owned(); + let s_round = String::from_utf8_lossy(&bytes_round).into_owned(); + return Err(format!( + "canonical bytes differ.\noriginal: {s_orig}\nround: {s_round}" + )); + } + Ok(()) +} + +#[test] +fn handwritten_exhibits_match_json_fixtures() { + let dir = examples_dir(); + let pairs: &[(&str, &str)] = &[ + ("hello.ailx", "hello.ail.json"), + ("box.ailx", "box.ail.json"), + ("list_map_poly.ailx", "list_map_poly.ail.json"), + ]; + let mut failures = Vec::new(); + for (ailx, json) in pairs { + let ailx_path = dir.join(ailx); + let json_path = dir.join(json); + let text = std::fs::read_to_string(&ailx_path) + .unwrap_or_else(|e| panic!("read {}: {e}", ailx_path.display())); + let parsed = match ailang_surface::parse(&text) { + Ok(m) => m, + Err(e) => { + failures.push(format!("{ailx}: parse failed: {e}")); + continue; + } + }; + let original = ailang_core::load_module(&json_path) + .unwrap_or_else(|e| panic!("load {}: {e}", json_path.display())); + let bytes_orig = ailang_core::canonical::to_bytes(&original); + let bytes_parsed = ailang_core::canonical::to_bytes(&parsed); + if bytes_orig != bytes_parsed { + let s_orig = String::from_utf8_lossy(&bytes_orig).into_owned(); + let s_round = String::from_utf8_lossy(&bytes_parsed).into_owned(); + failures.push(format!( + "{ailx} vs {json}: canonical bytes differ.\noriginal: {s_orig}\nparsed: {s_round}" + )); + } + } + if !failures.is_empty() { + panic!( + "{} hand-written exhibit(s) failed:\n{}", + failures.len(), + failures.join("\n\n") + ); + } +} diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 115d0d4..523bde6 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -391,6 +391,44 @@ on the shelf for a future iter only if both fail. `build`, `run`, `manifest`, `deps`, `diff`, `workspace`, `builtins`) keeps its current `.ail.json` interface. +### Form refinements during Iter 14c implementation + +Two productions in the original 14b sketch had to be widened +during implementation to round-trip the existing AST faithfully. +Captured here for the spec record: + +1. **`lam-term` carries types and effects.** The AST's `Term::Lam` + stores parallel `params`, `param_tys`, `ret_ty`, and `effects` + fields. The 14b sketch had only names. The implemented form is + + ``` + lam-term ::= "(" "lam" "(" "params" typed-param* ")" + "(" "ret" type ")" + effects-clause? body-attr ")" + typed-param ::= "(" "typed" ident type ")" + ``` + + This keeps the no-precedence / one-construct-per-token-list + invariants and adds no new lexical rules. + +2. **`import-clause` admits an optional alias.** The AST's + `Import.alias` is `Option`; the original sketch only + supported the `None` case. The implemented form is + + ``` + import-clause ::= "(" "import" ident ("as" ident)? ")" + ``` + + `as` is a bare ident token in this position; no special lexical + rule is needed. + +Neither change extends the grammar's rule budget meaningfully: +the 30-production ceiling of constraint 1 is intact (the parser +implements ~28 named productions). All 17 `examples/*.ail.json` +fixtures round-trip identically through `print → parse → canonical +JSON`; the three hand-written `.ailx` exhibits parse to canonical +JSON identical to their corresponding `.ail.json` files. + ## Mangling scheme (Iter 5c) All AILang functions are mangled to `@ail__` — even in the diff --git a/docs/JOURNAL.md b/docs/JOURNAL.md index 71d408d..e106ed3 100644 --- a/docs/JOURNAL.md +++ b/docs/JOURNAL.md @@ -1627,4 +1627,122 @@ with `length`, `filter`, `fold`, `concat`, `reverse`, `head`, for the still-young parameterised-ADT pipeline. Written in the new surface from day one — no JSON authoring of stdlib. +## Iter 14c — `ailang-surface` parser + pretty-printer ships + +Implements Decision 6's form (A). New crate `ailang-surface` +(~1843 LOC across `lex.rs`, `parse.rs`, `print.rs`, lib root, +plus tests) ships as a strictly additive producer of +`ailang-core::ast::Module` values. `ailang-check` and +`ailang-codegen` were not modified — projection-agnostic, as +the architectural pin requires. + +**Implementer dispatch went clean.** Brief gave the EBNF, the +fixtures to round-trip, the lexer rule, and the architectural +constraints. Implementer hand-wrote a recursive-descent parser +(one Rust fn per EBNF production), a deterministic pretty- +printer, an integration test that runs the round-trip gate on +every fixture, and the `ail parse` CLI subcommand. No +parser-combinator dependency. No AST-shape changes. + +**Two AST-driven form refinements** that were not in the 14b +sketch (both folded into DESIGN.md Decision 6 by the +implementer before commit): + +1. `lam-term` had to carry `param_tys`, `ret_ty`, and + `effects` because the AST's `Term::Lam` stores parallel + typed-param data. Production became `(lam (params (typed + x Int) ...) (ret T) (effects ...) (body ...))`. Same + shape-style as the rest of the form; no new lex rules. +2. `import-clause` had to admit `Option` aliases. + Production became `(import name (as alias)?)` with `as` + as a bare ident token. No new lex rules. + +The 14b 30-production budget held: implementer reports ~28 +named productions in the parser. Constraint 1 (formalisable +for foreign LLM) intact. + +**Verification gates, all green:** + +- `cargo build --workspace`: 0 warnings, finished. +- `cargo test --workspace`: 76 tests pass (was 64; +9 unit + tests in `ailang-surface`, +2 integration tests in + `tests/round_trip.rs`, +1 e2e regression preserved). All + 17 `examples/*.ail.json` fixtures round-trip + byte-identical through `print → parse → canonical JSON`; + 3 hand-written `.ailx` exhibits parse to canonical JSON + byte-identical to their `.ail.json` siblings. +- `cargo doc --no-deps`: 0 warnings (workspace invariant + from 13d/e/f preserved; new crate's rustdoc landed + correctly with crate-root `//!` plus all `pub` items + documented). + +**Manual smoke test (orchestrator-side after agent reported +done):** `ail parse -o ` followed +by `ail run ` for all three exhibits: + +``` +hello.ailx → "Hello, AILang." +box.ailx → "42" +list_map_poly.ailx → "2\n3\n4" +``` + +End-to-end pipeline form (A) → AST → typecheck → codegen → +clang → binary works on all three. + +**Important non-issue.** `examples/*.ail.json` fixtures on +disk are hand-formatted JSON with non-canonical key order; +the parser produces canonical (lex-sorted) JSON. Diff at +the file-byte level is not zero. Diff at the canonical-byte +level is zero — which is the only thing that matters per +Decision 1, since hashing uses canonical form. The +round-trip test gates on canonical bytes, not file bytes. +This is correct behaviour; flagged here so a future reader +who runs `diff` doesn't think the surface is broken. + +**Files created:** + +- `crates/ailang-surface/Cargo.toml` +- `crates/ailang-surface/src/lib.rs` (~40 LOC, rustdoc heavy) +- `crates/ailang-surface/src/lex.rs` (~264 LOC) +- `crates/ailang-surface/src/parse.rs` (~1041 LOC, one fn + per production) +- `crates/ailang-surface/src/print.rs` (~371 LOC) +- `crates/ailang-surface/tests/round_trip.rs` (~128 LOC) + +**Files modified:** + +- `Cargo.toml` (workspace) — `ailang-surface` member + + workspace dep. +- `Cargo.lock` — refresh. +- `crates/ail/Cargo.toml` — `ailang-surface` dep. +- `crates/ail/src/main.rs` — new `Parse { path, output }` + subcommand (~36 LOC). +- `docs/DESIGN.md` — form refinements appendix to + Decision 6. +- `examples/list_map_poly.ailx` — implementer added doc + strings to match the JSON original (was a 14b design + exhibit, not byte-aligned). + +**Known debt:** none reported. `ailang-check` / +`ailang-codegen` untouched per the architectural pin. + +**Plan 14d (next, queued).** With form (A) now ergonomic +and round-trip-verified, the std-lib path opens up. First +target: `examples/std/std_list.ailx` with `length`, +`filter`, `fold` (left and right), `concat`, `reverse`, +`head`, `tail`. Each combinator a fresh test vector for +the parameterised-ADT pipeline (which 14a opened end-to- +end). Authored in form (A); the resulting `.ail.json` is +what tests consume. If 14d surfaces more codegen bugs in +the parameterised-ADT path (it likely will — 14a found +one already), debugger handles them inline. + +The 14b/c form-A hypothesis has held under the empirical +test of round-tripping every existing fixture. The +documented rollback to form (C) is now off the table for +this iter cycle, though the architectural pin keeps it +open for future replacement of `ailang-surface` should +the form prove inadequate at stdlib scale. + + diff --git a/examples/list_map_poly.ailx b/examples/list_map_poly.ailx index 75f4ffd..72ce0c1 100644 --- a/examples/list_map_poly.ailx +++ b/examples/list_map_poly.ailx @@ -1,20 +1,21 @@ -; Iter 14b design exhibit — Decision 6 form (A). -; Companion to box.ailx. Same hash-equivalence gate in Iter 14c. +; Iter 14b design exhibit, finalised in Iter 14c. +; Hash-equivalent to list_map_poly.ail.json (round-trip-tested). (module list_map_poly (data List (vars a) - (doc "Singly-linked polymorphic list.") + (doc "Polymorphic singly-linked list.") (ctor Nil) (ctor Cons a (con List a))) (fn inc + (doc "Add 1 to an Int.") (type (fn-type (params (con Int)) (ret (con Int)))) (params x) (body (app + x 1))) (fn map - (doc "Apply f to every element. Recursive on the tail.") + (doc "Polymorphic map: apply f to every element, recursing on the tail.") (type (forall (vars a b) (fn-type @@ -32,6 +33,7 @@ (app map f t)))))) (fn print_list + (doc "Print each Int on its own line.") (type (fn-type (params (con List (con Int))) @@ -48,6 +50,7 @@ (app print_list t)))))) (fn main + (doc "Map inc over [1,2,3] via polymorphic List, then print: 2,3,4.") (type (fn-type (params) (ret (con Unit)) (effects IO))) (params) (body