Brummel/RustAst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add AST emitter and comment support

Browse Source 
Introduces a new AST emitter module responsible for serializing AST
nodes back into Myc source code. This emitter preserves leading comments
and whitespace, enabling a round-trip guarantee for the parser and
emitter.

The documentation in `docs/BNF.md` has been updated to reflect the new
syntax rules for comments, including single-line comments (`;`) and
documentation comments (`;;`). The rules clarify that comments must be
whole lines and that inline comments are disallowed.

The lexer and AST node structures have been updated to accommodate
`CommentLine` variants, allowing for different types of comments
(regular, documentation, and blank lines) to be stored and processed.

A new test suite `tests/comment_roundtrip.rs` has been added to verify
the round-trip property of the parser and emitter across all example
`.myc` files. This test ensures that parsing source code, emitting it,
and then parsing the emitted code again results in the same output
string.

The `ast.rs` binary now includes a `--emit` flag that uses the new
emitter to output the parsed source code, facilitating debugging and
testing of the emitter itself.
This commit is contained in:
Brummel
2026-03-25 19:15:40 +01:00
parent ee89d2330d
commit e757b453a4
10 changed files with 501 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ast/compiler/emitter.rs
+258
View File
@@ -0,0 +1,258 @@
use crate::ast::nodes::{NodeKind, SyntaxNode};
use crate::ast::types::{CommentLine, Value};
use std::rc::Rc;
/// Emits a [`SyntaxNode`] back to Myc source code, preserving leading comments.
///
/// Round-trip guarantee: `emit(parse(emit(parse(source)))) == emit(parse(source))`.
/// In other words, the emitter is idempotent after the first pass.
pub fn emit(node: &SyntaxNode) -> String {
let mut out = String::new();
// The top-level node is always at block level (owns its own line).
emit_block(node, 0, &mut out);
out
}
// ── Block-level emission ────────────────────────────────────────────────────
/// Emits a node in **block-level** context: writes leading comments and
/// the appropriate indentation prefix, then the code.
///
/// Callers must only write a `\n` separator before calling this — never `\n + pad`.
fn emit_block(node: &SyntaxNode, indent: usize, out: &mut String) {
emit_comments(&node.comments, indent, out);
out.push_str(&" ".repeat(indent));
emit_kind(node, indent, out);
}
/// Emits a node in **inline** context: no indentation, no comments.
/// Used for sub-expressions that are part of a larger expression on the same line.
fn emit_inline(node: &SyntaxNode, indent: usize, out: &mut String) {
emit_kind(node, indent, out);
}
// ── Comment emission ────────────────────────────────────────────────────────
fn emit_comments(comments: &Rc<[CommentLine]>, indent: usize, out: &mut String) {
if comments.is_empty() {
return;
}
let pad = " ".repeat(indent);
for line in comments.iter() {
match line {
CommentLine::Comment(text) => {
out.push_str(&pad);
out.push(';');
if !text.is_empty() {
out.push(' ');
out.push_str(text);
}
out.push('\n');
}
CommentLine::Doc(text) => {
out.push_str(&pad);
out.push_str(";;");
if !text.is_empty() {
out.push(' ');
out.push_str(text);
}
out.push('\n');
}
CommentLine::Blank => {
out.push('\n');
}
}
}
}
// ── Kind emission ───────────────────────────────────────────────────────────
fn emit_kind(node: &SyntaxNode, indent: usize, out: &mut String) {
match &node.kind {
NodeKind::Nop => out.push_str("..."),
NodeKind::Constant(val) => emit_value(val, out),
NodeKind::Identifier { symbol, .. } => out.push_str(&symbol.name),
NodeKind::FieldAccessor(k) => {
out.push('.');
out.push_str(&k.name());
}
NodeKind::If { cond, then_br, else_br } => {
out.push_str("(if ");
emit_inline(cond, indent + 1, out);
out.push('\n');
emit_block(then_br, indent + 1, out);
if let Some(else_node) = else_br {
out.push('\n');
emit_block(else_node, indent + 1, out);
}
out.push(')');
}
NodeKind::Def { pattern, value, .. } => {
out.push_str("(def ");
emit_inline(pattern, indent, out);
out.push('\n');
emit_block(value, indent + 1, out);
out.push(')');
}
NodeKind::Assign { target, value, .. } => {
out.push_str("(assign ");
emit_inline(target, indent, out);
out.push(' ');
emit_inline(value, indent + 1, out);
out.push(')');
}
NodeKind::Lambda { params, body, .. } => {
out.push_str("(fn ");
emit_inline(params, indent, out);
out.push('\n');
emit_block(body, indent + 1, out);
out.push(')');
}
NodeKind::Call { callee, args } => {
out.push('(');
emit_inline(callee, indent, out);
if let NodeKind::Tuple { elements } = &args.kind {
for arg in elements {
out.push(' ');
emit_inline(arg, indent + 1, out);
}
}
out.push(')');
}
NodeKind::Again { args } => {
out.push_str("(again");
if let NodeKind::Tuple { elements } = &args.kind {
for arg in elements {
out.push(' ');
emit_inline(arg, indent + 1, out);
}
}
out.push(')');
}
NodeKind::Block { exprs } => {
out.push_str("(do");
for expr in exprs {
out.push('\n');
emit_block(expr, indent + 1, out);
}
out.push(')');
}
NodeKind::Tuple { elements } => {
out.push('[');
for (i, el) in elements.iter().enumerate() {
if i > 0 {
out.push(' ');
}
emit_inline(el, indent, out);
}
out.push(']');
}
NodeKind::Record { fields, .. } => {
if fields.is_empty() {
out.push_str("{}");
return;
}
// Always multi-line so that any leading comments on keys are valid.
out.push('{');
for (key, val) in fields.iter() {
out.push('\n');
emit_block(key, indent + 1, out);
out.push(' ');
emit_inline(val, indent + 1, out);
}
out.push('\n');
out.push_str(&" ".repeat(indent));
out.push('}');
}
NodeKind::MacroDecl { name, params, body } => {
out.push_str("(macro ");
out.push_str(&name.name);
out.push(' ');
emit_inline(params, indent, out);
out.push('\n');
emit_block(body, indent + 1, out);
out.push(')');
}
NodeKind::Template(inner) => {
out.push('`');
emit_inline(inner, indent, out);
}
NodeKind::Placeholder(inner) => {
out.push('~');
emit_inline(inner, indent, out);
}
NodeKind::Splice(inner) => {
out.push_str("~@");
emit_inline(inner, indent, out);
}
NodeKind::GetField { rec, field } => {
out.push_str("(.");
out.push_str(&field.name());
out.push(' ');
emit_inline(rec, indent, out);
out.push(')');
}
NodeKind::Expansion { original_call, .. } => {
// Round-trip uses the original pre-expansion form.
emit_inline(original_call, indent, out);
}
NodeKind::Error | NodeKind::Extension(_) => {
out.push_str("<error>");
}
}
}
fn emit_value(val: &Value, out: &mut String) {
match val {
Value::Int(n) => out.push_str(&n.to_string()),
Value::Float(f) => {
let s = format!("{}", f);
// Ensure a decimal point is present so it round-trips as a float.
if s.contains('.') || s.contains('e') {
out.push_str(&s);
} else {
out.push_str(&s);
out.push_str(".0");
}
}
Value::Text(t) => {
out.push('"');
for c in t.chars() {
match c {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
c => out.push(c),
}
}
out.push('"');
}
Value::Keyword(k) => {
out.push(':');
out.push_str(&k.name());
}
Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
other => out.push_str(&other.to_string()),
}
}
src/ast/compiler/macros.rs
+1 -1
View File
@@ -624,7 +624,7 @@ mod tests {
use crate::ast::diagnostics::Diagnostics;
use crate::ast::nodes::{Address, VirtualId};
use crate::ast::parser::Parser;
use crate::ast::types::{NodeIdentity, Object, Purity, SourceLocation, StaticType, Value};
use crate::ast::types::{NodeIdentity, Purity, SourceLocation, StaticType, Value};
struct SimpleEvaluator;
impl MacroEvaluator for SimpleEvaluator {
src/ast/compiler/mod.rs
+1
View File
@@ -2,6 +2,7 @@ pub mod analyzer;
pub mod binder;
pub mod captures;
pub mod dumper;
pub mod emitter;
pub mod lambda_collector;
pub mod macros;
pub mod optimizer;
src/ast/environment.rs
+80 -13
View File
@@ -22,7 +22,7 @@ use crate::ast::compiler::optimizer::Optimizer;
use crate::ast::compiler::specializer::{FunctionRegistry, MonoCache, RtlLookupFunc, Specializer};
use crate::ast::rtl::{self, intrinsics};
use crate::ast::types::{
NativeFunction, NodeIdentity, Purity, SourceLocation, StaticType, Value,
CommentLine, NativeFunction, NodeIdentity, Purity, SourceLocation, StaticType, Value,
};
use crate::ast::diagnostics::{DiagnosticLevel, Diagnostics};
@@ -148,6 +148,9 @@ pub struct Environment {
pub search_paths: Rc<RefCell<Vec<PathBuf>>>,
pub loaded_modules: Rc<RefCell<HashSet<PathBuf>>>,
pub rtl_docs: Rc<RefCell<Vec<RtlDocEntry>>>,
/// Documentation for symbols defined in Myc source (via `;;` comments before `def`/`macro`).
/// Key: symbol name. Value: concatenated doc lines joined by newlines.
pub myc_docs: Rc<RefCell<HashMap<String, String>>>,
}
struct EnvFunctionRegistry {
@@ -235,6 +238,7 @@ impl Environment {
search_paths: Rc::new(RefCell::new(Vec::new())),
loaded_modules: Rc::new(RefCell::new(HashSet::new())),
rtl_docs: Rc::new(RefCell::new(Vec::new())),
myc_docs: Rc::new(RefCell::new(HashMap::new())),
};
rtl::register(&env);
@@ -561,7 +565,9 @@ impl Environment {
comments: Rc::from([]),
}
};
Some(checker.check(&wrapped_ast, &[], diagnostics))
let typed = checker.check(&wrapped_ast, &[], diagnostics);
self.collect_doc_comments(&typed);
Some(typed)
}
fn compile_syntax(&self, syntax_ast: SyntaxNode) -> Result<TypedNode, String> {
@@ -719,6 +725,17 @@ impl Environment {
out
}).collect();
// Include Myc-defined symbol docs (from `;;` comments in source)
let myc_docs = self.myc_docs.borrow();
for (name, doc_text) in myc_docs.iter() {
let sig = scope
.locals
.get(&Symbol::from(name.as_str()))
.map(|info| info._ty.to_doc_string())
.unwrap_or_else(|| "unknown".to_string());
entries.push(format!("{} : {}\n {}", name, sig, doc_text));
}
entries.sort_by_key(|a| a.to_lowercase());
entries
}
@@ -727,9 +744,6 @@ impl Environment {
/// or `None` if the symbol has no doc entry.
pub fn get_rtl_doc(&self, name: &str) -> Option<String> {
use crate::ast::nodes::Symbol;
let docs = self.rtl_docs.borrow();
let entry = docs.iter().find(|e| e.name == name)?;
let root_scopes = self.root_scopes.borrow();
let sig = root_scopes[0]
.locals
@@ -737,17 +751,70 @@ impl Environment {
.map(|info| info._ty.to_doc_string())
.unwrap_or_else(|| "unknown".to_string());
let mut out = format!("{} : {}\n {}", entry.name, sig, entry.one_liner);
if let Some(desc) = entry.description {
out.push_str(&format!("\n {}", desc));
// Check RTL docs first
let docs = self.rtl_docs.borrow();
if let Some(entry) = docs.iter().find(|e| e.name == name) {
let mut out = format!("{} : {}\n {}", entry.name, sig, entry.one_liner);
if let Some(desc) = entry.description {
out.push_str(&format!("\n {}", desc));
}
if let Some(examples) = entry.examples {
out.push_str("\n Examples:");
for ex in examples.iter() {
out.push_str(&format!("\n {}", ex));
}
}
return Some(out);
}
if let Some(examples) = entry.examples {
out.push_str("\n Examples:");
for ex in examples.iter() {
out.push_str(&format!("\n {}", ex));
// Fall back to Myc-source docs
let myc_docs = self.myc_docs.borrow();
myc_docs
.get(name)
.map(|doc_text| format!("{} : {}\n {}", name, sig, doc_text))
}
/// Walks a typed AST and registers `;;` doc comments on `def`/`macro` nodes
/// whose target is a plain identifier into the `myc_docs` registry.
fn collect_doc_comments(&self, node: &TypedNode) {
// Extract the symbol name this node defines, if any.
let sym_name: Option<Rc<str>> = match &node.kind {
NodeKind::Def { pattern, .. } => {
if let NodeKind::Identifier { symbol, .. } = &pattern.kind {
Some(symbol.name.clone())
} else {
None
}
}
NodeKind::MacroDecl { name, .. } => Some(name.name.clone()),
NodeKind::Block { exprs } => {
for expr in exprs {
self.collect_doc_comments(expr);
}
return;
}
NodeKind::Lambda { body, .. } => {
self.collect_doc_comments(body);
return;
}
_ => None,
};
if let Some(name) = sym_name {
let doc_text: String = node
.comments
.iter()
.filter_map(|line| match line {
CommentLine::Doc(text) => Some(text.as_ref()),
_ => None,
})
.collect::<Vec<_>>()
.join("\n");
if !doc_text.is_empty() {
self.myc_docs.borrow_mut().insert(name.to_string(), doc_text);
}
}
Some(out)
}
pub fn dump_ast(&self, source: &str) -> Result<String, String> {
src/ast/lexer.rs
+28 -10
View File
@@ -1,4 +1,4 @@
use crate::ast::types::SourceLocation;
use crate::ast::types::{CommentLine, SourceLocation};
use std::iter::Peekable;
use std::rc::Rc;
use std::str::Chars;
@@ -27,7 +27,7 @@ pub enum TokenKind {
pub struct Token {
pub kind: TokenKind,
pub location: SourceLocation,
pub comments: Rc<[Rc<str>]>,
pub comments: Rc<[CommentLine]>,
}
pub struct Lexer<'a> {
@@ -35,7 +35,7 @@ pub struct Lexer<'a> {
line: u32,
col: u32,
at_line_start: bool,
comments_buffer: Vec<Rc<str>>,
comments_buffer: Vec<CommentLine>,
}
impl<'a> Lexer<'a> {
@@ -52,18 +52,20 @@ impl<'a> Lexer<'a> {
pub fn next_token(&mut self) -> Result<Token, String> {
self.skip_whitespace_and_comments()?;
let comments: Rc<[Rc<str>]> = if self.comments_buffer.is_empty() {
let comments: Rc<[CommentLine]> = if self.comments_buffer.is_empty() {
Rc::from([])
} else {
let mut start = 0;
while start < self.comments_buffer.len() && self.comments_buffer[start].is_empty() {
while start < self.comments_buffer.len()
&& self.comments_buffer[start] == CommentLine::Blank
{
start += 1;
}
let mut end = self.comments_buffer.len();
while end > start && self.comments_buffer[end - 1].is_empty() {
while end > start && self.comments_buffer[end - 1] == CommentLine::Blank {
end -= 1;
}
let res: Rc<[Rc<str>]> = self.comments_buffer[start..end].iter().cloned().collect();
let res: Rc<[CommentLine]> = self.comments_buffer[start..end].iter().cloned().collect();
self.comments_buffer.clear();
res
};
@@ -168,14 +170,24 @@ impl<'a> Lexer<'a> {
}
if !self.comments_buffer.is_empty() && consecutive_newlines > 1 {
for _ in 0..(consecutive_newlines - 1) {
self.comments_buffer.push(Rc::from(""));
self.comments_buffer.push(CommentLine::Blank);
}
}
consecutive_newlines = 0;
self.input.next(); // consume ';'
self.input.next(); // consume first ';'
self.col += 1;
// Check for a second ';' → Doc level
let is_doc = if let Some(&';') = self.peek() {
self.input.next(); // consume second ';'
self.col += 1;
true
} else {
false
};
// Consume optional single space after the semicolon(s)
if let Some(&' ') = self.peek() {
self.input.next();
self.col += 1;
@@ -189,7 +201,13 @@ impl<'a> Lexer<'a> {
comment_text.push(self.input.next().unwrap());
self.col += 1;
}
self.comments_buffer.push(Rc::from(comment_text));
let line = if is_doc {
CommentLine::Doc(Rc::from(comment_text))
} else {
CommentLine::Comment(Rc::from(comment_text))
};
self.comments_buffer.push(line);
} else if c == '#' {
if !self.at_line_start {
return Err(format!("Inline directives are not allowed at {}:{}", self.line, self.col));
src/ast/nodes.rs
+2 -2
View File
@@ -1,4 +1,4 @@
use crate::ast::types::{Identity, Keyword, Purity, RecordLayout, StaticType, Value};
use crate::ast::types::{CommentLine, Identity, Keyword, Purity, RecordLayout, StaticType, Value};
use std::fmt::Debug;
use std::rc::Rc;
use std::sync::Arc;
@@ -250,7 +250,7 @@ pub struct Node<P: CompilerPhase = BoundPhase> {
pub identity: Identity,
pub kind: NodeKind<P>,
pub ty: P::Metadata,
pub comments: Rc<[Rc<str>]>,
pub comments: Rc<[CommentLine]>,
}
impl<P: CompilerPhase> Clone for Node<P> {
src/ast/types.rs
+15
View File
@@ -9,6 +9,21 @@ use std::rc::Rc;
use std::sync::Mutex; // Still needed for global keyword registry
use std::sync::OnceLock;
/// A single line within a leading comment block attached to an AST node.
///
/// Comment blocks are attached to the *next* expression in the source.
/// The first and last line of a block are never `Blank` (enforced by the lexer).
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CommentLine {
/// `; text` — regular comment, no semantic effect.
Comment(Rc<str>),
/// `;; text` — documentation comment.
/// When attached to a `def` or `macro` node, the text is registered in the symbol registry.
Doc(Rc<str>),
/// A blank separator line within a comment block (preserves visual spacing).
Blank,
}
/// Simple source location
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct SourceLocation {
src/bin/ast.rs
+17 -1
View File
@@ -1,5 +1,7 @@
use clap::Parser;
use myc::ast::compiler::emitter;
use myc::ast::environment::Environment;
use myc::ast::parser::Parser as MycParser;
use myc::utils::tester;
use std::fs;
use std::io::{self, IsTerminal, Read};
@@ -43,6 +45,10 @@ struct Cli {
/// Start MCP server (stdio transport, for LLM integration)
#[arg(long)]
mcp: bool,
/// Emit the parsed source back as Myc code (round-trip test)
#[arg(long)]
emit: bool,
}
fn main() {
@@ -108,7 +114,17 @@ fn main() {
};
if let Some(content) = source {
if cli.dump {
if cli.emit {
let mut parser = MycParser::new(&content);
let syntax_ast = parser.parse_expression();
if parser.diagnostics.has_errors() {
for diag in &parser.diagnostics.items {
eprintln!("ERROR: {}", diag.message);
}
std::process::exit(1);
}
print!("{}", emitter::emit(&syntax_ast));
} else if cli.dump {
match env.dump_ast(&content) {
Ok(dump) => println!("{}", dump),
Err(e) => eprintln!("Error dumping AST: {}", e),