feat: Add AST emitter and comment support
Introduces a new AST emitter module responsible for serializing AST nodes back into Myc source code. This emitter preserves leading comments and whitespace, enabling a round-trip guarantee for the parser and emitter. The documentation in `docs/BNF.md` has been updated to reflect the new syntax rules for comments, including single-line comments (`;`) and documentation comments (`;;`). The rules clarify that comments must be whole lines and that inline comments are disallowed. The lexer and AST node structures have been updated to accommodate `CommentLine` variants, allowing for different types of comments (regular, documentation, and blank lines) to be stored and processed. A new test suite `tests/comment_roundtrip.rs` has been added to verify the round-trip property of the parser and emitter across all example `.myc` files. This test ensures that parsing source code, emitting it, and then parsing the emitted code again results in the same output string. The `ast.rs` binary now includes a `--emit` flag that uses the new emitter to output the parsed source code, facilitating debugging and testing of the emitter itself.
This commit is contained in:
+36
-5
@@ -66,6 +66,19 @@ This document defines the syntax (BNF) and semantics of the Myc language, a Lisp
|
|||||||
```
|
```
|
||||||
*(Lexical Note: `<ident-start>` consists of alphabetic characters or `+-*/<=>!$%&_?.`. `<ident-char>` includes digits as well. Identifiers evaluating to a field accessor begin with a `.` followed by one or more valid identifier characters.)*
|
*(Lexical Note: `<ident-start>` consists of alphabetic characters or `+-*/<=>!$%&_?.`. `<ident-char>` includes digits as well. Identifiers evaluating to a field accessor begin with a `.` followed by one or more valid identifier characters.)*
|
||||||
|
|
||||||
|
### Comments
|
||||||
|
|
||||||
|
```ebnf
|
||||||
|
<comment> ::= ";" " "? <line-text> <newline>
|
||||||
|
<doc-comment> ::= ";;" " "? <line-text> <newline>
|
||||||
|
```
|
||||||
|
|
||||||
|
**Rules:**
|
||||||
|
- Comments must be **whole lines** — they start at the beginning of a line. Inline comments (after code on the same line) are a compile error.
|
||||||
|
- Comment lines directly preceding an expression form a **comment block** attached to that expression.
|
||||||
|
- Blank lines between comment lines are preserved as separators within the block. The first and last line of a block must not be blank.
|
||||||
|
- `;;` (doc comment) before a `def` or `macro` registers the text in the symbol documentation registry (available via `--dump-docs`). Before other expressions it is allowed but has no registry effect.
|
||||||
|
|
||||||
## 2. Core Semantics & Data Types
|
## 2. Core Semantics & Data Types
|
||||||
|
|
||||||
- **Integers and Floats:** Standard 64-bit numeric types.
|
- **Integers and Floats:** Standard 64-bit numeric types.
|
||||||
@@ -138,7 +151,8 @@ The Myc runtime environment provides a collection of built-in functions and macr
|
|||||||
```clojure
|
```clojure
|
||||||
(do
|
(do
|
||||||
(def user {:id 101 :name "Alice" :role :admin})
|
(def user {:id 101 :name "Alice" :role :admin})
|
||||||
(def role (.role user)) ; Evaluates to :admin
|
; Evaluates to :admin
|
||||||
|
(def role (.role user))
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -156,7 +170,8 @@ The Myc runtime environment provides a collection of built-in functions and macr
|
|||||||
(def prices (.price my_ticks))
|
(def prices (.price my_ticks))
|
||||||
|
|
||||||
;; Series indexing: 0 is the newest (15.5), 1 is the previous (11.2)
|
;; Series indexing: 0 is the newest (15.5), 1 is the previous (11.2)
|
||||||
(+ (prices 0) (prices 1)) ;; Output: 26.7
|
;; Output: 26.7
|
||||||
|
(+ (prices 0) (prices 1))
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -223,6 +238,19 @@ The `check_syntax` tool uses the single-expression compiler pass. Wrap multiple
|
|||||||
(do (def x 1) (+ x 2))
|
(do (def x 1) (+ x 2))
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Inline comments are not allowed
|
||||||
|
|
||||||
|
Comments must be **whole lines** starting with `;` or `;;`. Placing a comment after code on the same line is a compile error.
|
||||||
|
|
||||||
|
```clojure
|
||||||
|
;; WRONG — inline comment causes a compile error:
|
||||||
|
(def x 42) ; this is x
|
||||||
|
|
||||||
|
;; CORRECT — comment on its own line before the expression:
|
||||||
|
; this is x
|
||||||
|
(def x 42)
|
||||||
|
```
|
||||||
|
|
||||||
### Field accessors are functions, not property syntax
|
### Field accessors are functions, not property syntax
|
||||||
|
|
||||||
```clojure
|
```clojure
|
||||||
@@ -247,7 +275,8 @@ user.name
|
|||||||
acc
|
acc
|
||||||
(again (- n 1) (* acc n)))))
|
(again (- n 1) (* acc n)))))
|
||||||
|
|
||||||
(factorial 10 1) ;; Output: 3628800
|
;; Output: 3628800
|
||||||
|
(factorial 10 1)
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -258,7 +287,8 @@ user.name
|
|||||||
(macro unless [c body]
|
(macro unless [c body]
|
||||||
`(if ~c ... ~body))
|
`(if ~c ... ~body))
|
||||||
|
|
||||||
(unless false "executed") ;; Output: "executed"
|
;; Output: "executed"
|
||||||
|
(unless false "executed")
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -276,6 +306,7 @@ user.name
|
|||||||
|
|
||||||
;; Read back the two most recent prices
|
;; Read back the two most recent prices
|
||||||
(def p (.price ticks))
|
(def p (.price ticks))
|
||||||
(+ (p 0) (p 1)) ;; newest + second-newest
|
;; newest + second-newest
|
||||||
|
(+ (p 0) (p 1))
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -0,0 +1,258 @@
|
|||||||
|
use crate::ast::nodes::{NodeKind, SyntaxNode};
|
||||||
|
use crate::ast::types::{CommentLine, Value};
|
||||||
|
use std::rc::Rc;
|
||||||
|
|
||||||
|
/// Emits a [`SyntaxNode`] back to Myc source code, preserving leading comments.
|
||||||
|
///
|
||||||
|
/// Round-trip guarantee: `emit(parse(emit(parse(source)))) == emit(parse(source))`.
|
||||||
|
/// In other words, the emitter is idempotent after the first pass.
|
||||||
|
pub fn emit(node: &SyntaxNode) -> String {
|
||||||
|
let mut out = String::new();
|
||||||
|
// The top-level node is always at block level (owns its own line).
|
||||||
|
emit_block(node, 0, &mut out);
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Block-level emission ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Emits a node in **block-level** context: writes leading comments and
|
||||||
|
/// the appropriate indentation prefix, then the code.
|
||||||
|
///
|
||||||
|
/// Callers must only write a `\n` separator before calling this — never `\n + pad`.
|
||||||
|
fn emit_block(node: &SyntaxNode, indent: usize, out: &mut String) {
|
||||||
|
emit_comments(&node.comments, indent, out);
|
||||||
|
out.push_str(&" ".repeat(indent));
|
||||||
|
emit_kind(node, indent, out);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Emits a node in **inline** context: no indentation, no comments.
|
||||||
|
/// Used for sub-expressions that are part of a larger expression on the same line.
|
||||||
|
fn emit_inline(node: &SyntaxNode, indent: usize, out: &mut String) {
|
||||||
|
emit_kind(node, indent, out);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Comment emission ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
fn emit_comments(comments: &Rc<[CommentLine]>, indent: usize, out: &mut String) {
|
||||||
|
if comments.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let pad = " ".repeat(indent);
|
||||||
|
for line in comments.iter() {
|
||||||
|
match line {
|
||||||
|
CommentLine::Comment(text) => {
|
||||||
|
out.push_str(&pad);
|
||||||
|
out.push(';');
|
||||||
|
if !text.is_empty() {
|
||||||
|
out.push(' ');
|
||||||
|
out.push_str(text);
|
||||||
|
}
|
||||||
|
out.push('\n');
|
||||||
|
}
|
||||||
|
CommentLine::Doc(text) => {
|
||||||
|
out.push_str(&pad);
|
||||||
|
out.push_str(";;");
|
||||||
|
if !text.is_empty() {
|
||||||
|
out.push(' ');
|
||||||
|
out.push_str(text);
|
||||||
|
}
|
||||||
|
out.push('\n');
|
||||||
|
}
|
||||||
|
CommentLine::Blank => {
|
||||||
|
out.push('\n');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Kind emission ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
fn emit_kind(node: &SyntaxNode, indent: usize, out: &mut String) {
|
||||||
|
match &node.kind {
|
||||||
|
NodeKind::Nop => out.push_str("..."),
|
||||||
|
|
||||||
|
NodeKind::Constant(val) => emit_value(val, out),
|
||||||
|
|
||||||
|
NodeKind::Identifier { symbol, .. } => out.push_str(&symbol.name),
|
||||||
|
|
||||||
|
NodeKind::FieldAccessor(k) => {
|
||||||
|
out.push('.');
|
||||||
|
out.push_str(&k.name());
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::If { cond, then_br, else_br } => {
|
||||||
|
out.push_str("(if ");
|
||||||
|
emit_inline(cond, indent + 1, out);
|
||||||
|
out.push('\n');
|
||||||
|
emit_block(then_br, indent + 1, out);
|
||||||
|
if let Some(else_node) = else_br {
|
||||||
|
out.push('\n');
|
||||||
|
emit_block(else_node, indent + 1, out);
|
||||||
|
}
|
||||||
|
out.push(')');
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::Def { pattern, value, .. } => {
|
||||||
|
out.push_str("(def ");
|
||||||
|
emit_inline(pattern, indent, out);
|
||||||
|
out.push('\n');
|
||||||
|
emit_block(value, indent + 1, out);
|
||||||
|
out.push(')');
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::Assign { target, value, .. } => {
|
||||||
|
out.push_str("(assign ");
|
||||||
|
emit_inline(target, indent, out);
|
||||||
|
out.push(' ');
|
||||||
|
emit_inline(value, indent + 1, out);
|
||||||
|
out.push(')');
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::Lambda { params, body, .. } => {
|
||||||
|
out.push_str("(fn ");
|
||||||
|
emit_inline(params, indent, out);
|
||||||
|
out.push('\n');
|
||||||
|
emit_block(body, indent + 1, out);
|
||||||
|
out.push(')');
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::Call { callee, args } => {
|
||||||
|
out.push('(');
|
||||||
|
emit_inline(callee, indent, out);
|
||||||
|
if let NodeKind::Tuple { elements } = &args.kind {
|
||||||
|
for arg in elements {
|
||||||
|
out.push(' ');
|
||||||
|
emit_inline(arg, indent + 1, out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.push(')');
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::Again { args } => {
|
||||||
|
out.push_str("(again");
|
||||||
|
if let NodeKind::Tuple { elements } = &args.kind {
|
||||||
|
for arg in elements {
|
||||||
|
out.push(' ');
|
||||||
|
emit_inline(arg, indent + 1, out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.push(')');
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::Block { exprs } => {
|
||||||
|
out.push_str("(do");
|
||||||
|
for expr in exprs {
|
||||||
|
out.push('\n');
|
||||||
|
emit_block(expr, indent + 1, out);
|
||||||
|
}
|
||||||
|
out.push(')');
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::Tuple { elements } => {
|
||||||
|
out.push('[');
|
||||||
|
for (i, el) in elements.iter().enumerate() {
|
||||||
|
if i > 0 {
|
||||||
|
out.push(' ');
|
||||||
|
}
|
||||||
|
emit_inline(el, indent, out);
|
||||||
|
}
|
||||||
|
out.push(']');
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::Record { fields, .. } => {
|
||||||
|
if fields.is_empty() {
|
||||||
|
out.push_str("{}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Always multi-line so that any leading comments on keys are valid.
|
||||||
|
out.push('{');
|
||||||
|
for (key, val) in fields.iter() {
|
||||||
|
out.push('\n');
|
||||||
|
emit_block(key, indent + 1, out);
|
||||||
|
out.push(' ');
|
||||||
|
emit_inline(val, indent + 1, out);
|
||||||
|
}
|
||||||
|
out.push('\n');
|
||||||
|
out.push_str(&" ".repeat(indent));
|
||||||
|
out.push('}');
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::MacroDecl { name, params, body } => {
|
||||||
|
out.push_str("(macro ");
|
||||||
|
out.push_str(&name.name);
|
||||||
|
out.push(' ');
|
||||||
|
emit_inline(params, indent, out);
|
||||||
|
out.push('\n');
|
||||||
|
emit_block(body, indent + 1, out);
|
||||||
|
out.push(')');
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::Template(inner) => {
|
||||||
|
out.push('`');
|
||||||
|
emit_inline(inner, indent, out);
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::Placeholder(inner) => {
|
||||||
|
out.push('~');
|
||||||
|
emit_inline(inner, indent, out);
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::Splice(inner) => {
|
||||||
|
out.push_str("~@");
|
||||||
|
emit_inline(inner, indent, out);
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::GetField { rec, field } => {
|
||||||
|
out.push_str("(.");
|
||||||
|
out.push_str(&field.name());
|
||||||
|
out.push(' ');
|
||||||
|
emit_inline(rec, indent, out);
|
||||||
|
out.push(')');
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::Expansion { original_call, .. } => {
|
||||||
|
// Round-trip uses the original pre-expansion form.
|
||||||
|
emit_inline(original_call, indent, out);
|
||||||
|
}
|
||||||
|
|
||||||
|
NodeKind::Error | NodeKind::Extension(_) => {
|
||||||
|
out.push_str("<error>");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn emit_value(val: &Value, out: &mut String) {
|
||||||
|
match val {
|
||||||
|
Value::Int(n) => out.push_str(&n.to_string()),
|
||||||
|
Value::Float(f) => {
|
||||||
|
let s = format!("{}", f);
|
||||||
|
// Ensure a decimal point is present so it round-trips as a float.
|
||||||
|
if s.contains('.') || s.contains('e') {
|
||||||
|
out.push_str(&s);
|
||||||
|
} else {
|
||||||
|
out.push_str(&s);
|
||||||
|
out.push_str(".0");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Value::Text(t) => {
|
||||||
|
out.push('"');
|
||||||
|
for c in t.chars() {
|
||||||
|
match c {
|
||||||
|
'"' => out.push_str("\\\""),
|
||||||
|
'\\' => out.push_str("\\\\"),
|
||||||
|
'\n' => out.push_str("\\n"),
|
||||||
|
'\r' => out.push_str("\\r"),
|
||||||
|
'\t' => out.push_str("\\t"),
|
||||||
|
c => out.push(c),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.push('"');
|
||||||
|
}
|
||||||
|
Value::Keyword(k) => {
|
||||||
|
out.push(':');
|
||||||
|
out.push_str(&k.name());
|
||||||
|
}
|
||||||
|
Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
|
||||||
|
other => out.push_str(&other.to_string()),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -624,7 +624,7 @@ mod tests {
|
|||||||
use crate::ast::diagnostics::Diagnostics;
|
use crate::ast::diagnostics::Diagnostics;
|
||||||
use crate::ast::nodes::{Address, VirtualId};
|
use crate::ast::nodes::{Address, VirtualId};
|
||||||
use crate::ast::parser::Parser;
|
use crate::ast::parser::Parser;
|
||||||
use crate::ast::types::{NodeIdentity, Object, Purity, SourceLocation, StaticType, Value};
|
use crate::ast::types::{NodeIdentity, Purity, SourceLocation, StaticType, Value};
|
||||||
|
|
||||||
struct SimpleEvaluator;
|
struct SimpleEvaluator;
|
||||||
impl MacroEvaluator for SimpleEvaluator {
|
impl MacroEvaluator for SimpleEvaluator {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ pub mod analyzer;
|
|||||||
pub mod binder;
|
pub mod binder;
|
||||||
pub mod captures;
|
pub mod captures;
|
||||||
pub mod dumper;
|
pub mod dumper;
|
||||||
|
pub mod emitter;
|
||||||
pub mod lambda_collector;
|
pub mod lambda_collector;
|
||||||
pub mod macros;
|
pub mod macros;
|
||||||
pub mod optimizer;
|
pub mod optimizer;
|
||||||
|
|||||||
+73
-6
@@ -22,7 +22,7 @@ use crate::ast::compiler::optimizer::Optimizer;
|
|||||||
use crate::ast::compiler::specializer::{FunctionRegistry, MonoCache, RtlLookupFunc, Specializer};
|
use crate::ast::compiler::specializer::{FunctionRegistry, MonoCache, RtlLookupFunc, Specializer};
|
||||||
use crate::ast::rtl::{self, intrinsics};
|
use crate::ast::rtl::{self, intrinsics};
|
||||||
use crate::ast::types::{
|
use crate::ast::types::{
|
||||||
NativeFunction, NodeIdentity, Purity, SourceLocation, StaticType, Value,
|
CommentLine, NativeFunction, NodeIdentity, Purity, SourceLocation, StaticType, Value,
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::ast::diagnostics::{DiagnosticLevel, Diagnostics};
|
use crate::ast::diagnostics::{DiagnosticLevel, Diagnostics};
|
||||||
@@ -148,6 +148,9 @@ pub struct Environment {
|
|||||||
pub search_paths: Rc<RefCell<Vec<PathBuf>>>,
|
pub search_paths: Rc<RefCell<Vec<PathBuf>>>,
|
||||||
pub loaded_modules: Rc<RefCell<HashSet<PathBuf>>>,
|
pub loaded_modules: Rc<RefCell<HashSet<PathBuf>>>,
|
||||||
pub rtl_docs: Rc<RefCell<Vec<RtlDocEntry>>>,
|
pub rtl_docs: Rc<RefCell<Vec<RtlDocEntry>>>,
|
||||||
|
/// Documentation for symbols defined in Myc source (via `;;` comments before `def`/`macro`).
|
||||||
|
/// Key: symbol name. Value: concatenated doc lines joined by newlines.
|
||||||
|
pub myc_docs: Rc<RefCell<HashMap<String, String>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct EnvFunctionRegistry {
|
struct EnvFunctionRegistry {
|
||||||
@@ -235,6 +238,7 @@ impl Environment {
|
|||||||
search_paths: Rc::new(RefCell::new(Vec::new())),
|
search_paths: Rc::new(RefCell::new(Vec::new())),
|
||||||
loaded_modules: Rc::new(RefCell::new(HashSet::new())),
|
loaded_modules: Rc::new(RefCell::new(HashSet::new())),
|
||||||
rtl_docs: Rc::new(RefCell::new(Vec::new())),
|
rtl_docs: Rc::new(RefCell::new(Vec::new())),
|
||||||
|
myc_docs: Rc::new(RefCell::new(HashMap::new())),
|
||||||
};
|
};
|
||||||
rtl::register(&env);
|
rtl::register(&env);
|
||||||
|
|
||||||
@@ -561,7 +565,9 @@ impl Environment {
|
|||||||
comments: Rc::from([]),
|
comments: Rc::from([]),
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
Some(checker.check(&wrapped_ast, &[], diagnostics))
|
let typed = checker.check(&wrapped_ast, &[], diagnostics);
|
||||||
|
self.collect_doc_comments(&typed);
|
||||||
|
Some(typed)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn compile_syntax(&self, syntax_ast: SyntaxNode) -> Result<TypedNode, String> {
|
fn compile_syntax(&self, syntax_ast: SyntaxNode) -> Result<TypedNode, String> {
|
||||||
@@ -719,6 +725,17 @@ impl Environment {
|
|||||||
out
|
out
|
||||||
}).collect();
|
}).collect();
|
||||||
|
|
||||||
|
// Include Myc-defined symbol docs (from `;;` comments in source)
|
||||||
|
let myc_docs = self.myc_docs.borrow();
|
||||||
|
for (name, doc_text) in myc_docs.iter() {
|
||||||
|
let sig = scope
|
||||||
|
.locals
|
||||||
|
.get(&Symbol::from(name.as_str()))
|
||||||
|
.map(|info| info._ty.to_doc_string())
|
||||||
|
.unwrap_or_else(|| "unknown".to_string());
|
||||||
|
entries.push(format!("{} : {}\n {}", name, sig, doc_text));
|
||||||
|
}
|
||||||
|
|
||||||
entries.sort_by_key(|a| a.to_lowercase());
|
entries.sort_by_key(|a| a.to_lowercase());
|
||||||
entries
|
entries
|
||||||
}
|
}
|
||||||
@@ -727,9 +744,6 @@ impl Environment {
|
|||||||
/// or `None` if the symbol has no doc entry.
|
/// or `None` if the symbol has no doc entry.
|
||||||
pub fn get_rtl_doc(&self, name: &str) -> Option<String> {
|
pub fn get_rtl_doc(&self, name: &str) -> Option<String> {
|
||||||
use crate::ast::nodes::Symbol;
|
use crate::ast::nodes::Symbol;
|
||||||
let docs = self.rtl_docs.borrow();
|
|
||||||
let entry = docs.iter().find(|e| e.name == name)?;
|
|
||||||
|
|
||||||
let root_scopes = self.root_scopes.borrow();
|
let root_scopes = self.root_scopes.borrow();
|
||||||
let sig = root_scopes[0]
|
let sig = root_scopes[0]
|
||||||
.locals
|
.locals
|
||||||
@@ -737,6 +751,9 @@ impl Environment {
|
|||||||
.map(|info| info._ty.to_doc_string())
|
.map(|info| info._ty.to_doc_string())
|
||||||
.unwrap_or_else(|| "unknown".to_string());
|
.unwrap_or_else(|| "unknown".to_string());
|
||||||
|
|
||||||
|
// Check RTL docs first
|
||||||
|
let docs = self.rtl_docs.borrow();
|
||||||
|
if let Some(entry) = docs.iter().find(|e| e.name == name) {
|
||||||
let mut out = format!("{} : {}\n {}", entry.name, sig, entry.one_liner);
|
let mut out = format!("{} : {}\n {}", entry.name, sig, entry.one_liner);
|
||||||
if let Some(desc) = entry.description {
|
if let Some(desc) = entry.description {
|
||||||
out.push_str(&format!("\n {}", desc));
|
out.push_str(&format!("\n {}", desc));
|
||||||
@@ -747,7 +764,57 @@ impl Environment {
|
|||||||
out.push_str(&format!("\n {}", ex));
|
out.push_str(&format!("\n {}", ex));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Some(out)
|
return Some(out);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fall back to Myc-source docs
|
||||||
|
let myc_docs = self.myc_docs.borrow();
|
||||||
|
myc_docs
|
||||||
|
.get(name)
|
||||||
|
.map(|doc_text| format!("{} : {}\n {}", name, sig, doc_text))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Walks a typed AST and registers `;;` doc comments on `def`/`macro` nodes
|
||||||
|
/// whose target is a plain identifier into the `myc_docs` registry.
|
||||||
|
fn collect_doc_comments(&self, node: &TypedNode) {
|
||||||
|
// Extract the symbol name this node defines, if any.
|
||||||
|
let sym_name: Option<Rc<str>> = match &node.kind {
|
||||||
|
NodeKind::Def { pattern, .. } => {
|
||||||
|
if let NodeKind::Identifier { symbol, .. } = &pattern.kind {
|
||||||
|
Some(symbol.name.clone())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
NodeKind::MacroDecl { name, .. } => Some(name.name.clone()),
|
||||||
|
NodeKind::Block { exprs } => {
|
||||||
|
for expr in exprs {
|
||||||
|
self.collect_doc_comments(expr);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
NodeKind::Lambda { body, .. } => {
|
||||||
|
self.collect_doc_comments(body);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(name) = sym_name {
|
||||||
|
let doc_text: String = node
|
||||||
|
.comments
|
||||||
|
.iter()
|
||||||
|
.filter_map(|line| match line {
|
||||||
|
CommentLine::Doc(text) => Some(text.as_ref()),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n");
|
||||||
|
|
||||||
|
if !doc_text.is_empty() {
|
||||||
|
self.myc_docs.borrow_mut().insert(name.to_string(), doc_text);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn dump_ast(&self, source: &str) -> Result<String, String> {
|
pub fn dump_ast(&self, source: &str) -> Result<String, String> {
|
||||||
|
|||||||
+28
-10
@@ -1,4 +1,4 @@
|
|||||||
use crate::ast::types::SourceLocation;
|
use crate::ast::types::{CommentLine, SourceLocation};
|
||||||
use std::iter::Peekable;
|
use std::iter::Peekable;
|
||||||
use std::rc::Rc;
|
use std::rc::Rc;
|
||||||
use std::str::Chars;
|
use std::str::Chars;
|
||||||
@@ -27,7 +27,7 @@ pub enum TokenKind {
|
|||||||
pub struct Token {
|
pub struct Token {
|
||||||
pub kind: TokenKind,
|
pub kind: TokenKind,
|
||||||
pub location: SourceLocation,
|
pub location: SourceLocation,
|
||||||
pub comments: Rc<[Rc<str>]>,
|
pub comments: Rc<[CommentLine]>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct Lexer<'a> {
|
pub struct Lexer<'a> {
|
||||||
@@ -35,7 +35,7 @@ pub struct Lexer<'a> {
|
|||||||
line: u32,
|
line: u32,
|
||||||
col: u32,
|
col: u32,
|
||||||
at_line_start: bool,
|
at_line_start: bool,
|
||||||
comments_buffer: Vec<Rc<str>>,
|
comments_buffer: Vec<CommentLine>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> Lexer<'a> {
|
impl<'a> Lexer<'a> {
|
||||||
@@ -52,18 +52,20 @@ impl<'a> Lexer<'a> {
|
|||||||
pub fn next_token(&mut self) -> Result<Token, String> {
|
pub fn next_token(&mut self) -> Result<Token, String> {
|
||||||
self.skip_whitespace_and_comments()?;
|
self.skip_whitespace_and_comments()?;
|
||||||
|
|
||||||
let comments: Rc<[Rc<str>]> = if self.comments_buffer.is_empty() {
|
let comments: Rc<[CommentLine]> = if self.comments_buffer.is_empty() {
|
||||||
Rc::from([])
|
Rc::from([])
|
||||||
} else {
|
} else {
|
||||||
let mut start = 0;
|
let mut start = 0;
|
||||||
while start < self.comments_buffer.len() && self.comments_buffer[start].is_empty() {
|
while start < self.comments_buffer.len()
|
||||||
|
&& self.comments_buffer[start] == CommentLine::Blank
|
||||||
|
{
|
||||||
start += 1;
|
start += 1;
|
||||||
}
|
}
|
||||||
let mut end = self.comments_buffer.len();
|
let mut end = self.comments_buffer.len();
|
||||||
while end > start && self.comments_buffer[end - 1].is_empty() {
|
while end > start && self.comments_buffer[end - 1] == CommentLine::Blank {
|
||||||
end -= 1;
|
end -= 1;
|
||||||
}
|
}
|
||||||
let res: Rc<[Rc<str>]> = self.comments_buffer[start..end].iter().cloned().collect();
|
let res: Rc<[CommentLine]> = self.comments_buffer[start..end].iter().cloned().collect();
|
||||||
self.comments_buffer.clear();
|
self.comments_buffer.clear();
|
||||||
res
|
res
|
||||||
};
|
};
|
||||||
@@ -168,14 +170,24 @@ impl<'a> Lexer<'a> {
|
|||||||
}
|
}
|
||||||
if !self.comments_buffer.is_empty() && consecutive_newlines > 1 {
|
if !self.comments_buffer.is_empty() && consecutive_newlines > 1 {
|
||||||
for _ in 0..(consecutive_newlines - 1) {
|
for _ in 0..(consecutive_newlines - 1) {
|
||||||
self.comments_buffer.push(Rc::from(""));
|
self.comments_buffer.push(CommentLine::Blank);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
consecutive_newlines = 0;
|
consecutive_newlines = 0;
|
||||||
|
|
||||||
self.input.next(); // consume ';'
|
self.input.next(); // consume first ';'
|
||||||
self.col += 1;
|
self.col += 1;
|
||||||
|
|
||||||
|
// Check for a second ';' → Doc level
|
||||||
|
let is_doc = if let Some(&';') = self.peek() {
|
||||||
|
self.input.next(); // consume second ';'
|
||||||
|
self.col += 1;
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
};
|
||||||
|
|
||||||
|
// Consume optional single space after the semicolon(s)
|
||||||
if let Some(&' ') = self.peek() {
|
if let Some(&' ') = self.peek() {
|
||||||
self.input.next();
|
self.input.next();
|
||||||
self.col += 1;
|
self.col += 1;
|
||||||
@@ -189,7 +201,13 @@ impl<'a> Lexer<'a> {
|
|||||||
comment_text.push(self.input.next().unwrap());
|
comment_text.push(self.input.next().unwrap());
|
||||||
self.col += 1;
|
self.col += 1;
|
||||||
}
|
}
|
||||||
self.comments_buffer.push(Rc::from(comment_text));
|
|
||||||
|
let line = if is_doc {
|
||||||
|
CommentLine::Doc(Rc::from(comment_text))
|
||||||
|
} else {
|
||||||
|
CommentLine::Comment(Rc::from(comment_text))
|
||||||
|
};
|
||||||
|
self.comments_buffer.push(line);
|
||||||
} else if c == '#' {
|
} else if c == '#' {
|
||||||
if !self.at_line_start {
|
if !self.at_line_start {
|
||||||
return Err(format!("Inline directives are not allowed at {}:{}", self.line, self.col));
|
return Err(format!("Inline directives are not allowed at {}:{}", self.line, self.col));
|
||||||
|
|||||||
+2
-2
@@ -1,4 +1,4 @@
|
|||||||
use crate::ast::types::{Identity, Keyword, Purity, RecordLayout, StaticType, Value};
|
use crate::ast::types::{CommentLine, Identity, Keyword, Purity, RecordLayout, StaticType, Value};
|
||||||
use std::fmt::Debug;
|
use std::fmt::Debug;
|
||||||
use std::rc::Rc;
|
use std::rc::Rc;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -250,7 +250,7 @@ pub struct Node<P: CompilerPhase = BoundPhase> {
|
|||||||
pub identity: Identity,
|
pub identity: Identity,
|
||||||
pub kind: NodeKind<P>,
|
pub kind: NodeKind<P>,
|
||||||
pub ty: P::Metadata,
|
pub ty: P::Metadata,
|
||||||
pub comments: Rc<[Rc<str>]>,
|
pub comments: Rc<[CommentLine]>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<P: CompilerPhase> Clone for Node<P> {
|
impl<P: CompilerPhase> Clone for Node<P> {
|
||||||
|
|||||||
@@ -9,6 +9,21 @@ use std::rc::Rc;
|
|||||||
use std::sync::Mutex; // Still needed for global keyword registry
|
use std::sync::Mutex; // Still needed for global keyword registry
|
||||||
use std::sync::OnceLock;
|
use std::sync::OnceLock;
|
||||||
|
|
||||||
|
/// A single line within a leading comment block attached to an AST node.
|
||||||
|
///
|
||||||
|
/// Comment blocks are attached to the *next* expression in the source.
|
||||||
|
/// The first and last line of a block are never `Blank` (enforced by the lexer).
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum CommentLine {
|
||||||
|
/// `; text` — regular comment, no semantic effect.
|
||||||
|
Comment(Rc<str>),
|
||||||
|
/// `;; text` — documentation comment.
|
||||||
|
/// When attached to a `def` or `macro` node, the text is registered in the symbol registry.
|
||||||
|
Doc(Rc<str>),
|
||||||
|
/// A blank separator line within a comment block (preserves visual spacing).
|
||||||
|
Blank,
|
||||||
|
}
|
||||||
|
|
||||||
/// Simple source location
|
/// Simple source location
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||||
pub struct SourceLocation {
|
pub struct SourceLocation {
|
||||||
|
|||||||
+17
-1
@@ -1,5 +1,7 @@
|
|||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
|
use myc::ast::compiler::emitter;
|
||||||
use myc::ast::environment::Environment;
|
use myc::ast::environment::Environment;
|
||||||
|
use myc::ast::parser::Parser as MycParser;
|
||||||
use myc::utils::tester;
|
use myc::utils::tester;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::io::{self, IsTerminal, Read};
|
use std::io::{self, IsTerminal, Read};
|
||||||
@@ -43,6 +45,10 @@ struct Cli {
|
|||||||
/// Start MCP server (stdio transport, for LLM integration)
|
/// Start MCP server (stdio transport, for LLM integration)
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
mcp: bool,
|
mcp: bool,
|
||||||
|
|
||||||
|
/// Emit the parsed source back as Myc code (round-trip test)
|
||||||
|
#[arg(long)]
|
||||||
|
emit: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
@@ -108,7 +114,17 @@ fn main() {
|
|||||||
};
|
};
|
||||||
|
|
||||||
if let Some(content) = source {
|
if let Some(content) = source {
|
||||||
if cli.dump {
|
if cli.emit {
|
||||||
|
let mut parser = MycParser::new(&content);
|
||||||
|
let syntax_ast = parser.parse_expression();
|
||||||
|
if parser.diagnostics.has_errors() {
|
||||||
|
for diag in &parser.diagnostics.items {
|
||||||
|
eprintln!("ERROR: {}", diag.message);
|
||||||
|
}
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
print!("{}", emitter::emit(&syntax_ast));
|
||||||
|
} else if cli.dump {
|
||||||
match env.dump_ast(&content) {
|
match env.dump_ast(&content) {
|
||||||
Ok(dump) => println!("{}", dump),
|
Ok(dump) => println!("{}", dump),
|
||||||
Err(e) => eprintln!("Error dumping AST: {}", e),
|
Err(e) => eprintln!("Error dumping AST: {}", e),
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
use myc::ast::compiler::emitter;
|
||||||
|
use myc::ast::parser::Parser;
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
/// Verifies the round-trip property for all example scripts:
|
||||||
|
///
|
||||||
|
/// 1. Parse source → `ast1`
|
||||||
|
/// 2. Emit `ast1` → `code1`
|
||||||
|
/// 3. Parse `code1` → `ast2`
|
||||||
|
/// 4. Emit `ast2` → `code2`
|
||||||
|
/// 5. Assert `code1 == code2`
|
||||||
|
///
|
||||||
|
/// If the emitter is idempotent (a second pass produces the same output),
|
||||||
|
/// the AST is structurally stable across round-trips.
|
||||||
|
#[test]
|
||||||
|
fn test_roundtrip_examples() {
|
||||||
|
let entries = fs::read_dir("examples").expect("examples/ directory not found");
|
||||||
|
|
||||||
|
let mut tested = 0;
|
||||||
|
for entry in entries.filter_map(|e| e.ok()) {
|
||||||
|
let path = entry.path();
|
||||||
|
if path.extension().is_none_or(|ext| ext != "myc") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let name = path.file_name().unwrap().to_string_lossy().to_string();
|
||||||
|
let source = fs::read_to_string(&path)
|
||||||
|
.unwrap_or_else(|e| panic!("Failed to read {}: {}", name, e));
|
||||||
|
|
||||||
|
// Pass 1: parse original source, emit to code1
|
||||||
|
let mut parser1 = Parser::new(&source);
|
||||||
|
let ast1 = parser1.parse_expression();
|
||||||
|
assert!(
|
||||||
|
!parser1.diagnostics.has_errors(),
|
||||||
|
"{}: parse errors on original source: {:?}",
|
||||||
|
name,
|
||||||
|
parser1.diagnostics.items
|
||||||
|
);
|
||||||
|
let code1 = emitter::emit(&ast1);
|
||||||
|
|
||||||
|
// Pass 2: parse emitted code1, emit to code2
|
||||||
|
let mut parser2 = Parser::new(&code1);
|
||||||
|
let ast2 = parser2.parse_expression();
|
||||||
|
assert!(
|
||||||
|
!parser2.diagnostics.has_errors(),
|
||||||
|
"{}: parse errors on round-tripped code:\n---\n{}\n---\nErrors: {:?}",
|
||||||
|
name,
|
||||||
|
code1,
|
||||||
|
parser2.diagnostics.items
|
||||||
|
);
|
||||||
|
let code2 = emitter::emit(&ast2);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
code1, code2,
|
||||||
|
"{}: emitter is not idempotent (AST changed after second round-trip)",
|
||||||
|
name
|
||||||
|
);
|
||||||
|
|
||||||
|
tested += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(tested > 0, "No .myc example files found in examples/");
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user