feat: Implement lexer and parser for AST

Introduces the lexer and parser modules, enabling the conversion of
source code into an Abstract Syntax Tree (AST).

This includes:
- Defining token kinds and structures.
- Implementing lexer logic to tokenize input.
- Defining AST node kinds and structures.
- Implementing parser logic to construct the AST from tokens.
- Adding support for basic expressions, lists, keywords, and
  identifiers.
- Adding the `lazy_static` dependency for keyword interning.
- Refactoring `Value::Nil` to `Value::Void`.

Cargo.lock

@@ -7,6 +7,7 @@ name = "Ast"
 version = "0.1.0"
 dependencies = [
  "eframe",
+ "lazy_static",
 ]
 
 [[package]]
@@ -1590,6 +1591,12 @@ version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc"
 
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
 [[package]]
 name = "leb128fmt"
 version = "0.1.0"

Cargo.toml

@@ -5,3 +5,4 @@ edition = "2024"
 
 [dependencies]
 eframe = "0.33.3"
+lazy_static = "1.4.0"

src/ast/lexer.rs

@@ -0,0 +1,164 @@
+use std::iter::Peekable;
+use std::str::Chars;
+use std::sync::Arc;
+use crate::ast::types::SourceLocation;
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum TokenKind {
+    LeftParen, RightParen,
+    LeftBracket, RightBracket,
+    LeftBrace, RightBrace,
+    Quote, Backtick, Tilde, At,
+    Identifier(Arc<str>),
+    Keyword(Arc<str>),
+    Number(f64),
+    String(Arc<str>),
+    EOF,
+}
+
+#[derive(Debug, Clone)]
+pub struct Token {
+    pub kind: TokenKind,
+    pub location: SourceLocation,
+}
+
+pub struct Lexer<'a> {
+    input: Peekable<Chars<'a>>,
+    line: u32,
+    col: u32,
+}
+
+impl<'a> Lexer<'a> {
+    pub fn new(input: &'a str) -> Self {
+        Self {
+            input: input.chars().peekable(),
+            line: 1,
+            col: 1,
+        }
+    }
+
+    pub fn next_token(&mut self) -> Result<Token, String> {
+        self.skip_whitespace();
+
+        let start_location = SourceLocation { line: self.line, col: self.col };
+        
+        let char = match self.input.next() {
+            Some(c) => {
+                let current_char = c;
+                self.col += 1;
+                current_char
+            },
+            None => return Ok(Token { kind: TokenKind::EOF, location: start_location }),
+        };
+
+        let kind = match char {
+            '(' => TokenKind::LeftParen,
+            ')' => TokenKind::RightParen,
+            '[' => TokenKind::LeftBracket,
+            ']' => TokenKind::RightBracket,
+            '{' => TokenKind::LeftBrace,
+            '}' => TokenKind::RightBrace,
+            '\'' => TokenKind::Quote,
+            '`' => TokenKind::Backtick,
+            '~' => TokenKind::Tilde,
+            '@' => TokenKind::At,
+            ':' => {
+                let id = self.read_while(|c| !c.is_whitespace() && !"()[]{}\"'`~@;".contains(c));
+                TokenKind::Keyword(Arc::from(id))
+            }
+            '"' => TokenKind::String(Arc::from(self.read_string()?)),
+            c if c.is_ascii_digit() || (c == '-' && self.peek().map_or(false, |p| p.is_ascii_digit())) => {
+                let mut num_str = String::from(c);
+                num_str.push_str(&self.read_while(|c| c.is_ascii_digit() || c == '.'));
+                TokenKind::Number(num_str.parse().map_err(|_| "Invalid number format")?)
+            }
+            c if is_ident_start(c) => {
+                let mut id = String::from(c);
+                id.push_str(&self.read_while(is_ident_char));
+                TokenKind::Identifier(Arc::from(id))
+            }
+            _ => return Err(format!("Unexpected character: {} at {}:{}", char, self.line, self.col - 1)),
+        };
+
+        Ok(Token { kind, location: start_location })
+    }
+
+    fn peek(&mut self) -> Option<&char> {
+        self.input.peek()
+    }
+
+    fn skip_whitespace(&mut self) {
+        while let Some(&c) = self.peek() {
+            if c.is_whitespace() {
+                if c == '\n' {
+                    self.line += 1;
+                    self.col = 1;
+                } else {
+                    self.col += 1;
+                }
+                self.input.next();
+            } else if c == ';' {
+                while let Some(c) = self.input.next() {
+                    if c == '\n' {
+                        self.line += 1;
+                        self.col = 1;
+                        break;
+                    }
+                }
+            } else {
+                break;
+            }
+        }
+    }
+
+    fn read_while<F>(&mut self, mut predicate: F) -> String 
+    where F: FnMut(char) -> bool {
+        let mut s = String::new();
+        while let Some(&c) = self.peek() {
+            if predicate(c) {
+                s.push(self.input.next().unwrap());
+                self.col += 1;
+            } else {
+                break;
+            }
+        }
+        s
+    }
+
+    fn read_string(&mut self) -> Result<String, String> {
+        let mut s = String::new();
+        while let Some(c) = self.input.next() {
+            self.col += 1;
+            match c {
+                '"' => return Ok(s),
+                '\\' => {
+                    let next = self.input.next().ok_or("Unterminated string escape")?;
+                    self.col += 1;
+                    match next {
+                        'n' => s.push('\n'),
+                        'r' => s.push('\r'),
+                        't' => s.push('\t'),
+                        '\\' => s.push('\\'),
+                        '"' => s.push('"'),
+                        _ => s.push(next),
+                    }
+                }
+                '\n' => {
+                    self.line += 1;
+                    self.col = 1;
+                    s.push(c);
+                }
+                _ => s.push(c),
+            }
+        }
+        Err("Unterminated string".to_string())
+    }
+}
+
+fn is_ident_start(c: char) -> bool {
+    c.is_alphabetic() || "+-*/<=>!$%&_?.".contains(c)
+}
+
+fn is_ident_char(c: char) -> bool {
+    is_ident_start(c) || c.is_ascii_digit()
+}

src/ast/mod.rs

@@ -1,5 +1,9 @@
 pub mod types;
 pub mod nodes;
+pub mod lexer;
+pub mod parser;
 
 pub use types::*;
 pub use nodes::*;
+pub use lexer::*;
+pub use parser::*;

src/ast/nodes.rs

@@ -23,6 +23,8 @@ pub struct Context {
 /// The core AST variant enum for performance and structure
 #[derive(Debug)]
 pub enum UntypedKind {
+    /// The "..." placeholder for incomplete code
+    Nop,
     Constant(Value),
     Identifier(Arc<str>),
     If {
@@ -41,6 +43,7 @@ pub enum UntypedKind {
 impl Node<UntypedKind> {
     pub fn eval(&self, ctx: &mut Context) -> Value {
         match &self.kind {
+            UntypedKind::Nop => Value::Void,
             UntypedKind::Constant(v) => v.clone(),
             UntypedKind::Identifier(_) => todo!("Lookup in Context"),
             UntypedKind::If { cond, then_br, else_br } => {
@@ -49,7 +52,7 @@ impl Node<UntypedKind> {
                 } else if let Some(eb) = else_br {
                     eb.eval(ctx)
                 } else {
-                    Value::Nil
+                    Value::Void
                 }
             }
             UntypedKind::Call { callee, args } => {

src/ast/parser.rs

@@ -0,0 +1,168 @@
+use std::sync::Arc;
+use crate::ast::lexer::{Lexer, Token, TokenKind};
+use crate::ast::types::{Identity, NodeIdentity, Value, Keyword};
+use crate::ast::nodes::{Node, UntypedKind};
+
+pub struct Parser<'a> {
+    lexer: Lexer<'a>,
+    current_token: Token,
+}
+
+impl<'a> Parser<'a> {
+    pub fn new(input: &'a str) -> Result<Self, String> {
+        let mut lexer = Lexer::new(input);
+        let current_token = lexer.next_token()?;
+        Ok(Self { lexer, current_token })
+    }
+
+    fn advance(&mut self) -> Result<Token, String> {
+        let prev = std::mem::replace(&mut self.current_token, self.lexer.next_token()?);
+        Ok(prev)
+    }
+
+    fn peek(&self) -> &TokenKind {
+        &self.current_token.kind
+    }
+
+    pub fn parse_expression(&mut self) -> Result<Node<UntypedKind>, String> {
+        match self.peek() {
+            TokenKind::LeftParen => self.parse_list(),
+            TokenKind::LeftBracket => self.parse_vector(),
+            // TokenKind::LeftBrace => self.parse_map(), // TODO
+            TokenKind::Quote => {
+                let identity = Arc::new(NodeIdentity { location: self.current_token.location });
+                self.parse_reader_macro(UntypedKind::Call {
+                    callee: Box::new(self.make_id_node("quote", identity)),
+                    args: vec![], // will be filled
+                })
+            }
+            _ => self.parse_atom(),
+        }
+    }
+
+    fn parse_atom(&mut self) -> Result<Node<UntypedKind>, String> {
+        let token = self.advance()?;
+        let identity = Arc::new(NodeIdentity { location: token.location });
+
+        let kind = match token.kind {
+            TokenKind::Number(n) => UntypedKind::Constant(Value::Float(n)),
+            TokenKind::String(s) => UntypedKind::Constant(Value::Text(s)),
+            TokenKind::Keyword(k) => UntypedKind::Constant(Value::Keyword(Keyword::intern(&k))),
+            TokenKind::Identifier(id) => {
+                match id.as_ref() {
+                    "..." => UntypedKind::Nop,
+                    "true" => UntypedKind::Constant(Value::Bool(true)),
+                    "false" => UntypedKind::Constant(Value::Bool(false)),
+                    "void" => UntypedKind::Constant(Value::Void),
+                    _ => UntypedKind::Identifier(id),
+                }
+            }
+            _ => return Err(format!("Unexpected token in atom: {:?} at {:?}", token.kind, token.location)),
+        };
+
+        Ok(Node { identity, kind })
+    }
+
+    fn parse_list(&mut self) -> Result<Node<UntypedKind>, String> {
+        let start_loc = self.advance()?.location; // consume '('
+        
+        if *self.peek() == TokenKind::RightParen {
+            return Err(format!("Empty list () is not a valid expression at {:?}", start_loc));
+        }
+
+        // Peek at head to check for special forms
+        let head = self.parse_expression()?;
+        
+        let result = if let UntypedKind::Identifier(name) = &head.kind {
+            match name.as_ref() {
+                "if" => self.parse_if(head.identity.clone()),
+                _ => self.parse_call(head),
+            }
+        } else {
+            self.parse_call(head)
+        };
+
+        self.expect(TokenKind::RightParen)?;
+        result
+    }
+
+    fn parse_if(&mut self, identity: Identity) -> Result<Node<UntypedKind>, String> {
+        let cond = Box::new(self.parse_expression()?);
+        let then_br = Box::new(self.parse_expression()?);
+        let mut else_br = None;
+
+        if *self.peek() != TokenKind::RightParen {
+            else_br = Some(Box::new(self.parse_expression()?));
+        }
+
+        Ok(Node {
+            identity,
+            kind: UntypedKind::If { cond, then_br, else_br },
+        })
+    }
+
+    fn parse_call(&mut self, callee: Node<UntypedKind>) -> Result<Node<UntypedKind>, String> {
+        let mut args = Vec::new();
+        let identity = callee.identity.clone();
+
+        while *self.peek() != TokenKind::RightParen && *self.peek() != TokenKind::EOF {
+            args.push(self.parse_expression()?);
+        }
+
+        Ok(Node {
+            identity,
+            kind: UntypedKind::Call { callee: Box::new(callee), args },
+        })
+    }
+
+    fn parse_vector(&mut self) -> Result<Node<UntypedKind>, String> {
+        let token = self.advance()?; // consume '['
+        let mut elements = Vec::new();
+
+        while *self.peek() != TokenKind::RightBracket && *self.peek() != TokenKind::EOF {
+            // Vector elements in Myc are usually just constants in a list
+            let expr = self.parse_expression()?;
+            elements.push(expr.eval(&mut crate::ast::nodes::Context {})); // Simple direct eval for literals
+        }
+        
+        self.expect(TokenKind::RightBracket)?;
+        
+        Ok(Node {
+            identity: Arc::new(NodeIdentity { location: token.location }),
+            kind: UntypedKind::Constant(Value::List(Arc::new(elements))),
+        })
+    }
+
+    fn parse_reader_macro(&mut self, _template: UntypedKind) -> Result<Node<UntypedKind>, String> {
+        let token = self.advance()?; // consume '
+        let expr = self.parse_expression()?;
+        
+        // 'x -> (quote x)
+        let identity = Arc::new(NodeIdentity { location: token.location });
+        let quote_id = self.make_id_node("quote", identity.clone());
+        
+        Ok(Node {
+            identity,
+            kind: UntypedKind::Call {
+                callee: Box::new(quote_id),
+                args: vec![expr],
+            },
+        })
+    }
+
+    fn expect(&mut self, kind: TokenKind) -> Result<(), String> {
+        let token = self.advance()?;
+        if token.kind == kind {
+            Ok(())
+        } else {
+            Err(format!("Expected {:?}, but found {:?} at {:?}", kind, token.kind, token.location))
+        }
+    }
+
+    fn make_id_node(&self, name: &str, identity: Identity) -> Node<UntypedKind> {
+        Node {
+            identity,
+            kind: UntypedKind::Identifier(Arc::from(name)),
+        }
+    }
+}

src/ast/types.rs

@@ -1,12 +1,20 @@
 use std::sync::Arc;
 use std::collections::HashMap;
 use std::fmt;
+use std::sync::Mutex;
+use lazy_static::lazy_static;
+
+/// Simple source location
+#[derive(Debug, Clone, Copy)]
+pub struct SourceLocation {
+    pub line: u32,
+    pub col: u32,
+}
 
 /// Shared identity for nodes (Location, etc.)
 #[derive(Debug, Clone)]
 pub struct NodeIdentity {
-    pub line: u32,
+    pub location: SourceLocation,
-    pub col: u32,
 }
 
 pub type Identity = Arc<NodeIdentity>;
@@ -15,10 +23,35 @@ pub type Identity = Arc<NodeIdentity>;
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub struct Keyword(pub u32);
 
+lazy_static! {
+    static ref KEYWORD_REGISTRY: Mutex<HashMap<String, u32>> = Mutex::new(HashMap::new());
+    static ref KEYWORD_REVERSE: Mutex<Vec<String>> = Mutex::new(Vec::new());
+}
+
+impl Keyword {
+    pub fn intern(name: &str) -> Self {
+        let mut reg = KEYWORD_REGISTRY.lock().unwrap();
+        if let Some(&id) = reg.get(name) {
+            Keyword(id)
+        } else {
+            let mut rev = KEYWORD_REVERSE.lock().unwrap();
+            let id = rev.len() as u32;
+            reg.insert(name.to_string(), id);
+            rev.push(name.to_string());
+            Keyword(id)
+        }
+    }
+
+    pub fn name(&self) -> String {
+        let rev = KEYWORD_REVERSE.lock().unwrap();
+        rev[self.0 as usize].clone()
+    }
+}
+
 /// Core data value in Myc Script (similar to TDataValue)
 #[derive(Clone)]
 pub enum Value {
-    Nil,
+    Void, // Maps to vkVoid
     Bool(bool),
     Int(i64),
     Float(f64),
@@ -32,7 +65,7 @@ pub enum Value {
 impl Value {
     pub fn is_truthy(&self) -> bool {
         match self {
-            Value::Nil => false,
+            Value::Void => false,
             Value::Bool(b) => *b,
             _ => true,
         }
@@ -42,12 +75,12 @@ impl Value {
 impl fmt::Display for Value {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
-            Value::Nil => write!(f, "nil"),
+            Value::Void => write!(f, "void"),
             Value::Bool(b) => write!(f, "{}", b),
             Value::Int(i) => write!(f, "{}", i),
             Value::Float(fl) => write!(f, "{}", fl),
             Value::Text(t) => write!(f, "\"{}\"", t),
-            Value::Keyword(k) => write!(f, ":kw#{}", k.0),
+            Value::Keyword(k) => write!(f, ":{}", k.name()),
             Value::List(l) => write!(f, "<list[{}]>", l.len()),
             Value::Record(r) => write!(f, "<record[{}]>", r.len()),
             Value::Function(_) => write!(f, "<fn>"),

src/main.rs

@@ -40,40 +40,31 @@ impl eframe::App for CompilerApp {
 
                 // Compile Button (at the top of the bottom panel)
                 if ui.button("Compile").clicked() {
-                    use crate::ast::*;
+                    use crate::ast::parser::Parser;
-                    use std::sync::Arc;
+                    use crate::ast::nodes::Context;
 
-                    // 1. Create shared identity
+                    let mut parser = match Parser::new(&self.source_code) {
-                    let identity = Arc::new(NodeIdentity { line: 1, col: 1 });
+                        Ok(p) => p,
-
+                        Err(e) => {
-                    // 2. Build AST: (if true 42 nil)
+                            self.output_log = format!("Lexer Error: {}", e);
-                    let ast = Node {
+                            return;
-                        identity: identity.clone(),
+                        }
-                        kind: UntypedKind::If {
-                            cond: Box::new(Node {
-                                identity: identity.clone(),
-                                kind: UntypedKind::Constant(Value::Bool(true)),
-                            }),
-                            then_br: Box::new(Node {
-                                identity: identity.clone(),
-                                kind: UntypedKind::Constant(Value::Int(42)),
-                            }),
-                            else_br: Some(Box::new(Node {
-                                identity: identity.clone(),
-                                kind: UntypedKind::Constant(Value::Nil),
-                            })),
-                        },
                     };
 
-                    // 3. Evaluate
+                    match parser.parse_expression() {
-                    let mut context = Context {};
+                        Ok(ast) => {
-                    let result = ast.eval(&mut context);
+                            let mut context = Context {};
-
+                            let result = ast.eval(&mut context);
-                    self.output_log = format!(
+                            self.output_log = format!(
-                        "AST Test:\nSource: (if true 42 nil)\nResult: {}\n\nCompiler log:\nStarted at {:?}",
+                                "AST Parsed Successfully.\nResult: {}\n\nFinished at {:?}",
-                        result,
+                                result,
-                        std::time::SystemTime::now(),
+                                std::time::SystemTime::now(),
-                    );
+                            );
+                        }
+                        Err(e) => {
+                            self.output_log = format!("Parser Error: {}", e);
+                        }
+                    }
                 }
 
                 ui.add_space(5.0);