diff --git a/Cargo.lock b/Cargo.lock index 58ec89c..0803178 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7,6 +7,7 @@ name = "Ast" version = "0.1.0" dependencies = [ "eframe", + "lazy_static", ] [[package]] @@ -1590,6 +1591,12 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "leb128fmt" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index af271ab..e4a964c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,3 +5,4 @@ edition = "2024" [dependencies] eframe = "0.33.3" +lazy_static = "1.4.0" diff --git a/src/ast/lexer.rs b/src/ast/lexer.rs new file mode 100644 index 0000000..8ef313e --- /dev/null +++ b/src/ast/lexer.rs @@ -0,0 +1,164 @@ +use std::iter::Peekable; +use std::str::Chars; +use std::sync::Arc; +use crate::ast::types::SourceLocation; + +#[derive(Debug, Clone, PartialEq)] +pub enum TokenKind { + LeftParen, RightParen, + LeftBracket, RightBracket, + LeftBrace, RightBrace, + Quote, Backtick, Tilde, At, + Identifier(Arc), + Keyword(Arc), + Number(f64), + String(Arc), + EOF, +} + +#[derive(Debug, Clone)] +pub struct Token { + pub kind: TokenKind, + pub location: SourceLocation, +} + +pub struct Lexer<'a> { + input: Peekable>, + line: u32, + col: u32, +} + +impl<'a> Lexer<'a> { + pub fn new(input: &'a str) -> Self { + Self { + input: input.chars().peekable(), + line: 1, + col: 1, + } + } + + pub fn next_token(&mut self) -> Result { + self.skip_whitespace(); + + let start_location = SourceLocation { line: self.line, col: self.col }; + + let char = match self.input.next() { + Some(c) => { + let current_char = c; + self.col += 1; + current_char + }, + None => return Ok(Token { kind: TokenKind::EOF, location: start_location }), + }; + + let kind = match char { + '(' => TokenKind::LeftParen, + ')' => TokenKind::RightParen, + '[' => TokenKind::LeftBracket, + ']' => TokenKind::RightBracket, + '{' => TokenKind::LeftBrace, + '}' => TokenKind::RightBrace, + '\'' => TokenKind::Quote, + '`' => TokenKind::Backtick, + '~' => TokenKind::Tilde, + '@' => TokenKind::At, + ':' => { + let id = self.read_while(|c| !c.is_whitespace() && !"()[]{}\"'`~@;".contains(c)); + TokenKind::Keyword(Arc::from(id)) + } + '"' => TokenKind::String(Arc::from(self.read_string()?)), + c if c.is_ascii_digit() || (c == '-' && self.peek().map_or(false, |p| p.is_ascii_digit())) => { + let mut num_str = String::from(c); + num_str.push_str(&self.read_while(|c| c.is_ascii_digit() || c == '.')); + TokenKind::Number(num_str.parse().map_err(|_| "Invalid number format")?) + } + c if is_ident_start(c) => { + let mut id = String::from(c); + id.push_str(&self.read_while(is_ident_char)); + TokenKind::Identifier(Arc::from(id)) + } + _ => return Err(format!("Unexpected character: {} at {}:{}", char, self.line, self.col - 1)), + }; + + Ok(Token { kind, location: start_location }) + } + + fn peek(&mut self) -> Option<&char> { + self.input.peek() + } + + fn skip_whitespace(&mut self) { + while let Some(&c) = self.peek() { + if c.is_whitespace() { + if c == '\n' { + self.line += 1; + self.col = 1; + } else { + self.col += 1; + } + self.input.next(); + } else if c == ';' { + while let Some(c) = self.input.next() { + if c == '\n' { + self.line += 1; + self.col = 1; + break; + } + } + } else { + break; + } + } + } + + fn read_while(&mut self, mut predicate: F) -> String + where F: FnMut(char) -> bool { + let mut s = String::new(); + while let Some(&c) = self.peek() { + if predicate(c) { + s.push(self.input.next().unwrap()); + self.col += 1; + } else { + break; + } + } + s + } + + fn read_string(&mut self) -> Result { + let mut s = String::new(); + while let Some(c) = self.input.next() { + self.col += 1; + match c { + '"' => return Ok(s), + '\\' => { + let next = self.input.next().ok_or("Unterminated string escape")?; + self.col += 1; + match next { + 'n' => s.push('\n'), + 'r' => s.push('\r'), + 't' => s.push('\t'), + '\\' => s.push('\\'), + '"' => s.push('"'), + _ => s.push(next), + } + } + '\n' => { + self.line += 1; + self.col = 1; + s.push(c); + } + _ => s.push(c), + } + } + Err("Unterminated string".to_string()) + } +} + +fn is_ident_start(c: char) -> bool { + c.is_alphabetic() || "+-*/<=>!$%&_?.".contains(c) +} + +fn is_ident_char(c: char) -> bool { + is_ident_start(c) || c.is_ascii_digit() +} diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 690fa04..d4f61b9 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -1,5 +1,9 @@ pub mod types; pub mod nodes; +pub mod lexer; +pub mod parser; pub use types::*; pub use nodes::*; +pub use lexer::*; +pub use parser::*; diff --git a/src/ast/nodes.rs b/src/ast/nodes.rs index f1c9d4b..80aad4a 100644 --- a/src/ast/nodes.rs +++ b/src/ast/nodes.rs @@ -23,6 +23,8 @@ pub struct Context { /// The core AST variant enum for performance and structure #[derive(Debug)] pub enum UntypedKind { + /// The "..." placeholder for incomplete code + Nop, Constant(Value), Identifier(Arc), If { @@ -41,6 +43,7 @@ pub enum UntypedKind { impl Node { pub fn eval(&self, ctx: &mut Context) -> Value { match &self.kind { + UntypedKind::Nop => Value::Void, UntypedKind::Constant(v) => v.clone(), UntypedKind::Identifier(_) => todo!("Lookup in Context"), UntypedKind::If { cond, then_br, else_br } => { @@ -49,7 +52,7 @@ impl Node { } else if let Some(eb) = else_br { eb.eval(ctx) } else { - Value::Nil + Value::Void } } UntypedKind::Call { callee, args } => { diff --git a/src/ast/parser.rs b/src/ast/parser.rs new file mode 100644 index 0000000..c5a1664 --- /dev/null +++ b/src/ast/parser.rs @@ -0,0 +1,168 @@ +use std::sync::Arc; +use crate::ast::lexer::{Lexer, Token, TokenKind}; +use crate::ast::types::{Identity, NodeIdentity, Value, Keyword}; +use crate::ast::nodes::{Node, UntypedKind}; + +pub struct Parser<'a> { + lexer: Lexer<'a>, + current_token: Token, +} + +impl<'a> Parser<'a> { + pub fn new(input: &'a str) -> Result { + let mut lexer = Lexer::new(input); + let current_token = lexer.next_token()?; + Ok(Self { lexer, current_token }) + } + + fn advance(&mut self) -> Result { + let prev = std::mem::replace(&mut self.current_token, self.lexer.next_token()?); + Ok(prev) + } + + fn peek(&self) -> &TokenKind { + &self.current_token.kind + } + + pub fn parse_expression(&mut self) -> Result, String> { + match self.peek() { + TokenKind::LeftParen => self.parse_list(), + TokenKind::LeftBracket => self.parse_vector(), + // TokenKind::LeftBrace => self.parse_map(), // TODO + TokenKind::Quote => { + let identity = Arc::new(NodeIdentity { location: self.current_token.location }); + self.parse_reader_macro(UntypedKind::Call { + callee: Box::new(self.make_id_node("quote", identity)), + args: vec![], // will be filled + }) + } + _ => self.parse_atom(), + } + } + + fn parse_atom(&mut self) -> Result, String> { + let token = self.advance()?; + let identity = Arc::new(NodeIdentity { location: token.location }); + + let kind = match token.kind { + TokenKind::Number(n) => UntypedKind::Constant(Value::Float(n)), + TokenKind::String(s) => UntypedKind::Constant(Value::Text(s)), + TokenKind::Keyword(k) => UntypedKind::Constant(Value::Keyword(Keyword::intern(&k))), + TokenKind::Identifier(id) => { + match id.as_ref() { + "..." => UntypedKind::Nop, + "true" => UntypedKind::Constant(Value::Bool(true)), + "false" => UntypedKind::Constant(Value::Bool(false)), + "void" => UntypedKind::Constant(Value::Void), + _ => UntypedKind::Identifier(id), + } + } + _ => return Err(format!("Unexpected token in atom: {:?} at {:?}", token.kind, token.location)), + }; + + Ok(Node { identity, kind }) + } + + fn parse_list(&mut self) -> Result, String> { + let start_loc = self.advance()?.location; // consume '(' + + if *self.peek() == TokenKind::RightParen { + return Err(format!("Empty list () is not a valid expression at {:?}", start_loc)); + } + + // Peek at head to check for special forms + let head = self.parse_expression()?; + + let result = if let UntypedKind::Identifier(name) = &head.kind { + match name.as_ref() { + "if" => self.parse_if(head.identity.clone()), + _ => self.parse_call(head), + } + } else { + self.parse_call(head) + }; + + self.expect(TokenKind::RightParen)?; + result + } + + fn parse_if(&mut self, identity: Identity) -> Result, String> { + let cond = Box::new(self.parse_expression()?); + let then_br = Box::new(self.parse_expression()?); + let mut else_br = None; + + if *self.peek() != TokenKind::RightParen { + else_br = Some(Box::new(self.parse_expression()?)); + } + + Ok(Node { + identity, + kind: UntypedKind::If { cond, then_br, else_br }, + }) + } + + fn parse_call(&mut self, callee: Node) -> Result, String> { + let mut args = Vec::new(); + let identity = callee.identity.clone(); + + while *self.peek() != TokenKind::RightParen && *self.peek() != TokenKind::EOF { + args.push(self.parse_expression()?); + } + + Ok(Node { + identity, + kind: UntypedKind::Call { callee: Box::new(callee), args }, + }) + } + + fn parse_vector(&mut self) -> Result, String> { + let token = self.advance()?; // consume '[' + let mut elements = Vec::new(); + + while *self.peek() != TokenKind::RightBracket && *self.peek() != TokenKind::EOF { + // Vector elements in Myc are usually just constants in a list + let expr = self.parse_expression()?; + elements.push(expr.eval(&mut crate::ast::nodes::Context {})); // Simple direct eval for literals + } + + self.expect(TokenKind::RightBracket)?; + + Ok(Node { + identity: Arc::new(NodeIdentity { location: token.location }), + kind: UntypedKind::Constant(Value::List(Arc::new(elements))), + }) + } + + fn parse_reader_macro(&mut self, _template: UntypedKind) -> Result, String> { + let token = self.advance()?; // consume ' + let expr = self.parse_expression()?; + + // 'x -> (quote x) + let identity = Arc::new(NodeIdentity { location: token.location }); + let quote_id = self.make_id_node("quote", identity.clone()); + + Ok(Node { + identity, + kind: UntypedKind::Call { + callee: Box::new(quote_id), + args: vec![expr], + }, + }) + } + + fn expect(&mut self, kind: TokenKind) -> Result<(), String> { + let token = self.advance()?; + if token.kind == kind { + Ok(()) + } else { + Err(format!("Expected {:?}, but found {:?} at {:?}", kind, token.kind, token.location)) + } + } + + fn make_id_node(&self, name: &str, identity: Identity) -> Node { + Node { + identity, + kind: UntypedKind::Identifier(Arc::from(name)), + } + } +} diff --git a/src/ast/types.rs b/src/ast/types.rs index 81fc648..cae0732 100644 --- a/src/ast/types.rs +++ b/src/ast/types.rs @@ -1,12 +1,20 @@ use std::sync::Arc; use std::collections::HashMap; use std::fmt; +use std::sync::Mutex; +use lazy_static::lazy_static; + +/// Simple source location +#[derive(Debug, Clone, Copy)] +pub struct SourceLocation { + pub line: u32, + pub col: u32, +} /// Shared identity for nodes (Location, etc.) #[derive(Debug, Clone)] pub struct NodeIdentity { - pub line: u32, - pub col: u32, + pub location: SourceLocation, } pub type Identity = Arc; @@ -15,10 +23,35 @@ pub type Identity = Arc; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Keyword(pub u32); +lazy_static! { + static ref KEYWORD_REGISTRY: Mutex> = Mutex::new(HashMap::new()); + static ref KEYWORD_REVERSE: Mutex> = Mutex::new(Vec::new()); +} + +impl Keyword { + pub fn intern(name: &str) -> Self { + let mut reg = KEYWORD_REGISTRY.lock().unwrap(); + if let Some(&id) = reg.get(name) { + Keyword(id) + } else { + let mut rev = KEYWORD_REVERSE.lock().unwrap(); + let id = rev.len() as u32; + reg.insert(name.to_string(), id); + rev.push(name.to_string()); + Keyword(id) + } + } + + pub fn name(&self) -> String { + let rev = KEYWORD_REVERSE.lock().unwrap(); + rev[self.0 as usize].clone() + } +} + /// Core data value in Myc Script (similar to TDataValue) #[derive(Clone)] pub enum Value { - Nil, + Void, // Maps to vkVoid Bool(bool), Int(i64), Float(f64), @@ -32,7 +65,7 @@ pub enum Value { impl Value { pub fn is_truthy(&self) -> bool { match self { - Value::Nil => false, + Value::Void => false, Value::Bool(b) => *b, _ => true, } @@ -42,12 +75,12 @@ impl Value { impl fmt::Display for Value { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Value::Nil => write!(f, "nil"), + Value::Void => write!(f, "void"), Value::Bool(b) => write!(f, "{}", b), Value::Int(i) => write!(f, "{}", i), Value::Float(fl) => write!(f, "{}", fl), Value::Text(t) => write!(f, "\"{}\"", t), - Value::Keyword(k) => write!(f, ":kw#{}", k.0), + Value::Keyword(k) => write!(f, ":{}", k.name()), Value::List(l) => write!(f, "", l.len()), Value::Record(r) => write!(f, "", r.len()), Value::Function(_) => write!(f, ""), diff --git a/src/main.rs b/src/main.rs index 08b9ab7..0fbc661 100644 --- a/src/main.rs +++ b/src/main.rs @@ -40,40 +40,31 @@ impl eframe::App for CompilerApp { // Compile Button (at the top of the bottom panel) if ui.button("Compile").clicked() { - use crate::ast::*; - use std::sync::Arc; + use crate::ast::parser::Parser; + use crate::ast::nodes::Context; - // 1. Create shared identity - let identity = Arc::new(NodeIdentity { line: 1, col: 1 }); - - // 2. Build AST: (if true 42 nil) - let ast = Node { - identity: identity.clone(), - kind: UntypedKind::If { - cond: Box::new(Node { - identity: identity.clone(), - kind: UntypedKind::Constant(Value::Bool(true)), - }), - then_br: Box::new(Node { - identity: identity.clone(), - kind: UntypedKind::Constant(Value::Int(42)), - }), - else_br: Some(Box::new(Node { - identity: identity.clone(), - kind: UntypedKind::Constant(Value::Nil), - })), - }, + let mut parser = match Parser::new(&self.source_code) { + Ok(p) => p, + Err(e) => { + self.output_log = format!("Lexer Error: {}", e); + return; + } }; - // 3. Evaluate - let mut context = Context {}; - let result = ast.eval(&mut context); - - self.output_log = format!( - "AST Test:\nSource: (if true 42 nil)\nResult: {}\n\nCompiler log:\nStarted at {:?}", - result, - std::time::SystemTime::now(), - ); + match parser.parse_expression() { + Ok(ast) => { + let mut context = Context {}; + let result = ast.eval(&mut context); + self.output_log = format!( + "AST Parsed Successfully.\nResult: {}\n\nFinished at {:?}", + result, + std::time::SystemTime::now(), + ); + } + Err(e) => { + self.output_log = format!("Parser Error: {}", e); + } + } } ui.add_space(5.0);