feat: Implement lexer and parser for AST

Introduces the lexer and parser modules, enabling the conversion of
source code into an Abstract Syntax Tree (AST).

This includes:
- Defining token kinds and structures.
- Implementing lexer logic to tokenize input.
- Defining AST node kinds and structures.
- Implementing parser logic to construct the AST from tokens.
- Adding support for basic expressions, lists, keywords, and
  identifiers.
- Adding the `lazy_static` dependency for keyword interning.
- Refactoring `Value::Nil` to `Value::Void`.
This commit is contained in:
Michael Schimmel
2026-02-17 00:15:08 +01:00
parent 3fdfd01982
commit c05c74bb65
8 changed files with 409 additions and 38 deletions
+39 -6
View File
@@ -1,12 +1,20 @@
use std::sync::Arc;
use std::collections::HashMap;
use std::fmt;
use std::sync::Mutex;
use lazy_static::lazy_static;
/// Simple source location
#[derive(Debug, Clone, Copy)]
pub struct SourceLocation {
pub line: u32,
pub col: u32,
}
/// Shared identity for nodes (Location, etc.)
#[derive(Debug, Clone)]
pub struct NodeIdentity {
pub line: u32,
pub col: u32,
pub location: SourceLocation,
}
pub type Identity = Arc<NodeIdentity>;
@@ -15,10 +23,35 @@ pub type Identity = Arc<NodeIdentity>;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Keyword(pub u32);
lazy_static! {
static ref KEYWORD_REGISTRY: Mutex<HashMap<String, u32>> = Mutex::new(HashMap::new());
static ref KEYWORD_REVERSE: Mutex<Vec<String>> = Mutex::new(Vec::new());
}
impl Keyword {
pub fn intern(name: &str) -> Self {
let mut reg = KEYWORD_REGISTRY.lock().unwrap();
if let Some(&id) = reg.get(name) {
Keyword(id)
} else {
let mut rev = KEYWORD_REVERSE.lock().unwrap();
let id = rev.len() as u32;
reg.insert(name.to_string(), id);
rev.push(name.to_string());
Keyword(id)
}
}
pub fn name(&self) -> String {
let rev = KEYWORD_REVERSE.lock().unwrap();
rev[self.0 as usize].clone()
}
}
/// Core data value in Myc Script (similar to TDataValue)
#[derive(Clone)]
pub enum Value {
Nil,
Void, // Maps to vkVoid
Bool(bool),
Int(i64),
Float(f64),
@@ -32,7 +65,7 @@ pub enum Value {
impl Value {
pub fn is_truthy(&self) -> bool {
match self {
Value::Nil => false,
Value::Void => false,
Value::Bool(b) => *b,
_ => true,
}
@@ -42,12 +75,12 @@ impl Value {
impl fmt::Display for Value {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Value::Nil => write!(f, "nil"),
Value::Void => write!(f, "void"),
Value::Bool(b) => write!(f, "{}", b),
Value::Int(i) => write!(f, "{}", i),
Value::Float(fl) => write!(f, "{}", fl),
Value::Text(t) => write!(f, "\"{}\"", t),
Value::Keyword(k) => write!(f, ":kw#{}", k.0),
Value::Keyword(k) => write!(f, ":{}", k.name()),
Value::List(l) => write!(f, "<list[{}]>", l.len()),
Value::Record(r) => write!(f, "<record[{}]>", r.len()),
Value::Function(_) => write!(f, "<fn>"),