511 lines
18 KiB
Rust
511 lines
18 KiB
Rust
//! Lexer for form (A).
|
|
//!
|
|
//! The lexical layer has three rules:
|
|
//!
|
|
//! 1. Whitespace (` `, `\t`, `\n`, `\r`) and parens (`(`, `)`) are the
|
|
//! only delimiters. Any maximal non-whitespace, non-paren run is a
|
|
//! single token.
|
|
//! 2. `;` introduces a comment to end-of-line. Comments produce no
|
|
//! tokens.
|
|
//! 3. After tokenisation, classify by first character:
|
|
//! - `"` ⇒ string literal (consume until closing `"`; `\"` and
|
|
//! `\n` escapes supported).
|
|
//! - digit, or `-` followed by digit ⇒ integer or float literal
|
|
//! (float if the token contains `.`, `e`, or `E`; integer
|
|
//! otherwise).
|
|
//! - otherwise ⇒ ident.
|
|
//!
|
|
//! Operators (`+`, `==`, `<=`, `**`), qualified names like
|
|
//! `io/print_int`, and dotted cross-module references like
|
|
//! `std_list.map` are all single-token idents — there is no special
|
|
//! lexical rule for any of them. `(`, `)`, and whitespace are the only
|
|
//! reserved tokens. Bool literals (`true`, `false`) and unit
|
|
//! (`(lit-unit)`) are disambiguated by the parser via context.
|
|
|
|
use thiserror::Error;
|
|
|
|
/// Source byte offset (start, end] into the original input. `end` is
|
|
/// one past the last byte. Used for error reporting only.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub struct Span {
|
|
pub start: usize,
|
|
pub end: usize,
|
|
}
|
|
|
|
/// One lexical token.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub enum Tok {
|
|
/// `(`
|
|
LParen,
|
|
/// `)`
|
|
RParen,
|
|
/// Integer literal. Original textual form preserved for error messages;
|
|
/// parsed `i64` value carried for direct use.
|
|
Int(i64),
|
|
/// IEEE-754 binary64 literal. The
|
|
/// payload is the bit pattern produced by `f64::to_bits`. The
|
|
/// surface forms recognised by the lexer are
|
|
/// `<digits>.<digits>(e[+-]?<digits>)?` and
|
|
/// `<digits>e[+-]?<digits>` (per spec A2). Hex floats and bare
|
|
/// leading/trailing dots are not recognised.
|
|
Float(u64),
|
|
/// String literal contents (after escape processing).
|
|
Str(String),
|
|
/// Identifier. Anything that isn't a paren, integer, float, or string.
|
|
Ident(String),
|
|
}
|
|
|
|
/// A token plus its span.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Token {
|
|
pub tok: Tok,
|
|
pub span: Span,
|
|
}
|
|
|
|
/// Lexer error.
|
|
#[derive(Debug, Error)]
|
|
pub enum LexError {
|
|
#[error("unterminated string literal at byte {start}")]
|
|
UnterminatedString { start: usize },
|
|
#[error("invalid integer literal {literal:?} at byte {start}")]
|
|
InvalidInteger { literal: String, start: usize },
|
|
#[error("invalid float literal {literal:?} at byte {start}")]
|
|
InvalidFloat { literal: String, start: usize },
|
|
#[error("invalid escape sequence \\{ch} in string literal at byte {pos}")]
|
|
InvalidEscape { ch: char, pos: usize },
|
|
}
|
|
|
|
/// Tokenise an input string into a flat token stream.
|
|
///
|
|
/// Whitespace and comments are dropped. Returns a [`LexError`] on
|
|
/// unterminated string literals or numeric tokens that fail to parse.
|
|
pub fn tokenize(input: &str) -> Result<Vec<Token>, LexError> {
|
|
let bytes = input.as_bytes();
|
|
let mut out = Vec::new();
|
|
let mut i = 0usize;
|
|
while i < bytes.len() {
|
|
let b = bytes[i];
|
|
// Whitespace.
|
|
if b == b' ' || b == b'\t' || b == b'\n' || b == b'\r' {
|
|
i += 1;
|
|
continue;
|
|
}
|
|
// Comment to end-of-line.
|
|
if b == b';' {
|
|
while i < bytes.len() && bytes[i] != b'\n' {
|
|
i += 1;
|
|
}
|
|
continue;
|
|
}
|
|
// Parens.
|
|
if b == b'(' {
|
|
out.push(Token { tok: Tok::LParen, span: Span { start: i, end: i + 1 } });
|
|
i += 1;
|
|
continue;
|
|
}
|
|
if b == b')' {
|
|
out.push(Token { tok: Tok::RParen, span: Span { start: i, end: i + 1 } });
|
|
i += 1;
|
|
continue;
|
|
}
|
|
// String literal. Iterate over UTF-8 chars (not raw bytes) so
|
|
// multi-byte glyphs in the source survive round-trip.
|
|
if b == b'"' {
|
|
let start = i;
|
|
i += 1;
|
|
let mut value = String::new();
|
|
// Walk via char_indices on the unread tail to keep i a byte offset.
|
|
loop {
|
|
if i >= bytes.len() {
|
|
return Err(LexError::UnterminatedString { start });
|
|
}
|
|
// Quick byte-level checks for ASCII control bytes that
|
|
// can never start a multi-byte UTF-8 sequence.
|
|
let c0 = bytes[i];
|
|
if c0 == b'"' {
|
|
i += 1;
|
|
break;
|
|
}
|
|
if c0 == b'\\' {
|
|
if i + 1 >= bytes.len() {
|
|
return Err(LexError::UnterminatedString { start });
|
|
}
|
|
let esc = bytes[i + 1];
|
|
match esc {
|
|
b'"' => value.push('"'),
|
|
b'\\' => value.push('\\'),
|
|
b'n' => value.push('\n'),
|
|
b't' => value.push('\t'),
|
|
b'r' => value.push('\r'),
|
|
other => {
|
|
return Err(LexError::InvalidEscape {
|
|
ch: other as char,
|
|
pos: i,
|
|
});
|
|
}
|
|
}
|
|
i += 2;
|
|
continue;
|
|
}
|
|
// Decode one full UTF-8 character starting at i. The
|
|
// input is guaranteed to be valid UTF-8 by Rust's
|
|
// `&str` invariant, so a single `chars()` step on the
|
|
// tail gives us the next scalar.
|
|
if let Some(ch) = input[i..].chars().next() {
|
|
value.push(ch);
|
|
i += ch.len_utf8();
|
|
} else {
|
|
return Err(LexError::UnterminatedString { start });
|
|
}
|
|
}
|
|
out.push(Token {
|
|
tok: Tok::Str(value),
|
|
span: Span { start, end: i },
|
|
});
|
|
continue;
|
|
}
|
|
// Otherwise: maximal non-paren / non-whitespace / non-comment run.
|
|
let start = i;
|
|
while i < bytes.len() {
|
|
let c = bytes[i];
|
|
if c == b' '
|
|
|| c == b'\t'
|
|
|| c == b'\n'
|
|
|| c == b'\r'
|
|
|| c == b'('
|
|
|| c == b')'
|
|
|| c == b';'
|
|
{
|
|
break;
|
|
}
|
|
i += 1;
|
|
}
|
|
let raw = &input[start..i];
|
|
// Classify.
|
|
let first = raw.as_bytes()[0];
|
|
let is_int = first.is_ascii_digit()
|
|
|| (first == b'-' && raw.len() > 1 && raw.as_bytes()[1].is_ascii_digit());
|
|
if is_int {
|
|
let has_dot = raw.contains('.');
|
|
let has_exp = raw.contains('e') || raw.contains('E');
|
|
if has_dot || has_exp {
|
|
if !looks_like_float(raw) {
|
|
return Err(LexError::InvalidFloat {
|
|
literal: raw.to_string(),
|
|
start,
|
|
});
|
|
}
|
|
match raw.parse::<f64>() {
|
|
Ok(f) => out.push(Token {
|
|
tok: Tok::Float(f.to_bits()),
|
|
span: Span { start, end: i },
|
|
}),
|
|
Err(_) => {
|
|
return Err(LexError::InvalidFloat {
|
|
literal: raw.to_string(),
|
|
start,
|
|
});
|
|
}
|
|
}
|
|
} else {
|
|
// Pure integer: the existing path.
|
|
match raw.parse::<i64>() {
|
|
Ok(v) => out.push(Token {
|
|
tok: Tok::Int(v),
|
|
span: Span { start, end: i },
|
|
}),
|
|
Err(_) => {
|
|
return Err(LexError::InvalidInteger {
|
|
literal: raw.to_string(),
|
|
start,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
out.push(Token {
|
|
tok: Tok::Ident(raw.to_string()),
|
|
span: Span { start, end: i },
|
|
});
|
|
}
|
|
}
|
|
Ok(out)
|
|
}
|
|
|
|
/// Spec A2 grammar validator for a digit-lead token that already
|
|
/// contains `.` or `e`/`E`. Returns `true` iff `raw` matches one of
|
|
/// - `[-]?<digits>.<digits>`
|
|
/// - `[-]?<digits>.<digits>(e|E)[+-]?<digits>`
|
|
/// - `[-]?<digits>(e|E)[+-]?<digits>`
|
|
/// and contains at least one of `.`, `e`, `E` (otherwise the caller
|
|
/// would dispatch to the integer path). Rejects bare leading/trailing
|
|
/// dots (`5.`, `5.e10`), missing exponent digits (`1.5e`), and any
|
|
/// trailing junk.
|
|
///
|
|
/// Hex floats (`0x1.8p4`) are naturally rejected because the
|
|
/// digit-lead caller arm only runs for tokens whose first digit
|
|
/// character is decimal; an `x` mid-token would fail validation here.
|
|
fn looks_like_float(raw: &str) -> bool {
|
|
let body = raw.strip_prefix('-').unwrap_or(raw);
|
|
let bytes = body.as_bytes();
|
|
if bytes.is_empty() || !bytes[0].is_ascii_digit() {
|
|
return false;
|
|
}
|
|
let mut i = 0;
|
|
while i < bytes.len() && bytes[i].is_ascii_digit() {
|
|
i += 1;
|
|
}
|
|
let mut saw_dot = false;
|
|
if i < bytes.len() && bytes[i] == b'.' {
|
|
saw_dot = true;
|
|
i += 1;
|
|
let frac_start = i;
|
|
while i < bytes.len() && bytes[i].is_ascii_digit() {
|
|
i += 1;
|
|
}
|
|
if i == frac_start {
|
|
return false; // `5.` or `5.e10` — trailing dot
|
|
}
|
|
}
|
|
let mut saw_exp = false;
|
|
if i < bytes.len() && (bytes[i] == b'e' || bytes[i] == b'E') {
|
|
saw_exp = true;
|
|
i += 1;
|
|
if i < bytes.len() && (bytes[i] == b'+' || bytes[i] == b'-') {
|
|
i += 1;
|
|
}
|
|
let exp_start = i;
|
|
while i < bytes.len() && bytes[i].is_ascii_digit() {
|
|
i += 1;
|
|
}
|
|
if i == exp_start {
|
|
return false; // `1.5e` or `1.5e+`
|
|
}
|
|
}
|
|
i == bytes.len() && (saw_dot || saw_exp)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn parens_and_idents() {
|
|
let toks = tokenize("(foo bar)").unwrap();
|
|
assert_eq!(toks.len(), 4);
|
|
assert!(matches!(toks[0].tok, Tok::LParen));
|
|
assert!(matches!(&toks[1].tok, Tok::Ident(s) if s == "foo"));
|
|
assert!(matches!(&toks[2].tok, Tok::Ident(s) if s == "bar"));
|
|
assert!(matches!(toks[3].tok, Tok::RParen));
|
|
}
|
|
|
|
#[test]
|
|
fn comments_are_dropped() {
|
|
let toks = tokenize("(a ; this is a comment\n b)").unwrap();
|
|
assert_eq!(toks.len(), 4);
|
|
}
|
|
|
|
#[test]
|
|
fn integers() {
|
|
let toks = tokenize("42 -7 0").unwrap();
|
|
assert_eq!(toks.len(), 3);
|
|
assert!(matches!(toks[0].tok, Tok::Int(42)));
|
|
assert!(matches!(toks[1].tok, Tok::Int(-7)));
|
|
assert!(matches!(toks[2].tok, Tok::Int(0)));
|
|
}
|
|
|
|
#[test]
|
|
fn operators_are_idents() {
|
|
let toks = tokenize("+ == <= io/print_int std_list.map").unwrap();
|
|
let names: Vec<_> = toks
|
|
.iter()
|
|
.map(|t| match &t.tok {
|
|
Tok::Ident(s) => s.clone(),
|
|
_ => panic!("not ident"),
|
|
})
|
|
.collect();
|
|
assert_eq!(names, vec!["+", "==", "<=", "io/print_int", "std_list.map"]);
|
|
}
|
|
|
|
#[test]
|
|
fn string_with_escapes() {
|
|
let toks = tokenize(r#""hello\nworld""#).unwrap();
|
|
assert_eq!(toks.len(), 1);
|
|
assert!(matches!(&toks[0].tok, Tok::Str(s) if s == "hello\nworld"));
|
|
}
|
|
|
|
#[test]
|
|
fn negative_lone_minus_is_ident() {
|
|
// `-` alone (no digit follows) is an ident, not a (failed) integer.
|
|
let toks = tokenize("-").unwrap();
|
|
assert_eq!(toks.len(), 1);
|
|
assert!(matches!(&toks[0].tok, Tok::Ident(s) if s == "-"));
|
|
}
|
|
|
|
#[test]
|
|
fn string_preserves_multibyte_utf8() {
|
|
let toks = tokenize(r#""em — dash and naïve""#).unwrap();
|
|
assert_eq!(toks.len(), 1);
|
|
assert!(matches!(&toks[0].tok, Tok::Str(s) if s == "em — dash and naïve"));
|
|
}
|
|
|
|
/// A `<digits>.<digits>` token lexes as
|
|
/// `Tok::Float` carrying the IEEE-754 binary64 bit pattern of
|
|
/// `f64::from_str(raw).unwrap().to_bits()`. Pinned literal:
|
|
/// `1.5_f64` → bit pattern `0x3FF8000000000000`.
|
|
#[test]
|
|
fn float_basic_decimal() {
|
|
let toks = tokenize("1.5").unwrap();
|
|
assert_eq!(toks.len(), 1);
|
|
assert!(matches!(toks[0].tok, Tok::Float(0x3ff8_0000_0000_0000)));
|
|
}
|
|
|
|
/// Spec A2 grammar: scientific-notation forms with a fractional
|
|
/// part (`1.5e3`, `2.0e-10`).
|
|
#[test]
|
|
fn float_scientific_with_fraction() {
|
|
let toks = tokenize("1.5e3 2.0e-10").unwrap();
|
|
assert_eq!(toks.len(), 2);
|
|
let bits1500 = 1.5e3_f64.to_bits();
|
|
let bits_neg10 = 2.0e-10_f64.to_bits();
|
|
match toks[0].tok {
|
|
Tok::Float(b) => assert_eq!(b, bits1500),
|
|
ref other => panic!("expected Tok::Float, got {other:?}"),
|
|
}
|
|
match toks[1].tok {
|
|
Tok::Float(b) => assert_eq!(b, bits_neg10),
|
|
ref other => panic!("expected Tok::Float, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
/// Spec A2 grammar: exponent-only forms (`1e10`, `3e-5`).
|
|
#[test]
|
|
fn float_pure_exponent() {
|
|
let toks = tokenize("1e10 3e-5").unwrap();
|
|
assert_eq!(toks.len(), 2);
|
|
match toks[0].tok {
|
|
Tok::Float(b) => assert_eq!(b, 1e10_f64.to_bits()),
|
|
ref other => panic!("expected Tok::Float, got {other:?}"),
|
|
}
|
|
match toks[1].tok {
|
|
Tok::Float(b) => assert_eq!(b, 3e-5_f64.to_bits()),
|
|
ref other => panic!("expected Tok::Float, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
/// Negative float literals follow the existing negative-int rule
|
|
/// (`-` followed by a digit is part of the numeric token), so
|
|
/// `-1.5` and `-1.5e-3` lex as single Tok::Float values.
|
|
#[test]
|
|
fn float_negative() {
|
|
let toks = tokenize("-1.5 -1.5e-3").unwrap();
|
|
assert_eq!(toks.len(), 2);
|
|
match toks[0].tok {
|
|
Tok::Float(b) => assert_eq!(b, (-1.5_f64).to_bits()),
|
|
ref other => panic!("expected Tok::Float, got {other:?}"),
|
|
}
|
|
match toks[1].tok {
|
|
Tok::Float(b) => assert_eq!(b, (-1.5e-3_f64).to_bits()),
|
|
ref other => panic!("expected Tok::Float, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
/// Edge case: `0.0` and `-0.0` lex as distinct Tok::Float bit
|
|
/// patterns (per spec A5: -0 and +0 are distinct at the bit level).
|
|
#[test]
|
|
fn float_signed_zero() {
|
|
let toks = tokenize("0.0 -0.0").unwrap();
|
|
assert_eq!(toks.len(), 2);
|
|
match toks[0].tok {
|
|
Tok::Float(b) => assert_eq!(b, 0u64),
|
|
ref other => panic!("expected Tok::Float, got {other:?}"),
|
|
}
|
|
match toks[1].tok {
|
|
Tok::Float(b) => assert_eq!(b, 0x8000_0000_0000_0000u64),
|
|
ref other => panic!("expected Tok::Float, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
/// Spec A2 rejects bare trailing dot — `5.` is not a float literal.
|
|
#[test]
|
|
fn float_rejects_trailing_dot() {
|
|
let err = tokenize("5.").unwrap_err();
|
|
assert!(
|
|
matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "5."),
|
|
"expected InvalidFloat for `5.`, got {err:?}"
|
|
);
|
|
}
|
|
|
|
/// Spec A2 rejects missing fractional part before exponent — `5.e10`
|
|
/// has the `<digits>.` shape with no fraction digits.
|
|
#[test]
|
|
fn float_rejects_missing_fraction_before_exponent() {
|
|
let err = tokenize("5.e10").unwrap_err();
|
|
assert!(
|
|
matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "5.e10"),
|
|
"expected InvalidFloat for `5.e10`, got {err:?}"
|
|
);
|
|
}
|
|
|
|
/// Spec A2 rejects empty exponent — `1.5e` has no exponent digits.
|
|
#[test]
|
|
fn float_rejects_empty_exponent() {
|
|
let err = tokenize("1.5e").unwrap_err();
|
|
assert!(
|
|
matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "1.5e"),
|
|
"expected InvalidFloat for `1.5e`, got {err:?}"
|
|
);
|
|
}
|
|
|
|
/// Spec A2 rejects sign-only exponent — `1.5e+` has no exponent
|
|
/// digits after the sign.
|
|
#[test]
|
|
fn float_rejects_sign_only_exponent() {
|
|
let err = tokenize("1.5e+").unwrap_err();
|
|
assert!(
|
|
matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "1.5e+"),
|
|
"expected InvalidFloat for `1.5e+`, got {err:?}"
|
|
);
|
|
}
|
|
|
|
/// Spec A2 rejects double dot — `1..5`.
|
|
#[test]
|
|
fn float_rejects_double_dot() {
|
|
let err = tokenize("1..5").unwrap_err();
|
|
assert!(
|
|
matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "1..5"),
|
|
"expected InvalidFloat for `1..5`, got {err:?}"
|
|
);
|
|
}
|
|
|
|
/// Spec A2 grammar accepts both `e` and `E` for the exponent
|
|
/// marker. The `looks_like_float` validator handles both bytes;
|
|
/// pin the uppercase form explicitly.
|
|
#[test]
|
|
fn float_uppercase_exponent() {
|
|
let toks = tokenize("1.5E3 2E-5").unwrap();
|
|
assert_eq!(toks.len(), 2);
|
|
match toks[0].tok {
|
|
Tok::Float(b) => assert_eq!(b, 1.5e3_f64.to_bits()),
|
|
ref other => panic!("expected Tok::Float, got {other:?}"),
|
|
}
|
|
match toks[1].tok {
|
|
Tok::Float(b) => assert_eq!(b, 2e-5_f64.to_bits()),
|
|
ref other => panic!("expected Tok::Float, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
/// Spec A2 rejects bare leading dot at the *float* level — `.5` is
|
|
/// not a float literal. The lexer's digit-lead rule does not fire
|
|
/// (first byte `.` is not a digit and not `-`-followed-by-digit),
|
|
/// so `.5` falls through to the maximal-non-paren-non-whitespace
|
|
/// run and lexes as `Tok::Ident(".5")`. Downstream parser /
|
|
/// typecheck stages produce the appropriate diagnostic.
|
|
#[test]
|
|
fn float_leading_dot_lexes_as_ident() {
|
|
let toks = tokenize(".5").unwrap();
|
|
assert_eq!(toks.len(), 1);
|
|
assert!(matches!(&toks[0].tok, Tok::Ident(s) if s == ".5"));
|
|
}
|
|
}
|