diff --git a/crates/ailang-surface/src/lex.rs b/crates/ailang-surface/src/lex.rs index fa68b83..a67df2e 100644 --- a/crates/ailang-surface/src/lex.rs +++ b/crates/ailang-surface/src/lex.rs @@ -40,6 +40,13 @@ pub enum Tok { /// Integer literal. Original textual form preserved for error messages; /// parsed `i64` value carried for direct use. Int(i64), + /// IEEE-754 binary64 literal (Floats milestone iter 2). The + /// payload is the bit pattern produced by `f64::to_bits`. The + /// surface forms recognised by the lexer are + /// `.(e[+-]?)?` and + /// `e[+-]?` (per spec A2). Hex floats and bare + /// leading/trailing dots are not recognised. + Float(u64), /// String literal contents (after escape processing). Str(String), /// Identifier. Anything that isn't a paren, integer, or string. @@ -60,6 +67,8 @@ pub enum LexError { UnterminatedString { start: usize }, #[error("invalid integer literal {literal:?} at byte {start}")] InvalidInteger { literal: String, start: usize }, + #[error("invalid float literal {literal:?} at byte {start}")] + InvalidFloat { literal: String, start: usize }, #[error("invalid escape sequence \\{ch} in string literal at byte {pos}")] InvalidEscape { ch: char, pos: usize }, } @@ -175,18 +184,41 @@ pub fn tokenize(input: &str) -> Result, LexError> { let is_int = first.is_ascii_digit() || (first == b'-' && raw.len() > 1 && raw.as_bytes()[1].is_ascii_digit()); if is_int { - // Numeric parse must succeed; else parse error. - match raw.parse::() { - Ok(v) => out.push(Token { - tok: Tok::Int(v), - span: Span { start, end: i }, - }), - Err(_) => { - return Err(LexError::InvalidInteger { + let has_dot = raw.contains('.'); + let has_exp = raw.contains('e') || raw.contains('E'); + if has_dot || has_exp { + if !looks_like_float(raw) { + return Err(LexError::InvalidFloat { literal: raw.to_string(), start, }); } + match raw.parse::() { + Ok(f) => out.push(Token { + tok: Tok::Float(f.to_bits()), + span: Span { start, end: i }, + }), + Err(_) => { + return Err(LexError::InvalidFloat { + literal: raw.to_string(), + start, + }); + } + } + } else { + // Pure integer: the existing path. + match raw.parse::() { + Ok(v) => out.push(Token { + tok: Tok::Int(v), + span: Span { start, end: i }, + }), + Err(_) => { + return Err(LexError::InvalidInteger { + literal: raw.to_string(), + start, + }); + } + } } } else { out.push(Token { @@ -198,6 +230,59 @@ pub fn tokenize(input: &str) -> Result, LexError> { Ok(out) } +/// Spec A2 grammar validator for a digit-lead token that already +/// contains `.` or `e`/`E`. Returns `true` iff `raw` matches one of +/// - `[-]?.` +/// - `[-]?.(e|E)[+-]?` +/// - `[-]?(e|E)[+-]?` +/// and contains at least one of `.`, `e`, `E` (otherwise the caller +/// would dispatch to the integer path). Rejects bare leading/trailing +/// dots (`5.`, `5.e10`), missing exponent digits (`1.5e`), and any +/// trailing junk. +/// +/// Hex floats (`0x1.8p4`) are naturally rejected because the +/// digit-lead caller arm only runs for tokens whose first digit +/// character is decimal; an `x` mid-token would fail validation here. +fn looks_like_float(raw: &str) -> bool { + let body = raw.strip_prefix('-').unwrap_or(raw); + let bytes = body.as_bytes(); + if bytes.is_empty() || !bytes[0].is_ascii_digit() { + return false; + } + let mut i = 0; + while i < bytes.len() && bytes[i].is_ascii_digit() { + i += 1; + } + let mut saw_dot = false; + if i < bytes.len() && bytes[i] == b'.' { + saw_dot = true; + i += 1; + let frac_start = i; + while i < bytes.len() && bytes[i].is_ascii_digit() { + i += 1; + } + if i == frac_start { + return false; // `5.` or `5.e10` — trailing dot + } + } + let mut saw_exp = false; + if i < bytes.len() && (bytes[i] == b'e' || bytes[i] == b'E') { + saw_exp = true; + i += 1; + if i < bytes.len() && (bytes[i] == b'+' || bytes[i] == b'-') { + i += 1; + } + let exp_start = i; + while i < bytes.len() && bytes[i].is_ascii_digit() { + i += 1; + } + if i == exp_start { + return false; // `1.5e` or `1.5e+` + } + } + i == bytes.len() && (saw_dot || saw_exp) +} + #[cfg(test)] mod tests { use super::*; @@ -261,4 +346,133 @@ mod tests { assert_eq!(toks.len(), 1); assert!(matches!(&toks[0].tok, Tok::Str(s) if s == "em — dash and naïve")); } + + /// Iter 22-floats.2 RED: a `.` token lexes as + /// `Tok::Float` carrying the IEEE-754 binary64 bit pattern of + /// `f64::from_str(raw).unwrap().to_bits()`. Pinned literal: + /// `1.5_f64` → bit pattern `0x3FF8000000000000`. + #[test] + fn float_basic_decimal() { + let toks = tokenize("1.5").unwrap(); + assert_eq!(toks.len(), 1); + assert!(matches!(toks[0].tok, Tok::Float(0x3ff8_0000_0000_0000))); + } + + /// Spec A2 grammar: scientific-notation forms with a fractional + /// part (`1.5e3`, `2.0e-10`). + #[test] + fn float_scientific_with_fraction() { + let toks = tokenize("1.5e3 2.0e-10").unwrap(); + assert_eq!(toks.len(), 2); + let bits1500 = 1.5e3_f64.to_bits(); + let bits_neg10 = 2.0e-10_f64.to_bits(); + match toks[0].tok { + Tok::Float(b) => assert_eq!(b, bits1500), + ref other => panic!("expected Tok::Float, got {other:?}"), + } + match toks[1].tok { + Tok::Float(b) => assert_eq!(b, bits_neg10), + ref other => panic!("expected Tok::Float, got {other:?}"), + } + } + + /// Spec A2 grammar: exponent-only forms (`1e10`, `3e-5`). + #[test] + fn float_pure_exponent() { + let toks = tokenize("1e10 3e-5").unwrap(); + assert_eq!(toks.len(), 2); + match toks[0].tok { + Tok::Float(b) => assert_eq!(b, 1e10_f64.to_bits()), + ref other => panic!("expected Tok::Float, got {other:?}"), + } + match toks[1].tok { + Tok::Float(b) => assert_eq!(b, 3e-5_f64.to_bits()), + ref other => panic!("expected Tok::Float, got {other:?}"), + } + } + + /// Negative float literals follow the existing negative-int rule + /// (`-` followed by a digit is part of the numeric token), so + /// `-1.5` and `-1.5e-3` lex as single Tok::Float values. + #[test] + fn float_negative() { + let toks = tokenize("-1.5 -1.5e-3").unwrap(); + assert_eq!(toks.len(), 2); + match toks[0].tok { + Tok::Float(b) => assert_eq!(b, (-1.5_f64).to_bits()), + ref other => panic!("expected Tok::Float, got {other:?}"), + } + match toks[1].tok { + Tok::Float(b) => assert_eq!(b, (-1.5e-3_f64).to_bits()), + ref other => panic!("expected Tok::Float, got {other:?}"), + } + } + + /// Edge case: `0.0` and `-0.0` lex as distinct Tok::Float bit + /// patterns (per spec A5: -0 and +0 are distinct at the bit level). + #[test] + fn float_signed_zero() { + let toks = tokenize("0.0 -0.0").unwrap(); + assert_eq!(toks.len(), 2); + match toks[0].tok { + Tok::Float(b) => assert_eq!(b, 0u64), + ref other => panic!("expected Tok::Float, got {other:?}"), + } + match toks[1].tok { + Tok::Float(b) => assert_eq!(b, 0x8000_0000_0000_0000u64), + ref other => panic!("expected Tok::Float, got {other:?}"), + } + } + + /// Spec A2 rejects bare trailing dot — `5.` is not a float literal. + #[test] + fn float_rejects_trailing_dot() { + let err = tokenize("5.").unwrap_err(); + assert!( + matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "5."), + "expected InvalidFloat for `5.`, got {err:?}" + ); + } + + /// Spec A2 rejects missing fractional part before exponent — `5.e10` + /// has the `.` shape with no fraction digits. + #[test] + fn float_rejects_missing_fraction_before_exponent() { + let err = tokenize("5.e10").unwrap_err(); + assert!( + matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "5.e10"), + "expected InvalidFloat for `5.e10`, got {err:?}" + ); + } + + /// Spec A2 rejects empty exponent — `1.5e` has no exponent digits. + #[test] + fn float_rejects_empty_exponent() { + let err = tokenize("1.5e").unwrap_err(); + assert!( + matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "1.5e"), + "expected InvalidFloat for `1.5e`, got {err:?}" + ); + } + + /// Spec A2 rejects sign-only exponent — `1.5e+` has no exponent + /// digits after the sign. + #[test] + fn float_rejects_sign_only_exponent() { + let err = tokenize("1.5e+").unwrap_err(); + assert!( + matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "1.5e+"), + "expected InvalidFloat for `1.5e+`, got {err:?}" + ); + } + + /// Spec A2 rejects double dot — `1..5`. + #[test] + fn float_rejects_double_dot() { + let err = tokenize("1..5").unwrap_err(); + assert!( + matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "1..5"), + "expected InvalidFloat for `1..5`, got {err:?}" + ); + } }