floats iter 2.1: Tok::Float + LexError::InvalidFloat + spec-A2 grammar validator

crates/ailang-surface/src/lex.rs

@@ -40,6 +40,13 @@ pub enum Tok {
     /// Integer literal. Original textual form preserved for error messages;
     /// parsed `i64` value carried for direct use.
     Int(i64),
+    /// IEEE-754 binary64 literal (Floats milestone iter 2). The
+    /// payload is the bit pattern produced by `f64::to_bits`. The
+    /// surface forms recognised by the lexer are
+    /// `<digits>.<digits>(e[+-]?<digits>)?` and
+    /// `<digits>e[+-]?<digits>` (per spec A2). Hex floats and bare
+    /// leading/trailing dots are not recognised.
+    Float(u64),
     /// String literal contents (after escape processing).
     Str(String),
     /// Identifier. Anything that isn't a paren, integer, or string.
@@ -60,6 +67,8 @@ pub enum LexError {
     UnterminatedString { start: usize },
     #[error("invalid integer literal {literal:?} at byte {start}")]
     InvalidInteger { literal: String, start: usize },
+    #[error("invalid float literal {literal:?} at byte {start}")]
+    InvalidFloat { literal: String, start: usize },
     #[error("invalid escape sequence \\{ch} in string literal at byte {pos}")]
     InvalidEscape { ch: char, pos: usize },
 }
@@ -175,18 +184,41 @@ pub fn tokenize(input: &str) -> Result<Vec<Token>, LexError> {
         let is_int = first.is_ascii_digit()
             || (first == b'-' && raw.len() > 1 && raw.as_bytes()[1].is_ascii_digit());
         if is_int {
-            // Numeric parse must succeed; else parse error.
-            match raw.parse::<i64>() {
-                Ok(v) => out.push(Token {
-                    tok: Tok::Int(v),
-                    span: Span { start, end: i },
-                }),
-                Err(_) => {
-                    return Err(LexError::InvalidInteger {
+            let has_dot = raw.contains('.');
+            let has_exp = raw.contains('e') || raw.contains('E');
+            if has_dot || has_exp {
+                if !looks_like_float(raw) {
+                    return Err(LexError::InvalidFloat {
                         literal: raw.to_string(),
                         start,
                     });
                 }
+                match raw.parse::<f64>() {
+                    Ok(f) => out.push(Token {
+                        tok: Tok::Float(f.to_bits()),
+                        span: Span { start, end: i },
+                    }),
+                    Err(_) => {
+                        return Err(LexError::InvalidFloat {
+                            literal: raw.to_string(),
+                            start,
+                        });
+                    }
+                }
+            } else {
+                // Pure integer: the existing path.
+                match raw.parse::<i64>() {
+                    Ok(v) => out.push(Token {
+                        tok: Tok::Int(v),
+                        span: Span { start, end: i },
+                    }),
+                    Err(_) => {
+                        return Err(LexError::InvalidInteger {
+                            literal: raw.to_string(),
+                            start,
+                        });
+                    }
+                }
             }
         } else {
             out.push(Token {
@@ -198,6 +230,59 @@ pub fn tokenize(input: &str) -> Result<Vec<Token>, LexError> {
     Ok(out)
 }
 
+/// Spec A2 grammar validator for a digit-lead token that already
+/// contains `.` or `e`/`E`. Returns `true` iff `raw` matches one of
+///   - `[-]?<digits>.<digits>`
+///   - `[-]?<digits>.<digits>(e|E)[+-]?<digits>`
+///   - `[-]?<digits>(e|E)[+-]?<digits>`
+/// and contains at least one of `.`, `e`, `E` (otherwise the caller
+/// would dispatch to the integer path). Rejects bare leading/trailing
+/// dots (`5.`, `5.e10`), missing exponent digits (`1.5e`), and any
+/// trailing junk.
+///
+/// Hex floats (`0x1.8p4`) are naturally rejected because the
+/// digit-lead caller arm only runs for tokens whose first digit
+/// character is decimal; an `x` mid-token would fail validation here.
+fn looks_like_float(raw: &str) -> bool {
+    let body = raw.strip_prefix('-').unwrap_or(raw);
+    let bytes = body.as_bytes();
+    if bytes.is_empty() || !bytes[0].is_ascii_digit() {
+        return false;
+    }
+    let mut i = 0;
+    while i < bytes.len() && bytes[i].is_ascii_digit() {
+        i += 1;
+    }
+    let mut saw_dot = false;
+    if i < bytes.len() && bytes[i] == b'.' {
+        saw_dot = true;
+        i += 1;
+        let frac_start = i;
+        while i < bytes.len() && bytes[i].is_ascii_digit() {
+            i += 1;
+        }
+        if i == frac_start {
+            return false; // `5.` or `5.e10` — trailing dot
+        }
+    }
+    let mut saw_exp = false;
+    if i < bytes.len() && (bytes[i] == b'e' || bytes[i] == b'E') {
+        saw_exp = true;
+        i += 1;
+        if i < bytes.len() && (bytes[i] == b'+' || bytes[i] == b'-') {
+            i += 1;
+        }
+        let exp_start = i;
+        while i < bytes.len() && bytes[i].is_ascii_digit() {
+            i += 1;
+        }
+        if i == exp_start {
+            return false; // `1.5e` or `1.5e+`
+        }
+    }
+    i == bytes.len() && (saw_dot || saw_exp)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -261,4 +346,133 @@ mod tests {
         assert_eq!(toks.len(), 1);
         assert!(matches!(&toks[0].tok, Tok::Str(s) if s == "em — dash and naïve"));
     }
+
+    /// Iter 22-floats.2 RED: a `<digits>.<digits>` token lexes as
+    /// `Tok::Float` carrying the IEEE-754 binary64 bit pattern of
+    /// `f64::from_str(raw).unwrap().to_bits()`. Pinned literal:
+    /// `1.5_f64` → bit pattern `0x3FF8000000000000`.
+    #[test]
+    fn float_basic_decimal() {
+        let toks = tokenize("1.5").unwrap();
+        assert_eq!(toks.len(), 1);
+        assert!(matches!(toks[0].tok, Tok::Float(0x3ff8_0000_0000_0000)));
+    }
+
+    /// Spec A2 grammar: scientific-notation forms with a fractional
+    /// part (`1.5e3`, `2.0e-10`).
+    #[test]
+    fn float_scientific_with_fraction() {
+        let toks = tokenize("1.5e3 2.0e-10").unwrap();
+        assert_eq!(toks.len(), 2);
+        let bits1500 = 1.5e3_f64.to_bits();
+        let bits_neg10 = 2.0e-10_f64.to_bits();
+        match toks[0].tok {
+            Tok::Float(b) => assert_eq!(b, bits1500),
+            ref other => panic!("expected Tok::Float, got {other:?}"),
+        }
+        match toks[1].tok {
+            Tok::Float(b) => assert_eq!(b, bits_neg10),
+            ref other => panic!("expected Tok::Float, got {other:?}"),
+        }
+    }
+
+    /// Spec A2 grammar: exponent-only forms (`1e10`, `3e-5`).
+    #[test]
+    fn float_pure_exponent() {
+        let toks = tokenize("1e10 3e-5").unwrap();
+        assert_eq!(toks.len(), 2);
+        match toks[0].tok {
+            Tok::Float(b) => assert_eq!(b, 1e10_f64.to_bits()),
+            ref other => panic!("expected Tok::Float, got {other:?}"),
+        }
+        match toks[1].tok {
+            Tok::Float(b) => assert_eq!(b, 3e-5_f64.to_bits()),
+            ref other => panic!("expected Tok::Float, got {other:?}"),
+        }
+    }
+
+    /// Negative float literals follow the existing negative-int rule
+    /// (`-` followed by a digit is part of the numeric token), so
+    /// `-1.5` and `-1.5e-3` lex as single Tok::Float values.
+    #[test]
+    fn float_negative() {
+        let toks = tokenize("-1.5 -1.5e-3").unwrap();
+        assert_eq!(toks.len(), 2);
+        match toks[0].tok {
+            Tok::Float(b) => assert_eq!(b, (-1.5_f64).to_bits()),
+            ref other => panic!("expected Tok::Float, got {other:?}"),
+        }
+        match toks[1].tok {
+            Tok::Float(b) => assert_eq!(b, (-1.5e-3_f64).to_bits()),
+            ref other => panic!("expected Tok::Float, got {other:?}"),
+        }
+    }
+
+    /// Edge case: `0.0` and `-0.0` lex as distinct Tok::Float bit
+    /// patterns (per spec A5: -0 and +0 are distinct at the bit level).
+    #[test]
+    fn float_signed_zero() {
+        let toks = tokenize("0.0 -0.0").unwrap();
+        assert_eq!(toks.len(), 2);
+        match toks[0].tok {
+            Tok::Float(b) => assert_eq!(b, 0u64),
+            ref other => panic!("expected Tok::Float, got {other:?}"),
+        }
+        match toks[1].tok {
+            Tok::Float(b) => assert_eq!(b, 0x8000_0000_0000_0000u64),
+            ref other => panic!("expected Tok::Float, got {other:?}"),
+        }
+    }
+
+    /// Spec A2 rejects bare trailing dot — `5.` is not a float literal.
+    #[test]
+    fn float_rejects_trailing_dot() {
+        let err = tokenize("5.").unwrap_err();
+        assert!(
+            matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "5."),
+            "expected InvalidFloat for `5.`, got {err:?}"
+        );
+    }
+
+    /// Spec A2 rejects missing fractional part before exponent — `5.e10`
+    /// has the `<digits>.` shape with no fraction digits.
+    #[test]
+    fn float_rejects_missing_fraction_before_exponent() {
+        let err = tokenize("5.e10").unwrap_err();
+        assert!(
+            matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "5.e10"),
+            "expected InvalidFloat for `5.e10`, got {err:?}"
+        );
+    }
+
+    /// Spec A2 rejects empty exponent — `1.5e` has no exponent digits.
+    #[test]
+    fn float_rejects_empty_exponent() {
+        let err = tokenize("1.5e").unwrap_err();
+        assert!(
+            matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "1.5e"),
+            "expected InvalidFloat for `1.5e`, got {err:?}"
+        );
+    }
+
+    /// Spec A2 rejects sign-only exponent — `1.5e+` has no exponent
+    /// digits after the sign.
+    #[test]
+    fn float_rejects_sign_only_exponent() {
+        let err = tokenize("1.5e+").unwrap_err();
+        assert!(
+            matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "1.5e+"),
+            "expected InvalidFloat for `1.5e+`, got {err:?}"
+        );
+    }
+
+    /// Spec A2 rejects double dot — `1..5`.
+    #[test]
+    fn float_rejects_double_dot() {
+        let err = tokenize("1..5").unwrap_err();
+        assert!(
+            matches!(err, LexError::InvalidFloat { ref literal, .. } if literal == "1..5"),
+            "expected InvalidFloat for `1..5`, got {err:?}"
+        );
+    }
 }