Brummel/MycLib
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Tests for Parser

Browse Source
This commit is contained in:
Michael Schimmel
2025-11-25 14:40:04 +01:00
parent 29c36c7ae0
commit d84509c034
6 changed files with 172 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Src/AST/Myc.Ast.Script.pas
+102 -29
View File
@@ -7,10 +7,21 @@ uses
Myc.Ast.Nodes,
Myc.Ast.Visitor;
type
// Exception for parsing errors containing location information
EParserException = class(EAstException)
strict private
FLine: Integer;
FCol: Integer;
public
constructor Create(const AMessage: string; ALine, ACol: Integer);
property Line: Integer read FLine;
property Col: Integer read FCol;
end;
var
// This BNF should always be kept up to date and valid. It is used to comunicate with LLMs.
// (Nop isn't part of the language syntax.)
BNF: String =
'''
(* ---------------------------------------------------------------------- *)
@@ -46,7 +57,7 @@ var
list ::= "(" list_content ")"
(* The parser distinguishes between special forms (like 'if') and
general function calls based on the first element. *)
general function calls based on the first element. *)
list_content ::= special_form | function_call
special_form ::= "if" expression expression expression?
@@ -61,7 +72,7 @@ var
| dot_identifier expression
(* A function call is any list not recognized as a special form.
The first 'expression' is the callee, the rest are the arguments. *)
The first 'expression' is the callee, the rest are the arguments. *)
function_call ::= expression expression*
(* ---------------------------------------------------------------------- *)
@@ -86,7 +97,7 @@ var
dot_identifier ::= "." identifier
(* An identifier is a sequence of characters that are not delimiters.
The lexer is very permissive. *)
The lexer is very permissive. *)
identifier ::= ? any sequence of chars not containing whitespace, '()[]{}'`~:; ?
digit ::= "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
@@ -142,20 +153,29 @@ type
TToken = record
Kind: TTokenKind;
Text: string;
Line: Integer;
Col: Integer;
end;
TLexer = class
private
FSource: string;
FCurrentPos: Integer;
FLine: Integer;
FLineStart: Integer;
function Peek: Char;
procedure Advance;
function ReadNumber: string;
function ReadString: string;
function ReadIdentifier: string;
function GetCurrentLine: Integer;
function GetCurrentCol: Integer;
procedure Error(const Msg: string);
public
constructor Create(const ASource: string);
function GetNextToken: TToken;
property CurrentLine: Integer read GetCurrentLine;
property CurrentCol: Integer read GetCurrentCol;
end;
TExpr = record
@@ -168,6 +188,8 @@ type
private
FLexer: TLexer;
FCurrentToken: TToken;
procedure Error(const Msg: string);
procedure ErrorFmt(const Msg: string; const Args: array of const);
procedure Consume(AExpectedKind: TTokenKind);
procedure NextToken;
function ParseList: IAstNode;
@@ -223,6 +245,15 @@ type
procedure VisitNop(const Node: INopNode); override;
end;
{ EParserException }
constructor EParserException.Create(const AMessage: string; ALine, ACol: Integer);
begin
inherited CreateFmt('[Line %d, Col %d] %s', [ALine, ACol, AMessage]);
FLine := ALine;
FCol := ACol;
end;
{ TLexer }
constructor TLexer.Create(const ASource: string);
@@ -230,6 +261,23 @@ begin
inherited Create;
FSource := ASource;
FCurrentPos := 1;
FLine := 1;
FLineStart := 1;
end;
function TLexer.GetCurrentLine: Integer;
begin
Result := FLine;
end;
function TLexer.GetCurrentCol: Integer;
begin
Result := FCurrentPos - FLineStart + 1;
end;
procedure TLexer.Error(const Msg: string);
begin
raise EParserException.Create(Msg, FLine, GetCurrentCol);
end;
function TLexer.Peek: Char;
@@ -241,8 +289,19 @@ begin
end;
procedure TLexer.Advance;
var
c: Char;
begin
inc(FCurrentPos);
if FCurrentPos <= Length(FSource) then
begin
c := FSource[FCurrentPos];
if c = #10 then
begin
Inc(FLine);
FLineStart := FCurrentPos + 1;
end;
end;
Inc(FCurrentPos);
end;
function TLexer.ReadIdentifier: string;
@@ -293,7 +352,7 @@ begin
begin
Advance; // Consume '\'
if FCurrentPos > Length(FSource) then
raise Exception.Create('Syntax Error: String literal ends with an escape character.');
Error('Syntax Error: String literal ends with an escape character.');
c := Peek; // Get escaped char
case c of
@@ -319,7 +378,7 @@ begin
end;
// If we fall out of the loop, we hit EOF without a closing quote.
raise Exception.Create('Syntax Error: Unterminated string literal.');
Error('Syntax Error: Unterminated string literal.');
finally
builder.Free;
@@ -349,6 +408,10 @@ begin
break;
end;
// Capture location for the start of the token
Result.Line := CurrentLine;
Result.Col := CurrentCol;
if FCurrentPos > Length(FSource) then
begin
Result.Kind := tkEOF;
@@ -449,6 +512,16 @@ begin
inherited;
end;
procedure TParser.Error(const Msg: string);
begin
raise EParserException.Create(Msg, FCurrentToken.Line, FCurrentToken.Col);
end;
procedure TParser.ErrorFmt(const Msg: string; const Args: array of const);
begin
raise EParserException.Create(Format(Msg, Args), FCurrentToken.Line, FCurrentToken.Col);
end;
procedure TParser.NextToken;
begin
FCurrentToken := FLexer.GetNextToken;
@@ -457,7 +530,7 @@ end;
procedure TParser.Consume(AExpectedKind: TTokenKind);
begin
if FCurrentToken.Kind <> AExpectedKind then
raise Exception.CreateFmt('Syntax Error: Expected token %s, but found %s', [AExpectedKind.ToString, FCurrentToken.Kind.ToString]);
ErrorFmt('Syntax Error: Expected token %s, but found %s', [AExpectedKind.ToString, FCurrentToken.Kind.ToString]);
NextToken;
end;
@@ -471,9 +544,9 @@ begin
while FCurrentToken.Kind <> tkRightBracket do
begin
if FCurrentToken.Kind = tkEOF then
raise Exception.Create('Syntax Error: Unexpected end of file.');
Error('Syntax Error: Unexpected end of file.');
if FCurrentToken.Kind <> tkIdentifier then
raise Exception.Create('Syntax Error: Expected identifier in parameter list.');
Error('Syntax Error: Expected identifier in parameter list.');
params.Add(TAst.Identifier(FCurrentToken.Text));
NextToken;
end;
@@ -496,17 +569,17 @@ begin
while FCurrentToken.Kind <> tkRightBrace do
begin
if FCurrentToken.Kind = tkEOF then
raise Exception.Create('Syntax Error: Unexpected end of file in record literal.');
Error('Syntax Error: Unexpected end of file in record literal.');
// Key must be a keyword
if FCurrentToken.Kind <> tkKeyword then
raise Exception.Create('Syntax Error: Expected keyword (e.g., :key) as field name in record literal.');
Error('Syntax Error: Expected keyword (e.g., :key) as field name in record literal.');
fieldName := FCurrentToken.Text;
NextToken;
// Value can be any expression
if FCurrentToken.Kind = tkRightBrace then
raise Exception.Create('Syntax Error: Missing value for key ' + fieldName + ' in record literal.');
Error('Syntax Error: Missing value for key ' + fieldName + ' in record literal.');
fieldValue := ParseExpression.Node;
fields.Add(TRecordFieldLiteral.Create(TAst.Keyword(fieldName), fieldValue));
@@ -520,7 +593,6 @@ begin
end;
function TParser.ParseList: IAstNode;
var
elements: TList<TExpr>;
head: TExpr;
@@ -530,14 +602,14 @@ var
begin
Consume(tkLeftParen);
if FCurrentToken.Kind = tkRightParen then
raise Exception.Create('Syntax Error: Empty list () is not a valid expression.');
Error('Syntax Error: Empty list () is not a valid expression.');
elements := TList<TExpr>.Create;
try
while FCurrentToken.Kind <> tkRightParen do
begin
if FCurrentToken.Kind = tkEOF then
raise Exception.Create('Syntax Error: Unexpected end of file.');
Error('Syntax Error: Unexpected end of file.');
elements.Add(ParseExpression);
end;
@@ -557,7 +629,7 @@ begin
if SameText(head.Token.Text, 'if') then
begin
if not (Length(tailNodes) in [2, 3]) then
raise Exception.Create('Syntax Error in if statement.');
Error('Syntax Error in if statement.');
var elseBranch: IAstNode := nil;
if Length(tailNodes) = 3 then
@@ -568,7 +640,7 @@ begin
else if SameText(head.Token.Text, '?') then
begin
if Length(tailNodes) <> 3 then
raise Exception.Create('Syntax Error in ternary statement.');
Error('Syntax Error in ternary statement.');
Result := TAst.TernaryExpr(tailNodes[0], tailNodes[1], tailNodes[2]);
end
@@ -576,7 +648,7 @@ begin
begin
// Identifier check removed to support macros (e.g. def ~x 1)
if not (Length(tailNodes) in [1, 2]) then
raise Exception.CreateFmt('Syntax Error: ''def'' requires a target and an optional initializer.', []);
Error('Syntax Error: ''def'' requires a target and an optional initializer.');
initializer := nil;
if Length(tailNodes) = 2 then
@@ -587,13 +659,13 @@ begin
else if SameText(head.Token.Text, 'defmacro') then
begin
if (Length(tailNodes) <> 3) or (tailTokens[0].Kind <> tkIdentifier) then
raise Exception.Create('Syntax Error: ''defmacro'' requires a name, a parameter list, and a body.');
Error('Syntax Error: ''defmacro'' requires a name, a parameter list, and a body.');
var macroName := tailNodes[0].AsIdentifier;
var macroParams := elements[2].Params;
if tailTokens[2].Kind <> tkBacktick then
raise Exception.Create('Syntax Error: Expected a quasiquote as macro body.');
Error('Syntax Error: Expected a quasiquote as macro body.');
var macroBody := IQuasiQuoteNode(tailNodes[2]);
@@ -603,14 +675,14 @@ begin
begin
// Identifier check removed
if Length(tailNodes) <> 2 then
raise Exception.Create('Syntax Error: ''assign'' requires exactly 2 arguments (target and value).');
Error('Syntax Error: ''assign'' requires exactly 2 arguments (target and value).');
Result := TAst.Assign(tailNodes[0], tailNodes[1]);
end
else if SameText(head.Token.Text, 'fn') then
begin
if Length(tailNodes) <> 2 then
raise Exception.Create('Syntax Error: ''fn'' requires a parameter list and a body.');
Error('Syntax Error: ''fn'' requires a parameter list and a body.');
Result := TAst.LambdaExpr(elements[1].Params, tailNodes[1]);
end
else if SameText(head.Token.Text, 'do') then
@@ -620,15 +692,16 @@ begin
else if SameText(head.Token.Text, 'get') then
begin
if Length(tailNodes) <> 2 then
raise Exception.Create('Syntax Error: ''get'' requires exactly 2 arguments (base and index).');
Error('Syntax Error: ''get'' requires exactly 2 arguments (base and index).');
Result := TAst.Indexer(tailNodes[0], tailNodes[1])
end
else if (Length(head.Token.Text) > 1) and (head.Token.Text.StartsWith('.')) then
begin
if Length(tailNodes) <> 1 then
raise Exception.CreateFmt(
ErrorFmt(
'Syntax Error: Member access (e.g., ''%s'') requires exactly 1 argument (the base object).',
[head.Token.Text]);
[head.Token.Text]
);
Result := TAst.MemberAccess(tailNodes[0], TAst.Keyword(head.Token.Text.Substring(1)));
end;
end;
@@ -696,7 +769,7 @@ begin
else if TryStrToFloat(FCurrentToken.Text, dbl, TFormatSettings.Invariant) then
Result.Node := TAst.Constant(dbl)
else
raise Exception.CreateFmt('Syntax Error: Invalid number format "%s"', [FCurrentToken.Text]);
ErrorFmt('Syntax Error: Invalid number format "%s"', [FCurrentToken.Text]);
NextToken;
end;
tkString:
@@ -736,7 +809,7 @@ begin
tkLeftBrace: Result.Node := ParseRecordLiteral;
tkEOF: {nop};
else
raise Exception.CreateFmt('Syntax Error: Unexpected token %s', [FCurrentToken.Kind.ToString]);
ErrorFmt('Syntax Error: Unexpected token %s', [FCurrentToken.Kind.ToString]);
end;
end;
@@ -744,7 +817,7 @@ function TParser.Parse: IAstNode;
begin
var expr := ParseExpression;
if FCurrentToken.Kind <> tkEOF then
raise Exception.Create('Syntax Error: Unexpected characters after end of expression.');
Error('Syntax Error: Unexpected characters after end of expression.');
Result := expr.Node;
end;