Json Schema for LLMs

This commit is contained in:
Michael Schimmel
2026-01-06 18:02:05 +01:00
parent 51d0b8d9b7
commit 60952494c1
4 changed files with 1061 additions and 939 deletions
File diff suppressed because one or more lines are too long
+185 -102
View File
@@ -3,129 +3,144 @@
interface
uses
System.SysUtils,
System.Classes,
System.Generics.Collections,
System.Generics.Defaults,
System.Rtti,
System.TypInfo,
system.sysutils,
system.classes,
system.generics.collections,
system.generics.defaults,
system.rtti,
system.typinfo,
Myc.Ast.Nodes,
Myc.Ast.Attributes;
Myc.Ast.Attributes,
Myc.Ast.RTL,
Myc.Ast.RTL.Core;
type
{ Scans AST and RTL metadata to generate a high-precision LLM system prompt. }
TAstSchema = class
strict private
type
TFieldInfo = record
Idx: Integer;
Name: string;
Kind: TFieldKind;
end;
class function GetTsType(Kind: TFieldKind): string; static;
public
// Generates the TypeScript definition by scanning INTERFACES via RTTI
class function GenerateTypeScriptDefinition: string;
// Generates the TypeScript definition including JSDoc and multiple @example entries
class function GenerateTypeScriptDefinition: string; static;
// Static rules for the format
class function GenerateGenerationRules: string;
// Generates documented RTL function signatures with all overloads
class function GenerateRtlDocumentation: string; static;
// Combined prompt
class function GenerateFullSystemPrompt: string;
// Detailed generation rules to eliminate ambiguity
class function GenerateGenerationRules: string; static;
// The final consolidated system prompt
class function GenerateFullSystemPrompt: string; static;
end;
implementation
type
// Helper structure to collect fields per type
TFieldInfo = record
Idx: Integer;
Def: string;
end;
{ TAstSchema }
class function TAstSchema.GetTsType(Kind: TFieldKind): string;
begin
case Kind of
fkNode: Result := 'AstNode';
fkTuple: Result := 'Tuple';
fkString: Result := 'string';
fkValue: Result := 'number | string | boolean';
fkNullableNode: Result := 'AstNode | null';
fkArrayOfPairs: Result := 'Array<[AstNode, AstNode]>';
else
Result := 'any';
end;
end;
class function TAstSchema.GenerateTypeScriptDefinition: string;
var
ctx: TRttiContext;
types: TArray<TRttiType>;
typ: TRttiType;
prop: TRttiProperty;
attr: TCustomAttribute;
fieldAttr: AstFieldAttribute;
nodeDefs: TDictionary<string, TList<TFieldInfo>>;
nodeDocs: TDictionary<string, string>;
nodeExamples: TDictionary<string, TList<string>>;
unionTypes: TList<string>;
fieldsList: TList<TFieldInfo>;
sb: TStringBuilder;
tag: string;
fieldInfo: TFieldInfo;
tag, doc: string;
fieldList: TList<TFieldInfo>;
exampleList: TList<string>;
i: Integer;
function GetTsType(Kind: TFieldKind): string;
begin
case Kind of
fkNode: Result := 'AstNode';
fkTuple: Result := 'Tuple';
fkString: Result := 'string';
fkValue: Result := 'number | string | boolean';
fkNullableNode: Result := 'AstNode | null';
fkArrayOfPairs: Result := 'Array<[AstNode, AstNode]>';
else
Result := 'any';
end;
end;
begin
ctx := TRttiContext.Create;
nodeDefs := TDictionary<string, TList<TFieldInfo>>.Create;
nodeDocs := TDictionary<string, string>.Create;
nodeExamples := TDictionary<string, TList<string>>.Create;
unionTypes := TList<string>.Create;
sb := TStringBuilder.Create;
try
// 1. Scan all types (looking for INTERFACES now)
types := ctx.GetTypes;
for typ in types do
for typ in ctx.GetTypes do
begin
// CHANGE: We now look for Interfaces, not Classes
if typ.TypeKind <> tkInterface then
Continue;
continue;
// Check for [AstTag] attribute on the interface
tag := '';
doc := '';
fieldList := nil;
exampleList := nil;
for attr in typ.GetAttributes do
begin
if attr is AstTagAttribute then
begin
tag := AstTagAttribute(attr).Tag;
Break;
if attr is AstDocAttribute then
doc := AstDocAttribute(attr).Description;
if attr is AstExampleAttribute then
begin
if exampleList = nil then
exampleList := TList<string>.Create;
exampleList.Add(AstExampleAttribute(attr).Example);
end;
if tag = '' then
Continue;
if not nodeDefs.ContainsKey(tag) then
begin
nodeDefs.Add(tag, TList<TFieldInfo>.Create);
unionTypes.Add(tag);
if attr is AstFieldAttribute then
begin
if fieldList = nil then
fieldList := TList<TFieldInfo>.Create;
var f: TFieldInfo;
f.Idx := AstFieldAttribute(attr).Index;
f.Name := AstFieldAttribute(attr).Name;
f.Kind := AstFieldAttribute(attr).Kind;
fieldList.Add(f);
end;
end;
fieldsList := nodeDefs[tag];
// CHANGE: Scan Properties of the interface
for prop in typ.GetProperties do
if tag <> '' then
begin
for attr in prop.GetAttributes do
if attr is AstFieldAttribute then
begin
fieldAttr := AstFieldAttribute(attr);
fieldInfo.Idx := fieldAttr.Index;
fieldInfo.Def := GetTsType(fieldAttr.Kind) + ' /*' + fieldAttr.Name + '*/';
fieldsList.Add(fieldInfo);
end;
unionTypes.Add(tag);
if doc <> '' then
nodeDocs.Add(tag, doc);
if exampleList <> nil then
nodeExamples.Add(tag, exampleList);
if fieldList = nil then
fieldList := TList<TFieldInfo>.Create;
nodeDefs.Add(tag, fieldList);
end
else
begin
fieldList.Free;
exampleList.Free;
end;
end;
// 2. Build the output
sb.AppendLine('// AST Schema Definition (Auto-Generated via RTTI from Interfaces)');
sb.AppendLine('// AST Schema Definition (Auto-Generated)');
sb.AppendLine('// Format: Compact JSON Arrays [Tag, Arg1, Arg2, ...]');
sb.AppendLine;
unionTypes.Sort;
// Union Type Definition
sb.AppendLine('type AstNode = ');
for i := 0 to unionTypes.Count - 1 do
begin
@@ -136,63 +151,131 @@ begin
sb.AppendLine(';');
sb.AppendLine;
// Special Handling for Tuple (it's recursive array structure)
sb.AppendLine('/** Explicit list structure */');
sb.AppendLine('type Tuple = ["Tuple", AstNode[]];');
sb.AppendLine;
// Node Definitions
for tag in unionTypes do
begin
// Skip Tuple in automatic generation as we defined it manually above
// to represent the array structure correctly in TypeScript
if tag = 'Tuple' then
Continue;
continue;
fieldsList := nodeDefs[tag];
sb.Append('/** ');
if nodeDocs.TryGetValue(tag, doc) then
sb.Append(doc);
// Sort fields by Index to ensure correct JSON array order
fieldsList.Sort(TComparer<TFieldInfo>.Construct(function(const L, R: TFieldInfo): Integer begin Result := L.Idx - R.Idx; end));
if nodeExamples.TryGetValue(tag, exampleList) then
begin
for var ex in exampleList do
sb.Append(sLineBreak + ' @example ' + ex);
end;
sb.AppendLine(' */');
fieldList := nodeDefs[tag];
fieldList.Sort(TComparer<TFieldInfo>.Construct(function(const L, R: TFieldInfo): Integer begin Result := L.Idx - R.Idx; end));
sb.Append('type ' + tag + ' = ["' + tag + '"');
for fieldInfo in fieldsList do
sb.Append(', ' + fieldInfo.Def);
for var f in fieldList do
sb.Append(Format(', %s /* %s */', [GetTsType(f.Kind), f.Name]));
sb.AppendLine('];');
end;
Result := sb.ToString;
finally
// Clean up lists in dictionary
for tag in nodeDefs.Keys do
nodeDefs[tag].Free;
for fieldList in nodeDefs.Values do
fieldList.Free;
for exampleList in nodeExamples.Values do
exampleList.Free;
nodeDefs.Free;
nodeDocs.Free;
nodeExamples.Free;
unionTypes.Free;
sb.Free;
ctx.Free;
end;
end;
class function TAstSchema.GenerateRtlDocumentation: string;
var
ctx: TRttiContext;
typ: TRttiType;
meth: TRttiMethod;
attr: TCustomAttribute;
doc, rtlName: string;
sigs: TList<string>;
sb: TStringBuilder;
processedNames: THashSet<string>;
begin
ctx := TRttiContext.Create;
sb := TStringBuilder.Create;
sigs := TList<string>.Create;
processedNames := THashSet<string>.Create;
try
typ := ctx.GetType(TypeInfo(TRtlFunctions));
sb.AppendLine('### 3. Runtime Library (Available Functions)');
sb.AppendLine;
for meth in typ.GetMethods do
begin
doc := '';
rtlName := '';
sigs.Clear;
for attr in meth.GetAttributes do
begin
if attr is AstDocAttribute then
doc := AstDocAttribute(attr).Description;
if attr is TRtlExportAttribute then
rtlName := TRtlExportAttribute(attr).Name;
if attr is AstSignatureAttribute then
sigs.Add(AstSignatureAttribute(attr).Signature);
end;
if (rtlName <> '') and (not processedNames.Contains(rtlName)) then
begin
sb.Append(Format('- `%s`', [rtlName]));
if sigs.Count > 0 then
sb.Append(': ' + string.Join(' | ', sigs.ToArray));
if doc <> '' then
sb.Append(' — ' + doc);
sb.AppendLine;
processedNames.Add(rtlName);
end;
end;
Result := sb.ToString;
finally
processedNames.Free;
sigs.Free;
sb.Free;
ctx.Free;
end;
end;
class function TAstSchema.GenerateGenerationRules: string;
var
sb: TStringBuilder;
begin
// These rules describe the PROTOCOL (Compact Arrays), not the content.
// They remain static unless you change TJsonAstConverter logic significantly.
sb := TStringBuilder.Create;
try
sb.AppendLine('**⚠️ CRITICAL GENERATION RULES:**');
sb.AppendLine;
sb.AppendLine('1. **NO JSON OBJECTS:** Do not use `{ "key": "value" }`. Every node MUST be a JSON Array `["Tag", Arg1, Arg2]`.');
sb.AppendLine('1. **NO JSON OBJECTS:** Do not use `{}`. Every node MUST be a JSON Array `["Tag", Arg1, Arg2]`.');
sb.AppendLine('2. **EXPLICIT TUPLES:** Fields defined as `Tuple` MUST be wrapped in `["Tuple", [...]]`.');
sb.AppendLine('3. **EXPLICIT NULLS:** Optional fields (`| null`) MUST be explicit `null` if not present.');
sb.AppendLine('4. **STRICT ORDER:** Array elements must follow the schema index exactly. Never swap arguments.');
sb.AppendLine(
'2. **EXPLICIT TUPLES:** Whenever a field in the schema is defined as `Tuple`, you MUST wrap the list in a `["Tuple", [...]]` node.'
'5. **OPERATORS:** Use `Call` with an `Id` for operators: `["Call", ["Id", "+"], ["Tuple", [["Const", 1], ["Const", 2]]]]`.'
);
sb.AppendLine(
'3. **EXPLICIT NULLS:** Optional fields (marked as `| null`) MUST be explicit `null` if not present. Do not omit them.'
'6. **ID VS KEY:** Use `Key` for record keys (e.g. `["Field", ["Key", "p"], ...]`). Use `Id` for variables, functions, and operators.'
);
sb.AppendLine('4. **STRICT ORDER:** The order of elements in the array is fixed defined by the schema. Never swap arguments.');
sb.AppendLine('7. **BOOLEANS:** The literals `true` and `false` are identifiers: `["Id", "true"]` or `["Id", "false"]`.');
sb.AppendLine(
'8. **SCALAR TYPES:** Valid types for series definitions are: `:Ordinal`, `:Float`, `:Boolean`, `:Text`, `:DateTime`.'
);
sb.AppendLine('9. **NESTED ARRAYS:** In "Cond", the pairs are an array of arrays: `[ [cond1, branch1], [cond2, branch2] ]`.');
sb.AppendLine('10. **RECURSION:** Use `Recur` for tail-recursive calls to enable optimization.');
sb.AppendLine('11. **NO COMMENTS:** Output raw JSON only. No explanations.');
Result := sb.ToString;
finally
@@ -202,9 +285,8 @@ end;
class function TAstSchema.GenerateFullSystemPrompt: string;
begin
// Combine everything
Result :=
'You are a compiler frontend for a scripting language.'
'You are a compiler frontend for a domain-specific scripting language.'
+ sLineBreak
+ 'Your task is to generate a valid Abstract Syntax Tree (AST) in a specific Compact JSON format.'
+ sLineBreak
@@ -215,8 +297,9 @@ begin
+ sLineBreak
+ '### 2. Constraints & Rules'
+ sLineBreak
+ GenerateGenerationRules;
//TODO add 3. RTL
+ GenerateGenerationRules
+ sLineBreak
+ GenerateRtlDocumentation;
end;
end.
+796 -801
View File
File diff suppressed because it is too large Load Diff
+79 -35
View File
@@ -3,7 +3,7 @@ unit Myc.Ast.RTL.Core;
interface
uses
System.SysUtils,
system.sysutils,
Myc.Utils,
Myc.Data.Scalar,
Myc.Data.Value,
@@ -11,140 +11,184 @@ uses
Myc.Ast.Attributes;
type
// Contains the "pure" native implementations.
{ Contains the native implementations for the Myc standard library. }
TRtlFunctions = record
public
// Arithmetic
// --- Arithmetic ---
[TRtlExport('+', Pure)]
[AstDoc('Adds two numbers or concatenates two strings.')]
[AstSignature('(number, number) -> number')]
[AstSignature('(string, string) -> string')]
class function Add(const Args: TArray<TDataValue>): TDataValue; overload; static;
[TRtlExport('-', Pure)]
[AstDoc('Subtracts B from A, or negates A if called with one argument.')]
[AstSignature('(number, number) -> number')]
[AstSignature('(number) -> number')]
class function Subtract(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('*', Pure)]
[AstDoc('Multiplies two numbers.')]
[AstSignature('(number, number) -> number')]
class function Multiply(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('/', Pure)]
[AstDoc('Divides A by B. Returns a Float.')]
[AstDoc('Divides A by B. Always returns a floating point number.')]
[AstSignature('(number, number) -> number')]
class function Divide(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('div', Pure)]
[AstDoc('Integer division.')]
[AstDoc('Performs integer division.')]
[AstSignature('(number, number) -> number')]
class function IntDivide(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('mod', Pure)]
[AstDoc('Remainder of integer division.')]
[AstDoc('Returns the remainder of integer division.')]
[AstSignature('(number, number) -> number')]
class function Modulus(const Args: TArray<TDataValue>): TDataValue; static;
// Comparison
// --- Comparison ---
[TRtlExport('=', Pure)]
[AstDoc('Returns true if A equals B.')]
[AstDoc('Checks if A is equal to B.')]
[AstSignature('(any, any) -> boolean')]
class function Equal(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('<>', Pure)]
[AstDoc('Returns true if A is not equal to B.')]
[AstDoc('Checks if A is not equal to B.')]
[AstSignature('(any, any) -> boolean')]
class function NotEqual(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('<', Pure)]
[AstDoc('Returns true if A is less than B.')]
[AstDoc('Checks if A is less than B.')]
[AstSignature('(number, number) -> boolean')]
class function LessThan(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('<=', Pure)]
[AstDoc('Returns true if A is less than or equal to B.')]
[AstDoc('Checks if A is less than or equal to B.')]
[AstSignature('(number, number) -> boolean')]
class function LessThanOrEqual(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('>', Pure)]
[AstDoc('Returns true if A is greater than B.')]
[AstDoc('Checks if A is greater than B.')]
[AstSignature('(number, number) -> boolean')]
class function GreaterThan(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('>=', Pure)]
[AstDoc('Returns true if A is greater than or equal to B.')]
[AstDoc('Checks if A is greater than or equal to B.')]
[AstSignature('(number, number) -> boolean')]
class function GreaterThanOrEqual(const Args: TArray<TDataValue>): TDataValue; static;
// Logic / Bitwise
// --- Logic / Bitwise ---
[TRtlExport('not', Pure)]
[AstDoc('Logical or bitwise NOT.')]
[AstDoc('Logical or bitwise NOT operation.')]
[AstSignature('(boolean) -> boolean')]
[AstSignature('(number) -> number')]
class function LogicalNot(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('and', Pure)]
[AstDoc('Logical or bitwise AND.')]
[AstDoc('Logical or bitwise AND operation.')]
[AstSignature('(boolean, boolean) -> boolean')]
[AstSignature('(number, number) -> number')]
class function BitwiseAnd(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('or', Pure)]
[AstDoc('Logical or bitwise OR.')]
[AstDoc('Logical or bitwise OR operation.')]
[AstSignature('(boolean, boolean) -> boolean')]
[AstSignature('(number, number) -> number')]
class function BitwiseOr(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('xor', Pure)]
[AstDoc('Logical or bitwise XOR.')]
[AstDoc('Logical or bitwise XOR operation.')]
[AstSignature('(boolean, boolean) -> boolean')]
[AstSignature('(number, number) -> number')]
class function BitwiseXor(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('shl', Pure)]
[AstDoc('Bitwise shift left.')]
[AstSignature('(number, number) -> number')]
class function LeftShift(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('shr', Pure)]
[AstDoc('Bitwise shift right.')]
[AstSignature('(number, number) -> number')]
class function RightShift(const Args: TArray<TDataValue>): TDataValue; static;
// Math Functions
// --- Math Functions ---
[TRtlExport('Abs', Pure)]
[AstDoc('Returns the absolute value.')]
[AstDoc('Returns the absolute value of a number.')]
[AstSignature('(number) -> number')]
class function Abs(const Arg: TScalar): TScalar; static;
[TRtlExport('Round', Pure)]
[AstDoc('Rounds a float to the nearest integer.')]
[AstSignature('(number) -> number')]
class function Round(const Arg: TScalar): TScalar; static;
[TRtlExport('Trunc', Pure)]
[AstDoc('Truncates a float to an integer.')]
[AstDoc('Returns the integer part of a float.')]
[AstSignature('(number) -> number')]
class function Trunc(const Arg: TScalar): TScalar; static;
[TRtlExport('Ceil', Pure)]
[AstDoc('Returns the smallest integer greater than or equal to the argument.')]
[AstDoc('Returns the smallest integer >= the argument.')]
[AstSignature('(number) -> number')]
class function Ceil(const Arg: TScalar): TScalar; static;
[TRtlExport('Floor', Pure)]
[AstDoc('Returns the largest integer less than or equal to the argument.')]
[AstDoc('Returns the largest integer <= the argument.')]
[AstSignature('(number) -> number')]
class function Floor(const Arg: TScalar): TScalar; static;
[TRtlExport('Sign', Pure)]
[AstDoc('Returns -1 for negative numbers, 1 for positive numbers, and 0 for zero.')]
[AstDoc('Returns -1 for negative, 1 for positive, 0 for zero.')]
[AstSignature('(number) -> number')]
class function Sign(const Arg: TScalar): TScalar; static;
// DateTime
// --- DateTime ---
[TRtlExport('Now', Impure)]
[AstDoc('Returns the current system date and time.')]
[AstDoc('Returns the current system timestamp.')]
[AstSignature('() -> datetime')]
class function Now(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('Date', Impure)]
[AstDoc('Returns the current date or constructs a date from (Year, Month, Day).')]
[AstDoc('Returns the current date or constructs a date from Y, M, D.')]
[AstSignature('() -> datetime')]
[AstSignature('(number, number, number) -> datetime')]
class function Date(const Args: TArray<TDataValue>): TDataValue; static;
// High-Order / Series
// --- Functional / Series ---
[TRtlExport('Memoize', Pure)]
[AstDoc('Returns a memoized version of the provided function.')]
[AstDoc('Creates a memoized version of a function for performance.')]
[AstSignature('((any) -> any) -> ((any) -> any)')]
class function Memoize(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('Map', Pure)]
[AstDoc('Maps a function over a series.')]
[AstDoc('Applies a function to every element of a series.')]
[AstSignature('(series, (any) -> any) -> series')]
class function Map(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('Reduce', Pure)]
[AstDoc('Reduces a series using a combining function and an accumulator.')]
[AstDoc('Collapses a series into a single value using a reducer function.')]
[AstSignature('(series, any, (any, any) -> any) -> any')]
class function Reduce(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('Where', Pure)]
[AstDoc('Filters a series using a predicate function.')]
[AstDoc('Filters a series based on a predicate function.')]
[AstSignature('(series, (any) -> boolean) -> series')]
class function Where(const Args: TArray<TDataValue>): TDataValue; static;
[TRtlExport('Any', Pure)]
[AstDoc('Returns true if any element in the series satisfies the predicate.')]
[AstDoc('Returns true if at least one element satisfies the predicate.')]
[AstSignature('(series, (any) -> boolean) -> boolean')]
class function Any(const Args: TArray<TDataValue>): TDataValue; static;
// --- Static Specializations (Picked up by TRtlRegistry via attributes) ---
// --- Static Specializations (Targets for TypeChecker/Specializer) ---
[TRtlExport('+', Pure)]
class function Add_Ordinal_Ordinal_Ordinal(A, B: Int64): Int64; static;
@@ -155,7 +199,7 @@ type
[TRtlExport('+', Pure)]
class function Add_Float_Ordinal_Float(A: Double; B: Int64): Double; static;
[TRtlExport('+', Pure)]
class function Add_Text_Text_Text(const A: String; const B: String): String; static;
class function Add_Text_Text_Text(const A: string; const B: string): string; static;
[TRtlExport('-', Pure)]
class function Subtract_Ordinal_Ordinal_Ordinal(A, B: Int64): Int64; static;