diff --git a/experiments/2026-05-12-cross-model-authoring/format-findings.md b/experiments/2026-05-12-cross-model-authoring/format-findings.md index d6e6c73..44cbaee 100644 --- a/experiments/2026-05-12-cross-model-authoring/format-findings.md +++ b/experiments/2026-05-12-cross-model-authoring/format-findings.md @@ -127,6 +127,17 @@ So the "modern" structured-output channel is the worst surface here, for exactly the reason the rule predicts: AST JSON is maximal extra bookkeeping with no redundant cue. +**Re-run with max_tokens raised 2500 → 8000** (the bad-json cases at 2500 were +truncation, so this is the fair correction): it did NOT help — 1/8, slightly +worse. The full budget just gives the model more room to degenerate. At L2 +("print 7", a trivial program) Qwen emitted **8000 tokens — ~20 KB of AST JSON +— and ran off the end** (a repetition loop on a one-line program); at L7 it +produced JSON nested so deep the parser hit its recursion limit. So the AST-JSON +surface is not budget-limited in a recoverable way: given more room it loops or +nests without end. The verbose surface invites the same degeneration the +annotated-paren run hit at its hardest level — but here it strikes even the +trivial tasks. + ## Overall finding across all six surfaces | surface | green | structure marking | diff --git a/experiments/2026-05-12-cross-model-authoring/qwen-tool-8k.md b/experiments/2026-05-12-cross-model-authoring/qwen-tool-8k.md new file mode 100644 index 0000000..9638adb --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/qwen-tool-8k.md @@ -0,0 +1,631 @@ +# Qwen format probe — tool-calling / structured AST + +**Model:** Qwen/Qwen3-Coder-Next; submit_program tool, forced; AST written directly to .ail.json. + +--- + +## L0_const — ✅ green + +tokens: prompt=1078 completion=162 + +submitted AST: +```json +{ + "schema": "ailang/v0", + "name": "m", + "imports": [], + "defs": [ + { + "kind": "fn", + "name": "answer", + "params": [], + "type": { + "k": "fn", + "param_modes": [], + "params": [], + "ret": { + "k": "con", + "name": "Int", + "effects": [] + }, + "ret_mode": "own", + "effects": [] + }, + "body": { + "lit": { + "kind": "int", + "value": 42 + }, + "t": "lit" + } + } + ] +} +``` + + +## L1_arith — ❌ check + +tokens: prompt=1079 completion=170 + +submitted AST: +```json +{ + "schema": "ailang/v0", + "name": "m", + "imports": [], + "defs": [ + { + "kind": "fn", + "name": "triple", + "params": [ + "x" + ], + "type": { + "k": "fn", + "param_modes": [ + "own" + ], + "params": [ + { + "k": "con", + "name": "Int" + } + ], + "ret": { + "k": "con", + "name": "Int" + }, + "ret_mode": "own", + "effects": [] + }, + "body": { + "t": "app", + "fn": { + "t": "var", + "name": "*" + }, + "args": [ + { + "t": "var", + "name": "x" + }, + { + "t": "var", + "name": "x" + }, + { + "t": "var", + "name": "x" + } + ] + } + } + ] +} +``` + +**check:** +``` +error: [arity-mismatch] triple: arity mismatch for `*`: expected 2 args, got 3 +``` + +## L2_print — ❌ bad-json + +tokens: prompt=1075 completion=8000 + +submitted AST: +```json +{"module": {"schema": "ailang/v0", "name": "m", "imports": [], "defs": [{"kind": "fn", "name": "main", "params": [], "type": {"k": "fn", "param_modes": [], "params": [], "ret": {"k": "con", "name": "Unit"}, "effects": ["IO"]}, "body": {"t": "seq", "lhs": {"t": "app", "fn": {"t": "var", "name": "print"}, "args": [{"t": "app", "fn": {"t": "var", "name": "print"}, "args": [{"t": "var", "name": "x"}]} +``` + +**bad-json:** +``` +tool arguments not valid JSON: Unterminated string starting at: line 1 column 20034 (char 20033) +``` + +## L3_adt — ❌ check + +tokens: prompt=1092 completion=253 + +submitted AST: +```json +{ + "schema": "ailang/v0", + "name": "m", + "imports": [], + "defs": [ + { + "kind": "type", + "name": "Light", + "ctors": [ + { + "name": "Red", + "fields": [] + }, + { + "name": "Green", + "fields": [] + } + ] + }, + { + "kind": "fn", + "name": "code", + "params": [ + "light" + ], + "type": { + "k": "fn", + "param_modes": [ + "own" + ], + "params": [ + { + "k": "con", + "name": "Light" + } + ], + "ret": { + "k": "con", + "name": "Int" + }, + "ret_mode": "own", + "effects": [] + }, + "body": { + "kind": "match", + "scrutinee": { + "name": "light", + "t": "var" + }, + "arms": [ + { + "pat": { + "p": "ctor", + "ctor": "Red" + }, + "body": { + "lit": { + "kind": "int", + "value": 0 + }, + "t": "lit" + } + }, + { + "pat": { + "p": "ctor", + "ctor": "Green" + }, + "body": { + "lit": { + "kind": "int", + "value": 1 + }, + "t": "lit" + } + } + ] + } + } + ] +} +``` + +**check:** +``` +Error: schema/parse error in /tmp/abl/m.ail.json: json: missing field `t` at line 80 column 3 + +Caused by: + 0: json: missing field `t` at line 80 column 3 + 1: missing field `t` at line 80 column 3 +``` + +## L4_rec — ❌ check + +tokens: prompt=1104 completion=1031 + +submitted AST: +```json +{ + "schema": "ailang/v0", + "name": "m", + "imports": [], + "defs": [ + { + "kind": "type", + "name": "IntList", + "ctors": [ + { + "name": "Nil", + "fields": [] + }, + { + "name": "Cons", + "fields": [ + { + "k": "con", + "name": "Int" + }, + { + "k": "con", + "name": "IntList" + } + ] + } + ] + }, + { + "kind": "fn", + "name": "length", + "params": [ + "xs" + ], + "type": { + "k": "fn", + "param_modes": [ + "own" + ], + "params": [ + { + "k": "con", + "name": "IntList" + } + ], + "ret": { + "k": "con", + "name": "Int" + }, + "ret_mode": "own", + "effects": [] + }, + "body": { + "t": "match", + "scrutinee": { + "t": "var", + "name": "xs" + }, + "arms": [ + { + "t": "arm", + "pat": { + "p": "ctor", + "ctor": "Nil", + "fields": [] + }, + "body": { + "t": "lit", + "lit": { + "kind": "int", + "value": 0 + } + }, + "arms": [ + { + "t": "arm", + "pat": { + "p": "ctor", + "ctor": "Cons", + "fields": [ + { + "p": "var", + "name": "h" + }, + { + "p": "var", + "name": "t" + } + ] + }, + "body": { + "t": "app", + "fn": { + "t": "var", + "name": " +``` + +**check:** +``` +Error: schema/parse error in /tmp/abl/m.ail.json: json: missing field `lit` at line 227 column 3 + +Caused by: + 0: json: missing field `lit` at line 227 column 3 + 1: missing field `lit` at line 227 column 3 +``` + +## L5_series_len — ❌ check + +tokens: prompt=1494 completion=651 + +submitted AST: +```json +{ + "schema": "ailang/v0", + "name": "m", + "imports": [], + "defs": [ + { + "kind": "fn", + "name": "s", + "params": [], + "type": { + "k": "fn", + "param_modes": [], + "params": [], + "ret": { + "k": "con", + "name": "Unit" + }, + "ret_mode": "own", + "effects": [ + "IO" + ] + }, + "body": { + "body": { + "body": { + "body": { + "lhs": { + "args": [ + { + "name": "s3", + "t": "var" + } + ], + "fn": { + "name": "Series.len", + "t": "var" + }, + "t": "app" + }, + "fn": { + "name": "print", + "t": "var" + }, + "t": "app" + }, + "rhs": { + "args": [ + { + "kind": "str", + "value": "\n", + "t": "lit" + } + ], + "op": "io/print_str", + "t": "do" + }, + "t": "seq" + }, + "name": "s3", + "t": "let", + "value": { + "args": [ + { + "name": "s2", + "t": "var" + }, + { + "kind": "float", + "bits": "400000000000000", + "t": "lit" + } + ], + "fn": { + "name": "Series.push", + "t": "var" + }, + "t": "app" + } + }, + "name": "s2", + "t": "let", + "value": { + "args": [ + { + "name": "s", + "t": "var" + }, + { + "kind": "float", + "bits": "3ff00 +``` + +**check:** +``` +Error: schema/parse error in /tmp/abl/m.ail.json: json: missing field `lit` at line 124 column 3 + +Caused by: + 0: json: missing field `lit` at line 124 column 3 + 1: missing field `lit` at line 124 column 3 +``` + +## L6_series_at — ❌ check + +tokens: prompt=1502 completion=740 + +submitted AST: +```json +{ + "schema": "ailang/v0", + "name": "m", + "imports": [], + "defs": [ + { + "kind": "fn", + "name": "main", + "params": [], + "type": { + "k": "fn", + "param_modes": [], + "params": [], + "ret": { + "k": "con", + "name": "Unit" + }, + "ret_mode": "own", + "effects": [ + "IO" + ] + }, + "body": { + "body": { + "body": { + "body": { + "lhs": { + "args": [ + { + "args": [ + { + "name": "s3", + "t": "var" + }, + { + "kind": "value", + "value": { + "lit": { + "kind": "int", + "value": 0 + }, + "t": "lit" + } + } + ], + "fn": { + "name": "Series.at", + "t": "var" + }, + "t": "app" + } + ], + "fn": { + "name": "print", + "t": "var" + }, + "t": "app" + }, + "rhs": { + "args": [ + { + "lit": { + "kind": "str", + "value": "\n" + }, + "t": "lit" + }, + { + "op": "io/print_str", + "t": "do" + } + ], + "t": "seq" + }, + "name": "s3", + "t": "let", + "value": { + "args": [ + { + "name": "s2", + "t": "var" + }, + { + +``` + +**check:** +``` +Error: schema/parse error in /tmp/abl/m.ail.json: json: missing field `t` at line 149 column 3 + +Caused by: + 0: json: missing field `t` at line 149 column 3 + 1: missing field `t` at line 149 column 3 +``` + +## L7_sma — ❌ check + +tokens: prompt=1588 completion=4382 + +submitted AST: +```json +{ + "schema": "ailang/v0", + "name": "m", + "imports": [], + "defs": [ + { + "kind": "fn", + "name": "main", + "params": [], + "type": { + "k": "fn", + "param_modes": [], + "params": [], + "ret": { + "k": "con", + "name": "Unit" + }, + "ret_mode": "own", + "effects": [ + "IO" + ] + }, + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + "body": { + +``` + +**check:** +``` +Error: schema/parse error in /tmp/abl/m.ail.json: json: recursion limit exceeded at line 147 column 263 + +Caused by: + 0: json: recursion limit exceeded at line 147 column 263 + 1: recursion limit exceeded at line 147 column 263 +``` + + +--- +## Ladder + +- L0_const: ✅ green +- L1_arith: ❌ check +- L2_print: ❌ bad-json +- L3_adt: ❌ check +- L4_rec: ❌ check +- L5_series_len: ❌ check +- L6_series_at: ❌ check +- L7_sma: ❌ check \ No newline at end of file diff --git a/experiments/2026-05-12-cross-model-authoring/qwen_tool.py b/experiments/2026-05-12-cross-model-authoring/qwen_tool.py index 7639d16..3869b03 100644 --- a/experiments/2026-05-12-cross-model-authoring/qwen_tool.py +++ b/experiments/2026-05-12-cross-model-authoring/qwen_tool.py @@ -16,7 +16,7 @@ ENDPOINT="https://openai.inference.de-txl.ionos.com/v1/chat/completions" MODEL="Qwen/Qwen3-Coder-Next" REPO=Path(__file__).resolve().parents[2] AIL=os.environ.get("AIL_BIN", str(REPO/"target/debug/ail")) -DOC=REPO/"experiments/2026-05-12-cross-model-authoring/qwen-tool.md" +DOC=REPO/"experiments/2026-05-12-cross-model-authoring/qwen-tool-8k.md" def ail_ast(src): p=Path("/tmp/abl/_fs.ail"); p.write_text(src+"\n") @@ -84,7 +84,7 @@ STAGES=[ ] def call(system,task): - body=json.dumps({"model":MODEL,"temperature":0.2,"max_tokens":2500, + body=json.dumps({"model":MODEL,"temperature":0.2,"max_tokens":8000, "messages":[{"role":"system","content":system},{"role":"user","content":task}], "tools":[TOOL],"tool_choice":{"type":"function","function":{"name":"submit_program"}}}).encode() req=urllib.request.Request(ENDPOINT,data=body,headers={