diff --git a/experiments/2026-05-12-cross-model-authoring/format-findings.md b/experiments/2026-05-12-cross-model-authoring/format-findings.md new file mode 100644 index 0000000..1c5e6cc --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/format-findings.md @@ -0,0 +1,72 @@ +# Do alternative surface formats help Qwen? — format 4 (annotated parens) + +**Date:** 2026-06-02 **Model:** Qwen/Qwen3-Coder-Next (IONOS) +**Question (user):** the fully-parenthesised surface is only one encoding of +the tree. If no human must read it, try formats that lift the paren-counting +burden — indent, annotated indent, keywords, annotated parens. Does any help? + +**Discipline (the "watch out"):** these formats have NO `ail` parser, so +`ail check` cannot judge them directly. For each, a converter (format → Form-A) +is built and **round-trip-verified** (Form-A → format → back must be +parse-identical to the original, on the known-good demos) BEFORE any Qwen +output is judged. Format 4's converter passed: parse-identity + green check on +all three demos, and end-to-end on a fenced annotated program. + +## Format 4 tested: annotated parens `(#N ... #N)` + +Every paren carries its nesting depth; opener `(#N ` pairs with closer ` #N)`. +Qwen writes the annotated dialect; the verified stripper removes the +annotations back to plain Form-A, then `ail check`/`run`. Same ablation ladder +as the plain-Form-A baseline, so results are comparable. Raw: `qwen-fmt4.md`, +`qwen-fmt4-seqhint.md`. + +### Result: it helps, measurably + +| Level | plain Form-A | **format 4** | +|---|---|---| +| L0–L3 | ✅ | ✅ | +| **L4** build-list + recursion | ❌ paren misplaced | **✅** | +| L5, L6 Series | ✅ | ✅ | +| L7 SMA | ❌ | ❌ (but different — see below) | + +Plain Form-A: 6/8. **Format 4: 7/8.** The annotation cracked the +bracket-balance wall that broke L4 — the per-paren depth keeps openers and +closers paired. + +### What format 4 does to L7 (the hardest level) + +In plain Form-A, L7 failed on bracket imbalance. In format 4, Qwen produced +**perfectly balanced brackets even at the deepest nesting** (the depth +annotations all matched through the whole SMA) AND the **correct SMA logic** +(`(/ (+ (at 0) (+ (at 1) (at 2))) 3.0)`). Proof the building blocks were +right: taking Qwen's exact logic and only fixing the one remaining issue makes +it run and emit the exact expected output (`3.0 / 5.33333 / 5.66667 / 5.33333`) +— see `/tmp/abl/qwen_seqfix.ail` reconstruction in session. + +The one remaining issue was **not a format problem**: Qwen treats `seq` as +variadic (Lisp `progn`) — `(seq A B C)` — but AILang `seq` is strictly binary. + +### Pushing one more level (format 4 + an explicit `seq`-is-binary hint) + +Gave Qwen the binary-`seq` rule plus a nested-`seq` example, re-ran L7. It +**fixed the seq nesting** (correct `(seq (seq print nl) (let …))`), but L7 +still failed — now on two model-behaviour limits, not surface encoding: +1. it **simplified the logic** (printed `at 0` instead of the average), and +2. a **termination-degeneration tail**: hundreds of repeated `#0)` — Qwen + *knows* via `#0` it is at the outermost level, yet keeps closing. A pure + repetition loop, the same failure mode the ~80k-token full-SMA run hit. + +## Takeaway + +- **Annotated parens (format 4) measurably help**: they crack the + bracket-balance wall (L4 green; L7 brackets perfect). For everything up to + medium-deep nesting, the format lifts the exact burden the ablation isolated. +- **The hardest level (L7) is no longer a bracketing problem** — it is (a) a + construct-arity quirk (`seq` binary vs variadic; fixable by an example or by + making `seq` n-ary) and (b) a single-shot complexity ceiling where the model + simplifies and falls into a closing-token repetition loop. +- The repetition tail is paren-driven (`#0)` repeated). The natural next test + is a **bracket-free** format (indent, or keyword-delimited blocks) where + there is no closing token to loop on — that directly targets the one limit + format 4 does not remove. It needs a new round-trip-verified converter + (an offside / keyword parser), so it is a deliberate next step, not run here. diff --git a/experiments/2026-05-12-cross-model-authoring/qwen-fmt4-seqhint.md b/experiments/2026-05-12-cross-model-authoring/qwen-fmt4-seqhint.md new file mode 100644 index 0000000..27eb241 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/qwen-fmt4-seqhint.md @@ -0,0 +1,29 @@ +# Format 4 + seq-binary hint — L7 SMA only + +Result: ❌ check + +stripped Form-A: +``` +(module m + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body + (let s (new Series (con Float) 3) + (let s (app Series.push s 1.0) + (let s (app Series.push s 5.0) + (seq (seq (app print (app Series.at s 0)) (do io/print_str "\n")) + (let s (app Series.push s 3.0) + (seq (seq (app print (app Series.at s 0)) (do io/print_str "\n")) + (let s (app Series.push s 8.0) + (seq (seq (app print (app Series.at s 0)) (do io/print_str "\n")) + (let s (app Series.push s 6.0) + (seq (seq (app print (app Series.at s 0)) (do io/print_str "\n")) + (let s (app Series.push s 2.0) + (seq (app print (app Series.at s 0)) (do io/print_str "\n"))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))) #0 +``` + +error (check): +``` +error: [surface-parse-error] parse error: expected end of input, got `)` at byte 872 +``` diff --git a/experiments/2026-05-12-cross-model-authoring/qwen-fmt4.md b/experiments/2026-05-12-cross-model-authoring/qwen-fmt4.md new file mode 100644 index 0000000..6ceeb4f --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/qwen-fmt4.md @@ -0,0 +1,268 @@ +# Qwen format probe 4 — annotated parens `(#N ... #N)` + +**Model:** Qwen/Qwen3-Coder-Next; single-shot per level; annotated-paren dialect; output stripped via round-trip-verified converter then ail-checked. + +--- + +## L0_const — ✅ green + +tokens: prompt=675 completion=74 + +annotated output: +``` +(#0 module m + (#1 fn answer + (#2 type (#3 fn-type (#4 params #4) (#4 ret (#5 own (#6 con Int #6) #5) #4) #3) #2) + (#2 params #2) + (#2 body 42 #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (fn answer + (type (fn-type (params) (ret (own (con Int))))) + (params) + (body 42))) +``` + + +## L1_arith — ✅ green + +tokens: prompt=677 completion=95 + +annotated output: +``` +(#0 module m + (#1 fn triple + (#2 type (#3 fn-type (#4 params (#5 own (#6 con Int #6) #5) #4) (#4 ret (#5 own (#6 con Int #6) #5) #4) #3) #2) + (#2 params x #2) + (#2 body (#3 app * x 3 #3) #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (fn triple + (type (fn-type (params (own (con Int))) (ret (own (con Int))))) + (params x) + (body (app * x 3)))) +``` + + +## L2_print — ✅ green + +tokens: prompt=672 completion=105 + +annotated output: +``` +(#0 module m + (#1 fn main + (#2 type (#3 fn-type (#4 params #4) (#4 ret (#5 own (#6 con Unit #6) #5) #4) (#4 effects IO #4) #3) #2) + (#2 params #2) + (#2 body (#3 seq (#4 app print 7 #4) (#4 do io/print_str "\n" #4) #3) #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body (seq (app print 7) (do io/print_str "\n"))))) +``` + + +## L3_adt — ✅ green + +tokens: prompt=691 completion=151 + +annotated output: +``` +(#0 module m + (#1 data Light (#2 ctor Red #2) (#2 ctor Green #2) #1) + (#1 fn code + (#2 type (#3 fn-type (#4 params (#5 own (#6 con Light #6) #5) #4) (#4 ret (#5 own (#6 con Int #6) #5) #4) #3) #2) + (#2 params x #2) + (#2 body (#3 match x + (#4 case (#5 pat-ctor Red #5) 0 #4) + (#4 case (#5 pat-ctor Green #5) 1 #4) #3) #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (data Light (ctor Red) (ctor Green)) + (fn code + (type (fn-type (params (own (con Light))) (ret (own (con Int))))) + (params x) + (body (match x + (case (pat-ctor Red) 0) + (case (pat-ctor Green) 1))))) +``` + + +## L4_rec — ✅ green + +tokens: prompt=703 completion=349 + +annotated output: +``` +(#0 module m + (#1 data IntList (#2 ctor Nil #2) (#2 ctor Cons (#3 con Int #3) (#3 con IntList #3) #2) #1) + (#1 fn length + (#2 type (#3 fn-type (#4 params (#5 own (#6 con IntList #6) #5) #4) (#4 ret (#5 own (#6 con Int #6) #5) #4) #3) #2) + (#2 params xs #2) + (#2 body + (#3 match xs + (#4 case (#5 pat-ctor Nil #5) 0 #4) + (#4 case (#5 pat-ctor Cons h t #5) (#5 app + 1 (#6 app length t #6) #5) #4) #3) #2) #1) + (#1 fn main + (#2 type (#3 fn-type (#4 params #4) (#4 ret (#5 own (#6 con Unit #6) #5) #4) (#4 effects IO #4) #3) #2) + (#2 params #2) + (#2 body + (#3 let xs (#4 term-ctor IntList Cons 1 (#5 term-ctor IntList Cons 2 (#6 term-ctor IntList Cons 3 (#7 term-ctor IntList Nil #7) #6) #5) #4) + (#4 seq (#5 app print (#6 app length xs #6) #5) (#5 do io/print_str "\n" #5) #4) #3) #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (data IntList (ctor Nil) (ctor Cons (con Int) (con IntList))) + (fn length + (type (fn-type (params (own (con IntList))) (ret (own (con Int))))) + (params xs) + (body + (match xs + (case (pat-ctor Nil) 0) + (case (pat-ctor Cons h t) (app + 1 (app length t)))))) + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body + (let xs (term-ctor IntList Cons 1 (term-ctor IntList Cons 2 (term-ctor IntList Cons 3 (term-ctor IntList Nil)))) + (seq (app print (app length xs)) (do io/print_str "\n")))))) +``` + + +## L5_series_len — ✅ green + +tokens: prompt=964 completion=201 + +annotated output: +``` +(#0 module m + (#1 fn main + (#2 type (#3 fn-type (#4 params #4) (#4 ret (#5 own (#6 con Unit #6) #5) #4) (#4 effects IO #4) #3) #2) + (#2 params #2) + (#2 body + (#3 let s (#4 new Series (#5 con Float #5) 3 #4) + (#4 let s (#5 app Series.push s 1.0 #5) + (#5 let s (#6 app Series.push s 2.0 #6) + (#6 let s (#7 app Series.push s 3.0 #7) + (#7 seq (#8 app print (#9 app Series.len s #9) #8) (#8 do io/print_str "\n" #8) #7) #6) #5) #4) #3) #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body + (let s (new Series (con Float) 3) + (let s (app Series.push s 1.0) + (let s (app Series.push s 2.0) + (let s (app Series.push s 3.0) + (seq (app print (app Series.len s)) (do io/print_str "\n"))))))))) +``` + + +## L6_series_at — ✅ green + +tokens: prompt=972 completion=203 + +annotated output: +``` +(#0 module m + (#1 fn main + (#2 type (#3 fn-type (#4 params #4) (#4 ret (#5 own (#6 con Unit #6) #5) #4) (#4 effects IO #4) #3) #2) + (#2 params #2) + (#2 body + (#3 let s (#4 new Series (#5 con Float #5) 3 #4) + (#4 let s (#5 app Series.push s 1.0 #5) + (#5 let s (#6 app Series.push s 2.0 #6) + (#6 let s (#7 app Series.push s 3.0 #7) + (#7 seq (#8 app print (#9 app Series.at s 0 #9) #8) (#8 do io/print_str "\n" #8) #7) #6) #5) #4) #3) #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body + (let s (new Series (con Float) 3) + (let s (app Series.push s 1.0) + (let s (app Series.push s 2.0) + (let s (app Series.push s 3.0) + (seq (app print (app Series.at s 0)) (do io/print_str "\n"))))))))) +``` + + +## L7_sma — ❌ check + +tokens: prompt=1054 completion=1800 + +annotated output: +``` +(#0 module m + (#1 fn main + (#2 type (#3 fn-type (#4 params #4) (#4 ret (#5 own (#6 con Unit #6) #5) #4) (#4 effects IO #4) #3) #2) + (#2 params #2) + (#2 body + (#3 let s (#4 new Series (#5 con Float #5) 3 #4) + (#4 let s (#5 app Series.push s 1.0 #5) + (#5 let s (#6 app Series.push s 5.0 #6) + (#6 let s (#7 app Series.push s 3.0 #7) + (#7 seq (#8 app print (#9 app / (#10 app + (#11 app Series.at s 0 #11) (#12 app + (#13 app Series.at s 1 #13) (#14 app Series.at s 2 #14) #12) #11) 3.0 #9) #8) (#8 do io/print_str "\n" #8) (#8 let s (#9 app Series.push s 8.0 #9) + (#9 seq (#10 app print (#11 app / (#12 app + (#13 app Series.at s 0 #13) (#14 app + (#15 app Series.at s 1 #15) (#16 app Series.at s 2 #16) #14) #12) 3.0 #11) #10) (#10 do io/print_str "\n" #10) (#10 let s (#11 app Series.push s 6.0 #11) + (#11 seq (#12 app print (#13 app / (#14 app + (#15 app Series.at s 0 #15) (#16 app + (#17 app Series.at s 1 #17) (#18 app Series.at s 2 #18) #16) #14) 3.0 #13) #12) (#12 do io/print_str "\n" #12) (#12 let s (#13 app Series.push s 2.0 #13) + (#13 seq (#14 app print (#15 app / (#16 app + (#17 app Series.at s 0 #17) (#18 app + (#19 app Series.at s 1 #19) (#20 app Series.at s 2 #20) #18) #16) 3.0 #15) #14) (#14 do io/print_str "\n" #14) (#14 do io/print_str "\n" #14) #13) #12) #11) #10) #9) #8) #7) #6) #5) #4) #3) #2) #1) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) #0) +``` + +stripped to Form-A: +``` +(module m + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body + (let s (new Series (con Float) 3) + (let s (app Series.push s 1.0) + (let s (app Series.push s 5.0) + (let s (app Series.push s 3.0) + (seq (app print (app / (app + (app Series.at s 0) (app + (app Series.at s 1) (app Series.at s 2))) 3.0)) (do io/print_str "\n") (let s (app Series.push s 8.0) + (seq (app print (app / (app + (app Series.at s 0) (app + (app Series.at s 1) (app Series.at s 2))) 3.0)) (do io/print_str "\n") (let s (app Series.push s 6.0) + (seq (app print (app / (app + (app Series.at s 0) (app + (app Series.at s 1) (app Series.at s 2))) 3.0)) (do io/print_str "\n") (let s (app Series.push s 2.0) + (seq (app print (app / (app + (app Series.at s 0) (app + (app Series.at s 1) (app Series.at s 2))) 3.0)) (do io/print_str "\n") (do io/print_str "\n")))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))) +``` + +**`ail check`:** +``` +error: [surface-parse-error] parse error: expected `)` (end of seq-term), got `(` at byte 415 +``` + + +--- +## Ladder + +- L0_const: ✅ green +- L1_arith: ✅ green +- L2_print: ✅ green +- L3_adt: ✅ green +- L4_rec: ✅ green +- L5_series_len: ✅ green +- L6_series_at: ✅ green +- L7_sma: ❌ check \ No newline at end of file diff --git a/experiments/2026-05-12-cross-model-authoring/qwen_fmt4.py b/experiments/2026-05-12-cross-model-authoring/qwen_fmt4.py new file mode 100644 index 0000000..bf9e757 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/qwen_fmt4.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Format probe 4 — annotated parens. Does giving every paren its nesting +depth (`(#N ` ... ` #N)`) help Qwen keep brackets balanced at depth? + +The model writes the annotated dialect; we STRIP the annotations with a +round-trip-verified converter (see qwen_fmt4 verification in session) back to +plain Form-A, then `ail check` / `ail run`. Strip-variant: we trust the bare +parens, the annotation is the model's self-discipline scaffold. Same ladder as +qwen_ablation.py so results are comparable. + +Reads $IONOS_API_TOKEN (never printed). Live IONOS — run with consent. +""" +from __future__ import annotations +import json, os, re, subprocess, urllib.request +from pathlib import Path + +ENDPOINT = "https://openai.inference.de-txl.ionos.com/v1/chat/completions" +MODEL = "Qwen/Qwen3-Coder-Next" +REPO = Path(__file__).resolve().parents[2] +AIL = os.environ.get("AIL_BIN", str(REPO / "target/debug/ail")) +DOC = REPO / "experiments/2026-05-12-cross-model-authoring/qwen-fmt4.md" + +def annotate(src): + out=[]; depth=0; in_str=False; esc=False + for ch in src: + if in_str: + out.append(ch) + if esc: esc=False + elif ch=='\\': esc=True + elif ch=='"': in_str=False + continue + if ch=='"': in_str=True; out.append(ch) + elif ch=='(': out.append(f'(#{depth} '); depth+=1 + elif ch==')': depth-=1; out.append(f' #{depth})') + else: out.append(ch) + return ''.join(out) + +def strip_ann(src): + out=[]; i=0; in_str=False; esc=False + while iown series appends & returns; " + "Series.at (borrow series)(own Int)->own elem reads index back from " + "newest (0=newest); Series.len / Series.total_count " + "(borrow series)->own Int):\n" + annotate(S) + "\n") + +SMA_OUT = "3.0\n5.33333\n5.66667\n5.33333\n" +SMA_TASK = ("Write a module `m` computing a simple moving average, window 3, over " + "the float stream 1.0, 5.0, 3.0, 8.0, 6.0, 2.0 using the Series library: " + "create a Float Series lookback 3, push each value in turn, and after each " + "push once at least 3 values have been pushed, print the average of the " + "most recent 3 (their sum / 3.0) on its own line. Expected:\n" + SMA_OUT) + +STAGES = [ + ("L0_const", BASE, "Write a module `m` with a function `answer` taking no parameters that returns the Int 42.", None), + ("L1_arith", BASE, "Write a module `m` with a function `triple` taking one Int parameter, returning it multiplied by 3.", None), + ("L2_print", BASE, "Write a module `m` whose `main` prints the Int 7 on its own line.", "7\n"), + ("L3_adt", BASE, "Write a module `m` with an ADT `Light` (ctors Red, Green) and a function `code` returning Int 0 for Red and 1 for Green via match.", None), + ("L4_rec", BASE, "Write a module `m` with an IntList ADT (Nil/Cons), a function `length` counting elements by recursion, and a `main` that prints the length of the list [1,2,3] on its own line.", "3\n"), + ("L5_series_len", SERIES, "Write a module `m` using the Series library: create a Float Series lookback 3, push 1.0 then 2.0 then 3.0, then print its Series.len on its own line.", "3\n"), + ("L6_series_at", SERIES, "Write a module `m` using the Series library: create a Float Series lookback 3, push 1.0 then 2.0 then 3.0, then print (Series.at on index 0, the newest) on its own line.", "3.0\n"), + ("L7_sma", SERIES, SMA_TASK, SMA_OUT), +] + +def call(system, task): + body=json.dumps({"model":MODEL,"temperature":0.2,"max_tokens":1800, + "messages":[{"role":"system","content":system},{"role":"user","content":task}]}).encode() + req=urllib.request.Request(ENDPOINT,data=body,headers={ + "Authorization":f"Bearer {os.environ['IONOS_API_TOKEN']}","Content-Type":"application/json"}) + with urllib.request.urlopen(req,timeout=120) as r: + d=json.loads(r.read()) + return d["choices"][0]["message"]["content"], d.get("usage",{}) + +def extract(resp): + m=re.search(r"```(?:[a-zA-Z]*)?\n(.*?)```",resp,re.S) + code=m.group(1) if m else resp + i=code.find("(#0") + if i<0: i=code.find("(module") + return code[i:].strip() if i>=0 else code.strip() + +def evaluate(annotated_code, expected): + plain = strip_ann(annotated_code) # <-- verified converter + m=re.search(r"\(module\s+([A-Za-z_][A-Za-z0-9_]*)", plain) + name=m.group(1) if m else "m" + f=Path(f"/tmp/abl/{name}.ail"); f.parent.mkdir(exist_ok=True); f.write_text(plain+"\n") + chk=subprocess.run([AIL,"check",str(f)],capture_output=True,text=True) + if chk.returncode!=0: return "check",(chk.stderr or chk.stdout).strip(), plain + if expected is None: return "green","(check-only)", plain + run=subprocess.run([AIL,"run",str(f)],capture_output=True,text=True) + if run.returncode!=0: return "run",(run.stderr or run.stdout).strip(), plain + if run.stdout!=expected: return "output",f"got {run.stdout!r}, expected {expected!r}", plain + return "green",run.stdout, plain + +def main(): + log=["# Qwen format probe 4 — annotated parens `(#N ... #N)`\n", + f"**Model:** {MODEL}; single-shot per level; annotated-paren dialect; " + "output stripped via round-trip-verified converter then ail-checked.\n","---\n"] + summary=[] + for sid,ctx,task,exp in STAGES: + resp,usage=call(ctx,task) + code=extract(resp) + stage,fb,plain=evaluate(code,exp) + ok=stage=="green"; mark="✅ green" if ok else f"❌ {stage}" + summary.append((sid,mark)); print(f"{sid}: {mark}") + log+=[f"## {sid} — {mark}", + f"\ntokens: prompt={usage.get('prompt_tokens','?')} completion={usage.get('completion_tokens','?')}\n", + "annotated output:\n```\n"+code+"\n```\n", + "stripped to Form-A:\n```\n"+plain+"\n```\n", + ("" if ok else f"**`ail {stage}`:**\n```\n{fb[:700]}\n```\n")] + log+=["\n---\n## Ladder\n"]+[f"- {s}: {m}" for s,m in summary] + DOC.write_text("\n".join(log)); print("doc →",DOC) + +if __name__=="__main__": + main()