From ded28e2f75e6285ab8dc70f0198d6322ed699be9 Mon Sep 17 00:00:00 2001 From: Brummel Date: Tue, 2 Jun 2026 19:59:17 +0200 Subject: [PATCH] =?UTF-8?q?experiment(cma):=20large-model=20control=20on?= =?UTF-8?q?=20Llama-3.1-405B=20=E2=80=94=20the=20format=20effect=20REVERSE?= =?UTF-8?q?S=20(refs=20#68)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ran the two core formats (plain Form-A, annotated parens) on Llama-3.1-405B (405B dense, ~135x Qwen3-Coder-Next's 3B active) to separate format-effect from small-model-effect. Scripts parametrized by PROBE_MODEL/PROBE_TAG (Qwen default preserved). Watch-out caught: Llama-3.1 on IONOS leaks its chat template (repeats "assistant", generates endless variants to max_tokens — an IONOS-documented quirk). First run read 0/8; that was the instrument. Fixed with a stop sequence and a balanced-first-module extractor. Result reverses the effect: Qwen3-Coder (3B): plain 6/8, annotated 7/8 Llama-405B (405B): plain 7/8, annotated 6/8 Llama solves L4 (Qwen's bracket wall) with plain parens, so that wall was a small-model effect. Annotated parens help the small model and hurt the large one (at L1 Llama invented a spurious (lit 3) wrapper under the annotated dialect). Both still fail L7 (SMA) — a genuine complexity ceiling. Synthesis: redundant explicit structure is a crutch for weak models. A frontier model tracks plain brackets fine and the unfamiliar scaffolding only adds noise. The right surface is model-relative. Synthesis in format-findings.md; raw ladders llama405-*.md. --- .../format-findings.md | 40 +++ .../llama405-ablation-ctor.md | 169 ++++++++++ .../llama405-fmt4.md | 304 ++++++++++++++++++ .../qwen_ablation.py | 33 +- .../qwen_fmt4.py | 26 +- 5 files changed, 553 insertions(+), 19 deletions(-) create mode 100644 experiments/2026-05-12-cross-model-authoring/llama405-ablation-ctor.md create mode 100644 experiments/2026-05-12-cross-model-authoring/llama405-fmt4.md diff --git a/experiments/2026-05-12-cross-model-authoring/format-findings.md b/experiments/2026-05-12-cross-model-authoring/format-findings.md index cb96639..8d0c9ea 100644 --- a/experiments/2026-05-12-cross-model-authoring/format-findings.md +++ b/experiments/2026-05-12-cross-model-authoring/format-findings.md @@ -196,3 +196,43 @@ never tried here: `meta-llama/Meta-Llama-3.1-405B-Instruct-FP8` (405B dense — `mistralai/Mistral-Small-24B-Instruct`. The obvious next step to separate "format effect" from "small-model effect" is to re-run the annotated-paren vs plain-paren comparison on Llama-3.1-405B. + +## Large-model control: Llama-3.1-405B (researched + run 2026-06-02) + +To separate "format effect" from "small-model effect", re-ran the two core +formats on `meta-llama/Meta-Llama-3.1-405B-Instruct-FP8` (405B dense — ~135× +Qwen3-Coder-Next's 3B active; the 2024 open frontier model, [IONOS](https://docs.ionos.com/cloud/ai/ai-model-hub/models/llms/meta-llama-3-1-405b), +[Meta](https://ai.meta.com/blog/meta-llama-3-1/)). + +**Measurement watch-out (caught and fixed):** Llama-3.1 on IONOS leaks its chat +template — without the Llama-3 stop tokens it repeats `assistant` and generates +endless program variants until max_tokens (IONOS documents this "repetitive +output that does not terminate"). The first run looked like 0/8; that was the +instrument, not the model. Fixed with `stop` + a balanced-first-module +extractor before judging anything. + +**Result — the effect REVERSES:** + +| | plain Form-A | annotated parens | +|---|---|---| +| Qwen3-Coder-Next (3B active) | 6/8 | **7/8** | +| Llama-3.1-405B (405B active) | **7/8** | 6/8 | + +- Llama solves **L4 with plain parens** — Qwen's bracket wall. So that wall was + a *small-model effect*: the 405B model tracks plain brackets fine. +- The annotated parens **help the small model (6→7) and hurt the large one + (7→6)**. At L1 Llama, given the annotated dialect, invented a spurious + `(lit 3)` wrapper for the integer `3` (it wrote plain `3` correctly in plain + Form-A) — the unfamiliar surface induces structure-hallucination in a model + that doesn't need the redundant cue. +- Both models still fail L7 (SMA) — a genuine single-shot complexity ceiling, + not a surface problem. + +**Synthesis:** redundant explicit structure (depth-annotated brackets) is a +*crutch for weak models*. It lifts the structure-tracking load a 3B-active +model struggles with, but a large capable model already tracks plain brackets +and the extra, unfamiliar scaffolding only adds noise. The lever is +model-relative: for the LLM that AILang actually targets, the right surface +depends on how strong that authoring model is — a small local model benefits +from redundant structure, a frontier model is better served by the clean +canonical form. diff --git a/experiments/2026-05-12-cross-model-authoring/llama405-ablation-ctor.md b/experiments/2026-05-12-cross-model-authoring/llama405-ablation-ctor.md new file mode 100644 index 0000000..0349705 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/llama405-ablation-ctor.md @@ -0,0 +1,169 @@ +# Qwen3-Coder-Next ablation — where does Form-A authoring break? + +**Model:** meta-llama/Meta-Llama-3.1-405B-Instruct-FP8; single-shot per level; minimal few-shot context. + +Each level is independent (fresh context, one call, no retry). + +--- + +## L0_const — ✅ green + +ctx=BASE tokens: prompt=395 completion=38 + +``` +(module m + (fn answer + (type (fn-type (params) (ret (own (con Int))))) + (params) + (body 42))) +``` + + +## L1_arith — ✅ green + +ctx=BASE tokens: prompt=398 completion=48 + +``` +(module m + (fn triple + (type (fn-type (params (own (con Int))) (ret (own (con Int))))) + (params x) + (body (app * x 3)))) +``` + + +## L2_print — ✅ green + +ctx=BASE tokens: prompt=393 completion=56 + +``` +(module m + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body (seq (app print 7) (do io/print_str "\n"))))) +``` + + +## L3_adt — ✅ green + +ctx=BASE tokens: prompt=412 completion=84 + +``` +(module m + (data Light (ctor Red) (ctor Green)) + (fn code + (type (fn-type (params (own (con Light))) (ret (own (con Int))))) + (params x) + (body + (match x + (case (pat-ctor Red) 0) + (case (pat-ctor Green) 1))))) +``` + + +## L4_rec — ✅ green + +ctx=BASE tokens: prompt=424 completion=196 + +``` +(module m + (data IntList (ctor Nil) (ctor Cons (con Int) (con IntList))) + (fn length + (type (fn-type (params (own (con IntList))) (ret (own (con Int))))) + (params xs) + (body + (match xs + (case (pat-ctor Nil) 0) + (case (pat-ctor Cons _ t) (app + 1 (app length t)))))) + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body + (let xs (term-ctor IntList Cons 1 (term-ctor IntList Cons 2 (term-ctor IntList Cons 3 (term-ctor IntList Nil)))) + (seq (app print (app length xs)) (do io/print_str "\n")))))) +``` + + +## L5_series_len — ✅ green + +ctx=SERIES tokens: prompt=634 completion=124 + +``` +(module m + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body + (let s (new Series (con Float) 3) + (let s2 (app Series.push s 1.0) + (let s3 (app Series.push s2 2.0) + (let s4 (app Series.push s3 3.0) + (seq (app print (app Series.len s4)) (do io/print_str "\n"))))))))) +``` + + +## L6_series_at — ✅ green + +ctx=SERIES tokens: prompt=642 completion=126 + +``` +(module m + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body + (let s (new Series (con Float) 3) + (let s2 (app Series.push s 1.0) + (let s3 (app Series.push s2 2.0) + (let s4 (app Series.push s3 3.0) + (seq (app print (app Series.at s4 0)) (do io/print_str "\n"))))))))) +``` + + +## L7_sma — ❌ check + +ctx=SERIES tokens: prompt=714 completion=363 + +``` +(module m + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body + (let s (new Series (con Float) 3) + (let s2 (app Series.push s 1.0) + (let s3 (app Series.push s2 5.0) + (let s4 (app Series.push s3 3.0) + (seq + (app print (/ (app + (app Series.at s4 0) (app + (app Series.at s4 1) (app Series.at s4 2))) 3.0)) + (do io/print_str "\n") + (let s5 (app Series.push s4 8.0) + (seq + (app print (/ (app + (app Series.at s5 0) (app + (app Series.at s5 1) (app Series.at s5 2))) 3.0)) + (do io/print_str "\n") + (let s6 (app Series.push s5 6.0) + (seq + (app print (/ (app + (app Series.at s6 0) (app + (app Series.at s6 1) (app Series.at s6 2))) 3.0)) + (do io/print_str "\n") + (let s7 (app Series.push s6 2.0) + (app print (/ (app + (app Series.at s7 0) (app + (app Series.at s7 1) (app Series.at s7 2))) 3.0)))))))))))))) +``` + +**`ail check`:** +``` +error: [surface-parse-error] parse error in term: unknown term head `/`; expected one of `app`, `tail-app`, `lam`, `let`, `let-rec`, `if`, `match`, `do`, `tail-do`, `seq`, `term-ctor`, `clone`, `reuse-as`, `loop`, `recur`, `new`, `lit-unit` at byte 324 +``` + + +--- +## Ladder + +- L0_const: ✅ green +- L1_arith: ✅ green +- L2_print: ✅ green +- L3_adt: ✅ green +- L4_rec: ✅ green +- L5_series_len: ✅ green +- L6_series_at: ✅ green +- L7_sma: ❌ check \ No newline at end of file diff --git a/experiments/2026-05-12-cross-model-authoring/llama405-fmt4.md b/experiments/2026-05-12-cross-model-authoring/llama405-fmt4.md new file mode 100644 index 0000000..dca338d --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/llama405-fmt4.md @@ -0,0 +1,304 @@ +# Qwen format probe 4 — annotated parens `(#N ... #N)` + +**Model:** meta-llama/Meta-Llama-3.1-405B-Instruct-FP8; single-shot per level; annotated-paren dialect; output stripped via round-trip-verified converter then ail-checked. + +--- + +## L0_const — ✅ green + +tokens: prompt=695 completion=75 + +annotated output: +``` +(#0 module m + (#1 fn answer + (#2 type (#3 fn-type (#4 params #4) (#4 ret (#5 own (#6 con Int #6) #5) #4) #3) #2) + (#2 params #2) + (#2 body 42 #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (fn answer + (type (fn-type (params) (ret (own (con Int))))) + (params) + (body 42))) +``` + + +## L1_arith — ❌ check + +tokens: prompt=698 completion=103 + +annotated output: +``` +(#0 module m + (#1 fn triple + (#2 type (#3 fn-type (#4 params (#5 own (#6 con Int #6) #5) #4) (#4 ret (#5 own (#6 con Int #6) #5) #4) #3) #2) + (#2 params x #2) + (#2 body (#3 app * x (#4 lit 3 #4) #3) #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (fn triple + (type (fn-type (params (own (con Int))) (ret (own (con Int))))) + (params x) + (body (app * x (lit 3))))) +``` + +**`ail check`:** +``` +error: [surface-parse-error] parse error in term: unknown term head `lit`; expected one of `app`, `tail-app`, `lam`, `let`, `let-rec`, `if`, `match`, `do`, `tail-do`, `seq`, `term-ctor`, `clone`, `reuse-as`, `loop`, `recur`, `new`, `lit-unit` at byte 125 +``` + +## L2_print — ✅ green + +tokens: prompt=693 completion=107 + +annotated output: +``` +(#0 module m + (#1 fn main + (#2 type (#3 fn-type (#4 params #4) (#4 ret (#5 own (#6 con Unit #6) #5) #4) (#4 effects IO #4) #3) #2) + (#2 params #2) + (#2 body (#3 seq (#4 app print 7 #4) (#4 do io/print_str "\n" #4) #3) #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body (seq (app print 7) (do io/print_str "\n"))))) +``` + + +## L3_adt — ✅ green + +tokens: prompt=712 completion=155 + +annotated output: +``` +(#0 module m + (#1 data Light (#2 ctor Red #2) (#2 ctor Green #2) #1) + (#1 fn code + (#2 type (#3 fn-type (#4 params (#5 own (#6 con Light #6) #5) #4) (#4 ret (#5 own (#6 con Int #6) #5) #4) #3) #2) + (#2 params x #2) + (#2 body + (#3 match x + (#4 case (#5 pat-ctor Red #5) 0 #4) + (#4 case (#5 pat-ctor Green #5) 1 #4) #3) #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (data Light (ctor Red) (ctor Green)) + (fn code + (type (fn-type (params (own (con Light))) (ret (own (con Int))))) + (params x) + (body + (match x + (case (pat-ctor Red) 0) + (case (pat-ctor Green) 1))))) +``` + + +## L4_rec — ✅ green + +tokens: prompt=724 completion=351 + +annotated output: +``` +(#0 module m + (#1 data IntList (#2 ctor Nil #2) (#2 ctor Cons (#3 con Int #3) (#3 con IntList #3) #2) #1) + (#1 fn length + (#2 type (#3 fn-type (#4 params (#5 own (#6 con IntList #6) #5) #4) (#4 ret (#5 own (#6 con Int #6) #5) #4) #3) #2) + (#2 params xs #2) + (#2 body + (#3 match xs + (#4 case (#5 pat-ctor Nil #5) 0 #4) + (#4 case (#5 pat-ctor Cons _ t #5) (#5 app + 1 (#6 app length t #6) #5) #4) #3) #2) #1) + (#1 fn main + (#2 type (#3 fn-type (#4 params #4) (#4 ret (#5 own (#6 con Unit #6) #5) #4) (#4 effects IO #4) #3) #2) + (#2 params #2) + (#2 body + (#3 let xs (#4 term-ctor IntList Cons 1 (#5 term-ctor IntList Cons 2 (#6 term-ctor IntList Cons 3 (#7 term-ctor IntList Nil #7) #6) #5) #4) + (#4 seq (#5 app print (#6 app length xs #6) #5) (#5 do io/print_str "\n" #5) #4) #3) #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (data IntList (ctor Nil) (ctor Cons (con Int) (con IntList))) + (fn length + (type (fn-type (params (own (con IntList))) (ret (own (con Int))))) + (params xs) + (body + (match xs + (case (pat-ctor Nil) 0) + (case (pat-ctor Cons _ t) (app + 1 (app length t)))))) + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body + (let xs (term-ctor IntList Cons 1 (term-ctor IntList Cons 2 (term-ctor IntList Cons 3 (term-ctor IntList Nil)))) + (seq (app print (app length xs)) (do io/print_str "\n")))))) +``` + + +## L5_series_len — ✅ green + +tokens: prompt=985 completion=209 + +annotated output: +``` +(#0 module m + (#1 fn main + (#2 type (#3 fn-type (#4 params #4) (#4 ret (#5 own (#6 con Unit #6) #5) #4) (#4 effects IO #4) #3) #2) + (#2 params #2) + (#2 body + (#3 let s (#4 new Series (#5 con Float #5) 3 #4) + (#4 let s2 (#5 app Series.push s 1.0 #5) + (#5 let s3 (#6 app Series.push s2 2.0 #6) + (#6 let s4 (#7 app Series.push s3 3.0 #7) + (#7 seq (#8 app print (#9 app Series.len s4 #9) #8) (#8 do io/print_str "\n" #8) #7) #6) #5) #4) #3) #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body + (let s (new Series (con Float) 3) + (let s2 (app Series.push s 1.0) + (let s3 (app Series.push s2 2.0) + (let s4 (app Series.push s3 3.0) + (seq (app print (app Series.len s4)) (do io/print_str "\n"))))))))) +``` + + +## L6_series_at — ✅ green + +tokens: prompt=993 completion=211 + +annotated output: +``` +(#0 module m + (#1 fn main + (#2 type (#3 fn-type (#4 params #4) (#4 ret (#5 own (#6 con Unit #6) #5) #4) (#4 effects IO #4) #3) #2) + (#2 params #2) + (#2 body + (#3 let s (#4 new Series (#5 con Float #5) 3 #4) + (#4 let s2 (#5 app Series.push s 1.0 #5) + (#5 let s3 (#6 app Series.push s2 2.0 #6) + (#6 let s4 (#7 app Series.push s3 3.0 #7) + (#7 seq (#8 app print (#9 app Series.at s4 0 #9) #8) (#8 do io/print_str "\n" #8) #7) #6) #5) #4) #3) #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body + (let s (new Series (con Float) 3) + (let s2 (app Series.push s 1.0) + (let s3 (app Series.push s2 2.0) + (let s4 (app Series.push s3 3.0) + (seq (app print (app Series.at s4 0)) (do io/print_str "\n"))))))))) +``` + + +## L7_sma — ❌ check + +tokens: prompt=1065 completion=678 + +annotated output: +``` +(#0 module m + (#1 fn main + (#2 type (#3 fn-type (#4 params #4) (#4 ret (#5 own (#6 con Unit #6) #5) #4) (#4 effects IO #4) #3) #2) + (#2 params #2) + (#2 body + (#3 let s (#4 new Series (#5 con Float #5) 3 #4) + (#4 let s1 (#5 app Series.push s 1.0 #5) + (#5 let s2 (#6 app Series.push s1 5.0 #6) + (#6 let s3 (#7 app Series.push s2 3.0 #7) + (#7 seq + (#8 let sum (#9 app + (#10 app Series.at s3 0 #10) (#10 app + (#11 app Series.at s3 1 #11) (#11 app Series.at s3 2 #11) #10) #9) + (#9 do io/print_str (#10 app show_float (#11 app / sum 3.0 #11) #10) #9) + (#9 do io/print_str "\n" #9) #8) + (#8 let s4 (#9 app Series.push s3 8.0 #9) + (#9 let sum (#10 app + (#11 app Series.at s4 0 #11) (#11 app + (#12 app Series.at s4 1 #12) (#12 app Series.at s4 2 #12) #11) #10) + (#10 seq + (#11 do io/print_str (#12 app show_float (#13 app / sum 3.0 #13) #12) #11) + (#11 do io/print_str "\n" #11) #10) #9) #8) + (#8 let s5 (#9 app Series.push s4 6.0 #9) + (#9 let sum (#10 app + (#11 app Series.at s5 0 #11) (#11 app + (#12 app Series.at s5 1 #12) (#12 app Series.at s5 2 #12) #11) #10) + (#10 seq + (#11 do io/print_str (#12 app show_float (#13 app / sum 3.0 #13) #12) #11) + (#11 do io/print_str "\n" #11) #10) #9) #8) + (#8 let s6 (#9 app Series.push s5 2.0 #9) + (#9 let sum (#10 app + (#11 app Series.at s6 0 #11) (#11 app + (#12 app Series.at s6 1 #12) (#12 app Series.at s6 2 #12) #11) #10) + (#10 seq + (#11 do io/print_str (#12 app show_float (#13 app / sum 3.0 #13) #12) #11) + (#11 do io/print_str "\n" #11) #10) #9) #8) #7) #6) #5) #4) #3) #2) #1) #0) +``` + +stripped to Form-A: +``` +(module m + (fn main + (type (fn-type (params) (ret (own (con Unit))) (effects IO))) + (params) + (body + (let s (new Series (con Float) 3) + (let s1 (app Series.push s 1.0) + (let s2 (app Series.push s1 5.0) + (let s3 (app Series.push s2 3.0) + (seq + (let sum (app + (app Series.at s3 0) (app + (app Series.at s3 1) (app Series.at s3 2))) + (do io/print_str (app show_float (app / sum 3.0))) + (do io/print_str "\n")) + (let s4 (app Series.push s3 8.0) + (let sum (app + (app Series.at s4 0) (app + (app Series.at s4 1) (app Series.at s4 2))) + (seq + (do io/print_str (app show_float (app / sum 3.0))) + (do io/print_str "\n")))) + (let s5 (app Series.push s4 6.0) + (let sum (app + (app Series.at s5 0) (app + (app Series.at s5 1) (app Series.at s5 2))) + (seq + (do io/print_str (app show_float (app / sum 3.0))) + (do io/print_str "\n")))) + (let s6 (app Series.push s5 2.0) + (let sum (app + (app Series.at s6 0) (app + (app Series.at s6 1) (app Series.at s6 2))) + (seq + (do io/print_str (app show_float (app / sum 3.0))) + (do io/print_str "\n")))))))))))) +``` + +**`ail check`:** +``` +error: [surface-parse-error] parse error: expected `)` (end of let-term), got `(` at byte 488 +``` + + +--- +## Ladder + +- L0_const: ✅ green +- L1_arith: ❌ check +- L2_print: ✅ green +- L3_adt: ✅ green +- L4_rec: ✅ green +- L5_series_len: ✅ green +- L6_series_at: ✅ green +- L7_sma: ❌ check \ No newline at end of file diff --git a/experiments/2026-05-12-cross-model-authoring/qwen_ablation.py b/experiments/2026-05-12-cross-model-authoring/qwen_ablation.py index feb58dc..0bec6f0 100644 --- a/experiments/2026-05-12-cross-model-authoring/qwen_ablation.py +++ b/experiments/2026-05-12-cross-model-authoring/qwen_ablation.py @@ -14,10 +14,10 @@ import json, os, re, subprocess, urllib.request from pathlib import Path ENDPOINT = "https://openai.inference.de-txl.ionos.com/v1/chat/completions" -MODEL = "Qwen/Qwen3-Coder-Next" +MODEL = os.environ.get("PROBE_MODEL","Qwen/Qwen3-Coder-Next") REPO = Path(__file__).resolve().parents[2] AIL = os.environ.get("AIL_BIN", str(REPO / "target/debug/ail")) -DOC = REPO / "experiments/2026-05-12-cross-model-authoring/qwen-ablation-ctor.md" +DOC = REPO / f"experiments/2026-05-12-cross-model-authoring/{os.environ.get('PROBE_TAG','qwen')}-ablation-ctor.md" DEMO_A = """(module demo_a (fn double @@ -89,19 +89,27 @@ STAGES = [ ] def call(system, task): - body = json.dumps({"model": MODEL, "temperature": 0.2, "max_tokens": 1500, + body = json.dumps({"model": MODEL, "temperature": 0.2, "max_tokens": 1000, "stop": ["assistant"], "messages": [{"role":"system","content":system},{"role":"user","content":task}]}).encode() req = urllib.request.Request(ENDPOINT, data=body, headers={ "Authorization": f"Bearer {os.environ['IONOS_API_TOKEN']}", "Content-Type":"application/json"}) - with urllib.request.urlopen(req, timeout=120) as r: + with urllib.request.urlopen(req, timeout=300) as r: d = json.loads(r.read()) return d["choices"][0]["message"]["content"], d.get("usage", {}) def extract(resp): - m = re.search(r"```(?:[a-zA-Z]*)?\n(.*?)```", resp, re.S) - code = m.group(1) if m else resp - i = code.find("(module") - return code[i:].strip() if i >= 0 else code.strip() + m=re.search(r"```(?:[a-zA-Z]*)?\n(.*?)```",resp,re.S) + code=m.group(1) if m else resp + i=code.find("(#0") + if i<0: i=code.find("(module") + if i<0: return code.strip() + depth=0 + for j in range(i,len(code)): + if code[j]=="(": depth+=1 + elif code[j]==")": + depth-=1 + if depth==0: return code[i:j+1] + return code[i:].strip() def evaluate(code, expected): m = re.search(r"\(module\s+([A-Za-z_][A-Za-z0-9_]*)", code) @@ -121,9 +129,12 @@ def main(): "Each level is independent (fresh context, one call, no retry).\n", "---\n"] summary = [] for sid, ctx, task, exp in STAGES: - resp, usage = call(ctx, task) - code = extract(resp) - stage, fb = evaluate(code, exp) + try: + resp, usage = call(ctx, task) + code = extract(resp) + stage, fb = evaluate(code, exp) + except Exception as e: + usage={}; code=""; stage="api-error"; fb=str(e)[:200] ok = stage == "green" mark = "✅ green" if ok else f"❌ {stage}" summary.append((sid, mark)) diff --git a/experiments/2026-05-12-cross-model-authoring/qwen_fmt4.py b/experiments/2026-05-12-cross-model-authoring/qwen_fmt4.py index bf9e757..146797d 100644 --- a/experiments/2026-05-12-cross-model-authoring/qwen_fmt4.py +++ b/experiments/2026-05-12-cross-model-authoring/qwen_fmt4.py @@ -15,10 +15,10 @@ import json, os, re, subprocess, urllib.request from pathlib import Path ENDPOINT = "https://openai.inference.de-txl.ionos.com/v1/chat/completions" -MODEL = "Qwen/Qwen3-Coder-Next" +MODEL = os.environ.get("PROBE_MODEL","Qwen/Qwen3-Coder-Next") REPO = Path(__file__).resolve().parents[2] AIL = os.environ.get("AIL_BIN", str(REPO / "target/debug/ail")) -DOC = REPO / "experiments/2026-05-12-cross-model-authoring/qwen-fmt4.md" +DOC = REPO / f"experiments/2026-05-12-cross-model-authoring/{os.environ.get('PROBE_TAG','qwen')}-fmt4.md" def annotate(src): out=[]; depth=0; in_str=False; esc=False @@ -122,11 +122,11 @@ STAGES = [ ] def call(system, task): - body=json.dumps({"model":MODEL,"temperature":0.2,"max_tokens":1800, + body=json.dumps({"model":MODEL,"temperature":0.2,"max_tokens":1000,"stop":["assistant"], "messages":[{"role":"system","content":system},{"role":"user","content":task}]}).encode() req=urllib.request.Request(ENDPOINT,data=body,headers={ "Authorization":f"Bearer {os.environ['IONOS_API_TOKEN']}","Content-Type":"application/json"}) - with urllib.request.urlopen(req,timeout=120) as r: + with urllib.request.urlopen(req,timeout=300) as r: d=json.loads(r.read()) return d["choices"][0]["message"]["content"], d.get("usage",{}) @@ -135,7 +135,14 @@ def extract(resp): code=m.group(1) if m else resp i=code.find("(#0") if i<0: i=code.find("(module") - return code[i:].strip() if i>=0 else code.strip() + if i<0: return code.strip() + depth=0 + for j in range(i,len(code)): + if code[j]=="(": depth+=1 + elif code[j]==")": + depth-=1 + if depth==0: return code[i:j+1] + return code[i:].strip() def evaluate(annotated_code, expected): plain = strip_ann(annotated_code) # <-- verified converter @@ -156,9 +163,12 @@ def main(): "output stripped via round-trip-verified converter then ail-checked.\n","---\n"] summary=[] for sid,ctx,task,exp in STAGES: - resp,usage=call(ctx,task) - code=extract(resp) - stage,fb,plain=evaluate(code,exp) + try: + resp,usage=call(ctx,task) + code=extract(resp) + stage,fb,plain=evaluate(code,exp) + except Exception as e: + usage={}; code=""; plain=""; stage="api-error"; fb=str(e)[:200] ok=stage=="green"; mark="✅ green" if ok else f"❌ {stage}" summary.append((sid,mark)); print(f"{sid}: {mark}") log+=[f"## {sid} — {mark}",