fieldtest: naming A/B against Qwen3-Coder — refactor not justified
Three-cohort harness against IONOS-hosted Qwen3-Coder-Next, testing
whether application-head tag (`app` vs `apply` vs `call`) affects
LLM authoring success. Cohorts share lambda/constructor/typecon tag
renamings (the non-discriminator changes); only the head differs.
Result: 100 % cohort-treue — Qwen writes whatever the spec teaches,
no measurable natural preference. Pipeline pass-rate (classic 1/8,
apply 0/8, call 1/8) is cohort-independent. The four task failures
are general Form-A friction, not naming-related. Refactor not
justified per the feature-acceptance gate.
Tracked harness:
- 2026-05-12-cross-model-authoring/rename-spec.py — regex tag
rewriter for the two non-classic spec variants
- 2026-05-21-naming-ab/run.py, reprocess.py — three-cohort runner
+ post-processing with disjoint-discriminator counting and
module-name-matched temp dirs
Generated artefacts (runs/, rendered/ail-{renamed,call}.md) are
gitignored — regeneratable from rename-spec.py + run.py.
Side-effects filed against current spec/schema bugs surfaced during
the run:
- refs #28 spec teaches (ctor X) for term position; parser requires
(term-ctor X) — 100 % t4 failure across cohorts
- refs #29 io/print_str appends newline (de facto println);
spec now documents the behavior, code question still open
- refs #30 schema camelCase outlier: paramTypes/retType in Term::Lam
Spec patch in rendered/ail.md:
- replaces stale io/print_int references with io/print_str (bitrot
from the print-builtin consolidation)
- documents io/print_str's trailing-newline behavior
- corrects the prelude claim about int_to_str (IS in prelude)
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
rendered/ail-renamed.md
|
||||
rendered/ail-call.md
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Produce a renamed-tags variant of the rendered Form-A spec.
|
||||
|
||||
Two modes:
|
||||
- apply: full-form resolution of abbreviations
|
||||
app→apply, lam→lambda, ctor→constructor, con→typecon, tail-app→tail-apply
|
||||
- call: like apply but with call as the application head (semantic shift)
|
||||
app→call, tail-app→tail-call, rest as in apply
|
||||
|
||||
Each pattern is matched in two contexts:
|
||||
- S-expression head: `(NAME ` and `(NAME)`
|
||||
- Backtick mention: `` `NAME` ``
|
||||
|
||||
Plain-text occurrences ("application", "construction", etc.) are
|
||||
preserved by anchoring on the contexts above.
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PAIRS_APPLY = [
|
||||
("tail-app", "tail-apply"),
|
||||
("term-ctor", "term-constructor"),
|
||||
("pat-ctor", "pat-constructor"),
|
||||
("ctor", "constructor"),
|
||||
("app", "apply"),
|
||||
("lam", "lambda"),
|
||||
("con", "typecon"),
|
||||
]
|
||||
|
||||
PAIRS_CALL = [
|
||||
("tail-app", "tail-call"),
|
||||
("term-ctor", "term-constructor"),
|
||||
("pat-ctor", "pat-constructor"),
|
||||
("ctor", "constructor"),
|
||||
("app", "call"),
|
||||
("lam", "lambda"),
|
||||
("con", "typecon"),
|
||||
]
|
||||
|
||||
MODES = {"apply": PAIRS_APPLY, "call": PAIRS_CALL}
|
||||
|
||||
def transform(src: str, pairs) -> str:
|
||||
out = src
|
||||
for old, new in pairs:
|
||||
out = re.sub(rf"\({re.escape(old)}(?=[\s)])", f"({new}", out)
|
||||
out = re.sub(rf"`{re.escape(old)}`", f"`{new}`", out)
|
||||
return out
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 4 or sys.argv[1] not in MODES:
|
||||
print(f"usage: {sys.argv[0]} <apply|call> <input.md> <output.md>", file=sys.stderr)
|
||||
return 2
|
||||
mode = sys.argv[1]
|
||||
inp = Path(sys.argv[2])
|
||||
out = Path(sys.argv[3])
|
||||
out.write_text(transform(inp.read_text(), MODES[mode]))
|
||||
print(f"wrote {out} ({out.stat().st_size} bytes, mode={mode})")
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -349,13 +349,13 @@ arms or into code outside the `match`.
|
||||
|
||||
Every function type carries an effect row — a set of effect labels
|
||||
that the function may raise. The two currently wired effects are
|
||||
`IO` (required to call effect operations like `io/print_int`) and
|
||||
`IO` (required to call effect operations like `io/print_str`) and
|
||||
`Diverge` (used for functions that may not terminate). An empty
|
||||
effect row marks the function as pure.
|
||||
|
||||
Effect operations are reached through the `do` term, not through a
|
||||
regular function application. The `do` form names the operation
|
||||
(e.g. `io/print_int`) and lists the operation's arguments; the
|
||||
(e.g. `io/print_str`) and lists the operation's arguments; the
|
||||
typechecker resolves the operation against the prelude's effect-op
|
||||
table, checks the arguments, and accumulates the operation's
|
||||
effect label into the enclosing function's effect row.
|
||||
@@ -374,8 +374,9 @@ in canonical output.
|
||||
|
||||
Effect operation: `(do OP-NAME ARG*)` — the leading `do` head is
|
||||
the parser's signal that the next identifier is an effect op, not a
|
||||
function. Example: `(do io/print_int 42)` prints the integer 42
|
||||
and raises the `IO` effect.
|
||||
function. Example: `(do io/print_str "hi\n")` prints the string
|
||||
`hi` followed by a newline and raises the `IO` effect. To print a
|
||||
non-string value, first convert it: `(do io/print_str (app int_to_str 42))`.
|
||||
|
||||
Sequencing: `(seq LHS RHS)`. The value of `(seq X Y)` is `Y`'s
|
||||
value; `X`'s value is discarded. Chains of sequencing are written
|
||||
@@ -385,10 +386,10 @@ value.
|
||||
```ail
|
||||
(module fn_with_do_seq
|
||||
(fn main
|
||||
(doc "Print two Ints in sequence. Exercises TermDo, TermSeq, and the IO effect on a fn type. The trailing unit return is implicit in the last Do (op returns Unit).")
|
||||
(doc "Print two Ints in sequence. Exercises TermDo, TermSeq, the IO effect on a fn type, and int-to-str conversion via the prelude `int_to_str`. The trailing unit return is implicit in the last Do (op returns Unit).")
|
||||
(type (fn-type (params) (ret (con Unit)) (effects IO)))
|
||||
(params)
|
||||
(body (seq (do io/print_int 1) (do io/print_int 2)))))
|
||||
(body (seq (do io/print_str (app int_to_str 1)) (do io/print_str (app int_to_str 2))))))
|
||||
```
|
||||
|
||||
## 9. Literals
|
||||
@@ -547,14 +548,14 @@ Effect operations (reached through `do`, not `app`):
|
||||
|
||||
| Op | Signature | Effect |
|
||||
|----|----------|-------|
|
||||
| `io/print_int` | `(Int) -> Unit` | `IO` |
|
||||
| `io/print_bool` | `(Bool) -> Unit` | `IO` |
|
||||
| `io/print_str` | `(Str) -> Unit` | `IO` |
|
||||
| `io/print_float` | `(Float) -> Unit` | `IO` |
|
||||
|
||||
`int_to_str` is intentionally NOT in the prelude — it is type-
|
||||
installed in a future milestone but codegen-deferred pending the
|
||||
heap-Str ABI work. Do not call it.
|
||||
`io/print_str` is the only print operation, and it appends a
|
||||
trailing newline (the runtime calls C `puts`). To print a non-string
|
||||
value, convert it first via `int_to_str`, `float_to_str`, or
|
||||
`bool_to_str` — all three are in the prelude (see the prelude
|
||||
table above). To print two values on consecutive lines, sequence
|
||||
two `(do io/print_str ...)` calls with `seq`.
|
||||
|
||||
## 12. Content addressing
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
runs/
|
||||
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Re-process the raw responses from a finished run with fixes:
|
||||
- Per-call temp dir so file stem matches module name
|
||||
- Tag-disjoint counting (only count tags unique to a cohort,
|
||||
ignore overlapping ones like `lambda` shared by apply+call)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json, re, subprocess, sys
|
||||
from pathlib import Path
|
||||
|
||||
RUN_DIR = Path(sys.argv[1])
|
||||
RESULTS = json.loads((RUN_DIR / "results.json").read_text())
|
||||
|
||||
REVERSE_APPLY = [("tail-apply","tail-app"),("term-constructor","term-ctor"),
|
||||
("pat-constructor","pat-ctor"),("constructor","ctor"),
|
||||
("apply","app"),("lambda","lam"),("typecon","con")]
|
||||
REVERSE_CALL = [("tail-call","tail-app"),("term-constructor","term-ctor"),
|
||||
("pat-constructor","pat-ctor"),("constructor","ctor"),
|
||||
("call","app"),("lambda","lam"),("typecon","con")]
|
||||
|
||||
def reverse_sub(src, pairs):
|
||||
out = src
|
||||
for n,o in pairs:
|
||||
out = re.sub(rf"\({re.escape(n)}(?=[\s)])", f"({o}", out)
|
||||
return out
|
||||
|
||||
# Disjoint discriminator tags only — the head of function application
|
||||
# (which IS the test variable). Ignore lambda/constructor/typecon since
|
||||
# apply and call share those.
|
||||
DISCRIM = {
|
||||
"classic": ["app","tail-app"],
|
||||
"apply": ["apply","tail-apply"],
|
||||
"call": ["call","tail-call"],
|
||||
}
|
||||
|
||||
def count_tags(src, tags):
|
||||
return {t: len(re.findall(rf"\({re.escape(t)}(?=[\s)])", src)) for t in tags}
|
||||
|
||||
def extract_module_name(code):
|
||||
m = re.search(r"\(module\s+([A-Za-z_][A-Za-z0-9_]*)", code)
|
||||
return m.group(1) if m else None
|
||||
|
||||
def run_pipeline_fixed(code, module_name, expected_stdout, base):
|
||||
work = base / module_name
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
src = work / f"{module_name}.ail"
|
||||
src.write_text(code)
|
||||
out = {"check_ok": False, "check_stderr": "", "build_ok": False,
|
||||
"run_ok": False, "run_stdout": "", "matches_expected": False}
|
||||
c = subprocess.run(["ail","check",str(src)], capture_output=True, text=True, timeout=30)
|
||||
out["check_stderr"] = (c.stderr or c.stdout)[:2000]
|
||||
if c.returncode != 0:
|
||||
return out
|
||||
out["check_ok"] = True
|
||||
b = subprocess.run(["ail","build",str(src),"-o",str(work/"bin")],
|
||||
capture_output=True, text=True, timeout=60)
|
||||
if b.returncode != 0:
|
||||
out["check_stderr"] = b.stderr[:2000]
|
||||
return out
|
||||
out["build_ok"] = True
|
||||
r = subprocess.run([str(work/"bin")], capture_output=True, text=True, timeout=10)
|
||||
if r.returncode == 0:
|
||||
out["run_ok"] = True
|
||||
out["run_stdout"] = r.stdout
|
||||
out["matches_expected"] = (r.stdout == expected_stdout)
|
||||
return out
|
||||
|
||||
# need expected stdout per task
|
||||
TASKS = {}
|
||||
CMA_TASKS = Path("/home/brummel/dev/ailang/experiments/2026-05-12-cross-model-authoring/master/tasks")
|
||||
for f in CMA_TASKS.glob("*.task.json"):
|
||||
t = json.loads(f.read_text())
|
||||
TASKS[t["id"]] = t["expected_stdout"]
|
||||
|
||||
base = RUN_DIR / "work2"
|
||||
base.mkdir(exist_ok=True)
|
||||
|
||||
new_rows = []
|
||||
for r in RESULTS:
|
||||
code = r["extracted_code"]
|
||||
cohort = r["cohort"]
|
||||
pairs = {"classic":None,"apply":REVERSE_APPLY,"call":REVERSE_CALL}[cohort]
|
||||
pipeline_input = reverse_sub(code, pairs) if pairs else code
|
||||
mod_name = extract_module_name(pipeline_input) or r["task_id"]
|
||||
pipe = run_pipeline_fixed(pipeline_input, mod_name, TASKS[r["task_id"]], base)
|
||||
|
||||
own_disc = count_tags(code, DISCRIM[cohort])
|
||||
foreign_disc = {}
|
||||
for c, tags in DISCRIM.items():
|
||||
if c == cohort: continue
|
||||
foreign_disc[c] = count_tags(code, tags)
|
||||
|
||||
new_rows.append({
|
||||
"cohort": cohort, "task_id": r["task_id"], "trial": r["trial"],
|
||||
"module_name_in_output": mod_name,
|
||||
"module_name_expected": r["task_id"],
|
||||
"module_match": (mod_name == r["task_id"]),
|
||||
"own_disc": own_disc,
|
||||
"foreign_disc": foreign_disc,
|
||||
"check_ok": pipe["check_ok"],
|
||||
"build_ok": pipe["build_ok"],
|
||||
"run_ok": pipe["run_ok"],
|
||||
"matches_expected": pipe["matches_expected"],
|
||||
"run_stdout": pipe["run_stdout"],
|
||||
"first_err_line": pipe["check_stderr"].split("\n")[0][:140],
|
||||
})
|
||||
|
||||
(RUN_DIR / "results2.json").write_text(json.dumps(new_rows, indent=2))
|
||||
|
||||
print("\n=== final pipeline pass per cohort ===")
|
||||
by_cohort = {}
|
||||
for r in new_rows:
|
||||
c = by_cohort.setdefault(r["cohort"], {"n":0,"mod":0,"check":0,"build":0,"run":0,"correct":0})
|
||||
c["n"] += 1
|
||||
if r["module_match"]: c["mod"] += 1
|
||||
if r["check_ok"]: c["check"] += 1
|
||||
if r["build_ok"]: c["build"] += 1
|
||||
if r["run_ok"]: c["run"] += 1
|
||||
if r["matches_expected"]: c["correct"] += 1
|
||||
for cohort, c in by_cohort.items():
|
||||
print(f"{cohort:8s} n={c['n']} mod-name-OK={c['mod']} check={c['check']} build={c['build']} run={c['run']} stdout-correct={c['correct']}")
|
||||
|
||||
print("\n=== discriminator tag totals (head of function application only) ===")
|
||||
disc_totals = {}
|
||||
for r in new_rows:
|
||||
c = r["cohort"]
|
||||
own_sum = sum(r["own_disc"].values())
|
||||
foreign_sum = sum(sum(v.values()) for v in r["foreign_disc"].values())
|
||||
t = disc_totals.setdefault(c, {"own":0,"foreign":0,"by_foreign":{}})
|
||||
t["own"] += own_sum
|
||||
for fc, fv in r["foreign_disc"].items():
|
||||
t["by_foreign"][fc] = t["by_foreign"].get(fc,0) + sum(fv.values())
|
||||
t["foreign"] += foreign_sum
|
||||
for cohort, t in disc_totals.items():
|
||||
print(f"{cohort:8s} own={t['own']:3d} foreign={t['foreign']:3d} breakdown={t['by_foreign']}")
|
||||
|
||||
print("\n=== per-row failure mode ===")
|
||||
for r in new_rows:
|
||||
flag = "OK" if r["matches_expected"] else ("CORRECT-NOT" if r["run_ok"] else ("RUN" if r["build_ok"] else ("BUILD" if r["check_ok"] else "CHECK")))
|
||||
print(f"[{r['cohort']:8s}] {r['task_id']:18s} t{r['trial']} {flag:12s} {r['first_err_line']}")
|
||||
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A/B test: does Qwen3-Coder produce valid Form-A code more reliably
|
||||
when the spec uses long tag names (apply/lambda/constructor/typecon)
|
||||
vs. the current short ones (app/lam/ctor/con)?
|
||||
|
||||
Setup:
|
||||
- Two specs, identical except for tag names.
|
||||
- Same N tasks, fired against each cohort.
|
||||
- For each call: log raw response, run it through `ail check`
|
||||
(renamed outputs go through reverse substitution first), capture
|
||||
parser / typecheck / build / run results.
|
||||
|
||||
Token budget guard: aborts before exceeding --token-budget.
|
||||
Reads token from $IONOS_API_TOKEN (caller sources from ~/.ionos_token).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from dataclasses import dataclass, asdict, field
|
||||
from pathlib import Path
|
||||
|
||||
ENDPOINT = "https://openai.inference.de-txl.ionos.com/v1/chat/completions"
|
||||
MODEL = "Qwen/Qwen3-Coder-Next"
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
CMA = REPO / "experiments" / "2026-05-12-cross-model-authoring"
|
||||
SPEC_CLASSIC = CMA / "rendered" / "ail.md"
|
||||
SPEC_APPLY = CMA / "rendered" / "ail-renamed.md"
|
||||
SPEC_CALL = CMA / "rendered" / "ail-call.md"
|
||||
TASKS_DIR = CMA / "master" / "tasks"
|
||||
|
||||
# Reverse substitution per cohort so `ail check` (which only knows
|
||||
# the classic tags) can parse outputs. Order: longer first.
|
||||
REVERSE_APPLY = [
|
||||
("tail-apply", "tail-app"),
|
||||
("term-constructor","term-ctor"),
|
||||
("pat-constructor", "pat-ctor"),
|
||||
("constructor", "ctor"),
|
||||
("apply", "app"),
|
||||
("lambda", "lam"),
|
||||
("typecon", "con"),
|
||||
]
|
||||
REVERSE_CALL = [
|
||||
("tail-call", "tail-app"),
|
||||
("term-constructor","term-ctor"),
|
||||
("pat-constructor", "pat-ctor"),
|
||||
("constructor", "ctor"),
|
||||
("call", "app"),
|
||||
("lambda", "lam"),
|
||||
("typecon", "con"),
|
||||
]
|
||||
|
||||
def reverse_substitute(src: str, pairs) -> str:
|
||||
out = src
|
||||
for new, old in pairs:
|
||||
out = re.sub(rf"\({re.escape(new)}(?=[\s)])", f"({old}", out)
|
||||
return out
|
||||
|
||||
# Tag sets used to count cohort-consistency.
|
||||
TAGS_CLASSIC = ["tail-app","term-ctor","pat-ctor","ctor","app","lam","con"]
|
||||
TAGS_APPLY = ["tail-apply","term-constructor","pat-constructor","constructor","apply","lambda","typecon"]
|
||||
TAGS_CALL = ["tail-call","term-constructor","pat-constructor","constructor","call","lambda","typecon"]
|
||||
|
||||
def count_tags(src: str, tags: list[str]) -> dict[str,int]:
|
||||
counts: dict[str,int] = {}
|
||||
for t in tags:
|
||||
# match `(tag ` or `(tag)` only
|
||||
counts[t] = len(re.findall(rf"\({re.escape(t)}(?=[\s)])", src))
|
||||
return counts
|
||||
|
||||
@dataclass
|
||||
class CallResult:
|
||||
cohort: str
|
||||
task_id: str
|
||||
trial: int
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
raw_response: str = ""
|
||||
extracted_code: str = ""
|
||||
own_tag_counts: dict[str,int] = field(default_factory=dict)
|
||||
foreign_tag_counts: dict[str,dict[str,int]] = field(default_factory=dict)
|
||||
ail_check_ok: bool = False
|
||||
ail_check_stderr: str = ""
|
||||
ail_build_ok: bool = False
|
||||
ail_run_ok: bool = False
|
||||
ail_run_stdout: str = ""
|
||||
stdout_matches_expected: bool = False
|
||||
notes: str = ""
|
||||
|
||||
def load_tasks() -> list[dict]:
|
||||
tasks = []
|
||||
for f in sorted(TASKS_DIR.glob("*.task.json")):
|
||||
tasks.append(json.loads(f.read_text()))
|
||||
return tasks
|
||||
|
||||
def extract_form_a(response: str) -> str:
|
||||
# Prefer fenced ```ail blocks; else fall back to the longest
|
||||
# parenthesised expression starting with (module …).
|
||||
m = re.findall(r"```(?:ail)?\s*\n(.*?)```", response, re.DOTALL)
|
||||
for block in m:
|
||||
if "(module" in block:
|
||||
return block.strip()
|
||||
# Fallback: from first '(module' to matching close paren.
|
||||
idx = response.find("(module")
|
||||
if idx < 0:
|
||||
return ""
|
||||
depth = 0
|
||||
out = []
|
||||
for ch in response[idx:]:
|
||||
out.append(ch)
|
||||
if ch == "(": depth += 1
|
||||
elif ch == ")":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return "".join(out)
|
||||
return "".join(out)
|
||||
|
||||
def call_ionos(token: str, system_prompt: str, user_prompt: str, *, max_retries=3) -> dict:
|
||||
body = json.dumps({
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
"temperature": 0.2,
|
||||
"top_p": 0.95,
|
||||
}).encode("utf-8")
|
||||
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
|
||||
backoffs = [1, 4, 16]
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
req = urllib.request.Request(ENDPOINT, data=body, headers=headers, method="POST")
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
return json.loads(r.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
code = e.code
|
||||
if code in (429,) or 500 <= code < 600:
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(backoffs[attempt])
|
||||
continue
|
||||
raise
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(backoffs[attempt])
|
||||
continue
|
||||
raise
|
||||
raise RuntimeError("retries exhausted")
|
||||
|
||||
def run_pipeline(code: str, task_id: str, expected_stdout: str, work: Path) -> dict:
|
||||
src = work / f"{task_id}.ail"
|
||||
src.write_text(code)
|
||||
result = {"check_ok": False, "check_stderr": "", "build_ok": False,
|
||||
"run_ok": False, "run_stdout": "", "matches_expected": False}
|
||||
check = subprocess.run(["ail", "check", str(src)], capture_output=True, text=True, timeout=30)
|
||||
result["check_stderr"] = check.stderr + check.stdout if check.returncode != 0 else ""
|
||||
if check.returncode != 0:
|
||||
return result
|
||||
result["check_ok"] = True
|
||||
build = subprocess.run(["ail", "build", str(src), "-o", str(work / task_id)],
|
||||
capture_output=True, text=True, timeout=60)
|
||||
if build.returncode != 0:
|
||||
result["check_stderr"] = build.stderr
|
||||
return result
|
||||
result["build_ok"] = True
|
||||
run = subprocess.run([str(work / task_id)], capture_output=True, text=True, timeout=10)
|
||||
if run.returncode == 0:
|
||||
result["run_ok"] = True
|
||||
result["run_stdout"] = run.stdout
|
||||
result["matches_expected"] = (run.stdout == expected_stdout)
|
||||
return result
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", type=Path, required=True)
|
||||
ap.add_argument("--trials", type=int, default=2)
|
||||
ap.add_argument("--token-budget", type=int, default=300_000)
|
||||
ap.add_argument("--dry-run", action="store_true", help="don't call IONOS; use empty responses")
|
||||
args = ap.parse_args()
|
||||
|
||||
token = os.environ.get("IONOS_API_TOKEN", "").strip()
|
||||
if not args.dry_run and not token:
|
||||
print("ERROR: IONOS_API_TOKEN not set", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
tasks = load_tasks()
|
||||
|
||||
# (name, system_prompt, own_tags, reverse_pairs_or_None)
|
||||
cohorts = [
|
||||
("classic", SPEC_CLASSIC.read_text(), TAGS_CLASSIC, None),
|
||||
("apply", SPEC_APPLY.read_text(), TAGS_APPLY, REVERSE_APPLY),
|
||||
("call", SPEC_CALL.read_text(), TAGS_CALL, REVERSE_CALL),
|
||||
]
|
||||
all_tag_sets = {"classic": TAGS_CLASSIC, "apply": TAGS_APPLY, "call": TAGS_CALL}
|
||||
|
||||
rows: list[CallResult] = []
|
||||
tokens_used = 0
|
||||
work = args.out / "work"
|
||||
work.mkdir(exist_ok=True)
|
||||
|
||||
for cohort, system_prompt, own_tags, reverse_pairs in cohorts:
|
||||
for task in tasks:
|
||||
for trial in range(args.trials):
|
||||
if tokens_used >= args.token_budget:
|
||||
print(f"BUDGET HIT at {tokens_used} tokens — stopping", file=sys.stderr)
|
||||
break
|
||||
user_prompt = task["description"] + "\n\nReturn only the complete module as a single ```ail fenced block."
|
||||
row = CallResult(cohort=cohort, task_id=task["id"], trial=trial)
|
||||
if args.dry_run:
|
||||
row.raw_response = "(dry-run)"
|
||||
else:
|
||||
print(f"[{cohort}] {task['id']} trial {trial} …", file=sys.stderr)
|
||||
resp = call_ionos(token, system_prompt, user_prompt)
|
||||
row.raw_response = resp["choices"][0]["message"]["content"]
|
||||
row.prompt_tokens = resp["usage"]["prompt_tokens"]
|
||||
row.completion_tokens = resp["usage"]["completion_tokens"]
|
||||
tokens_used += resp["usage"]["total_tokens"]
|
||||
code = extract_form_a(row.raw_response)
|
||||
row.extracted_code = code
|
||||
row.own_tag_counts = count_tags(code, own_tags)
|
||||
for other_name, other_tags in all_tag_sets.items():
|
||||
if other_name != cohort:
|
||||
row.foreign_tag_counts[other_name] = count_tags(code, other_tags)
|
||||
if code:
|
||||
pipeline_input = reverse_substitute(code, reverse_pairs) if reverse_pairs else code
|
||||
pipe = run_pipeline(pipeline_input, f"{cohort}_{task['id']}_t{trial}",
|
||||
task["expected_stdout"], work)
|
||||
row.ail_check_ok = pipe["check_ok"]
|
||||
row.ail_check_stderr = pipe["check_stderr"][:2000]
|
||||
row.ail_build_ok = pipe["build_ok"]
|
||||
row.ail_run_ok = pipe["run_ok"]
|
||||
row.ail_run_stdout = pipe["run_stdout"]
|
||||
row.stdout_matches_expected = pipe["matches_expected"]
|
||||
else:
|
||||
row.notes = "no form-A extracted"
|
||||
rows.append(row)
|
||||
(args.out / f"{cohort}_{task['id']}_t{trial}.raw.txt").write_text(row.raw_response)
|
||||
|
||||
out_file = args.out / "results.json"
|
||||
out_file.write_text(json.dumps([asdict(r) for r in rows], indent=2))
|
||||
print(f"\nwrote {out_file} — {len(rows)} rows, {tokens_used} tokens used")
|
||||
|
||||
# Summary table.
|
||||
by_cohort: dict[str, dict] = {}
|
||||
for r in rows:
|
||||
c = by_cohort.setdefault(r.cohort, {"n":0, "extracted":0, "check":0, "build":0, "run":0, "correct":0,
|
||||
"own_tags":0, "foreign_tags":0})
|
||||
c["n"] += 1
|
||||
if r.extracted_code: c["extracted"] += 1
|
||||
if r.ail_check_ok: c["check"] += 1
|
||||
if r.ail_build_ok: c["build"] += 1
|
||||
if r.ail_run_ok: c["run"] += 1
|
||||
if r.stdout_matches_expected: c["correct"] += 1
|
||||
c["own_tags"] += sum(r.own_tag_counts.values())
|
||||
c["foreign_tags"] += sum(sum(v.values()) for v in r.foreign_tag_counts.values())
|
||||
|
||||
print("\n=== summary ===")
|
||||
for cohort, c in by_cohort.items():
|
||||
print(f"{cohort:8s} n={c['n']:2d} extracted={c['extracted']:2d} "
|
||||
f"check={c['check']:2d} build={c['build']:2d} run={c['run']:2d} "
|
||||
f"correct={c['correct']:2d} own-tags={c['own_tags']:3d} foreign-tags={c['foreign_tags']:3d}")
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user