fieldtest: naming A/B against Qwen3-Coder — refactor not justified
Three-cohort harness against IONOS-hosted Qwen3-Coder-Next, testing
whether application-head tag (`app` vs `apply` vs `call`) affects
LLM authoring success. Cohorts share lambda/constructor/typecon tag
renamings (the non-discriminator changes); only the head differs.
Result: 100 % cohort-treue — Qwen writes whatever the spec teaches,
no measurable natural preference. Pipeline pass-rate (classic 1/8,
apply 0/8, call 1/8) is cohort-independent. The four task failures
are general Form-A friction, not naming-related. Refactor not
justified per the feature-acceptance gate.
Tracked harness:
- 2026-05-12-cross-model-authoring/rename-spec.py — regex tag
rewriter for the two non-classic spec variants
- 2026-05-21-naming-ab/run.py, reprocess.py — three-cohort runner
+ post-processing with disjoint-discriminator counting and
module-name-matched temp dirs
Generated artefacts (runs/, rendered/ail-{renamed,call}.md) are
gitignored — regeneratable from rename-spec.py + run.py.
Side-effects filed against current spec/schema bugs surfaced during
the run:
- refs #28 spec teaches (ctor X) for term position; parser requires
(term-ctor X) — 100 % t4 failure across cohorts
- refs #29 io/print_str appends newline (de facto println);
spec now documents the behavior, code question still open
- refs #30 schema camelCase outlier: paramTypes/retType in Term::Lam
Spec patch in rendered/ail.md:
- replaces stale io/print_int references with io/print_str (bitrot
from the print-builtin consolidation)
- documents io/print_str's trailing-newline behavior
- corrects the prelude claim about int_to_str (IS in prelude)
This commit is contained in:
@@ -0,0 +1 @@
|
||||
runs/
|
||||
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Re-process the raw responses from a finished run with fixes:
|
||||
- Per-call temp dir so file stem matches module name
|
||||
- Tag-disjoint counting (only count tags unique to a cohort,
|
||||
ignore overlapping ones like `lambda` shared by apply+call)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json, re, subprocess, sys
|
||||
from pathlib import Path
|
||||
|
||||
RUN_DIR = Path(sys.argv[1])
|
||||
RESULTS = json.loads((RUN_DIR / "results.json").read_text())
|
||||
|
||||
REVERSE_APPLY = [("tail-apply","tail-app"),("term-constructor","term-ctor"),
|
||||
("pat-constructor","pat-ctor"),("constructor","ctor"),
|
||||
("apply","app"),("lambda","lam"),("typecon","con")]
|
||||
REVERSE_CALL = [("tail-call","tail-app"),("term-constructor","term-ctor"),
|
||||
("pat-constructor","pat-ctor"),("constructor","ctor"),
|
||||
("call","app"),("lambda","lam"),("typecon","con")]
|
||||
|
||||
def reverse_sub(src, pairs):
|
||||
out = src
|
||||
for n,o in pairs:
|
||||
out = re.sub(rf"\({re.escape(n)}(?=[\s)])", f"({o}", out)
|
||||
return out
|
||||
|
||||
# Disjoint discriminator tags only — the head of function application
|
||||
# (which IS the test variable). Ignore lambda/constructor/typecon since
|
||||
# apply and call share those.
|
||||
DISCRIM = {
|
||||
"classic": ["app","tail-app"],
|
||||
"apply": ["apply","tail-apply"],
|
||||
"call": ["call","tail-call"],
|
||||
}
|
||||
|
||||
def count_tags(src, tags):
|
||||
return {t: len(re.findall(rf"\({re.escape(t)}(?=[\s)])", src)) for t in tags}
|
||||
|
||||
def extract_module_name(code):
|
||||
m = re.search(r"\(module\s+([A-Za-z_][A-Za-z0-9_]*)", code)
|
||||
return m.group(1) if m else None
|
||||
|
||||
def run_pipeline_fixed(code, module_name, expected_stdout, base):
|
||||
work = base / module_name
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
src = work / f"{module_name}.ail"
|
||||
src.write_text(code)
|
||||
out = {"check_ok": False, "check_stderr": "", "build_ok": False,
|
||||
"run_ok": False, "run_stdout": "", "matches_expected": False}
|
||||
c = subprocess.run(["ail","check",str(src)], capture_output=True, text=True, timeout=30)
|
||||
out["check_stderr"] = (c.stderr or c.stdout)[:2000]
|
||||
if c.returncode != 0:
|
||||
return out
|
||||
out["check_ok"] = True
|
||||
b = subprocess.run(["ail","build",str(src),"-o",str(work/"bin")],
|
||||
capture_output=True, text=True, timeout=60)
|
||||
if b.returncode != 0:
|
||||
out["check_stderr"] = b.stderr[:2000]
|
||||
return out
|
||||
out["build_ok"] = True
|
||||
r = subprocess.run([str(work/"bin")], capture_output=True, text=True, timeout=10)
|
||||
if r.returncode == 0:
|
||||
out["run_ok"] = True
|
||||
out["run_stdout"] = r.stdout
|
||||
out["matches_expected"] = (r.stdout == expected_stdout)
|
||||
return out
|
||||
|
||||
# need expected stdout per task
|
||||
TASKS = {}
|
||||
CMA_TASKS = Path("/home/brummel/dev/ailang/experiments/2026-05-12-cross-model-authoring/master/tasks")
|
||||
for f in CMA_TASKS.glob("*.task.json"):
|
||||
t = json.loads(f.read_text())
|
||||
TASKS[t["id"]] = t["expected_stdout"]
|
||||
|
||||
base = RUN_DIR / "work2"
|
||||
base.mkdir(exist_ok=True)
|
||||
|
||||
new_rows = []
|
||||
for r in RESULTS:
|
||||
code = r["extracted_code"]
|
||||
cohort = r["cohort"]
|
||||
pairs = {"classic":None,"apply":REVERSE_APPLY,"call":REVERSE_CALL}[cohort]
|
||||
pipeline_input = reverse_sub(code, pairs) if pairs else code
|
||||
mod_name = extract_module_name(pipeline_input) or r["task_id"]
|
||||
pipe = run_pipeline_fixed(pipeline_input, mod_name, TASKS[r["task_id"]], base)
|
||||
|
||||
own_disc = count_tags(code, DISCRIM[cohort])
|
||||
foreign_disc = {}
|
||||
for c, tags in DISCRIM.items():
|
||||
if c == cohort: continue
|
||||
foreign_disc[c] = count_tags(code, tags)
|
||||
|
||||
new_rows.append({
|
||||
"cohort": cohort, "task_id": r["task_id"], "trial": r["trial"],
|
||||
"module_name_in_output": mod_name,
|
||||
"module_name_expected": r["task_id"],
|
||||
"module_match": (mod_name == r["task_id"]),
|
||||
"own_disc": own_disc,
|
||||
"foreign_disc": foreign_disc,
|
||||
"check_ok": pipe["check_ok"],
|
||||
"build_ok": pipe["build_ok"],
|
||||
"run_ok": pipe["run_ok"],
|
||||
"matches_expected": pipe["matches_expected"],
|
||||
"run_stdout": pipe["run_stdout"],
|
||||
"first_err_line": pipe["check_stderr"].split("\n")[0][:140],
|
||||
})
|
||||
|
||||
(RUN_DIR / "results2.json").write_text(json.dumps(new_rows, indent=2))
|
||||
|
||||
print("\n=== final pipeline pass per cohort ===")
|
||||
by_cohort = {}
|
||||
for r in new_rows:
|
||||
c = by_cohort.setdefault(r["cohort"], {"n":0,"mod":0,"check":0,"build":0,"run":0,"correct":0})
|
||||
c["n"] += 1
|
||||
if r["module_match"]: c["mod"] += 1
|
||||
if r["check_ok"]: c["check"] += 1
|
||||
if r["build_ok"]: c["build"] += 1
|
||||
if r["run_ok"]: c["run"] += 1
|
||||
if r["matches_expected"]: c["correct"] += 1
|
||||
for cohort, c in by_cohort.items():
|
||||
print(f"{cohort:8s} n={c['n']} mod-name-OK={c['mod']} check={c['check']} build={c['build']} run={c['run']} stdout-correct={c['correct']}")
|
||||
|
||||
print("\n=== discriminator tag totals (head of function application only) ===")
|
||||
disc_totals = {}
|
||||
for r in new_rows:
|
||||
c = r["cohort"]
|
||||
own_sum = sum(r["own_disc"].values())
|
||||
foreign_sum = sum(sum(v.values()) for v in r["foreign_disc"].values())
|
||||
t = disc_totals.setdefault(c, {"own":0,"foreign":0,"by_foreign":{}})
|
||||
t["own"] += own_sum
|
||||
for fc, fv in r["foreign_disc"].items():
|
||||
t["by_foreign"][fc] = t["by_foreign"].get(fc,0) + sum(fv.values())
|
||||
t["foreign"] += foreign_sum
|
||||
for cohort, t in disc_totals.items():
|
||||
print(f"{cohort:8s} own={t['own']:3d} foreign={t['foreign']:3d} breakdown={t['by_foreign']}")
|
||||
|
||||
print("\n=== per-row failure mode ===")
|
||||
for r in new_rows:
|
||||
flag = "OK" if r["matches_expected"] else ("CORRECT-NOT" if r["run_ok"] else ("RUN" if r["build_ok"] else ("BUILD" if r["check_ok"] else "CHECK")))
|
||||
print(f"[{r['cohort']:8s}] {r['task_id']:18s} t{r['trial']} {flag:12s} {r['first_err_line']}")
|
||||
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A/B test: does Qwen3-Coder produce valid Form-A code more reliably
|
||||
when the spec uses long tag names (apply/lambda/constructor/typecon)
|
||||
vs. the current short ones (app/lam/ctor/con)?
|
||||
|
||||
Setup:
|
||||
- Two specs, identical except for tag names.
|
||||
- Same N tasks, fired against each cohort.
|
||||
- For each call: log raw response, run it through `ail check`
|
||||
(renamed outputs go through reverse substitution first), capture
|
||||
parser / typecheck / build / run results.
|
||||
|
||||
Token budget guard: aborts before exceeding --token-budget.
|
||||
Reads token from $IONOS_API_TOKEN (caller sources from ~/.ionos_token).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from dataclasses import dataclass, asdict, field
|
||||
from pathlib import Path
|
||||
|
||||
ENDPOINT = "https://openai.inference.de-txl.ionos.com/v1/chat/completions"
|
||||
MODEL = "Qwen/Qwen3-Coder-Next"
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
CMA = REPO / "experiments" / "2026-05-12-cross-model-authoring"
|
||||
SPEC_CLASSIC = CMA / "rendered" / "ail.md"
|
||||
SPEC_APPLY = CMA / "rendered" / "ail-renamed.md"
|
||||
SPEC_CALL = CMA / "rendered" / "ail-call.md"
|
||||
TASKS_DIR = CMA / "master" / "tasks"
|
||||
|
||||
# Reverse substitution per cohort so `ail check` (which only knows
|
||||
# the classic tags) can parse outputs. Order: longer first.
|
||||
REVERSE_APPLY = [
|
||||
("tail-apply", "tail-app"),
|
||||
("term-constructor","term-ctor"),
|
||||
("pat-constructor", "pat-ctor"),
|
||||
("constructor", "ctor"),
|
||||
("apply", "app"),
|
||||
("lambda", "lam"),
|
||||
("typecon", "con"),
|
||||
]
|
||||
REVERSE_CALL = [
|
||||
("tail-call", "tail-app"),
|
||||
("term-constructor","term-ctor"),
|
||||
("pat-constructor", "pat-ctor"),
|
||||
("constructor", "ctor"),
|
||||
("call", "app"),
|
||||
("lambda", "lam"),
|
||||
("typecon", "con"),
|
||||
]
|
||||
|
||||
def reverse_substitute(src: str, pairs) -> str:
|
||||
out = src
|
||||
for new, old in pairs:
|
||||
out = re.sub(rf"\({re.escape(new)}(?=[\s)])", f"({old}", out)
|
||||
return out
|
||||
|
||||
# Tag sets used to count cohort-consistency.
|
||||
TAGS_CLASSIC = ["tail-app","term-ctor","pat-ctor","ctor","app","lam","con"]
|
||||
TAGS_APPLY = ["tail-apply","term-constructor","pat-constructor","constructor","apply","lambda","typecon"]
|
||||
TAGS_CALL = ["tail-call","term-constructor","pat-constructor","constructor","call","lambda","typecon"]
|
||||
|
||||
def count_tags(src: str, tags: list[str]) -> dict[str,int]:
|
||||
counts: dict[str,int] = {}
|
||||
for t in tags:
|
||||
# match `(tag ` or `(tag)` only
|
||||
counts[t] = len(re.findall(rf"\({re.escape(t)}(?=[\s)])", src))
|
||||
return counts
|
||||
|
||||
@dataclass
|
||||
class CallResult:
|
||||
cohort: str
|
||||
task_id: str
|
||||
trial: int
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
raw_response: str = ""
|
||||
extracted_code: str = ""
|
||||
own_tag_counts: dict[str,int] = field(default_factory=dict)
|
||||
foreign_tag_counts: dict[str,dict[str,int]] = field(default_factory=dict)
|
||||
ail_check_ok: bool = False
|
||||
ail_check_stderr: str = ""
|
||||
ail_build_ok: bool = False
|
||||
ail_run_ok: bool = False
|
||||
ail_run_stdout: str = ""
|
||||
stdout_matches_expected: bool = False
|
||||
notes: str = ""
|
||||
|
||||
def load_tasks() -> list[dict]:
|
||||
tasks = []
|
||||
for f in sorted(TASKS_DIR.glob("*.task.json")):
|
||||
tasks.append(json.loads(f.read_text()))
|
||||
return tasks
|
||||
|
||||
def extract_form_a(response: str) -> str:
|
||||
# Prefer fenced ```ail blocks; else fall back to the longest
|
||||
# parenthesised expression starting with (module …).
|
||||
m = re.findall(r"```(?:ail)?\s*\n(.*?)```", response, re.DOTALL)
|
||||
for block in m:
|
||||
if "(module" in block:
|
||||
return block.strip()
|
||||
# Fallback: from first '(module' to matching close paren.
|
||||
idx = response.find("(module")
|
||||
if idx < 0:
|
||||
return ""
|
||||
depth = 0
|
||||
out = []
|
||||
for ch in response[idx:]:
|
||||
out.append(ch)
|
||||
if ch == "(": depth += 1
|
||||
elif ch == ")":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return "".join(out)
|
||||
return "".join(out)
|
||||
|
||||
def call_ionos(token: str, system_prompt: str, user_prompt: str, *, max_retries=3) -> dict:
|
||||
body = json.dumps({
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
"temperature": 0.2,
|
||||
"top_p": 0.95,
|
||||
}).encode("utf-8")
|
||||
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
|
||||
backoffs = [1, 4, 16]
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
req = urllib.request.Request(ENDPOINT, data=body, headers=headers, method="POST")
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
return json.loads(r.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
code = e.code
|
||||
if code in (429,) or 500 <= code < 600:
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(backoffs[attempt])
|
||||
continue
|
||||
raise
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(backoffs[attempt])
|
||||
continue
|
||||
raise
|
||||
raise RuntimeError("retries exhausted")
|
||||
|
||||
def run_pipeline(code: str, task_id: str, expected_stdout: str, work: Path) -> dict:
|
||||
src = work / f"{task_id}.ail"
|
||||
src.write_text(code)
|
||||
result = {"check_ok": False, "check_stderr": "", "build_ok": False,
|
||||
"run_ok": False, "run_stdout": "", "matches_expected": False}
|
||||
check = subprocess.run(["ail", "check", str(src)], capture_output=True, text=True, timeout=30)
|
||||
result["check_stderr"] = check.stderr + check.stdout if check.returncode != 0 else ""
|
||||
if check.returncode != 0:
|
||||
return result
|
||||
result["check_ok"] = True
|
||||
build = subprocess.run(["ail", "build", str(src), "-o", str(work / task_id)],
|
||||
capture_output=True, text=True, timeout=60)
|
||||
if build.returncode != 0:
|
||||
result["check_stderr"] = build.stderr
|
||||
return result
|
||||
result["build_ok"] = True
|
||||
run = subprocess.run([str(work / task_id)], capture_output=True, text=True, timeout=10)
|
||||
if run.returncode == 0:
|
||||
result["run_ok"] = True
|
||||
result["run_stdout"] = run.stdout
|
||||
result["matches_expected"] = (run.stdout == expected_stdout)
|
||||
return result
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", type=Path, required=True)
|
||||
ap.add_argument("--trials", type=int, default=2)
|
||||
ap.add_argument("--token-budget", type=int, default=300_000)
|
||||
ap.add_argument("--dry-run", action="store_true", help="don't call IONOS; use empty responses")
|
||||
args = ap.parse_args()
|
||||
|
||||
token = os.environ.get("IONOS_API_TOKEN", "").strip()
|
||||
if not args.dry_run and not token:
|
||||
print("ERROR: IONOS_API_TOKEN not set", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
tasks = load_tasks()
|
||||
|
||||
# (name, system_prompt, own_tags, reverse_pairs_or_None)
|
||||
cohorts = [
|
||||
("classic", SPEC_CLASSIC.read_text(), TAGS_CLASSIC, None),
|
||||
("apply", SPEC_APPLY.read_text(), TAGS_APPLY, REVERSE_APPLY),
|
||||
("call", SPEC_CALL.read_text(), TAGS_CALL, REVERSE_CALL),
|
||||
]
|
||||
all_tag_sets = {"classic": TAGS_CLASSIC, "apply": TAGS_APPLY, "call": TAGS_CALL}
|
||||
|
||||
rows: list[CallResult] = []
|
||||
tokens_used = 0
|
||||
work = args.out / "work"
|
||||
work.mkdir(exist_ok=True)
|
||||
|
||||
for cohort, system_prompt, own_tags, reverse_pairs in cohorts:
|
||||
for task in tasks:
|
||||
for trial in range(args.trials):
|
||||
if tokens_used >= args.token_budget:
|
||||
print(f"BUDGET HIT at {tokens_used} tokens — stopping", file=sys.stderr)
|
||||
break
|
||||
user_prompt = task["description"] + "\n\nReturn only the complete module as a single ```ail fenced block."
|
||||
row = CallResult(cohort=cohort, task_id=task["id"], trial=trial)
|
||||
if args.dry_run:
|
||||
row.raw_response = "(dry-run)"
|
||||
else:
|
||||
print(f"[{cohort}] {task['id']} trial {trial} …", file=sys.stderr)
|
||||
resp = call_ionos(token, system_prompt, user_prompt)
|
||||
row.raw_response = resp["choices"][0]["message"]["content"]
|
||||
row.prompt_tokens = resp["usage"]["prompt_tokens"]
|
||||
row.completion_tokens = resp["usage"]["completion_tokens"]
|
||||
tokens_used += resp["usage"]["total_tokens"]
|
||||
code = extract_form_a(row.raw_response)
|
||||
row.extracted_code = code
|
||||
row.own_tag_counts = count_tags(code, own_tags)
|
||||
for other_name, other_tags in all_tag_sets.items():
|
||||
if other_name != cohort:
|
||||
row.foreign_tag_counts[other_name] = count_tags(code, other_tags)
|
||||
if code:
|
||||
pipeline_input = reverse_substitute(code, reverse_pairs) if reverse_pairs else code
|
||||
pipe = run_pipeline(pipeline_input, f"{cohort}_{task['id']}_t{trial}",
|
||||
task["expected_stdout"], work)
|
||||
row.ail_check_ok = pipe["check_ok"]
|
||||
row.ail_check_stderr = pipe["check_stderr"][:2000]
|
||||
row.ail_build_ok = pipe["build_ok"]
|
||||
row.ail_run_ok = pipe["run_ok"]
|
||||
row.ail_run_stdout = pipe["run_stdout"]
|
||||
row.stdout_matches_expected = pipe["matches_expected"]
|
||||
else:
|
||||
row.notes = "no form-A extracted"
|
||||
rows.append(row)
|
||||
(args.out / f"{cohort}_{task['id']}_t{trial}.raw.txt").write_text(row.raw_response)
|
||||
|
||||
out_file = args.out / "results.json"
|
||||
out_file.write_text(json.dumps([asdict(r) for r in rows], indent=2))
|
||||
print(f"\nwrote {out_file} — {len(rows)} rows, {tokens_used} tokens used")
|
||||
|
||||
# Summary table.
|
||||
by_cohort: dict[str, dict] = {}
|
||||
for r in rows:
|
||||
c = by_cohort.setdefault(r.cohort, {"n":0, "extracted":0, "check":0, "build":0, "run":0, "correct":0,
|
||||
"own_tags":0, "foreign_tags":0})
|
||||
c["n"] += 1
|
||||
if r.extracted_code: c["extracted"] += 1
|
||||
if r.ail_check_ok: c["check"] += 1
|
||||
if r.ail_build_ok: c["build"] += 1
|
||||
if r.ail_run_ok: c["run"] += 1
|
||||
if r.stdout_matches_expected: c["correct"] += 1
|
||||
c["own_tags"] += sum(r.own_tag_counts.values())
|
||||
c["foreign_tags"] += sum(sum(v.values()) for v in r.foreign_tag_counts.values())
|
||||
|
||||
print("\n=== summary ===")
|
||||
for cohort, c in by_cohort.items():
|
||||
print(f"{cohort:8s} n={c['n']:2d} extracted={c['extracted']:2d} "
|
||||
f"check={c['check']:2d} build={c['build']:2d} run={c['run']:2d} "
|
||||
f"correct={c['correct']:2d} own-tags={c['own_tags']:3d} foreign-tags={c['foreign_tags']:3d}")
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user