diff --git a/experiments/2026-05-12-cross-model-authoring/.gitignore b/experiments/2026-05-12-cross-model-authoring/.gitignore new file mode 100644 index 0000000..a8159c7 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/.gitignore @@ -0,0 +1,2 @@ +rendered/ail-renamed.md +rendered/ail-call.md diff --git a/experiments/2026-05-12-cross-model-authoring/rename-spec.py b/experiments/2026-05-12-cross-model-authoring/rename-spec.py new file mode 100644 index 0000000..3bb8836 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/rename-spec.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Produce a renamed-tags variant of the rendered Form-A spec. + +Two modes: +- apply: full-form resolution of abbreviations + app→apply, lam→lambda, ctor→constructor, con→typecon, tail-app→tail-apply +- call: like apply but with call as the application head (semantic shift) + app→call, tail-app→tail-call, rest as in apply + +Each pattern is matched in two contexts: +- S-expression head: `(NAME ` and `(NAME)` +- Backtick mention: `` `NAME` `` + +Plain-text occurrences ("application", "construction", etc.) are +preserved by anchoring on the contexts above. +""" +import re +import sys +from pathlib import Path + +PAIRS_APPLY = [ + ("tail-app", "tail-apply"), + ("term-ctor", "term-constructor"), + ("pat-ctor", "pat-constructor"), + ("ctor", "constructor"), + ("app", "apply"), + ("lam", "lambda"), + ("con", "typecon"), +] + +PAIRS_CALL = [ + ("tail-app", "tail-call"), + ("term-ctor", "term-constructor"), + ("pat-ctor", "pat-constructor"), + ("ctor", "constructor"), + ("app", "call"), + ("lam", "lambda"), + ("con", "typecon"), +] + +MODES = {"apply": PAIRS_APPLY, "call": PAIRS_CALL} + +def transform(src: str, pairs) -> str: + out = src + for old, new in pairs: + out = re.sub(rf"\({re.escape(old)}(?=[\s)])", f"({new}", out) + out = re.sub(rf"`{re.escape(old)}`", f"`{new}`", out) + return out + +def main() -> int: + if len(sys.argv) != 4 or sys.argv[1] not in MODES: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + return 2 + mode = sys.argv[1] + inp = Path(sys.argv[2]) + out = Path(sys.argv[3]) + out.write_text(transform(inp.read_text(), MODES[mode])) + print(f"wrote {out} ({out.stat().st_size} bytes, mode={mode})") + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/experiments/2026-05-12-cross-model-authoring/rendered/ail.md b/experiments/2026-05-12-cross-model-authoring/rendered/ail.md index b5620e4..2efd433 100644 --- a/experiments/2026-05-12-cross-model-authoring/rendered/ail.md +++ b/experiments/2026-05-12-cross-model-authoring/rendered/ail.md @@ -349,13 +349,13 @@ arms or into code outside the `match`. Every function type carries an effect row — a set of effect labels that the function may raise. The two currently wired effects are -`IO` (required to call effect operations like `io/print_int`) and +`IO` (required to call effect operations like `io/print_str`) and `Diverge` (used for functions that may not terminate). An empty effect row marks the function as pure. Effect operations are reached through the `do` term, not through a regular function application. The `do` form names the operation -(e.g. `io/print_int`) and lists the operation's arguments; the +(e.g. `io/print_str`) and lists the operation's arguments; the typechecker resolves the operation against the prelude's effect-op table, checks the arguments, and accumulates the operation's effect label into the enclosing function's effect row. @@ -374,8 +374,9 @@ in canonical output. Effect operation: `(do OP-NAME ARG*)` — the leading `do` head is the parser's signal that the next identifier is an effect op, not a -function. Example: `(do io/print_int 42)` prints the integer 42 -and raises the `IO` effect. +function. Example: `(do io/print_str "hi\n")` prints the string +`hi` followed by a newline and raises the `IO` effect. To print a +non-string value, first convert it: `(do io/print_str (app int_to_str 42))`. Sequencing: `(seq LHS RHS)`. The value of `(seq X Y)` is `Y`'s value; `X`'s value is discarded. Chains of sequencing are written @@ -385,10 +386,10 @@ value. ```ail (module fn_with_do_seq (fn main - (doc "Print two Ints in sequence. Exercises TermDo, TermSeq, and the IO effect on a fn type. The trailing unit return is implicit in the last Do (op returns Unit).") + (doc "Print two Ints in sequence. Exercises TermDo, TermSeq, the IO effect on a fn type, and int-to-str conversion via the prelude `int_to_str`. The trailing unit return is implicit in the last Do (op returns Unit).") (type (fn-type (params) (ret (con Unit)) (effects IO))) (params) - (body (seq (do io/print_int 1) (do io/print_int 2))))) + (body (seq (do io/print_str (app int_to_str 1)) (do io/print_str (app int_to_str 2)))))) ``` ## 9. Literals @@ -547,14 +548,14 @@ Effect operations (reached through `do`, not `app`): | Op | Signature | Effect | |----|----------|-------| -| `io/print_int` | `(Int) -> Unit` | `IO` | -| `io/print_bool` | `(Bool) -> Unit` | `IO` | | `io/print_str` | `(Str) -> Unit` | `IO` | -| `io/print_float` | `(Float) -> Unit` | `IO` | -`int_to_str` is intentionally NOT in the prelude — it is type- -installed in a future milestone but codegen-deferred pending the -heap-Str ABI work. Do not call it. +`io/print_str` is the only print operation, and it appends a +trailing newline (the runtime calls C `puts`). To print a non-string +value, convert it first via `int_to_str`, `float_to_str`, or +`bool_to_str` — all three are in the prelude (see the prelude +table above). To print two values on consecutive lines, sequence +two `(do io/print_str ...)` calls with `seq`. ## 12. Content addressing diff --git a/experiments/2026-05-21-naming-ab/.gitignore b/experiments/2026-05-21-naming-ab/.gitignore new file mode 100644 index 0000000..a1e0396 --- /dev/null +++ b/experiments/2026-05-21-naming-ab/.gitignore @@ -0,0 +1 @@ +runs/ diff --git a/experiments/2026-05-21-naming-ab/reprocess.py b/experiments/2026-05-21-naming-ab/reprocess.py new file mode 100644 index 0000000..b81b81e --- /dev/null +++ b/experiments/2026-05-21-naming-ab/reprocess.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Re-process the raw responses from a finished run with fixes: +- Per-call temp dir so file stem matches module name +- Tag-disjoint counting (only count tags unique to a cohort, + ignore overlapping ones like `lambda` shared by apply+call) +""" +from __future__ import annotations +import json, re, subprocess, sys +from pathlib import Path + +RUN_DIR = Path(sys.argv[1]) +RESULTS = json.loads((RUN_DIR / "results.json").read_text()) + +REVERSE_APPLY = [("tail-apply","tail-app"),("term-constructor","term-ctor"), + ("pat-constructor","pat-ctor"),("constructor","ctor"), + ("apply","app"),("lambda","lam"),("typecon","con")] +REVERSE_CALL = [("tail-call","tail-app"),("term-constructor","term-ctor"), + ("pat-constructor","pat-ctor"),("constructor","ctor"), + ("call","app"),("lambda","lam"),("typecon","con")] + +def reverse_sub(src, pairs): + out = src + for n,o in pairs: + out = re.sub(rf"\({re.escape(n)}(?=[\s)])", f"({o}", out) + return out + +# Disjoint discriminator tags only — the head of function application +# (which IS the test variable). Ignore lambda/constructor/typecon since +# apply and call share those. +DISCRIM = { + "classic": ["app","tail-app"], + "apply": ["apply","tail-apply"], + "call": ["call","tail-call"], +} + +def count_tags(src, tags): + return {t: len(re.findall(rf"\({re.escape(t)}(?=[\s)])", src)) for t in tags} + +def extract_module_name(code): + m = re.search(r"\(module\s+([A-Za-z_][A-Za-z0-9_]*)", code) + return m.group(1) if m else None + +def run_pipeline_fixed(code, module_name, expected_stdout, base): + work = base / module_name + work.mkdir(parents=True, exist_ok=True) + src = work / f"{module_name}.ail" + src.write_text(code) + out = {"check_ok": False, "check_stderr": "", "build_ok": False, + "run_ok": False, "run_stdout": "", "matches_expected": False} + c = subprocess.run(["ail","check",str(src)], capture_output=True, text=True, timeout=30) + out["check_stderr"] = (c.stderr or c.stdout)[:2000] + if c.returncode != 0: + return out + out["check_ok"] = True + b = subprocess.run(["ail","build",str(src),"-o",str(work/"bin")], + capture_output=True, text=True, timeout=60) + if b.returncode != 0: + out["check_stderr"] = b.stderr[:2000] + return out + out["build_ok"] = True + r = subprocess.run([str(work/"bin")], capture_output=True, text=True, timeout=10) + if r.returncode == 0: + out["run_ok"] = True + out["run_stdout"] = r.stdout + out["matches_expected"] = (r.stdout == expected_stdout) + return out + +# need expected stdout per task +TASKS = {} +CMA_TASKS = Path("/home/brummel/dev/ailang/experiments/2026-05-12-cross-model-authoring/master/tasks") +for f in CMA_TASKS.glob("*.task.json"): + t = json.loads(f.read_text()) + TASKS[t["id"]] = t["expected_stdout"] + +base = RUN_DIR / "work2" +base.mkdir(exist_ok=True) + +new_rows = [] +for r in RESULTS: + code = r["extracted_code"] + cohort = r["cohort"] + pairs = {"classic":None,"apply":REVERSE_APPLY,"call":REVERSE_CALL}[cohort] + pipeline_input = reverse_sub(code, pairs) if pairs else code + mod_name = extract_module_name(pipeline_input) or r["task_id"] + pipe = run_pipeline_fixed(pipeline_input, mod_name, TASKS[r["task_id"]], base) + + own_disc = count_tags(code, DISCRIM[cohort]) + foreign_disc = {} + for c, tags in DISCRIM.items(): + if c == cohort: continue + foreign_disc[c] = count_tags(code, tags) + + new_rows.append({ + "cohort": cohort, "task_id": r["task_id"], "trial": r["trial"], + "module_name_in_output": mod_name, + "module_name_expected": r["task_id"], + "module_match": (mod_name == r["task_id"]), + "own_disc": own_disc, + "foreign_disc": foreign_disc, + "check_ok": pipe["check_ok"], + "build_ok": pipe["build_ok"], + "run_ok": pipe["run_ok"], + "matches_expected": pipe["matches_expected"], + "run_stdout": pipe["run_stdout"], + "first_err_line": pipe["check_stderr"].split("\n")[0][:140], + }) + +(RUN_DIR / "results2.json").write_text(json.dumps(new_rows, indent=2)) + +print("\n=== final pipeline pass per cohort ===") +by_cohort = {} +for r in new_rows: + c = by_cohort.setdefault(r["cohort"], {"n":0,"mod":0,"check":0,"build":0,"run":0,"correct":0}) + c["n"] += 1 + if r["module_match"]: c["mod"] += 1 + if r["check_ok"]: c["check"] += 1 + if r["build_ok"]: c["build"] += 1 + if r["run_ok"]: c["run"] += 1 + if r["matches_expected"]: c["correct"] += 1 +for cohort, c in by_cohort.items(): + print(f"{cohort:8s} n={c['n']} mod-name-OK={c['mod']} check={c['check']} build={c['build']} run={c['run']} stdout-correct={c['correct']}") + +print("\n=== discriminator tag totals (head of function application only) ===") +disc_totals = {} +for r in new_rows: + c = r["cohort"] + own_sum = sum(r["own_disc"].values()) + foreign_sum = sum(sum(v.values()) for v in r["foreign_disc"].values()) + t = disc_totals.setdefault(c, {"own":0,"foreign":0,"by_foreign":{}}) + t["own"] += own_sum + for fc, fv in r["foreign_disc"].items(): + t["by_foreign"][fc] = t["by_foreign"].get(fc,0) + sum(fv.values()) + t["foreign"] += foreign_sum +for cohort, t in disc_totals.items(): + print(f"{cohort:8s} own={t['own']:3d} foreign={t['foreign']:3d} breakdown={t['by_foreign']}") + +print("\n=== per-row failure mode ===") +for r in new_rows: + flag = "OK" if r["matches_expected"] else ("CORRECT-NOT" if r["run_ok"] else ("RUN" if r["build_ok"] else ("BUILD" if r["check_ok"] else "CHECK"))) + print(f"[{r['cohort']:8s}] {r['task_id']:18s} t{r['trial']} {flag:12s} {r['first_err_line']}") diff --git a/experiments/2026-05-21-naming-ab/run.py b/experiments/2026-05-21-naming-ab/run.py new file mode 100644 index 0000000..1f329cb --- /dev/null +++ b/experiments/2026-05-21-naming-ab/run.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""A/B test: does Qwen3-Coder produce valid Form-A code more reliably +when the spec uses long tag names (apply/lambda/constructor/typecon) +vs. the current short ones (app/lam/ctor/con)? + +Setup: +- Two specs, identical except for tag names. +- Same N tasks, fired against each cohort. +- For each call: log raw response, run it through `ail check` + (renamed outputs go through reverse substitution first), capture + parser / typecheck / build / run results. + +Token budget guard: aborts before exceeding --token-budget. +Reads token from $IONOS_API_TOKEN (caller sources from ~/.ionos_token). +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import time +import urllib.request +import urllib.error +from dataclasses import dataclass, asdict, field +from pathlib import Path + +ENDPOINT = "https://openai.inference.de-txl.ionos.com/v1/chat/completions" +MODEL = "Qwen/Qwen3-Coder-Next" + +REPO = Path(__file__).resolve().parents[2] +CMA = REPO / "experiments" / "2026-05-12-cross-model-authoring" +SPEC_CLASSIC = CMA / "rendered" / "ail.md" +SPEC_APPLY = CMA / "rendered" / "ail-renamed.md" +SPEC_CALL = CMA / "rendered" / "ail-call.md" +TASKS_DIR = CMA / "master" / "tasks" + +# Reverse substitution per cohort so `ail check` (which only knows +# the classic tags) can parse outputs. Order: longer first. +REVERSE_APPLY = [ + ("tail-apply", "tail-app"), + ("term-constructor","term-ctor"), + ("pat-constructor", "pat-ctor"), + ("constructor", "ctor"), + ("apply", "app"), + ("lambda", "lam"), + ("typecon", "con"), +] +REVERSE_CALL = [ + ("tail-call", "tail-app"), + ("term-constructor","term-ctor"), + ("pat-constructor", "pat-ctor"), + ("constructor", "ctor"), + ("call", "app"), + ("lambda", "lam"), + ("typecon", "con"), +] + +def reverse_substitute(src: str, pairs) -> str: + out = src + for new, old in pairs: + out = re.sub(rf"\({re.escape(new)}(?=[\s)])", f"({old}", out) + return out + +# Tag sets used to count cohort-consistency. +TAGS_CLASSIC = ["tail-app","term-ctor","pat-ctor","ctor","app","lam","con"] +TAGS_APPLY = ["tail-apply","term-constructor","pat-constructor","constructor","apply","lambda","typecon"] +TAGS_CALL = ["tail-call","term-constructor","pat-constructor","constructor","call","lambda","typecon"] + +def count_tags(src: str, tags: list[str]) -> dict[str,int]: + counts: dict[str,int] = {} + for t in tags: + # match `(tag ` or `(tag)` only + counts[t] = len(re.findall(rf"\({re.escape(t)}(?=[\s)])", src)) + return counts + +@dataclass +class CallResult: + cohort: str + task_id: str + trial: int + prompt_tokens: int = 0 + completion_tokens: int = 0 + raw_response: str = "" + extracted_code: str = "" + own_tag_counts: dict[str,int] = field(default_factory=dict) + foreign_tag_counts: dict[str,dict[str,int]] = field(default_factory=dict) + ail_check_ok: bool = False + ail_check_stderr: str = "" + ail_build_ok: bool = False + ail_run_ok: bool = False + ail_run_stdout: str = "" + stdout_matches_expected: bool = False + notes: str = "" + +def load_tasks() -> list[dict]: + tasks = [] + for f in sorted(TASKS_DIR.glob("*.task.json")): + tasks.append(json.loads(f.read_text())) + return tasks + +def extract_form_a(response: str) -> str: + # Prefer fenced ```ail blocks; else fall back to the longest + # parenthesised expression starting with (module …). + m = re.findall(r"```(?:ail)?\s*\n(.*?)```", response, re.DOTALL) + for block in m: + if "(module" in block: + return block.strip() + # Fallback: from first '(module' to matching close paren. + idx = response.find("(module") + if idx < 0: + return "" + depth = 0 + out = [] + for ch in response[idx:]: + out.append(ch) + if ch == "(": depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + return "".join(out) + return "".join(out) + +def call_ionos(token: str, system_prompt: str, user_prompt: str, *, max_retries=3) -> dict: + body = json.dumps({ + "model": MODEL, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + "temperature": 0.2, + "top_p": 0.95, + }).encode("utf-8") + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + backoffs = [1, 4, 16] + for attempt in range(max_retries): + try: + req = urllib.request.Request(ENDPOINT, data=body, headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=120) as r: + return json.loads(r.read()) + except urllib.error.HTTPError as e: + code = e.code + if code in (429,) or 500 <= code < 600: + if attempt < max_retries - 1: + time.sleep(backoffs[attempt]) + continue + raise + except (urllib.error.URLError, TimeoutError) as e: + if attempt < max_retries - 1: + time.sleep(backoffs[attempt]) + continue + raise + raise RuntimeError("retries exhausted") + +def run_pipeline(code: str, task_id: str, expected_stdout: str, work: Path) -> dict: + src = work / f"{task_id}.ail" + src.write_text(code) + result = {"check_ok": False, "check_stderr": "", "build_ok": False, + "run_ok": False, "run_stdout": "", "matches_expected": False} + check = subprocess.run(["ail", "check", str(src)], capture_output=True, text=True, timeout=30) + result["check_stderr"] = check.stderr + check.stdout if check.returncode != 0 else "" + if check.returncode != 0: + return result + result["check_ok"] = True + build = subprocess.run(["ail", "build", str(src), "-o", str(work / task_id)], + capture_output=True, text=True, timeout=60) + if build.returncode != 0: + result["check_stderr"] = build.stderr + return result + result["build_ok"] = True + run = subprocess.run([str(work / task_id)], capture_output=True, text=True, timeout=10) + if run.returncode == 0: + result["run_ok"] = True + result["run_stdout"] = run.stdout + result["matches_expected"] = (run.stdout == expected_stdout) + return result + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--out", type=Path, required=True) + ap.add_argument("--trials", type=int, default=2) + ap.add_argument("--token-budget", type=int, default=300_000) + ap.add_argument("--dry-run", action="store_true", help="don't call IONOS; use empty responses") + args = ap.parse_args() + + token = os.environ.get("IONOS_API_TOKEN", "").strip() + if not args.dry_run and not token: + print("ERROR: IONOS_API_TOKEN not set", file=sys.stderr) + return 2 + + args.out.mkdir(parents=True, exist_ok=True) + tasks = load_tasks() + + # (name, system_prompt, own_tags, reverse_pairs_or_None) + cohorts = [ + ("classic", SPEC_CLASSIC.read_text(), TAGS_CLASSIC, None), + ("apply", SPEC_APPLY.read_text(), TAGS_APPLY, REVERSE_APPLY), + ("call", SPEC_CALL.read_text(), TAGS_CALL, REVERSE_CALL), + ] + all_tag_sets = {"classic": TAGS_CLASSIC, "apply": TAGS_APPLY, "call": TAGS_CALL} + + rows: list[CallResult] = [] + tokens_used = 0 + work = args.out / "work" + work.mkdir(exist_ok=True) + + for cohort, system_prompt, own_tags, reverse_pairs in cohorts: + for task in tasks: + for trial in range(args.trials): + if tokens_used >= args.token_budget: + print(f"BUDGET HIT at {tokens_used} tokens — stopping", file=sys.stderr) + break + user_prompt = task["description"] + "\n\nReturn only the complete module as a single ```ail fenced block." + row = CallResult(cohort=cohort, task_id=task["id"], trial=trial) + if args.dry_run: + row.raw_response = "(dry-run)" + else: + print(f"[{cohort}] {task['id']} trial {trial} …", file=sys.stderr) + resp = call_ionos(token, system_prompt, user_prompt) + row.raw_response = resp["choices"][0]["message"]["content"] + row.prompt_tokens = resp["usage"]["prompt_tokens"] + row.completion_tokens = resp["usage"]["completion_tokens"] + tokens_used += resp["usage"]["total_tokens"] + code = extract_form_a(row.raw_response) + row.extracted_code = code + row.own_tag_counts = count_tags(code, own_tags) + for other_name, other_tags in all_tag_sets.items(): + if other_name != cohort: + row.foreign_tag_counts[other_name] = count_tags(code, other_tags) + if code: + pipeline_input = reverse_substitute(code, reverse_pairs) if reverse_pairs else code + pipe = run_pipeline(pipeline_input, f"{cohort}_{task['id']}_t{trial}", + task["expected_stdout"], work) + row.ail_check_ok = pipe["check_ok"] + row.ail_check_stderr = pipe["check_stderr"][:2000] + row.ail_build_ok = pipe["build_ok"] + row.ail_run_ok = pipe["run_ok"] + row.ail_run_stdout = pipe["run_stdout"] + row.stdout_matches_expected = pipe["matches_expected"] + else: + row.notes = "no form-A extracted" + rows.append(row) + (args.out / f"{cohort}_{task['id']}_t{trial}.raw.txt").write_text(row.raw_response) + + out_file = args.out / "results.json" + out_file.write_text(json.dumps([asdict(r) for r in rows], indent=2)) + print(f"\nwrote {out_file} — {len(rows)} rows, {tokens_used} tokens used") + + # Summary table. + by_cohort: dict[str, dict] = {} + for r in rows: + c = by_cohort.setdefault(r.cohort, {"n":0, "extracted":0, "check":0, "build":0, "run":0, "correct":0, + "own_tags":0, "foreign_tags":0}) + c["n"] += 1 + if r.extracted_code: c["extracted"] += 1 + if r.ail_check_ok: c["check"] += 1 + if r.ail_build_ok: c["build"] += 1 + if r.ail_run_ok: c["run"] += 1 + if r.stdout_matches_expected: c["correct"] += 1 + c["own_tags"] += sum(r.own_tag_counts.values()) + c["foreign_tags"] += sum(sum(v.values()) for v in r.foreign_tag_counts.values()) + + print("\n=== summary ===") + for cohort, c in by_cohort.items(): + print(f"{cohort:8s} n={c['n']:2d} extracted={c['extracted']:2d} " + f"check={c['check']:2d} build={c['build']:2d} run={c['run']:2d} " + f"correct={c['correct']:2d} own-tags={c['own_tags']:3d} foreign-tags={c['foreign_tags']:3d}") + return 0 + +if __name__ == "__main__": + sys.exit(main())