From fe1fb6b4f0dd01a82137474943b9b17b8c74c393 Mon Sep 17 00:00:00 2001 From: Brummel Date: Tue, 12 May 2026 12:06:34 +0200 Subject: [PATCH] iter cma.2: harness binary + 4 tasks + reference solutions + 4 integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sibling standalone Cargo crate `harness/` under experiments/2026-05-12-cross-model-authoring/ (out-of-workspace idiom carried forward from cma.1 verbatim). Six modules: strip_locations (regex pass for form-asymmetric location info, calibrated against five real `ail check`/`ail parse` stderr captures), pipeline (subprocess wrapper for parse|check|build + 5s-timeout exec; preflight on ail+clang), ionos (blocking reqwest client + retry policy per spec), mock (canned-response loader), scoring (CSV + summary.md), tasks (definition struct + loader). main.rs ties them into the per-(cohort,task) loop with budget accounting and per-turn artefact recording. Four MVP tasks land with reference solutions that compile, build, and execute green through the actual ail+clang pipeline: t1_add_three (chained `+` + io/print_int), t2_length (polymorphic List + recursion), t3_main_prints (minimal IO module), t4_count_zeros (prelude Eq Int + branched if). Reference solutions stay in canonical AILang form — param_modes is omitted when every parameter is the Implicit default, consistent with the existing examples/ corpus. 13/13 tests green: 5 lib unit (strip_locations) + 5 integration (strip_locations against verbatim captured fixtures) + 1 verify_references (drives every reference through the real ail+clang pipeline) + 1 mock_full_run (full eight-row sweep with mixed green-on-turn-2 + turn-limit cycles) + 1 budget_abort (synthetic budget exhaustion with budget_abort rows + run_status). Two implementer-phase repairs beyond the plan, both small and surfaced in the iter journal Concerns: 1. pipeline.rs renames the program file to `.ail.json` between parse and check because `ail check` enforces filename stem == module name (compiler contract the plan did not anticipate). 2. main.rs fills synthetic budget_abort rows for tasks the outer loop never reached so scores.csv preserves the expected eight-row shape on budget exhaustion. One plan/text mismatch carried over: README "Total 8 passed" reflects the plan's four-suite count; actual sweep produces 13. Doc-fix candidate for cma.3. cma.3 (live IONOS run + DESIGN.md addendum + roadmap edits) remains out of scope. --- .../2026-05-12-iter-cma.2.json | 31 +++ docs/journals/2026-05-12-iter-cma.2.md | 212 +++++++++++++++++ docs/journals/INDEX.md | 1 + .../README.md | 36 +++ .../harness/.gitignore | 2 + .../harness/Cargo.toml | 34 +++ .../harness/src/ionos.rs | 148 ++++++++++++ .../harness/src/lib.rs | 9 + .../harness/src/main.rs | 211 +++++++++++++++++ .../harness/src/mock.rs | 61 +++++ .../harness/src/pipeline.rs | 219 ++++++++++++++++++ .../harness/src/scoring.rs | 101 ++++++++ .../harness/src/strip_locations.rs | 94 ++++++++ .../harness/src/tasks.rs | 42 ++++ .../harness/tests/budget_abort.rs | 39 ++++ .../tests/fixtures/check_bare_xmod.stderr | 1 + .../check_schema_missing_field.stderr | 5 + .../tests/fixtures/check_type_mismatch.stderr | 1 + .../tests/fixtures/check_unbound_var.stderr | 1 + .../harness/tests/fixtures/mock_full_run.json | 126 ++++++++++ .../tests/fixtures/parse_unclosed.stderr | 1 + .../harness/tests/mock_full_run.rs | 44 ++++ .../harness/tests/strip_locations.rs | 59 +++++ .../harness/tests/verify_references.rs | 35 +++ .../tasks/t1_add_three.reference.ail.json | 77 ++++++ .../master/tasks/t1_add_three.task.json | 7 + .../master/tasks/t2_length.reference.ail.json | 121 ++++++++++ .../master/tasks/t2_length.task.json | 7 + .../tasks/t3_main_prints.reference.ail.json | 31 +++ .../master/tasks/t3_main_prints.task.json | 7 + .../tasks/t4_count_zeros.reference.ail.json | 145 ++++++++++++ .../master/tasks/t4_count_zeros.task.json | 7 + 32 files changed, 1915 insertions(+) create mode 100644 bench/orchestrator-stats/2026-05-12-iter-cma.2.json create mode 100644 docs/journals/2026-05-12-iter-cma.2.md create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/.gitignore create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/Cargo.toml create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/src/ionos.rs create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/src/lib.rs create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/src/main.rs create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/src/mock.rs create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/src/pipeline.rs create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/src/scoring.rs create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/src/strip_locations.rs create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/src/tasks.rs create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/tests/budget_abort.rs create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_bare_xmod.stderr create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_schema_missing_field.stderr create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_type_mismatch.stderr create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_unbound_var.stderr create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/mock_full_run.json create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/parse_unclosed.stderr create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/tests/mock_full_run.rs create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/tests/strip_locations.rs create mode 100644 experiments/2026-05-12-cross-model-authoring/harness/tests/verify_references.rs create mode 100644 experiments/2026-05-12-cross-model-authoring/master/tasks/t1_add_three.reference.ail.json create mode 100644 experiments/2026-05-12-cross-model-authoring/master/tasks/t1_add_three.task.json create mode 100644 experiments/2026-05-12-cross-model-authoring/master/tasks/t2_length.reference.ail.json create mode 100644 experiments/2026-05-12-cross-model-authoring/master/tasks/t2_length.task.json create mode 100644 experiments/2026-05-12-cross-model-authoring/master/tasks/t3_main_prints.reference.ail.json create mode 100644 experiments/2026-05-12-cross-model-authoring/master/tasks/t3_main_prints.task.json create mode 100644 experiments/2026-05-12-cross-model-authoring/master/tasks/t4_count_zeros.reference.ail.json create mode 100644 experiments/2026-05-12-cross-model-authoring/master/tasks/t4_count_zeros.task.json diff --git a/bench/orchestrator-stats/2026-05-12-iter-cma.2.json b/bench/orchestrator-stats/2026-05-12-iter-cma.2.json new file mode 100644 index 0000000..a70731a --- /dev/null +++ b/bench/orchestrator-stats/2026-05-12-iter-cma.2.json @@ -0,0 +1,31 @@ +{ + "iter_id": "cma.2", + "date": "2026-05-12", + "mode": "standard", + "outcome": "DONE", + "tasks_total": 13, + "tasks_completed": 13, + "reloops_per_task": { + "1": 0, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0, + "10": 0, + "11": 0, + "12": 1, + "13": 0 + }, + "review_loops_spec": 0, + "review_loops_quality": 0, + "blocked_reason": null, + "notes": [ + "Task 3 + Task 12 each landed one implementer-phase repair (pipeline filename-matching; budget-abort row-filling) — both flagged as Concerns in the journal, not re-loops against the spec/quality checks. reloops_per_task['12'] = 1 records the budget_abort test going RED-then-GREEN after the main.rs repair.", + "All 13 tests pass: 5 lib unit (strip_locations) + 5 strip_locations integration + 1 verify_references + 1 mock_full_run + 1 budget_abort.", + "AIL_BIN env required for verify_references and mock_full_run tests when ail is not on PATH." + ] +} diff --git a/docs/journals/2026-05-12-iter-cma.2.md b/docs/journals/2026-05-12-iter-cma.2.md new file mode 100644 index 0000000..04cfba8 --- /dev/null +++ b/docs/journals/2026-05-12-iter-cma.2.md @@ -0,0 +1,212 @@ +# iter cma.2 — Harness binary + four tasks + reference solutions + three integration tests + +**Date:** 2026-05-12 +**Started from:** 21c2d2cfaa999980f727a069792e3bd073c1f1e7 +**Status:** DONE +**Tasks completed:** 13 of 13 + +## Summary + +Stands up the sibling `harness/` standalone Cargo crate under +`experiments/2026-05-12-cross-model-authoring/`, mirroring the +out-of-workspace idiom from cma.1's `render/`. Six modules under +`harness/src/`: `strip_locations` (regex pass for form-asymmetric +location info), `pipeline` (subprocess wrapper around +`ail parse|check|build` plus the model's compiled binary, with a +fail-fast preflight on `ail` + `clang`), `ionos` (blocking reqwest +client with retry policy), `mock` (canned-response loader keyed by +cohort/task/turn), `scoring` (CSV + summary.md writers), `tasks` +(task-definition struct + loader). `main.rs` ties them into the +per-(cohort, task) loop. Four task definitions (`t1_add_three`, +`t2_length`, `t3_main_prints`, `t4_count_zeros`) with reference +solutions that all reach green through the actual ail+clang +pipeline. 13/13 tests pass: 5 inline unit tests under `--lib` +(strip_locations), 5 strip_locations integration tests against +verbatim recon-captured stderr fixtures, 1 verify_references +integration test driving every reference through the real pipeline, +1 mock_full_run end-to-end with eight rows and per-cohort artefact +trees, 1 budget_abort with the harness aborting cleanly under a +1500-token cap. + +## Per-task notes + +- cma.2.1: Bootstrap harness Cargo project + skeleton — created + `harness/{Cargo.toml,.gitignore,src/lib.rs,src/main.rs}` plus + six stub module files. Empty `[workspace]` table at the top of + Cargo.toml keeps the crate out of the root workspace. Cargo.lock + gitignored per spec. First `cargo build` succeeded with the + expected unused-import warnings on the skeleton. +- cma.2.2: strip_locations module — written verbatim from the + plan with 5 inline unit tests (json_pointer, byte_offset, line/ + column, Caused-by chain, passthrough). All 5 green on first run. +- cma.2.3: pipeline module — written verbatim from the plan plus + a small implementer-phase repair: `ail check` enforces filename + stem == module name, so `pipeline::run_pipeline` extracts the + top-level `"name"` field from the JSON via a new + `module_name_from_json` helper and renames the working file to + `.ail.json` before invoking `ail check`. Without + this, every reference would have failed at the `ail check` step + with "module name in file does not match expected name from + path". See Concerns. +- cma.2.4: ionos module — written verbatim from the plan; added + `thiserror = "1"` to `[dependencies]`. Compile-time warnings + about three unused things on the `IonosClient` path (Auth, Usage, + `last_err` assignment) are expected and flagged in the plan. +- cma.2.5: mock module — written verbatim; compile clean. +- cma.2.6: scoring module — written verbatim; compile clean. +- cma.2.7: tasks module — written verbatim; compile clean. +- cma.2.8: Author four task definitions + reference solutions — + task.json files written verbatim from the plan text. Reference + `.ail.json` files authored fresh against the cma.1 templates + (hello.ail.json shape for t3; gc_stress.ail.json's polymorphic + `List a` shape for t2 and t4; canonical chained `+` for t1's + three-arg sum; canonical `eq` invocation pattern + branched `if` + for t4's count_zeros). All four references compile clean and + produce the expected stdout. Reference solutions intentionally + omit `param_modes` because every parameter is Implicit and the + canonical AILang form omits the field in that case — consistent + with hello.ail.json, list.ail.json, eq_primitives_smoke.ail.json, + and the explanatory doc in `master/examples/param_modes_all.ail.json`. + See Concerns. Added `tempfile = "3"` to `[dev-dependencies]` for + the `verify_references.rs` integration test. The test runs every + reference through `pipeline::run_pipeline(Cohort::Json, ...)` + end-to-end (ail parse-check-build-execute); passes when + `AIL_BIN` is set to the release binary. +- cma.2.9: Wire main.rs end-to-end — written verbatim from the + plan; added `tempfile = "3"` to `[dependencies]` (not only + dev-deps) because `main.rs::run_one` uses it for per-turn + workdir. Compile clean (modulo the three flagged warnings from + ionos). +- cma.2.10: Five stderr fixtures + strip_locations integration + test — all five fixture files written verbatim from the plan + (which itself captured them from recon's run against current HEAD). + Integration test asserts strip behaviour against each fixture; + 5/5 green on first run. +- cma.2.11: mock_full_run integration test + mock_full_run.json + fixture — fixture authored by inlining each + `master/tasks/.reference.ail.json` as the `content` field + for the green-path entries, plus `ail render` for the AILX-cohort + inlines (verbatim AILX serialisations of the same references). + Test runs the harness binary against the fixture; 8 rows total + (4 tasks × 2 cohorts), `(json, t3_main_prints)` reaches green on + turn 2 (matches the plan's intended cycle: broken-then-fixed), + `(ailx, t1_add_three)` runs to the 5-turn limit (`(module garbage)` + repeated). All artefacts (turn_N_program.ext, + turn_N_request.json, turn_N_response.json, + turn_N_check_stderr.txt, turn_N_run_stdout.txt) land in + `per_cohort///`. 1/1 green. +- cma.2.12: budget_abort integration test — test verbatim from + the plan, plus an implementer-phase repair to `main.rs`: with a + tiny budget (1500), the outer loop broke out after consuming + ~2400 tokens across tasks 1+2 (both green on turn 1), but + neither row carried `final_status = budget_abort` because the + inner `run_one` only marks BudgetAbort if the budget is exhausted + *during* the turn loop. The plan's test assertion expects at + least one row with `budget_abort`. Repair: after the outer loop + breaks with `run_status="budget_exceeded"`, fill in the not-yet-run + (cohort, task) pairs with synthetic `BudgetAbort` rows. mock_full_run + remains green (the new code only runs on budget_exceeded). + 1/1 green. See Concerns. +- cma.2.13: README update + final sweep — appended "Running the + harness" section to `experiments/.../README.md` verbatim from the + plan. Full `cargo test` sweep: 5 (lib unit, strip_locations) + 5 + (integ, strip_locations) + 1 (verify_references) + 1 + (mock_full_run) + 1 (budget_abort) = 13/13 green. `git status` + shows the README modification + the new harness/ tree + the new + master/tasks/ files, all unstaged. + +## Concerns + +- **Pipeline filename-matching: minimal repair landed in `pipeline.rs`, + not flagged in the plan.** `ail check` enforces filename stem == + module name; the plan's `run_pipeline` writes the model's program + to `prog.{ext}`, which would have been rejected at every `ail + check` invocation. The repair adds a 9-line helper + `module_name_from_json` (reads JSON, returns top-level `name`) plus + 14 lines in `run_pipeline` that rename `prog.ail.json` -> + `.ail.json` between parse and check. Behaviour + outside that rename is unchanged. The structural alternative + (telling the model to name its module `prog`) would have been + semantically wrong — the task description prompts say "module + named t1_add_three" etc. Recommend a forward-fix sweep in cma.3 + if the structural decision (whether the pipeline or the prompt + owns module naming) warrants spec attention. +- **`param_modes` omitted from every reference solution.** The plan's + Step 8.2 text says "mode annotations on every parameter", but the + templates the plan references (hello.ail.json, list.ail.json, + eq_primitives_smoke.ail.json, etc.) all omit `param_modes`, and + the `param_modes_all.ail.json` master example's own doc string + documents "Implicit mode is the legacy default — `param_modes` is + omitted from canonical JSON when every entry is Implicit". The + references stay in canonical form. If the plan's intent was + explicit `["implicit", "implicit", ...]` annotations on every + reference, that is a follow-on amendment. +- **Budget-abort row-filling, minimal repair landed in `main.rs`, + not flagged in the plan.** Without it, Task 12's test fails on + the assertion `expected at least one budget_abort row`. Repair + is gated on `run_status == "budget_exceeded"` so the green path + is unchanged. +- **README "Total 8 passed" text mismatches Step 13.2's "13 passed" + expectation.** The plan's README text (Step 13.1) advertises four + test suites with "Total 8 passed" — counts the four integration + suites but skips the 5 inline unit tests under `--lib`. Step 13.2 + expects 13 across `--lib` + 4 integration tests. The README text + is reproduced verbatim per plan; the actual sweep produces 13. + Recommend a docfix in cma.3 ("Total 13 passed across --lib + 4 + integration tests"), but the cma.2 plan locked the README copy. +- **`AIL_BIN` env required for `verify_references.rs` and + `mock_full_run.rs` to run from a fresh shell.** These tests shell + out to `ail`; without `ail` on PATH the preflight bails. The + README documents this. CI integration is out of scope. + +## Known debt + +- The harness has not yet been live-fired against IONOS (deferred + to cma.3 per spec). The retry/backoff path in + `ionos::IonosClient::post` has no test coverage in cma.2 (mock + mode bypasses it entirely). +- Three compile warnings in `ionos.rs` (`Auth` and one `Usage` are + read in tests/other code; `last_err` second assignment is + intentional dead-store for retry-loop symmetry). Not addressed. + +## Files touched + +Modified (1): +- `experiments/2026-05-12-cross-model-authoring/README.md` + +New — harness crate (8 source, 4 integration tests, 6 fixtures, 1 +manifest, 1 .gitignore): +- `experiments/2026-05-12-cross-model-authoring/harness/Cargo.toml` +- `experiments/2026-05-12-cross-model-authoring/harness/.gitignore` +- `experiments/2026-05-12-cross-model-authoring/harness/src/lib.rs` +- `experiments/2026-05-12-cross-model-authoring/harness/src/main.rs` +- `experiments/2026-05-12-cross-model-authoring/harness/src/strip_locations.rs` +- `experiments/2026-05-12-cross-model-authoring/harness/src/pipeline.rs` +- `experiments/2026-05-12-cross-model-authoring/harness/src/ionos.rs` +- `experiments/2026-05-12-cross-model-authoring/harness/src/mock.rs` +- `experiments/2026-05-12-cross-model-authoring/harness/src/scoring.rs` +- `experiments/2026-05-12-cross-model-authoring/harness/src/tasks.rs` +- `experiments/2026-05-12-cross-model-authoring/harness/tests/strip_locations.rs` +- `experiments/2026-05-12-cross-model-authoring/harness/tests/verify_references.rs` +- `experiments/2026-05-12-cross-model-authoring/harness/tests/mock_full_run.rs` +- `experiments/2026-05-12-cross-model-authoring/harness/tests/budget_abort.rs` +- `experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_unbound_var.stderr` +- `experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_type_mismatch.stderr` +- `experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_bare_xmod.stderr` +- `experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_schema_missing_field.stderr` +- `experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/parse_unclosed.stderr` +- `experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/mock_full_run.json` + +New — master tasks (4 task.json + 4 reference.ail.json): +- `experiments/2026-05-12-cross-model-authoring/master/tasks/t1_add_three.task.json` +- `experiments/2026-05-12-cross-model-authoring/master/tasks/t1_add_three.reference.ail.json` +- `experiments/2026-05-12-cross-model-authoring/master/tasks/t2_length.task.json` +- `experiments/2026-05-12-cross-model-authoring/master/tasks/t2_length.reference.ail.json` +- `experiments/2026-05-12-cross-model-authoring/master/tasks/t3_main_prints.task.json` +- `experiments/2026-05-12-cross-model-authoring/master/tasks/t3_main_prints.reference.ail.json` +- `experiments/2026-05-12-cross-model-authoring/master/tasks/t4_count_zeros.task.json` +- `experiments/2026-05-12-cross-model-authoring/master/tasks/t4_count_zeros.reference.ail.json` + +## Stats + +bench/orchestrator-stats/2026-05-12-iter-cma.2.json diff --git a/docs/journals/INDEX.md b/docs/journals/INDEX.md index 77abfc8..2d77dfc 100644 --- a/docs/journals/INDEX.md +++ b/docs/journals/INDEX.md @@ -21,3 +21,4 @@ - 2026-05-12 — audit-rt: milestone close (Roundtrip Invariant) — architect drift fixed in rt.tidy (3 items: Direction-2 5th test, roadmap P1 removed, wording sync); bench all-green → 2026-05-12-audit-rt.md - 2026-05-12 — iter boss: `/boss` skill — autonomous-mode discipline (Direction freedom, Notifications, WhatsNew procedure) extracted from CLAUDE.md into a user-invoked skill → 2026-05-12-iter-boss.md - 2026-05-12 — iter cma.1: master mini-spec + render binary for cross-model authoring experiment (13 fixtures cover 34/34 AST variants, 4 test gates green, rendered/{json,ailx}.md checked in; out-of-workspace nested crate) → 2026-05-12-iter-cma.1.md +- 2026-05-12 — iter cma.2: harness binary + 4 tasks + reference solutions + 4 integration tests (six modules, real-pipeline preflight via verify_references, 13/13 tests green; pipeline.rs renames program file to module-name for ail check) → 2026-05-12-iter-cma.2.md diff --git a/experiments/2026-05-12-cross-model-authoring/README.md b/experiments/2026-05-12-cross-model-authoring/README.md index a57c70b..8c592ce 100644 --- a/experiments/2026-05-12-cross-model-authoring/README.md +++ b/experiments/2026-05-12-cross-model-authoring/README.md @@ -40,3 +40,39 @@ prints to AILX, reparses to the same canonical bytes), `spec_completeness` (every AST variant in `ailang_core::ast` is exercised by at least one example), `token_balance` (form-only blocks balanced within ±5% across the two rendered files). + +## Running the harness + +Live mode (one full eight-run sweep, ~480k tokens budget by default): + +``` +export IONOS_API_TOKEN="" # see roadmap entry for token provenance +cargo run --manifest-path experiments/2026-05-12-cross-model-authoring/harness/Cargo.toml -- \ + --rendered experiments/2026-05-12-cross-model-authoring/rendered \ + --tasks experiments/2026-05-12-cross-model-authoring/master/tasks \ + --out experiments/2026-05-12-cross-model-authoring/runs \ + --model Qwen/Qwen3-Coder-Next +``` + +The harness pre-flights `ail --version` and `clang --version` before +the first API call. Set `AIL_BIN` if `ail` is not on PATH. + +Mock mode (offline; CI-friendly; bypasses the IONOS API): + +``` +cargo run --manifest-path experiments/2026-05-12-cross-model-authoring/harness/Cargo.toml -- \ + --rendered experiments/2026-05-12-cross-model-authoring/rendered \ + --tasks experiments/2026-05-12-cross-model-authoring/master/tasks \ + --out /tmp/mock-runs \ + --model mock \ + --mock experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/mock_full_run.json +``` + +Tests: + +``` +cargo test --manifest-path experiments/2026-05-12-cross-model-authoring/harness/Cargo.toml +``` + +Four test suites: `strip_locations` (5), `verify_references` (1), +`mock_full_run` (1), `budget_abort` (1). Total 8 passed. diff --git a/experiments/2026-05-12-cross-model-authoring/harness/.gitignore b/experiments/2026-05-12-cross-model-authoring/harness/.gitignore new file mode 100644 index 0000000..96ef6c0 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/experiments/2026-05-12-cross-model-authoring/harness/Cargo.toml b/experiments/2026-05-12-cross-model-authoring/harness/Cargo.toml new file mode 100644 index 0000000..44f3c52 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/Cargo.toml @@ -0,0 +1,34 @@ +# Standalone crate, intentionally outside the root workspace. +# The empty [workspace] table prevents Cargo's automatic +# workspace-discovery from attaching this crate to the root manifest. +# See parent spec §Architecture lines 90–95 ("not added to root +# workspace members; built locally inside the experiment directory"). +[workspace] + +[package] +name = "xmodel-harness" +version = "0.0.1" +edition = "2021" +publish = false + +[[bin]] +name = "xmodel-harness" +path = "src/main.rs" + +[lib] +name = "xmodel_harness" +path = "src/lib.rs" + +[dependencies] +clap = { version = "4", features = ["derive"] } +reqwest = { version = "0.12", features = ["blocking", "json", "rustls-tls"], default-features = false } +regex = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +anyhow = "1" +chrono = { version = "0.4", default-features = false, features = ["clock"] } +thiserror = "1" +tempfile = "3" + +[dev-dependencies] +tempfile = "3" diff --git a/experiments/2026-05-12-cross-model-authoring/harness/src/ionos.rs b/experiments/2026-05-12-cross-model-authoring/harness/src/ionos.rs new file mode 100644 index 0000000..662042e --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/src/ionos.rs @@ -0,0 +1,148 @@ +//! Blocking `reqwest` client for the IONOS OpenAI-compatible endpoint. +//! +//! Retry policy (parent spec §Error handling): +//! - HTTP 5xx / connection / read-timeout (30s): exp backoff 1s/4s/16s, give up after 3rd retry. +//! - HTTP 429: respect `Retry-After`, else 30s, give up after 5th retry. +//! - HTTP 4xx other than 429: hard fail (typically auth). + +use anyhow::{anyhow, Context, Result}; +use serde::{Deserialize, Serialize}; +use std::time::Duration; + +const ENDPOINT: &str = "https://openai.inference.de-txl.ionos.com/v1/chat/completions"; + +#[derive(Debug, Clone, Serialize)] +pub struct Message { + pub role: String, + pub content: String, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ChatRequest<'a> { + pub model: &'a str, + pub messages: &'a [Message], + pub temperature: f32, + pub top_p: f32, +} + +#[derive(Debug, Deserialize)] +pub struct ChatResponse { + pub choices: Vec, + pub usage: Usage, +} + +#[derive(Debug, Deserialize)] +pub struct Choice { + pub message: ChoiceMessage, +} + +#[derive(Debug, Deserialize)] +pub struct ChoiceMessage { + pub content: String, +} + +#[derive(Debug, Deserialize, Clone, Copy)] +pub struct Usage { + pub prompt_tokens: u64, + pub completion_tokens: u64, + pub total_tokens: u64, +} + +/// Errors the client treats as terminal (caller should mark the run as api_failure). +#[derive(Debug, thiserror::Error)] +pub enum IonosError { + #[error("authentication failure (HTTP {0}): check IONOS_API_TOKEN")] + Auth(u16), + #[error("retry budget exhausted after {tries} attempts: {last_err}")] + RetriesExhausted { tries: u32, last_err: String }, + #[error("transport error: {0}")] + Transport(String), +} + +pub struct IonosClient { + http: reqwest::blocking::Client, + token: String, +} + +impl IonosClient { + /// Build from `IONOS_API_TOKEN` env. Errors if the var is unset or empty. + pub fn from_env() -> Result { + let token = std::env::var("IONOS_API_TOKEN") + .context("IONOS_API_TOKEN env var is unset")?; + if token.trim().is_empty() { + return Err(anyhow!("IONOS_API_TOKEN is empty")); + } + let http = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(30)) + .build()?; + Ok(Self { http, token }) + } + + /// Post a chat request with the retry policy above. + pub fn post(&self, req: &ChatRequest<'_>) -> std::result::Result { + let backoffs_5xx = [Duration::from_secs(1), Duration::from_secs(4), Duration::from_secs(16)]; + let mut tries_5xx = 0u32; + let mut tries_429 = 0u32; + let mut last_err = String::from("no attempts"); + loop { + let result = self.http + .post(ENDPOINT) + .bearer_auth(&self.token) + .json(req) + .send(); + match result { + Ok(resp) => { + let status = resp.status(); + if status.is_success() { + return resp.json::().map_err(|e| IonosError::Transport(e.to_string())); + } + let code = status.as_u16(); + if code == 429 { + let wait = resp + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()) + .map(Duration::from_secs) + .unwrap_or(Duration::from_secs(30)); + if tries_429 >= 5 { + return Err(IonosError::RetriesExhausted { + tries: tries_429, + last_err: format!("429 rate limit; last wait {}s", wait.as_secs()), + }); + } + std::thread::sleep(wait); + tries_429 += 1; + continue; + } + if (400..500).contains(&code) { + return Err(IonosError::Auth(code)); + } + // 5xx + if let Some(backoff) = backoffs_5xx.get(tries_5xx as usize) { + std::thread::sleep(*backoff); + tries_5xx += 1; + last_err = format!("HTTP {code}"); + continue; + } + return Err(IonosError::RetriesExhausted { + tries: tries_5xx, + last_err: format!("HTTP {code}"), + }); + } + Err(e) => { + if let Some(backoff) = backoffs_5xx.get(tries_5xx as usize) { + std::thread::sleep(*backoff); + tries_5xx += 1; + last_err = e.to_string(); + continue; + } + return Err(IonosError::RetriesExhausted { + tries: tries_5xx, + last_err: e.to_string(), + }); + } + } + } + } +} diff --git a/experiments/2026-05-12-cross-model-authoring/harness/src/lib.rs b/experiments/2026-05-12-cross-model-authoring/harness/src/lib.rs new file mode 100644 index 0000000..8d25d9c --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/src/lib.rs @@ -0,0 +1,9 @@ +//! Library surface for xmodel-harness — exposed so integration tests +//! under tests/ can reach the per-module types. + +pub mod strip_locations; +pub mod pipeline; +pub mod ionos; +pub mod mock; +pub mod scoring; +pub mod tasks; diff --git a/experiments/2026-05-12-cross-model-authoring/harness/src/main.rs b/experiments/2026-05-12-cross-model-authoring/harness/src/main.rs new file mode 100644 index 0000000..74b815d --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/src/main.rs @@ -0,0 +1,211 @@ +//! xmodel-harness — drives the two-cohort cross-model authoring-form test. + +use anyhow::{anyhow, bail, Context, Result}; +use clap::Parser; +use std::collections::BTreeSet; +use std::path::PathBuf; +use xmodel_harness::ionos::{ChatRequest, ChatResponse, IonosClient, IonosError, Message}; +use xmodel_harness::mock::MockResponses; +use xmodel_harness::pipeline::{preflight, run_pipeline, Cohort}; +use xmodel_harness::scoring::{write_scores_csv, write_summary_md, FinalStatus, ScoreRow}; +use xmodel_harness::strip_locations::strip_locations; +use xmodel_harness::tasks::{self, Task}; + +#[derive(Parser, Debug)] +#[command(name = "xmodel-harness", version, about = "Two-cohort cross-model authoring-form test")] +struct Args { + #[arg(long)] rendered: PathBuf, + #[arg(long)] tasks: PathBuf, + #[arg(long)] out: PathBuf, + #[arg(long)] model: String, + #[arg(long, default_value_t = 5)] max_turns: u32, + #[arg(long, default_value_t = 500_000)] token_budget: u64, + #[arg(long)] mock: Option, +} + +enum Backend { + Live(IonosClient), + Mock(MockResponses), +} + +fn main() -> Result<()> { + let args = Args::parse(); + let run_dir = create_run_dir(&args.out)?; + + preflight().context("preflight: ail and clang must be on PATH")?; + + let backend = if let Some(mp) = &args.mock { + Backend::Mock(MockResponses::load(mp)?) + } else { + Backend::Live(IonosClient::from_env().context("IONOS_API_TOKEN")?) + }; + + let rendered_json = std::fs::read_to_string(args.rendered.join("json.md")) + .with_context(|| format!("reading {}", args.rendered.join("json.md").display()))?; + let rendered_ailx = std::fs::read_to_string(args.rendered.join("ailx.md")) + .with_context(|| format!("reading {}", args.rendered.join("ailx.md").display()))?; + let tasks = tasks::load_all(&args.tasks)?; + if tasks.is_empty() { bail!("no tasks found in {}", args.tasks.display()); } + + let mut tokens_used: u64 = 0; + let mut rows: Vec = Vec::new(); + let mut run_status = "ok"; + + let mut completed: std::collections::BTreeSet<(&'static str, String)> = Default::default(); + 'outer: for cohort in [Cohort::Json, Cohort::Ailx] { + let system_prompt = match cohort { Cohort::Json => &rendered_json, Cohort::Ailx => &rendered_ailx }; + for t in &tasks { + let (row, consumed) = run_one( + &backend, &args, cohort, system_prompt, t, + &run_dir, args.token_budget.saturating_sub(tokens_used), + )?; + tokens_used = tokens_used.saturating_add(consumed); + let aborted_budget = matches!(row.final_status, FinalStatus::BudgetAbort); + completed.insert((cohort.as_str(), t.id.clone())); + rows.push(row); + if tokens_used >= args.token_budget { + run_status = "budget_exceeded"; + break 'outer; + } + let _ = aborted_budget; // continue to next task regardless + } + } + + // If the budget ran out mid-run, the outer loop broke before + // walking every (cohort, task) pair. The pairs we never reached + // still need a row, with final_status = budget_abort. + if run_status == "budget_exceeded" { + for cohort in [Cohort::Json, Cohort::Ailx] { + for t in &tasks { + if !completed.contains(&(cohort.as_str(), t.id.clone())) { + rows.push(ScoreRow { + cohort: cohort.as_str().to_string(), + task_id: t.id.clone(), + first_attempt_green: false, + turns_to_green: None, + prompt_tokens: 0, + completion_tokens: 0, + error_classes: BTreeSet::new(), + final_status: FinalStatus::BudgetAbort, + }); + } + } + } + } + + write_scores_csv(&rows, &run_dir.join("scores.csv"))?; + write_summary_md(&rows, &run_dir.join("summary.md"))?; + std::fs::write(run_dir.join("RUN_STATUS"), run_status)?; + eprintln!("xmodel-harness: run complete at {}; status={run_status}", run_dir.display()); + Ok(()) +} + +fn create_run_dir(out: &std::path::Path) -> Result { + let date = chrono::Utc::now().format("%Y-%m-%d"); + let hash: String = (0..6).map(|_| { + let n = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().subsec_nanos(); + std::char::from_digit((n % 16) as u32, 16).unwrap() + }).collect(); + let dir = out.join(format!("{date}-{hash}")); + std::fs::create_dir_all(&dir)?; + Ok(dir) +} + +fn call_backend(backend: &Backend, model: &str, messages: &[Message], cohort: &str, task: &str, turn: u32) + -> std::result::Result +{ + match backend { + Backend::Live(c) => c.post(&ChatRequest { model, messages, temperature: 0.0, top_p: 1.0 }), + Backend::Mock(m) => m.response_for(cohort, task, turn).map_err(|e| IonosError::Transport(e.to_string())), + } +} + +fn run_one( + backend: &Backend, args: &Args, cohort: Cohort, system_prompt: &str, task: &Task, + run_dir: &std::path::Path, budget_remaining: u64, +) -> Result<(ScoreRow, u64)> { + let cohort_dir = run_dir.join("per_cohort").join(cohort.as_str()).join(&task.id); + std::fs::create_dir_all(&cohort_dir)?; + let mut messages = vec![ + Message { role: "system".into(), content: system_prompt.to_string() }, + Message { role: "user".into(), content: task.description.clone() }, + ]; + let mut prompt_total: u64 = 0; + let mut completion_total: u64 = 0; + let mut error_classes: BTreeSet = BTreeSet::new(); + let mut first_attempt_green = false; + let mut turns_to_green: Option = None; + let mut final_status = FinalStatus::TurnLimit; + + for turn in 1..=args.max_turns { + if prompt_total.saturating_add(completion_total) >= budget_remaining { + final_status = FinalStatus::BudgetAbort; + break; + } + let resp = match call_backend(backend, &args.model, &messages, cohort.as_str(), &task.id, turn) { + Ok(r) => r, + Err(IonosError::Auth(_)) | Err(IonosError::Transport(_)) | Err(IonosError::RetriesExhausted{..}) => { + final_status = FinalStatus::ApiFailure; + break; + } + }; + prompt_total += resp.usage.prompt_tokens; + completion_total += resp.usage.completion_tokens; + let program = resp.choices.first() + .ok_or_else(|| anyhow!("empty choices array"))? + .message.content.clone(); + + std::fs::write( + cohort_dir.join(format!("turn_{turn}_program.{}", cohort.extension())), + &program, + )?; + std::fs::write( + cohort_dir.join(format!("turn_{turn}_request.json")), + serde_json::to_string_pretty(&messages)?, + )?; + std::fs::write( + cohort_dir.join(format!("turn_{turn}_response.json")), + serde_json::to_string_pretty(&serde_json::json!({ + "usage": { "prompt_tokens": resp.usage.prompt_tokens, "completion_tokens": resp.usage.completion_tokens, "total_tokens": resp.usage.total_tokens }, + "content": program, + }))?, + )?; + + let workdir = tempfile::Builder::new().prefix("cma2-run-").tempdir()?; + let cap = run_pipeline(cohort, &program, &task.expected_stdout, workdir.path())?; + std::fs::write(cohort_dir.join(format!("turn_{turn}_check_stderr.txt")), &cap.stderr)?; + std::fs::write(cohort_dir.join(format!("turn_{turn}_run_stdout.txt")), &cap.stdout)?; + + match cap.error { + None => { + if turn == 1 { first_attempt_green = true; } + turns_to_green = Some(turn); + final_status = FinalStatus::Green; + break; + } + Some(err) => { + if let Some((class, _)) = err.split_once(':') { + error_classes.insert(class.to_string()); + } + let stripped = strip_locations(&err); + messages.push(Message { role: "assistant".into(), content: program }); + messages.push(Message { role: "user".into(), content: stripped }); + } + } + } + + let consumed = prompt_total + completion_total; + Ok(( + ScoreRow { + cohort: cohort.as_str().to_string(), + task_id: task.id.clone(), + first_attempt_green, + turns_to_green, + prompt_tokens: prompt_total, + completion_tokens: completion_total, + error_classes, + final_status, + }, + consumed, + )) +} diff --git a/experiments/2026-05-12-cross-model-authoring/harness/src/mock.rs b/experiments/2026-05-12-cross-model-authoring/harness/src/mock.rs new file mode 100644 index 0000000..9a41d57 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/src/mock.rs @@ -0,0 +1,61 @@ +//! `--mock ` canned-response loader. +//! +//! Mock file shape (JSON): +//! ```json +//! { +//! "json": { +//! "t1_add_three": { "1": { "content": "...", "usage": {...} } }, +//! "t2_length": { "1": {...}, "2": {...} } +//! }, +//! "ailx": { ... } +//! } +//! ``` +//! Where each `""` entry has `"content"` (the program the +//! mocked model would emit) and `"usage"` (matching IONOS Usage shape). + +use crate::ionos::{ChatResponse, Choice, ChoiceMessage, Usage}; +use anyhow::{anyhow, Context, Result}; +use serde::Deserialize; +use std::collections::BTreeMap; +use std::path::Path; + +#[derive(Debug, Deserialize)] +struct MockTurn { + content: String, + usage: Usage, +} + +#[derive(Debug, Deserialize)] +pub struct MockFile { + /// cohort_name -> task_id -> turn -> response + #[serde(flatten)] + by_cohort: BTreeMap>>, +} + +pub struct MockResponses(MockFile); + +impl MockResponses { + pub fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path) + .with_context(|| format!("reading mock file {}", path.display()))?; + let file: MockFile = serde_json::from_str(&text) + .with_context(|| format!("parsing mock file {}", path.display()))?; + Ok(MockResponses(file)) + } + + pub fn response_for(&self, cohort: &str, task: &str, turn: u32) -> Result { + let turn_str = turn.to_string(); + let mt = self + .0 + .by_cohort.get(cohort) + .and_then(|t| t.get(task)) + .and_then(|t| t.get(&turn_str)) + .ok_or_else(|| anyhow!("mock has no entry for {cohort}/{task}/turn={turn}"))?; + Ok(ChatResponse { + choices: vec![Choice { + message: ChoiceMessage { content: mt.content.clone() }, + }], + usage: mt.usage, + }) + } +} diff --git a/experiments/2026-05-12-cross-model-authoring/harness/src/pipeline.rs b/experiments/2026-05-12-cross-model-authoring/harness/src/pipeline.rs new file mode 100644 index 0000000..693adb0 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/src/pipeline.rs @@ -0,0 +1,219 @@ +//! Subprocess wrapper around `ail parse | check | build` plus the +//! model's compiled binary. The harness shells out to the system +//! `ail` binary (resolved via `AIL_BIN` env var or PATH lookup of +//! "ail") and to the binary that `ail build` emits. + +use anyhow::{anyhow, Context, Result}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +/// Which authoring form the program was produced in. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Cohort { + Json, + Ailx, +} + +impl Cohort { + pub fn extension(self) -> &'static str { + match self { + Cohort::Json => "ail.json", + Cohort::Ailx => "ailx", + } + } + pub fn as_str(self) -> &'static str { + match self { + Cohort::Json => "json", + Cohort::Ailx => "ailx", + } + } +} + +/// Outcome of one pipeline run; `Ok(None)` is the success path. +#[derive(Debug)] +pub struct PipelineCapture { + pub error: Option, + pub stdout: String, + pub stderr: String, +} + +/// Resolve the `ail` binary path: `AIL_BIN` env var, else bare "ail". +pub fn ail_bin() -> PathBuf { + std::env::var("AIL_BIN") + .ok() + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("ail")) +} + +/// Pre-flight check: `ail` and `clang` both runnable. +/// Returns an error with a clear "missing dependency" message if either is absent. +pub fn preflight() -> Result<()> { + for (label, bin) in [("ail", ail_bin()), ("clang", PathBuf::from("clang"))] { + let out = Command::new(&bin) + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + match out { + Ok(st) if st.success() => {} + Ok(st) => return Err(anyhow!("{label} ({}) --version exited {st}", bin.display())), + Err(e) => return Err(anyhow!("{label} ({}) not runnable: {e}", bin.display())), + } + } + Ok(()) +} + +/// Run the per-cohort pipeline on `program`. Returns `Ok(None)` on +/// success, `Ok(Some(error_string))` on a pipeline failure that the +/// harness will feed back to the model (un-stripped — stripping is +/// the caller's responsibility), `Err(...)` only on harness-level +/// faults (missing `ail` binary, IO error). +pub fn run_pipeline( + cohort: Cohort, + program: &str, + expected_stdout: &str, + workdir: &Path, +) -> Result { + std::fs::create_dir_all(workdir).with_context(|| format!("creating {}", workdir.display()))?; + let prog_path = workdir.join(format!("prog.{}", cohort.extension())); + std::fs::write(&prog_path, program).with_context(|| format!("writing {}", prog_path.display()))?; + + // For AILX cohort, parse to JSON first. + let json_path_initial = if matches!(cohort, Cohort::Ailx) { + let out = workdir.join("prog.ail.json"); + let parse_out = Command::new(ail_bin()) + .arg("parse").arg(&prog_path) + .arg("-o").arg(&out) + .output() + .with_context(|| format!("running ail parse on {}", prog_path.display()))?; + if !parse_out.status.success() { + return Ok(PipelineCapture { + error: Some(format!("parse: {}", String::from_utf8_lossy(&parse_out.stderr).trim())), + stdout: String::new(), + stderr: String::from_utf8_lossy(&parse_out.stderr).into_owned(), + }); + } + out + } else { + prog_path.clone() + }; + + // `ail check` enforces filename stem == module name. Extract the module + // name from the JSON and rename the file accordingly before checking. + // Faults reading/parsing the JSON are reported as a check-class error + // (the same diagnostic the model would see if it submitted a JSON with + // a malformed top-level shape). + let json_path = match module_name_from_json(&json_path_initial) { + Ok(name) => { + let renamed = workdir.join(format!("{name}.ail.json")); + if renamed != json_path_initial { + std::fs::rename(&json_path_initial, &renamed) + .with_context(|| format!("renaming {} -> {}", json_path_initial.display(), renamed.display()))?; + } + renamed + } + Err(e) => { + return Ok(PipelineCapture { + error: Some(format!("check: {e}")), + stdout: String::new(), + stderr: e.to_string(), + }); + } + }; + + // Type check. + let check_out = Command::new(ail_bin()).arg("check").arg(&json_path).output() + .with_context(|| format!("running ail check on {}", json_path.display()))?; + if !check_out.status.success() { + return Ok(PipelineCapture { + error: Some(format!("check: {}", String::from_utf8_lossy(&check_out.stderr).trim())), + stdout: String::new(), + stderr: String::from_utf8_lossy(&check_out.stderr).into_owned(), + }); + } + + // Build to a native binary. + let bin_path = workdir.join("prog.bin"); + let build_out = Command::new(ail_bin()) + .arg("build").arg(&json_path) + .arg("-o").arg(&bin_path) + .output() + .with_context(|| format!("running ail build on {}", json_path.display()))?; + if !build_out.status.success() { + return Ok(PipelineCapture { + error: Some(format!("build: {}", String::from_utf8_lossy(&build_out.stderr).trim())), + stdout: String::new(), + stderr: String::from_utf8_lossy(&build_out.stderr).into_owned(), + }); + } + + // Execute with a 5-second timeout. + let run_out = run_with_timeout(&bin_path, Duration::from_secs(5))?; + let actual_stdout = String::from_utf8_lossy(&run_out.stdout).into_owned(); + if !run_out.timed_out { + if actual_stdout != expected_stdout { + return Ok(PipelineCapture { + error: Some(format!( + "output: expected {:?}, got {:?}", + expected_stdout, actual_stdout + )), + stdout: actual_stdout, + stderr: String::from_utf8_lossy(&run_out.stderr).into_owned(), + }); + } + return Ok(PipelineCapture { + error: None, + stdout: actual_stdout, + stderr: String::from_utf8_lossy(&run_out.stderr).into_owned(), + }); + } + Ok(PipelineCapture { + error: Some("runtime: timeout after 5s".to_string()), + stdout: actual_stdout, + stderr: String::from_utf8_lossy(&run_out.stderr).into_owned(), + }) +} + +/// Extract the top-level `"name"` field from a JSON module file. +/// Returns a check-class error string if the JSON cannot be read or +/// the `name` field is absent / not a string. +fn module_name_from_json(path: &Path) -> Result { + let text = std::fs::read_to_string(path) + .with_context(|| format!("reading {}", path.display()))?; + let v: serde_json::Value = serde_json::from_str(&text) + .with_context(|| format!("parsing JSON in {}", path.display()))?; + let name = v.get("name") + .and_then(|n| n.as_str()) + .ok_or_else(|| anyhow!("top-level `name` field missing or not a string in {}", path.display()))?; + Ok(name.to_string()) +} + +struct RunOutput { + stdout: Vec, + stderr: Vec, + timed_out: bool, +} + +fn run_with_timeout(bin: &Path, timeout: Duration) -> Result { + use std::io::Read; + let mut child = Command::new(bin) + .stdout(Stdio::piped()).stderr(Stdio::piped()) + .spawn().with_context(|| format!("spawning {}", bin.display()))?; + let start = std::time::Instant::now(); + loop { + if let Some(_status) = child.try_wait()? { + let mut so = Vec::new(); + let mut se = Vec::new(); + if let Some(mut s) = child.stdout.take() { s.read_to_end(&mut so).ok(); } + if let Some(mut s) = child.stderr.take() { s.read_to_end(&mut se).ok(); } + return Ok(RunOutput { stdout: so, stderr: se, timed_out: false }); + } + if start.elapsed() >= timeout { + let _ = child.kill(); + let _ = child.wait(); + return Ok(RunOutput { stdout: Vec::new(), stderr: Vec::new(), timed_out: true }); + } + std::thread::sleep(Duration::from_millis(25)); + } +} diff --git a/experiments/2026-05-12-cross-model-authoring/harness/src/scoring.rs b/experiments/2026-05-12-cross-model-authoring/harness/src/scoring.rs new file mode 100644 index 0000000..29bdf8f --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/src/scoring.rs @@ -0,0 +1,101 @@ +//! `scores.csv` + `summary.md` emitters. +//! +//! Columns (parent spec §Scoring): +//! cohort, task_id, first_attempt_green, turns_to_green, +//! prompt_tokens, completion_tokens, error_classes, final_status + +use anyhow::{Context, Result}; +use serde::Serialize; +use std::collections::BTreeSet; +use std::path::Path; + +#[derive(Debug, Clone, Serialize)] +pub struct ScoreRow { + pub cohort: String, + pub task_id: String, + pub first_attempt_green: bool, + /// `None` means INF (never reached green). + pub turns_to_green: Option, + pub prompt_tokens: u64, + pub completion_tokens: u64, + pub error_classes: BTreeSet, + pub final_status: FinalStatus, +} + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum FinalStatus { + Green, + TurnLimit, + BudgetAbort, + ApiFailure, +} + +impl FinalStatus { + fn as_csv(self) -> &'static str { + match self { + FinalStatus::Green => "green", + FinalStatus::TurnLimit => "turn_limit", + FinalStatus::BudgetAbort => "budget_abort", + FinalStatus::ApiFailure => "api_failure", + } + } +} + +pub fn write_scores_csv(rows: &[ScoreRow], path: &Path) -> Result<()> { + use std::io::Write; + let mut f = std::fs::File::create(path) + .with_context(|| format!("creating {}", path.display()))?; + writeln!(f, "cohort,task_id,first_attempt_green,turns_to_green,prompt_tokens,completion_tokens,error_classes,final_status")?; + for r in rows { + let turns = match r.turns_to_green { + Some(n) => n.to_string(), + None => "INF".to_string(), + }; + let errs = r.error_classes.iter().cloned().collect::>().join(";"); + writeln!( + f, + "{},{},{},{},{},{},{},{}", + r.cohort, r.task_id, r.first_attempt_green, turns, r.prompt_tokens, + r.completion_tokens, errs, r.final_status.as_csv() + )?; + } + Ok(()) +} + +pub fn write_summary_md(rows: &[ScoreRow], path: &Path) -> Result<()> { + use std::io::Write; + let mut f = std::fs::File::create(path) + .with_context(|| format!("creating {}", path.display()))?; + writeln!(f, "# Cross-model authoring-form test — run summary\n")?; + for cohort in ["json", "ailx"] { + writeln!(f, "## Cohort: {cohort}\n")?; + let cohort_rows: Vec<&ScoreRow> = rows.iter().filter(|r| r.cohort == cohort).collect(); + if cohort_rows.is_empty() { + writeln!(f, "_(no rows)_\n")?; + continue; + } + let n_total = cohort_rows.len(); + let n_green = cohort_rows.iter().filter(|r| matches!(r.final_status, FinalStatus::Green)).count(); + let n_first = cohort_rows.iter().filter(|r| r.first_attempt_green).count(); + let mean_turns: f64 = { + let xs: Vec = cohort_rows.iter().filter_map(|r| r.turns_to_green.map(|n| n as f64)).collect(); + if xs.is_empty() { f64::NAN } else { xs.iter().sum::() / xs.len() as f64 } + }; + let total_prompt: u64 = cohort_rows.iter().map(|r| r.prompt_tokens).sum(); + let total_completion: u64 = cohort_rows.iter().map(|r| r.completion_tokens).sum(); + let mut error_freq: std::collections::BTreeMap = Default::default(); + for r in &cohort_rows { + for e in &r.error_classes { *error_freq.entry(e.clone()).or_default() += 1; } + } + let top_err = error_freq.iter().max_by_key(|(_, n)| **n).map(|(k, n)| format!("{} (x{})", k, n)).unwrap_or_else(|| "(none)".to_string()); + writeln!(f, "- tasks: {n_total}")?; + writeln!(f, "- reached green: {n_green}")?; + writeln!(f, "- first-attempt green: {n_first}")?; + writeln!(f, "- mean turns-to-green (green-only): {mean_turns:.2}")?; + writeln!(f, "- total prompt tokens: {total_prompt}")?; + writeln!(f, "- total completion tokens: {total_completion}")?; + writeln!(f, "- most common error class: {top_err}\n")?; + } + Ok(()) +} diff --git a/experiments/2026-05-12-cross-model-authoring/harness/src/strip_locations.rs b/experiments/2026-05-12-cross-model-authoring/harness/src/strip_locations.rs new file mode 100644 index 0000000..67c615c --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/src/strip_locations.rs @@ -0,0 +1,94 @@ +//! Regex pass that removes form-asymmetric location info from +//! compiler-error strings before they are fed back to the model. +//! +//! The intent (parent spec §strip_locations) is **symmetric +//! degradation**: both cohorts lose the localisation information +//! their compiler natively produces. The JSON cohort would +//! otherwise get JSON-pointer fragments; the AILX cohort would get +//! `at byte N` offsets. Neither survives this pass. + +use regex::Regex; +use std::sync::OnceLock; + +struct Patterns { + json_pointer: Regex, + byte_offset: Regex, + line: Regex, + column: Regex, + line_col: Regex, + file_prefix: Regex, + caused_by_chain: Regex, +} + +fn patterns() -> &'static Patterns { + static P: OnceLock = OnceLock::new(); + P.get_or_init(|| Patterns { + json_pointer: Regex::new(r"\$\.[A-Za-z0-9._\[\]]+").unwrap(), + byte_offset: Regex::new(r"\bat byte \d+\b").unwrap(), + line: Regex::new(r"\bline \d+\b").unwrap(), + column: Regex::new(r"\bcolumn \d+\b").unwrap(), + line_col: Regex::new(r"\bat \d+:\d+\b").unwrap(), + file_prefix: Regex::new(r"(?m)^[^:\s]+:\d+:\d+:\s*").unwrap(), + // Captures from "\nCaused by:" (inclusive) through end of string. + caused_by_chain: Regex::new(r"(?s)\nCaused by:.*$").unwrap(), + }) +} + +/// Strip form-asymmetric location info from a compiler-error string. +/// +/// Order matters: collapse the anyhow `Caused by:` chain first so the +/// per-line regexes operate only on the leading message line. +pub fn strip_locations(s: &str) -> String { + let p = patterns(); + let mut out = p.caused_by_chain.replace(s, "").into_owned(); + out = p.file_prefix.replace_all(&out, "").into_owned(); + out = p.json_pointer.replace_all(&out, "").into_owned(); + out = p.byte_offset.replace_all(&out, "").into_owned(); + out = p.line_col.replace_all(&out, "").into_owned(); + out = p.line.replace_all(&out, "").into_owned(); + out = p.column.replace_all(&out, "").into_owned(); + // Collapse runs of whitespace introduced by removed location tokens. + let ws = Regex::new(r" {2,}").unwrap(); + ws.replace_all(out.trim_end(), " ").into_owned() +} + +#[cfg(test)] +mod tests { + use super::strip_locations; + + #[test] + fn json_pointer_is_removed() { + let input = "type mismatch at $.defs[0].fn.body.app.fun"; + assert_eq!(strip_locations(input), "type mismatch at"); + } + + #[test] + fn byte_offset_is_removed() { + let input = "parse error: expected `)` (end of fn-def), got `(` at byte 28"; + assert_eq!( + strip_locations(input), + "parse error: expected `)` (end of fn-def), got `(`", + ); + } + + #[test] + fn line_and_column_are_removed() { + let input = "schema/parse error: missing field `type` at line 7 column 3"; + let stripped = strip_locations(input); + assert!(!stripped.contains("line ")); + assert!(!stripped.contains("column ")); + assert!(stripped.contains("missing field `type`")); + } + + #[test] + fn caused_by_chain_is_collapsed() { + let input = "Error: top message\n\nCaused by:\n 0: lower message\n 1: even lower"; + assert_eq!(strip_locations(input), "Error: top message"); + } + + #[test] + fn passthrough_when_no_locations() { + let input = "error: [unbound-var] main: unknown identifier: `does_not_exist`"; + assert_eq!(strip_locations(input), input); + } +} diff --git a/experiments/2026-05-12-cross-model-authoring/harness/src/tasks.rs b/experiments/2026-05-12-cross-model-authoring/harness/src/tasks.rs new file mode 100644 index 0000000..051cd1a --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/src/tasks.rs @@ -0,0 +1,42 @@ +//! Task definition struct + loader for `master/tasks/*.task.json`. + +use anyhow::{Context, Result}; +use serde::Deserialize; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, Deserialize)] +pub struct Task { + pub id: String, + pub title: String, + pub description: String, + pub expected_stdout: String, + pub reference_solution: PathBuf, +} + +impl Task { + pub fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path) + .with_context(|| format!("reading {}", path.display()))?; + let t: Task = serde_json::from_str(&text) + .with_context(|| format!("parsing {}", path.display()))?; + Ok(t) + } +} + +/// Load every `*.task.json` file from a directory, sorted by id. +pub fn load_all(dir: &Path) -> Result> { + let mut tasks: Vec = std::fs::read_dir(dir) + .with_context(|| format!("reading {}", dir.display()))? + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| { + p.file_name() + .and_then(|s| s.to_str()) + .map(|n| n.ends_with(".task.json")) + .unwrap_or(false) + }) + .map(|p| Task::load(&p)) + .collect::>>()?; + tasks.sort_by(|a, b| a.id.cmp(&b.id)); + Ok(tasks) +} diff --git a/experiments/2026-05-12-cross-model-authoring/harness/tests/budget_abort.rs b/experiments/2026-05-12-cross-model-authoring/harness/tests/budget_abort.rs new file mode 100644 index 0000000..cc8c202 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/tests/budget_abort.rs @@ -0,0 +1,39 @@ +//! Budget-abort test (parent spec §Error handling / Budget-level). +//! Mock run with deliberately tiny budget; asserts the harness +//! stops with RUN_STATUS=budget_exceeded and that the affected rows +//! carry final_status=budget_abort. + +use std::path::PathBuf; +use std::process::Command; + +fn harness_bin() -> PathBuf { PathBuf::from(env!("CARGO_BIN_EXE_xmodel-harness")) } + +fn experiment_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf() +} + +#[test] +fn tiny_budget_aborts_run_cleanly() { + let out = tempfile::Builder::new().prefix("cma2-budget-").tempdir().unwrap(); + let status = Command::new(harness_bin()) + .arg("--rendered").arg(experiment_root().join("rendered")) + .arg("--tasks").arg(experiment_root().join("master").join("tasks")) + .arg("--out").arg(out.path()) + .arg("--model").arg("mock") + .arg("--mock").arg(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests").join("fixtures").join("mock_full_run.json")) + .arg("--token-budget").arg("1500") // less than one task's worth (~1200 per turn) + .status() + .expect("running xmodel-harness"); + assert!(status.success(), "harness should exit 0 on budget exhaustion"); + + let mut entries = std::fs::read_dir(out.path()).unwrap(); + let run_dir = entries.next().unwrap().unwrap().path(); + let status_str = std::fs::read_to_string(run_dir.join("RUN_STATUS")).unwrap(); + assert_eq!(status_str, "budget_exceeded"); + let scores = std::fs::read_to_string(run_dir.join("scores.csv")).unwrap(); + // At least one row must carry budget_abort. + assert!( + scores.lines().any(|l| l.ends_with(",budget_abort")), + "expected at least one budget_abort row; got:\n{}", scores, + ); +} diff --git a/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_bare_xmod.stderr b/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_bare_xmod.stderr new file mode 100644 index 0000000..de77bd3 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_bare_xmod.stderr @@ -0,0 +1 @@ +Error: module `test_ct1_bare_xmod_rejected` contains bare type name `Ordering` that does not resolve to a local type. AILang's `.ail.json` requires cross-module type references to be qualified. Candidates from imports: ["prelude.Ordering"]. Run `ail migrate-canonical-types` to fix legacy fixtures. diff --git a/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_schema_missing_field.stderr b/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_schema_missing_field.stderr new file mode 100644 index 0000000..f44e18b --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_schema_missing_field.stderr @@ -0,0 +1,5 @@ +Error: schema/parse error in /tmp/airecon/missing_field.ail.json: json: missing field `type` at line 7 column 3 + +Caused by: + 0: json: missing field `type` at line 7 column 3 + 1: missing field `type` at line 7 column 3 diff --git a/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_type_mismatch.stderr b/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_type_mismatch.stderr new file mode 100644 index 0000000..5fb59ff --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_type_mismatch.stderr @@ -0,0 +1 @@ +error: [type-mismatch] main: type mismatch: expected Int, got Bool diff --git a/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_unbound_var.stderr b/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_unbound_var.stderr new file mode 100644 index 0000000..faae503 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/check_unbound_var.stderr @@ -0,0 +1 @@ +error: [unbound-var] main: unknown identifier: `does_not_exist` diff --git a/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/mock_full_run.json b/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/mock_full_run.json new file mode 100644 index 0000000..80565d9 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/mock_full_run.json @@ -0,0 +1,126 @@ +{ + "json": { + "t1_add_three": { + "1": { + "content": "{\n \"schema\": \"ailang/v0\",\n \"name\": \"t1_add_three\",\n \"imports\": [],\n \"defs\": [\n {\n \"kind\": \"fn\",\n \"name\": \"add_three\",\n \"type\": {\n \"k\": \"fn\",\n \"params\": [\n { \"k\": \"con\", \"name\": \"Int\" },\n { \"k\": \"con\", \"name\": \"Int\" },\n { \"k\": \"con\", \"name\": \"Int\" }\n ],\n \"ret\": { \"k\": \"con\", \"name\": \"Int\" },\n \"effects\": []\n },\n \"params\": [\"a\", \"b\", \"c\"],\n \"body\": {\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"+\" },\n \"args\": [\n { \"t\": \"var\", \"name\": \"a\" },\n {\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"+\" },\n \"args\": [\n { \"t\": \"var\", \"name\": \"b\" },\n { \"t\": \"var\", \"name\": \"c\" }\n ]\n }\n ]\n }\n },\n {\n \"kind\": \"fn\",\n \"name\": \"main\",\n \"type\": {\n \"k\": \"fn\",\n \"params\": [],\n \"ret\": { \"k\": \"con\", \"name\": \"Unit\" },\n \"effects\": [\"IO\"]\n },\n \"params\": [],\n \"body\": {\n \"t\": \"seq\",\n \"lhs\": {\n \"t\": \"do\",\n \"op\": \"io/print_int\",\n \"args\": [{\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"add_three\" },\n \"args\": [\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 1 } },\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 2 } },\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 3 } }\n ]\n }]\n },\n \"rhs\": {\n \"t\": \"do\",\n \"op\": \"io/print_int\",\n \"args\": [{\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"add_three\" },\n \"args\": [\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 10 } },\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 20 } },\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 30 } }\n ]\n }]\n }\n }\n }\n ]\n}\n", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "total_tokens": 1200 + } + } + }, + "t2_length": { + "1": { + "content": "{\n \"schema\": \"ailang/v0\",\n \"name\": \"t2_length\",\n \"imports\": [],\n \"defs\": [\n {\n \"kind\": \"type\",\n \"name\": \"List\",\n \"vars\": [\"a\"],\n \"ctors\": [\n { \"name\": \"Nil\", \"fields\": [] },\n {\n \"name\": \"Cons\",\n \"fields\": [\n { \"k\": \"var\", \"name\": \"a\" },\n { \"k\": \"con\", \"name\": \"List\", \"args\": [{ \"k\": \"var\", \"name\": \"a\" }] }\n ]\n }\n ]\n },\n {\n \"kind\": \"fn\",\n \"name\": \"length\",\n \"type\": {\n \"k\": \"forall\",\n \"vars\": [\"a\"],\n \"body\": {\n \"k\": \"fn\",\n \"params\": [\n { \"k\": \"con\", \"name\": \"List\", \"args\": [{ \"k\": \"var\", \"name\": \"a\" }] }\n ],\n \"ret\": { \"k\": \"con\", \"name\": \"Int\" },\n \"effects\": []\n }\n },\n \"params\": [\"xs\"],\n \"body\": {\n \"t\": \"match\",\n \"scrutinee\": { \"t\": \"var\", \"name\": \"xs\" },\n \"arms\": [\n {\n \"pat\": { \"p\": \"ctor\", \"ctor\": \"Nil\", \"fields\": [] },\n \"body\": { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 0 } }\n },\n {\n \"pat\": {\n \"p\": \"ctor\",\n \"ctor\": \"Cons\",\n \"fields\": [\n { \"p\": \"wild\" },\n { \"p\": \"var\", \"name\": \"rest\" }\n ]\n },\n \"body\": {\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"+\" },\n \"args\": [\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 1 } },\n {\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"length\" },\n \"args\": [{ \"t\": \"var\", \"name\": \"rest\" }]\n }\n ]\n }\n }\n ]\n }\n },\n {\n \"kind\": \"fn\",\n \"name\": \"main\",\n \"type\": {\n \"k\": \"fn\",\n \"params\": [],\n \"ret\": { \"k\": \"con\", \"name\": \"Unit\" },\n \"effects\": [\"IO\"]\n },\n \"params\": [],\n \"body\": {\n \"t\": \"seq\",\n \"lhs\": {\n \"t\": \"do\",\n \"op\": \"io/print_int\",\n \"args\": [{\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"length\" },\n \"args\": [{ \"t\": \"ctor\", \"type\": \"List\", \"ctor\": \"Nil\", \"args\": [] }]\n }]\n },\n \"rhs\": {\n \"t\": \"do\",\n \"op\": \"io/print_int\",\n \"args\": [{\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"length\" },\n \"args\": [{\n \"t\": \"ctor\", \"type\": \"List\", \"ctor\": \"Cons\",\n \"args\": [\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 7 } },\n {\n \"t\": \"ctor\", \"type\": \"List\", \"ctor\": \"Cons\",\n \"args\": [\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 8 } },\n {\n \"t\": \"ctor\", \"type\": \"List\", \"ctor\": \"Cons\",\n \"args\": [\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 9 } },\n { \"t\": \"ctor\", \"type\": \"List\", \"ctor\": \"Nil\", \"args\": [] }\n ]\n }\n ]\n }\n ]\n }]\n }]\n }\n }\n }\n ]\n}\n", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "total_tokens": 1200 + } + } + }, + "t3_main_prints": { + "1": { + "content": "{\"schema\":\"ailang/v0\",\"name\":\"broken\",\"imports\":[],\"defs\":[]}", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 50, + "total_tokens": 1050 + } + }, + "2": { + "content": "{\n \"schema\": \"ailang/v0\",\n \"name\": \"t3_main_prints\",\n \"imports\": [],\n \"defs\": [\n {\n \"kind\": \"fn\",\n \"name\": \"main\",\n \"type\": {\n \"k\": \"fn\",\n \"params\": [],\n \"ret\": { \"k\": \"con\", \"name\": \"Unit\" },\n \"effects\": [\"IO\"]\n },\n \"params\": [],\n \"body\": {\n \"t\": \"seq\",\n \"lhs\": {\n \"t\": \"do\",\n \"op\": \"io/print_int\",\n \"args\": [{ \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 42 } }]\n },\n \"rhs\": {\n \"t\": \"do\",\n \"op\": \"io/print_int\",\n \"args\": [{ \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 1337 } }]\n }\n }\n }\n ]\n}\n", + "usage": { + "prompt_tokens": 1200, + "completion_tokens": 200, + "total_tokens": 1400 + } + } + }, + "t4_count_zeros": { + "1": { + "content": "{\n \"schema\": \"ailang/v0\",\n \"name\": \"t4_count_zeros\",\n \"imports\": [],\n \"defs\": [\n {\n \"kind\": \"type\",\n \"name\": \"List\",\n \"vars\": [\"a\"],\n \"ctors\": [\n { \"name\": \"Nil\", \"fields\": [] },\n {\n \"name\": \"Cons\",\n \"fields\": [\n { \"k\": \"var\", \"name\": \"a\" },\n { \"k\": \"con\", \"name\": \"List\", \"args\": [{ \"k\": \"var\", \"name\": \"a\" }] }\n ]\n }\n ]\n },\n {\n \"kind\": \"fn\",\n \"name\": \"count_zeros\",\n \"type\": {\n \"k\": \"fn\",\n \"params\": [\n { \"k\": \"con\", \"name\": \"List\", \"args\": [{ \"k\": \"con\", \"name\": \"Int\" }] }\n ],\n \"ret\": { \"k\": \"con\", \"name\": \"Int\" },\n \"effects\": []\n },\n \"params\": [\"xs\"],\n \"body\": {\n \"t\": \"match\",\n \"scrutinee\": { \"t\": \"var\", \"name\": \"xs\" },\n \"arms\": [\n {\n \"pat\": { \"p\": \"ctor\", \"ctor\": \"Nil\", \"fields\": [] },\n \"body\": { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 0 } }\n },\n {\n \"pat\": {\n \"p\": \"ctor\",\n \"ctor\": \"Cons\",\n \"fields\": [\n { \"p\": \"var\", \"name\": \"h\" },\n { \"p\": \"var\", \"name\": \"rest\" }\n ]\n },\n \"body\": {\n \"t\": \"if\",\n \"cond\": {\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"eq\" },\n \"args\": [\n { \"t\": \"var\", \"name\": \"h\" },\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 0 } }\n ]\n },\n \"then\": {\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"+\" },\n \"args\": [\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 1 } },\n {\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"count_zeros\" },\n \"args\": [{ \"t\": \"var\", \"name\": \"rest\" }]\n }\n ]\n },\n \"else\": {\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"count_zeros\" },\n \"args\": [{ \"t\": \"var\", \"name\": \"rest\" }]\n }\n }\n }\n ]\n }\n },\n {\n \"kind\": \"fn\",\n \"name\": \"main\",\n \"type\": {\n \"k\": \"fn\",\n \"params\": [],\n \"ret\": { \"k\": \"con\", \"name\": \"Unit\" },\n \"effects\": [\"IO\"]\n },\n \"params\": [],\n \"body\": {\n \"t\": \"seq\",\n \"lhs\": {\n \"t\": \"do\",\n \"op\": \"io/print_int\",\n \"args\": [{\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"count_zeros\" },\n \"args\": [{ \"t\": \"ctor\", \"type\": \"List\", \"ctor\": \"Nil\", \"args\": [] }]\n }]\n },\n \"rhs\": {\n \"t\": \"do\",\n \"op\": \"io/print_int\",\n \"args\": [{\n \"t\": \"app\",\n \"fn\": { \"t\": \"var\", \"name\": \"count_zeros\" },\n \"args\": [{\n \"t\": \"ctor\", \"type\": \"List\", \"ctor\": \"Cons\",\n \"args\": [\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 0 } },\n {\n \"t\": \"ctor\", \"type\": \"List\", \"ctor\": \"Cons\",\n \"args\": [\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 5 } },\n {\n \"t\": \"ctor\", \"type\": \"List\", \"ctor\": \"Cons\",\n \"args\": [\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 0 } },\n {\n \"t\": \"ctor\", \"type\": \"List\", \"ctor\": \"Cons\",\n \"args\": [\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 3 } },\n {\n \"t\": \"ctor\", \"type\": \"List\", \"ctor\": \"Cons\",\n \"args\": [\n { \"t\": \"lit\", \"lit\": { \"kind\": \"int\", \"value\": 0 } },\n { \"t\": \"ctor\", \"type\": \"List\", \"ctor\": \"Nil\", \"args\": [] }\n ]\n }\n ]\n }\n ]\n }\n ]\n }\n ]\n }]\n }]\n }\n }\n }\n ]\n}\n", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "total_tokens": 1200 + } + } + } + }, + "ailx": { + "t1_add_three": { + "1": { + "content": "(module garbage", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 50, + "total_tokens": 1050 + } + }, + "2": { + "content": "(module garbage", + "usage": { + "prompt_tokens": 1100, + "completion_tokens": 50, + "total_tokens": 1150 + } + }, + "3": { + "content": "(module garbage", + "usage": { + "prompt_tokens": 1200, + "completion_tokens": 50, + "total_tokens": 1250 + } + }, + "4": { + "content": "(module garbage", + "usage": { + "prompt_tokens": 1300, + "completion_tokens": 50, + "total_tokens": 1350 + } + }, + "5": { + "content": "(module garbage", + "usage": { + "prompt_tokens": 1400, + "completion_tokens": 50, + "total_tokens": 1450 + } + } + }, + "t2_length": { + "1": { + "content": "(module t2_length\n (data List (vars a)\n (ctor Nil)\n (ctor Cons a (con List a)))\n (fn length\n (type (forall (vars a) (fn-type (params (con List a)) (ret (con Int)))))\n (params xs)\n (body (match xs\n (case (pat-ctor Nil) 0)\n (case (pat-ctor Cons _ rest) (app + 1 (app length rest))))))\n (fn main\n (type (fn-type (params) (ret (con Unit)) (effects IO)))\n (params)\n (body (seq (do io/print_int (app length (term-ctor List Nil))) (do io/print_int (app length (term-ctor List Cons 7 (term-ctor List Cons 8 (term-ctor List Cons 9 (term-ctor List Nil))))))))))\n", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "total_tokens": 1200 + } + } + }, + "t3_main_prints": { + "1": { + "content": "(module t3_main_prints\n (fn main\n (type (fn-type (params) (ret (con Unit)) (effects IO)))\n (params)\n (body (seq (do io/print_int 42) (do io/print_int 1337)))))\n", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "total_tokens": 1200 + } + } + }, + "t4_count_zeros": { + "1": { + "content": "(module t4_count_zeros\n (data List (vars a)\n (ctor Nil)\n (ctor Cons a (con List a)))\n (fn count_zeros\n (type (fn-type (params (con List (con Int))) (ret (con Int))))\n (params xs)\n (body (match xs\n (case (pat-ctor Nil) 0)\n (case (pat-ctor Cons h rest) (if (app eq h 0) (app + 1 (app count_zeros rest)) (app count_zeros rest))))))\n (fn main\n (type (fn-type (params) (ret (con Unit)) (effects IO)))\n (params)\n (body (seq (do io/print_int (app count_zeros (term-ctor List Nil))) (do io/print_int (app count_zeros (term-ctor List Cons 0 (term-ctor List Cons 5 (term-ctor List Cons 0 (term-ctor List Cons 3 (term-ctor List Cons 0 (term-ctor List Nil))))))))))))\n", + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 200, + "total_tokens": 1200 + } + } + } + } +} \ No newline at end of file diff --git a/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/parse_unclosed.stderr b/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/parse_unclosed.stderr new file mode 100644 index 0000000..2142ac8 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/tests/fixtures/parse_unclosed.stderr @@ -0,0 +1 @@ +parse error: parse error: expected `)` (end of fn-def), got `(` at byte 28 diff --git a/experiments/2026-05-12-cross-model-authoring/harness/tests/mock_full_run.rs b/experiments/2026-05-12-cross-model-authoring/harness/tests/mock_full_run.rs new file mode 100644 index 0000000..c62018d --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/tests/mock_full_run.rs @@ -0,0 +1,44 @@ +//! End-to-end mock-mode test (parent spec §Testing strategy). +//! Runs the harness binary against a canned response file that +//! makes (json, t3_main_prints) green on turn 2 and +//! (ailx, t1_add_three) run to the turn limit. Asserts the +//! scores.csv is well-formed and the right per-(cohort,task) +//! artefacts land on disk. + +use std::path::PathBuf; +use std::process::Command; + +fn harness_bin() -> PathBuf { PathBuf::from(env!("CARGO_BIN_EXE_xmodel-harness")) } + +fn experiment_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf() +} + +#[test] +fn mock_full_run_produces_scores_and_artefacts() { + let out = tempfile::Builder::new().prefix("cma2-mock-").tempdir().unwrap(); + let status = Command::new(harness_bin()) + .arg("--rendered").arg(experiment_root().join("rendered")) + .arg("--tasks").arg(experiment_root().join("master").join("tasks")) + .arg("--out").arg(out.path()) + .arg("--model").arg("mock") + .arg("--mock").arg(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests").join("fixtures").join("mock_full_run.json")) + .status() + .expect("running xmodel-harness"); + assert!(status.success(), "harness exited non-zero"); + + let mut entries = std::fs::read_dir(out.path()).unwrap(); + let run_dir = entries.next().unwrap().unwrap().path(); + let status_file = run_dir.join("RUN_STATUS"); + assert_eq!(std::fs::read_to_string(&status_file).unwrap(), "ok"); + let scores = std::fs::read_to_string(run_dir.join("scores.csv")).unwrap(); + assert!(scores.starts_with("cohort,task_id,first_attempt_green,")); + assert_eq!(scores.lines().count(), 1 + 8, "header + 8 rows expected"); + assert!(scores.contains("json,t3_main_prints,false,2")); + assert!(scores.contains("ailx,t1_add_three,false,INF")); + + // Spot-check one per-cohort artefact tree. + let pc = run_dir.join("per_cohort").join("ailx").join("t1_add_three"); + assert!(pc.join("turn_1_program.ailx").exists()); + assert!(pc.join("turn_5_program.ailx").exists()); +} diff --git a/experiments/2026-05-12-cross-model-authoring/harness/tests/strip_locations.rs b/experiments/2026-05-12-cross-model-authoring/harness/tests/strip_locations.rs new file mode 100644 index 0000000..7c3285a --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/tests/strip_locations.rs @@ -0,0 +1,59 @@ +//! Integration test for strip_locations against captured stderr from +//! real `ail check` / `ail parse` failures (parent spec §Testing strategy). +//! Each fixture is verbatim from a current-HEAD invocation; goldens +//! are the strip_locations output the model should see. + +use std::path::PathBuf; +use xmodel_harness::strip_locations::strip_locations; + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests").join("fixtures") +} + +fn read_fixture(name: &str) -> String { + let path = fixtures_dir().join(name); + std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +#[test] +fn check_unbound_var_strips_to_class_plus_message() { + let input = read_fixture("check_unbound_var.stderr"); + let out = strip_locations(input.trim_end()); + assert_eq!(out, "error: [unbound-var] main: unknown identifier: `does_not_exist`"); +} + +#[test] +fn check_type_mismatch_strips_to_class_plus_message() { + let input = read_fixture("check_type_mismatch.stderr"); + let out = strip_locations(input.trim_end()); + assert_eq!(out, "error: [type-mismatch] main: type mismatch: expected Int, got Bool"); +} + +#[test] +fn check_bare_xmod_passes_through_unchanged() { + let input = read_fixture("check_bare_xmod.stderr"); + let out = strip_locations(input.trim_end()); + // No location info to strip; passthrough. + assert!(out.contains("bare type name `Ordering`")); + assert!(!out.contains("Caused by:")); +} + +#[test] +fn check_schema_missing_field_strips_locations_and_collapses_chain() { + let input = read_fixture("check_schema_missing_field.stderr"); + let out = strip_locations(input.trim_end()); + assert!(!out.contains("line ")); + assert!(!out.contains("column ")); + assert!(!out.contains("Caused by:")); + assert!(out.contains("missing field `type`")); +} + +#[test] +fn parse_unclosed_strips_byte_offset() { + let input = read_fixture("parse_unclosed.stderr"); + let out = strip_locations(input.trim_end()); + assert!(!out.contains("at byte")); + assert!(out.contains("expected `)`")); +} diff --git a/experiments/2026-05-12-cross-model-authoring/harness/tests/verify_references.rs b/experiments/2026-05-12-cross-model-authoring/harness/tests/verify_references.rs new file mode 100644 index 0000000..d3de647 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/harness/tests/verify_references.rs @@ -0,0 +1,35 @@ +//! Pre-flight: every reference solution under master/tasks/ must +//! reach green through the harness pipeline locally (no API call). +//! This is the parent spec §Pre-flight item 4 enforced as a test. + +use std::path::PathBuf; +use xmodel_harness::pipeline::{run_pipeline, Cohort}; +use xmodel_harness::tasks; + +fn master_tasks_dir() -> PathBuf { + let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + manifest.parent().unwrap().join("master").join("tasks") +} + +#[test] +fn every_reference_solution_reaches_green() { + let tasks_dir = master_tasks_dir(); + let tasks = tasks::load_all(&tasks_dir).expect("loading tasks"); + assert!(!tasks.is_empty(), "no tasks loaded from {}", tasks_dir.display()); + + let mut failures: Vec = Vec::new(); + for t in &tasks { + let ref_path = tasks_dir.join(&t.reference_solution); + let program = std::fs::read_to_string(&ref_path) + .unwrap_or_else(|e| panic!("read {}: {e}", ref_path.display())); + let workdir = tempfile::Builder::new().prefix("cma2-ref-").tempdir().unwrap(); + let cap = run_pipeline(Cohort::Json, &program, &t.expected_stdout, workdir.path()) + .unwrap_or_else(|e| panic!("pipeline failed for {}: {e}", t.id)); + if let Some(err) = cap.error { + failures.push(format!("{}: {}", t.id, err)); + } + } + if !failures.is_empty() { + panic!("{} reference(s) failed pipeline:\n {}", failures.len(), failures.join("\n ")); + } +} diff --git a/experiments/2026-05-12-cross-model-authoring/master/tasks/t1_add_three.reference.ail.json b/experiments/2026-05-12-cross-model-authoring/master/tasks/t1_add_three.reference.ail.json new file mode 100644 index 0000000..2585774 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/master/tasks/t1_add_three.reference.ail.json @@ -0,0 +1,77 @@ +{ + "schema": "ailang/v0", + "name": "t1_add_three", + "imports": [], + "defs": [ + { + "kind": "fn", + "name": "add_three", + "type": { + "k": "fn", + "params": [ + { "k": "con", "name": "Int" }, + { "k": "con", "name": "Int" }, + { "k": "con", "name": "Int" } + ], + "ret": { "k": "con", "name": "Int" }, + "effects": [] + }, + "params": ["a", "b", "c"], + "body": { + "t": "app", + "fn": { "t": "var", "name": "+" }, + "args": [ + { "t": "var", "name": "a" }, + { + "t": "app", + "fn": { "t": "var", "name": "+" }, + "args": [ + { "t": "var", "name": "b" }, + { "t": "var", "name": "c" } + ] + } + ] + } + }, + { + "kind": "fn", + "name": "main", + "type": { + "k": "fn", + "params": [], + "ret": { "k": "con", "name": "Unit" }, + "effects": ["IO"] + }, + "params": [], + "body": { + "t": "seq", + "lhs": { + "t": "do", + "op": "io/print_int", + "args": [{ + "t": "app", + "fn": { "t": "var", "name": "add_three" }, + "args": [ + { "t": "lit", "lit": { "kind": "int", "value": 1 } }, + { "t": "lit", "lit": { "kind": "int", "value": 2 } }, + { "t": "lit", "lit": { "kind": "int", "value": 3 } } + ] + }] + }, + "rhs": { + "t": "do", + "op": "io/print_int", + "args": [{ + "t": "app", + "fn": { "t": "var", "name": "add_three" }, + "args": [ + { "t": "lit", "lit": { "kind": "int", "value": 10 } }, + { "t": "lit", "lit": { "kind": "int", "value": 20 } }, + { "t": "lit", "lit": { "kind": "int", "value": 30 } } + ] + }] + } + } + } + ] +} diff --git a/experiments/2026-05-12-cross-model-authoring/master/tasks/t1_add_three.task.json b/experiments/2026-05-12-cross-model-authoring/master/tasks/t1_add_three.task.json new file mode 100644 index 0000000..4205a37 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/master/tasks/t1_add_three.task.json @@ -0,0 +1,7 @@ +{ + "id": "t1_add_three", + "title": "Three-argument addition", + "description": "Write a complete AILang module named t1_add_three. It must export a top-level function add_three that takes three Int parameters and returns their sum. It must also export main : () -> () !IO that prints add_three(1, 2, 3) and then add_three(10, 20, 30), each on its own line.", + "expected_stdout": "6\n60\n", + "reference_solution": "t1_add_three.reference.ail.json" +} diff --git a/experiments/2026-05-12-cross-model-authoring/master/tasks/t2_length.reference.ail.json b/experiments/2026-05-12-cross-model-authoring/master/tasks/t2_length.reference.ail.json new file mode 100644 index 0000000..c4668d2 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/master/tasks/t2_length.reference.ail.json @@ -0,0 +1,121 @@ +{ + "schema": "ailang/v0", + "name": "t2_length", + "imports": [], + "defs": [ + { + "kind": "type", + "name": "List", + "vars": ["a"], + "ctors": [ + { "name": "Nil", "fields": [] }, + { + "name": "Cons", + "fields": [ + { "k": "var", "name": "a" }, + { "k": "con", "name": "List", "args": [{ "k": "var", "name": "a" }] } + ] + } + ] + }, + { + "kind": "fn", + "name": "length", + "type": { + "k": "forall", + "vars": ["a"], + "body": { + "k": "fn", + "params": [ + { "k": "con", "name": "List", "args": [{ "k": "var", "name": "a" }] } + ], + "ret": { "k": "con", "name": "Int" }, + "effects": [] + } + }, + "params": ["xs"], + "body": { + "t": "match", + "scrutinee": { "t": "var", "name": "xs" }, + "arms": [ + { + "pat": { "p": "ctor", "ctor": "Nil", "fields": [] }, + "body": { "t": "lit", "lit": { "kind": "int", "value": 0 } } + }, + { + "pat": { + "p": "ctor", + "ctor": "Cons", + "fields": [ + { "p": "wild" }, + { "p": "var", "name": "rest" } + ] + }, + "body": { + "t": "app", + "fn": { "t": "var", "name": "+" }, + "args": [ + { "t": "lit", "lit": { "kind": "int", "value": 1 } }, + { + "t": "app", + "fn": { "t": "var", "name": "length" }, + "args": [{ "t": "var", "name": "rest" }] + } + ] + } + } + ] + } + }, + { + "kind": "fn", + "name": "main", + "type": { + "k": "fn", + "params": [], + "ret": { "k": "con", "name": "Unit" }, + "effects": ["IO"] + }, + "params": [], + "body": { + "t": "seq", + "lhs": { + "t": "do", + "op": "io/print_int", + "args": [{ + "t": "app", + "fn": { "t": "var", "name": "length" }, + "args": [{ "t": "ctor", "type": "List", "ctor": "Nil", "args": [] }] + }] + }, + "rhs": { + "t": "do", + "op": "io/print_int", + "args": [{ + "t": "app", + "fn": { "t": "var", "name": "length" }, + "args": [{ + "t": "ctor", "type": "List", "ctor": "Cons", + "args": [ + { "t": "lit", "lit": { "kind": "int", "value": 7 } }, + { + "t": "ctor", "type": "List", "ctor": "Cons", + "args": [ + { "t": "lit", "lit": { "kind": "int", "value": 8 } }, + { + "t": "ctor", "type": "List", "ctor": "Cons", + "args": [ + { "t": "lit", "lit": { "kind": "int", "value": 9 } }, + { "t": "ctor", "type": "List", "ctor": "Nil", "args": [] } + ] + } + ] + } + ] + }] + }] + } + } + } + ] +} diff --git a/experiments/2026-05-12-cross-model-authoring/master/tasks/t2_length.task.json b/experiments/2026-05-12-cross-model-authoring/master/tasks/t2_length.task.json new file mode 100644 index 0000000..3036a85 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/master/tasks/t2_length.task.json @@ -0,0 +1,7 @@ +{ + "id": "t2_length", + "title": "List length (polymorphic, locally-defined List)", + "description": "Write a complete AILang module named t2_length. It must define a local algebraic data type List a with constructors Nil and Cons a (List a). It must export a top-level function length : forall a. (List a) -> Int that returns the number of elements in the list. It must also export main : () -> () !IO that prints length(Nil) and then length(Cons(7, Cons(8, Cons(9, Nil)))), each on its own line.", + "expected_stdout": "0\n3\n", + "reference_solution": "t2_length.reference.ail.json" +} diff --git a/experiments/2026-05-12-cross-model-authoring/master/tasks/t3_main_prints.reference.ail.json b/experiments/2026-05-12-cross-model-authoring/master/tasks/t3_main_prints.reference.ail.json new file mode 100644 index 0000000..b43d4d8 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/master/tasks/t3_main_prints.reference.ail.json @@ -0,0 +1,31 @@ +{ + "schema": "ailang/v0", + "name": "t3_main_prints", + "imports": [], + "defs": [ + { + "kind": "fn", + "name": "main", + "type": { + "k": "fn", + "params": [], + "ret": { "k": "con", "name": "Unit" }, + "effects": ["IO"] + }, + "params": [], + "body": { + "t": "seq", + "lhs": { + "t": "do", + "op": "io/print_int", + "args": [{ "t": "lit", "lit": { "kind": "int", "value": 42 } }] + }, + "rhs": { + "t": "do", + "op": "io/print_int", + "args": [{ "t": "lit", "lit": { "kind": "int", "value": 1337 } }] + } + } + } + ] +} diff --git a/experiments/2026-05-12-cross-model-authoring/master/tasks/t3_main_prints.task.json b/experiments/2026-05-12-cross-model-authoring/master/tasks/t3_main_prints.task.json new file mode 100644 index 0000000..19144ee --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/master/tasks/t3_main_prints.task.json @@ -0,0 +1,7 @@ +{ + "id": "t3_main_prints", + "title": "main prints two fixed integers", + "description": "Write a complete AILang module named t3_main_prints. It must export main : () -> () !IO that prints 42 and then 1337, each on its own line. No other definitions are required.", + "expected_stdout": "42\n1337\n", + "reference_solution": "t3_main_prints.reference.ail.json" +} diff --git a/experiments/2026-05-12-cross-model-authoring/master/tasks/t4_count_zeros.reference.ail.json b/experiments/2026-05-12-cross-model-authoring/master/tasks/t4_count_zeros.reference.ail.json new file mode 100644 index 0000000..400d3e4 --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/master/tasks/t4_count_zeros.reference.ail.json @@ -0,0 +1,145 @@ +{ + "schema": "ailang/v0", + "name": "t4_count_zeros", + "imports": [], + "defs": [ + { + "kind": "type", + "name": "List", + "vars": ["a"], + "ctors": [ + { "name": "Nil", "fields": [] }, + { + "name": "Cons", + "fields": [ + { "k": "var", "name": "a" }, + { "k": "con", "name": "List", "args": [{ "k": "var", "name": "a" }] } + ] + } + ] + }, + { + "kind": "fn", + "name": "count_zeros", + "type": { + "k": "fn", + "params": [ + { "k": "con", "name": "List", "args": [{ "k": "con", "name": "Int" }] } + ], + "ret": { "k": "con", "name": "Int" }, + "effects": [] + }, + "params": ["xs"], + "body": { + "t": "match", + "scrutinee": { "t": "var", "name": "xs" }, + "arms": [ + { + "pat": { "p": "ctor", "ctor": "Nil", "fields": [] }, + "body": { "t": "lit", "lit": { "kind": "int", "value": 0 } } + }, + { + "pat": { + "p": "ctor", + "ctor": "Cons", + "fields": [ + { "p": "var", "name": "h" }, + { "p": "var", "name": "rest" } + ] + }, + "body": { + "t": "if", + "cond": { + "t": "app", + "fn": { "t": "var", "name": "eq" }, + "args": [ + { "t": "var", "name": "h" }, + { "t": "lit", "lit": { "kind": "int", "value": 0 } } + ] + }, + "then": { + "t": "app", + "fn": { "t": "var", "name": "+" }, + "args": [ + { "t": "lit", "lit": { "kind": "int", "value": 1 } }, + { + "t": "app", + "fn": { "t": "var", "name": "count_zeros" }, + "args": [{ "t": "var", "name": "rest" }] + } + ] + }, + "else": { + "t": "app", + "fn": { "t": "var", "name": "count_zeros" }, + "args": [{ "t": "var", "name": "rest" }] + } + } + } + ] + } + }, + { + "kind": "fn", + "name": "main", + "type": { + "k": "fn", + "params": [], + "ret": { "k": "con", "name": "Unit" }, + "effects": ["IO"] + }, + "params": [], + "body": { + "t": "seq", + "lhs": { + "t": "do", + "op": "io/print_int", + "args": [{ + "t": "app", + "fn": { "t": "var", "name": "count_zeros" }, + "args": [{ "t": "ctor", "type": "List", "ctor": "Nil", "args": [] }] + }] + }, + "rhs": { + "t": "do", + "op": "io/print_int", + "args": [{ + "t": "app", + "fn": { "t": "var", "name": "count_zeros" }, + "args": [{ + "t": "ctor", "type": "List", "ctor": "Cons", + "args": [ + { "t": "lit", "lit": { "kind": "int", "value": 0 } }, + { + "t": "ctor", "type": "List", "ctor": "Cons", + "args": [ + { "t": "lit", "lit": { "kind": "int", "value": 5 } }, + { + "t": "ctor", "type": "List", "ctor": "Cons", + "args": [ + { "t": "lit", "lit": { "kind": "int", "value": 0 } }, + { + "t": "ctor", "type": "List", "ctor": "Cons", + "args": [ + { "t": "lit", "lit": { "kind": "int", "value": 3 } }, + { + "t": "ctor", "type": "List", "ctor": "Cons", + "args": [ + { "t": "lit", "lit": { "kind": "int", "value": 0 } }, + { "t": "ctor", "type": "List", "ctor": "Nil", "args": [] } + ] + } + ] + } + ] + } + ] + } + ] + }] + }] + } + } + } + ] +} diff --git a/experiments/2026-05-12-cross-model-authoring/master/tasks/t4_count_zeros.task.json b/experiments/2026-05-12-cross-model-authoring/master/tasks/t4_count_zeros.task.json new file mode 100644 index 0000000..564330d --- /dev/null +++ b/experiments/2026-05-12-cross-model-authoring/master/tasks/t4_count_zeros.task.json @@ -0,0 +1,7 @@ +{ + "id": "t4_count_zeros", + "title": "Count zeros in a list using Eq Int", + "description": "Write a complete AILang module named t4_count_zeros. It must define a local algebraic data type List a with constructors Nil and Cons a (List a). It must export a top-level function count_zeros : (List Int) -> Int that returns the number of elements in the list that equal 0, using the prelude Eq Int instance to compare against 0. It must also export main : () -> () !IO that prints count_zeros(Nil) and then count_zeros(Cons(0, Cons(5, Cons(0, Cons(3, Cons(0, Nil)))))), each on its own line.", + "expected_stdout": "0\n3\n", + "reference_solution": "t4_count_zeros.reference.ail.json" +}