fix(postmortem): anchor terminal_status on the structured Status line
The subagent terminal-status classifier scanned the whole final report text for status keywords, so `\bBLOCKED\b` fired on the implement end-report's own template lines (`BLOCKED file: BLOCKED.md`, `Blocked detail:`) and on prose narrating a surmounted blocker. Shipped implement runs whose end-report reads `Status: DONE` were flipped to BLOCKED — making the agent-effectiveness axis actively misleading. Anchor on the structured `Status:` line of the end-report instead (the implement-orchestrator emits a fixed `Status: DONE|PARTIAL|BLOCKED|...` header, optionally markdown-bold). Fall back to a status token standing alone on its own line for agents that emit a bare terminal token; prose mentions and `BLOCKED.md`-style substrings no longer match. Agents with no structured status now read `unknown` rather than a fabricated BLOCKED. Add an executable spec (postmortem/tests/test_terminal_status.py, plain asserts, no pytest dep) pinning the issue-#3 case plus the genuine BLOCKED / bare-token / no-signal guards. RED before, GREEN after. Verified on the real AILang-style session 2811d227: 3 BLOCKED (one a false positive) -> 2 genuine BLOCKED + 13 honest unknown. closes #3
This commit is contained in:
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Executable spec for postmortem.subagent_terminal_status.
|
||||
|
||||
Run directly: `python3 postmortem/tests/test_terminal_status.py`.
|
||||
Exits 0 on pass, 1 on failure. No pytest dependency — the plugin has
|
||||
no test harness, and a status-classifier is small enough to pin with
|
||||
plain asserts.
|
||||
|
||||
Pins issue #3: the terminal-status text-scan over-reports BLOCKED for
|
||||
shipped implement runs, because `\\bBLOCKED\\b` fires on the end-report's
|
||||
own template lines (`BLOCKED file: BLOCKED.md`, `Blocked detail:`)
|
||||
and on prose narrating a surmounted blocker. The authoritative signal is
|
||||
the structured `Status:` line of the end-report, not any keyword
|
||||
occurrence in the body.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
SCRIPT = os.path.join(HERE, "..", "scripts", "postmortem.py")
|
||||
|
||||
spec = importlib.util.spec_from_file_location("postmortem", SCRIPT)
|
||||
pm = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pm)
|
||||
|
||||
|
||||
def write_subagent_log(final_text):
|
||||
"""Write a one-assistant-turn subagent jsonl whose last text block is
|
||||
`final_text`, return its path. Caller cleans up."""
|
||||
fd, path = tempfile.mkstemp(suffix=".jsonl")
|
||||
rec = {"type": "assistant",
|
||||
"message": {"role": "assistant",
|
||||
"content": [{"type": "text", "text": final_text}]}}
|
||||
with os.fdopen(fd, "w") as fh:
|
||||
fh.write(json.dumps(rec) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
def status_of(final_text):
|
||||
path = write_subagent_log(final_text)
|
||||
try:
|
||||
return pm.subagent_terminal_status(path)
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
# A realistic implement-orchestrator end-report that SHIPPED (Status: DONE)
|
||||
# but whose body carries the template's BLOCKED-bearing lines. This is the
|
||||
# exact shape that issue #3 reports as misclassified.
|
||||
SHIPPED_DONE_REPORT = """\
|
||||
Status: DONE
|
||||
Iter: 0126-leg3
|
||||
Started from: a1b2c3d
|
||||
Tasks completed: 3 of 3
|
||||
- poly RawBuf elem-var mono fix
|
||||
- series kernel module
|
||||
- owned-param tail-recursion leak
|
||||
Working tree: dirty (7 files changed)
|
||||
BLOCKED file: BLOCKED.md (uncommitted; only on PARTIAL/BLOCKED)
|
||||
Stats: .claude/stats/0126.json (uncommitted)
|
||||
Files touched: 7
|
||||
Tests: 12 green, 0 red
|
||||
E2E coverage: none (mini mode)
|
||||
Blocked detail: (only if BLOCKED or PARTIAL — also written to BLOCKED.md)
|
||||
|
||||
Note: task 2 hit a blocker early (missing fixture) which was resolved
|
||||
in-phase; no BLOCKED.md was written.
|
||||
"""
|
||||
|
||||
GENUINE_BLOCKED_REPORT = """\
|
||||
Status: BLOCKED
|
||||
Iter: 0127
|
||||
Started from: d4e5f6a
|
||||
Tasks completed: 1 of 3
|
||||
Working tree: dirty (2 files changed)
|
||||
BLOCKED file: BLOCKED.md (uncommitted)
|
||||
Blocked detail: Task: 2
|
||||
Reason: review-loop-exhausted
|
||||
Worker says: spec-compliance never reached approved
|
||||
"""
|
||||
|
||||
PARTIAL_REPORT = """\
|
||||
Status: PARTIAL
|
||||
Iter: 0128
|
||||
Tasks completed: 2 of 3
|
||||
BLOCKED file: BLOCKED.md (uncommitted)
|
||||
"""
|
||||
|
||||
CASES = [
|
||||
# (name, final_text, expected)
|
||||
("shipped DONE with BLOCKED template lines (issue #3)",
|
||||
SHIPPED_DONE_REPORT, "DONE"),
|
||||
("genuine BLOCKED end-report",
|
||||
GENUINE_BLOCKED_REPORT, "BLOCKED"),
|
||||
("PARTIAL end-report mentioning BLOCKED.md",
|
||||
PARTIAL_REPORT, "PARTIAL"),
|
||||
("narrating agent whose final line is a bare status token",
|
||||
"Work complete, tests green.\n\nDONE", "DONE"),
|
||||
("prose mentioning a resolved blocker, no status line",
|
||||
"I was briefly BLOCKED by a missing import but fixed it. All good.",
|
||||
"unknown"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
failures = []
|
||||
for name, text, expected in CASES:
|
||||
got = status_of(text)
|
||||
ok = got == expected
|
||||
print(f"[{'PASS' if ok else 'FAIL'}] {name}: expected {expected!r}, got {got!r}")
|
||||
if not ok:
|
||||
failures.append(name)
|
||||
if failures:
|
||||
print(f"\n{len(failures)} failure(s).")
|
||||
return 1
|
||||
print(f"\nAll {len(CASES)} cases passed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user