fix(postmortem): anchor terminal_status on the structured Status line
The subagent terminal-status classifier scanned the whole final report text for status keywords, so `\bBLOCKED\b` fired on the implement end-report's own template lines (`BLOCKED file: BLOCKED.md`, `Blocked detail:`) and on prose narrating a surmounted blocker. Shipped implement runs whose end-report reads `Status: DONE` were flipped to BLOCKED — making the agent-effectiveness axis actively misleading. Anchor on the structured `Status:` line of the end-report instead (the implement-orchestrator emits a fixed `Status: DONE|PARTIAL|BLOCKED|...` header, optionally markdown-bold). Fall back to a status token standing alone on its own line for agents that emit a bare terminal token; prose mentions and `BLOCKED.md`-style substrings no longer match. Agents with no structured status now read `unknown` rather than a fabricated BLOCKED. Add an executable spec (postmortem/tests/test_terminal_status.py, plain asserts, no pytest dep) pinning the issue-#3 case plus the genuine BLOCKED / bare-token / no-signal guards. RED before, GREEN after. Verified on the real AILang-style session 2811d227: 3 BLOCKED (one a false positive) -> 2 genuine BLOCKED + 13 honest unknown. closes #3
This commit is contained in:
@@ -45,7 +45,22 @@ DEFAULT_PRICING = {
|
||||
}
|
||||
PRICING_AS_OF = "2026-01 list prices (estimate, override via POSTMORTEM_PRICING)"
|
||||
|
||||
STATUS_TOKENS = ["DONE_WITH_CONCERNS", "NEEDS_CONTEXT", "BLOCKED", "PARTIAL", "DONE"]
|
||||
_TOK_ALT = "DONE_WITH_CONCERNS|NEEDS_CONTEXT|BLOCKED|PARTIAL|DONE" # longest-first
|
||||
# Authoritative signal: the implement end-report leads with a structured
|
||||
# "Status: DONE" line (optionally wrapped in markdown bold or a list
|
||||
# marker). Anchoring here is what stops the scan from flipping a shipped
|
||||
# DONE run to BLOCKED on the strength of the report's own template lines
|
||||
# (`BLOCKED file: BLOCKED.md`, `Blocked detail:`) or prose narrating a
|
||||
# surmounted blocker. See issue #3.
|
||||
STATUS_LINE_RE = re.compile(
|
||||
r"^[\s>*#\-]*\**\s*status\s*\**\s*[:\-]\s*\**\s*(" + _TOK_ALT + r")\b",
|
||||
re.IGNORECASE | re.MULTILINE)
|
||||
# Fallback for agents that emit a bare status token as their final line
|
||||
# (no "Status:" prefix). Requires the token to BE the line — excludes
|
||||
# substrings like "BLOCKED.md" or "BLOCKED file:" buried in prose.
|
||||
STATUS_STANDALONE_RE = re.compile(
|
||||
r"^[\s>*#\-]*\**\s*(" + _TOK_ALT + r")\**\s*$",
|
||||
re.IGNORECASE | re.MULTILINE)
|
||||
|
||||
|
||||
def load_pricing(override_json):
|
||||
@@ -286,9 +301,15 @@ def subagent_terminal_status(path):
|
||||
for block in content:
|
||||
if block.get("type") == "text" and block.get("text", "").strip():
|
||||
last_text = block["text"]
|
||||
for tok in STATUS_TOKENS:
|
||||
if re.search(r"\b" + tok + r"\b", last_text):
|
||||
return tok
|
||||
# Primary: the structured "Status:" line. Take the LAST one so a
|
||||
# restated final status wins over any earlier mention.
|
||||
structured = STATUS_LINE_RE.findall(last_text)
|
||||
if structured:
|
||||
return structured[-1].upper()
|
||||
# Fallback: a status token standing alone on its own line.
|
||||
standalone = STATUS_STANDALONE_RE.findall(last_text)
|
||||
if standalone:
|
||||
return standalone[-1].upper()
|
||||
return "unknown"
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Executable spec for postmortem.subagent_terminal_status.
|
||||
|
||||
Run directly: `python3 postmortem/tests/test_terminal_status.py`.
|
||||
Exits 0 on pass, 1 on failure. No pytest dependency — the plugin has
|
||||
no test harness, and a status-classifier is small enough to pin with
|
||||
plain asserts.
|
||||
|
||||
Pins issue #3: the terminal-status text-scan over-reports BLOCKED for
|
||||
shipped implement runs, because `\\bBLOCKED\\b` fires on the end-report's
|
||||
own template lines (`BLOCKED file: BLOCKED.md`, `Blocked detail:`)
|
||||
and on prose narrating a surmounted blocker. The authoritative signal is
|
||||
the structured `Status:` line of the end-report, not any keyword
|
||||
occurrence in the body.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
SCRIPT = os.path.join(HERE, "..", "scripts", "postmortem.py")
|
||||
|
||||
spec = importlib.util.spec_from_file_location("postmortem", SCRIPT)
|
||||
pm = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(pm)
|
||||
|
||||
|
||||
def write_subagent_log(final_text):
|
||||
"""Write a one-assistant-turn subagent jsonl whose last text block is
|
||||
`final_text`, return its path. Caller cleans up."""
|
||||
fd, path = tempfile.mkstemp(suffix=".jsonl")
|
||||
rec = {"type": "assistant",
|
||||
"message": {"role": "assistant",
|
||||
"content": [{"type": "text", "text": final_text}]}}
|
||||
with os.fdopen(fd, "w") as fh:
|
||||
fh.write(json.dumps(rec) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
def status_of(final_text):
|
||||
path = write_subagent_log(final_text)
|
||||
try:
|
||||
return pm.subagent_terminal_status(path)
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
# A realistic implement-orchestrator end-report that SHIPPED (Status: DONE)
|
||||
# but whose body carries the template's BLOCKED-bearing lines. This is the
|
||||
# exact shape that issue #3 reports as misclassified.
|
||||
SHIPPED_DONE_REPORT = """\
|
||||
Status: DONE
|
||||
Iter: 0126-leg3
|
||||
Started from: a1b2c3d
|
||||
Tasks completed: 3 of 3
|
||||
- poly RawBuf elem-var mono fix
|
||||
- series kernel module
|
||||
- owned-param tail-recursion leak
|
||||
Working tree: dirty (7 files changed)
|
||||
BLOCKED file: BLOCKED.md (uncommitted; only on PARTIAL/BLOCKED)
|
||||
Stats: .claude/stats/0126.json (uncommitted)
|
||||
Files touched: 7
|
||||
Tests: 12 green, 0 red
|
||||
E2E coverage: none (mini mode)
|
||||
Blocked detail: (only if BLOCKED or PARTIAL — also written to BLOCKED.md)
|
||||
|
||||
Note: task 2 hit a blocker early (missing fixture) which was resolved
|
||||
in-phase; no BLOCKED.md was written.
|
||||
"""
|
||||
|
||||
GENUINE_BLOCKED_REPORT = """\
|
||||
Status: BLOCKED
|
||||
Iter: 0127
|
||||
Started from: d4e5f6a
|
||||
Tasks completed: 1 of 3
|
||||
Working tree: dirty (2 files changed)
|
||||
BLOCKED file: BLOCKED.md (uncommitted)
|
||||
Blocked detail: Task: 2
|
||||
Reason: review-loop-exhausted
|
||||
Worker says: spec-compliance never reached approved
|
||||
"""
|
||||
|
||||
PARTIAL_REPORT = """\
|
||||
Status: PARTIAL
|
||||
Iter: 0128
|
||||
Tasks completed: 2 of 3
|
||||
BLOCKED file: BLOCKED.md (uncommitted)
|
||||
"""
|
||||
|
||||
CASES = [
|
||||
# (name, final_text, expected)
|
||||
("shipped DONE with BLOCKED template lines (issue #3)",
|
||||
SHIPPED_DONE_REPORT, "DONE"),
|
||||
("genuine BLOCKED end-report",
|
||||
GENUINE_BLOCKED_REPORT, "BLOCKED"),
|
||||
("PARTIAL end-report mentioning BLOCKED.md",
|
||||
PARTIAL_REPORT, "PARTIAL"),
|
||||
("narrating agent whose final line is a bare status token",
|
||||
"Work complete, tests green.\n\nDONE", "DONE"),
|
||||
("prose mentioning a resolved blocker, no status line",
|
||||
"I was briefly BLOCKED by a missing import but fixed it. All good.",
|
||||
"unknown"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
failures = []
|
||||
for name, text, expected in CASES:
|
||||
got = status_of(text)
|
||||
ok = got == expected
|
||||
print(f"[{'PASS' if ok else 'FAIL'}] {name}: expected {expected!r}, got {got!r}")
|
||||
if not ok:
|
||||
failures.append(name)
|
||||
if failures:
|
||||
print(f"\n{len(failures)} failure(s).")
|
||||
return 1
|
||||
print(f"\nAll {len(CASES)} cases passed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user