feat(postmortem): aggregate Workflow run logs per stage
Sessions that ran Workflow-substrate pipelines read as near-zero subagent spend: analyze_subagents() globs one level only, while Workflow runs log under workflows/wf_*.json (per-agent summary) and subagents/workflows/<runId>/agent-*.jsonl (raw usage). New analyze_workflows() joins both — raw transcripts authoritative (dedup by requestId, as everywhere), summary supplies stage labels and models, label-prefix grouping (impl:3 -> impl), untracked transcripts isolated under '?'. Report gains workflows[] and workflow_totals; totals now include workflow spend. Live check against a real 17-agent workflow run: raw 1.30M vs reported 1.22M (the live counter's known ~7-11% under-tracking). closes #28
This commit is contained in:
@@ -2,8 +2,10 @@
|
||||
"""Deterministic aggregator for a single Claude Code session.
|
||||
|
||||
Reads the main transcript JSONL for one session plus its subagent
|
||||
sidechain logs, and emits a compact JSON summary on stdout that the
|
||||
post-mortem SKILL interprets into a written report.
|
||||
sidechain logs and its Workflow-substrate run logs (workflows/wf_*.json
|
||||
+ subagents/workflows/<runId>/agent-*.jsonl), and emits a compact JSON
|
||||
summary on stdout that the post-mortem SKILL interprets into a written
|
||||
report.
|
||||
|
||||
Why a script and not skill prose: transcripts run to >1 MB and a
|
||||
session can spawn dozens of subagent logs. Token accounting has two
|
||||
@@ -266,6 +268,77 @@ def subagent_terminal_status(path):
|
||||
return "unknown"
|
||||
|
||||
|
||||
def analyze_workflows(session_dir):
|
||||
"""Aggregate Workflow-substrate runs: workflows/wf_*.json (per-agent
|
||||
summary: stage label, model, live token counter) joined with the raw
|
||||
per-agent transcripts under subagents/workflows/<runId>/agent-*.jsonl.
|
||||
|
||||
The raw transcripts are authoritative for token counts (deduped by
|
||||
requestId, same as every other transcript here); the wf_*.json
|
||||
'tokens' counters under-track vs. the deduped raw usage (~11% in the
|
||||
206-run corpus that motivated this path, issue #28) and are reported
|
||||
only as a cross-check. Stage = the label up to the first ':'
|
||||
('impl:3' -> 'impl'), so loop stages aggregate across tasks/rounds."""
|
||||
runs = []
|
||||
for wf_path in sorted(glob.glob(os.path.join(session_dir, "workflows",
|
||||
"wf_*.json"))):
|
||||
try:
|
||||
with open(wf_path) as fh:
|
||||
data = json.load(fh)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
continue
|
||||
run_id = data.get("runId") or os.path.basename(wf_path)[:-len(".json")]
|
||||
raw_dir = os.path.join(session_dir, "subagents", "workflows", run_id)
|
||||
meta_by_agent = {}
|
||||
for entry in data.get("workflowProgress") or []:
|
||||
if entry.get("type") != "workflow_agent" or not entry.get("agentId"):
|
||||
continue
|
||||
meta_by_agent[entry["agentId"]] = entry
|
||||
|
||||
run_tokens = empty_tokens()
|
||||
stages = {}
|
||||
agent_count = 0
|
||||
untracked = 0
|
||||
for jl in sorted(glob.glob(os.path.join(raw_dir, "agent-*.jsonl"))):
|
||||
aid = os.path.basename(jl)[len("agent-"):-len(".jsonl")]
|
||||
agg = aggregate_transcript(jl)
|
||||
entry = meta_by_agent.get(aid)
|
||||
if entry is None:
|
||||
# Transcript with no progress entry (retried/dropped
|
||||
# attempt): keep it counted, grouped under '?', never
|
||||
# silently folded into a named stage.
|
||||
untracked += 1
|
||||
label = (entry or {}).get("label") or "?"
|
||||
stage = label.split(":", 1)[0]
|
||||
st = stages.setdefault(stage, {"count": 0, "tokens": empty_tokens(),
|
||||
"models": {}})
|
||||
st["count"] += 1
|
||||
agent_count += 1
|
||||
for k in run_tokens:
|
||||
run_tokens[k] += agg["tokens"][k]
|
||||
st["tokens"][k] += agg["tokens"][k]
|
||||
fallback_model = (entry or {}).get("model")
|
||||
for m, n in agg["models"].items():
|
||||
key = m or fallback_model
|
||||
st["models"][key] = st["models"].get(key, 0) + n
|
||||
|
||||
for st in stages.values():
|
||||
st["total_tokens"] = token_total(st["tokens"])
|
||||
|
||||
runs.append({
|
||||
"run_id": run_id,
|
||||
"name": data.get("workflowName"),
|
||||
"status": data.get("status"),
|
||||
"agents": agent_count,
|
||||
"agents_untracked": untracked,
|
||||
"reported_total_tokens": data.get("totalTokens"),
|
||||
"tokens": run_tokens,
|
||||
"total_tokens": token_total(run_tokens),
|
||||
"stages": stages,
|
||||
})
|
||||
return runs
|
||||
|
||||
|
||||
def analyze_subagents(session_dir):
|
||||
out = []
|
||||
sub_dir = os.path.join(session_dir, "subagents")
|
||||
@@ -335,17 +408,27 @@ def main():
|
||||
main_meta = analyze_main(main_path)
|
||||
|
||||
subagents = []
|
||||
workflows = []
|
||||
if session_dir and os.path.isdir(session_dir):
|
||||
subagents = analyze_subagents(session_dir)
|
||||
workflows = analyze_workflows(session_dir)
|
||||
|
||||
sub_tokens = empty_tokens()
|
||||
for s in subagents:
|
||||
for k in sub_tokens:
|
||||
sub_tokens[k] += s["tokens"][k]
|
||||
|
||||
wf_tokens = empty_tokens()
|
||||
wf_agents = 0
|
||||
for w in workflows:
|
||||
wf_agents += w["agents"]
|
||||
for k in wf_tokens:
|
||||
wf_tokens[k] += w["tokens"][k]
|
||||
|
||||
total_tokens = empty_tokens()
|
||||
for k in total_tokens:
|
||||
total_tokens[k] = main_agg["tokens"][k] + sub_tokens[k]
|
||||
total_tokens[k] = (main_agg["tokens"][k] + sub_tokens[k]
|
||||
+ wf_tokens[k])
|
||||
|
||||
warnings = []
|
||||
is_active = False
|
||||
@@ -396,6 +479,13 @@ def main():
|
||||
"tokens": sub_tokens,
|
||||
"total_tokens": token_total(sub_tokens),
|
||||
},
|
||||
"workflows": workflows,
|
||||
"workflow_totals": {
|
||||
"runs": len(workflows),
|
||||
"agents": wf_agents,
|
||||
"tokens": wf_tokens,
|
||||
"total_tokens": token_total(wf_tokens),
|
||||
},
|
||||
"totals": {
|
||||
"tokens": total_tokens,
|
||||
"total_tokens": token_total(total_tokens),
|
||||
|
||||
Reference in New Issue
Block a user