From ca352aafc7aa5f1df06042ce07d364c3df324d60 Mon Sep 17 00:00:00 2001 From: Brummel Date: Sun, 31 May 2026 18:02:38 +0200 Subject: [PATCH] =?UTF-8?q?docs:=20retire=20superpowers/=20layout=20?= =?UTF-8?q?=E2=80=94=20keep=20design=20spec=20under=20docs/specs/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The superpowers workflow methodology is no longer in use. Move the methodology-neutral design spec to docs/specs/ as the project's design ledger, drop the superpowers-bound implementation plan (it referenced superpowers sub-skills), and remove the now-dead /docs/superpowers/ .gitignore entry. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 2 - ...26-05-18-alpha-id-icd-suggestion-design.md | 0 .../2026-05-18-alpha-id-icd-suggestion.md | 2515 ----------------- 3 files changed, 2517 deletions(-) rename docs/{superpowers => }/specs/2026-05-18-alpha-id-icd-suggestion-design.md (100%) delete mode 100644 docs/superpowers/plans/2026-05-18-alpha-id-icd-suggestion.md diff --git a/.gitignore b/.gitignore index 20337d5..304c5e1 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,3 @@ # Lokale, persönliche Claude-Code-Overrides (nicht teamweit) /.claude/settings.local.json /.claude/worktrees/ - -/docs/superpowers/ diff --git a/docs/superpowers/specs/2026-05-18-alpha-id-icd-suggestion-design.md b/docs/specs/2026-05-18-alpha-id-icd-suggestion-design.md similarity index 100% rename from docs/superpowers/specs/2026-05-18-alpha-id-icd-suggestion-design.md rename to docs/specs/2026-05-18-alpha-id-icd-suggestion-design.md diff --git a/docs/superpowers/plans/2026-05-18-alpha-id-icd-suggestion.md b/docs/superpowers/plans/2026-05-18-alpha-id-icd-suggestion.md deleted file mode 100644 index f1b6ab7..0000000 --- a/docs/superpowers/plans/2026-05-18-alpha-id-icd-suggestion.md +++ /dev/null @@ -1,2515 +0,0 @@ -# alpha-id ICD Code Suggestion — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** A Rust core library + thin CLI that turns a list of caller-extracted diagnostic phrases (one per line) into one prioritized list of ≤10 ICD-10-GM codes, with tag filtering, validated by a noise+paraphrase eval harness. - -**Architecture:** Two modes behind one interface. Mode C = lexical baseline (tantivy BM25 over the ~90k Alpha-ID phrase corpus) + ClaML tag filter. Mode A = adds IONOS `bge-m3` semantic retrieval (brute-force cosine over precomputed embeddings) fused with lexical via Reciprocal Rank Fusion, then IONOS `Qwen3-VL-Reranker-8B` cross-encoder rerank. Cross-unit fusion (one unit = one extracted phrase) produces one global top-10. Embedding is one-time, hash-cached, resumable, cost-guarded. On IONOS failure mode A degrades to C. - -**Tech Stack:** Rust 1.94, `clap` (CLI), `tantivy` (lexical), `quick-xml` (ClaML streaming parse), `reqwest` blocking (IONOS HTTP), `serde`/`serde_json`/`toml`, `sha2` (cache keys), `rand` (seeded error injection). No ANN crate — brute-force exact cosine over 90k×1024 f32 is <50 ms and removes a native-build dependency (YAGNI). - -Spec: `docs/superpowers/specs/2026-05-18-alpha-id-icd-suggestion-design.md`. Data lives in gitignored `data/` and `icd-claml/` (already present). - -> **Revision 2026-05-18 (Input-Vertrag), spec §1a:** The input is no longer a blank-line-segmented dictation but a list of **caller-extracted diagnostic phrases, one per non-empty line** (extraction is doctate's LLM, never this project). Evidence-driven: terse corpus strings need terse query input; whole-paragraph embedding dilutes the diagnosis (see spec §1a). **Scope of code change is deliberately minimal:** only Task 6 (`segment.rs`) changes — split on newlines instead of blank lines; function name `split_segments` and the `segment_idx`/`source_segments` fields are retained (a "segment" now denotes one extracted phrase). The pipeline core (Tasks 9/10/18) is unchanged. Eval Tasks 11 & 19 gain paraphrase injection + a hand-labeled phrase-list set + separated reporting of extraction-recall ceiling vs. matcher recall. Where this plan still says "dictation/blank lines/Befund segment", read "phrase list / newlines / extracted-phrase unit". - ---- - -## File Structure - -``` -Cargo.toml # single binary+lib crate -config/default.toml # IONOS endpoint, model names, pool sizes, paths -src/lib.rs # library root, module decls, re-exports -src/model.rs # core types: AlphaIdEntry, IcdMeta, Tags, Candidate, Suggestion, Filter, Mode, Config, Diagnostics, AppError -src/corpus.rs # parse Alpha-ID-SE -> Vec + helpers -src/claml.rs # stream-parse ClaML XML -> HashMap -src/normalize.rs # text normalization + abbreviation expansion -src/segment.rs # split phrase list: one non-empty line -> one unit (rev 2026-05-18) -src/lexical.rs # tantivy index build/load/query over Alpha-ID texts -src/tags.rs # build Tags from IcdMeta+validity; Filter predicate -src/fusion.rs # RRF within segment; cross-segment dedupe(max) -src/ionos.rs # shared IONOS HTTP client: token, retry/backoff -src/embed.rs # bge-m3 client + sha256-keyed resumable disk cache + cost estimate -src/vector.rs # load embeddings; brute-force cosine top-k -src/rerank.rs # Qwen3-VL-Reranker-8B pair scoring client -src/pipeline.rs # orchestrate [1]-[7] for Mode C and Mode A (+ degrade) -src/eval.rs # error injector + gold-set generator + Recall@k -src/bin/alpha_id.rs # CLI: index | suggest | eval -tests/corpus_tests.rs # integration: real data/ file -tests/claml_tests.rs # integration: real icd-claml/ file -tests/pipeline_mode_c_tests.rs # integration: end-to-end Mode C -``` - -Each file = one responsibility. The library is CLI-agnostic so `doctate` can later embed it unchanged. IONOS-touching tests use a local mock HTTP server (`tiny_http` dev-dependency), never the real endpoint. - ---- - -## Phase 0 — Scaffold - -### Task 1: Cargo project skeleton - -**Files:** -- Create: `Cargo.toml` -- Create: `src/lib.rs` -- Create: `src/bin/alpha_id.rs` -- Create: `config/default.toml` - -- [ ] **Step 1: Create `Cargo.toml`** - -```toml -[package] -name = "alpha-id" -version = "0.1.0" -edition = "2021" - -[lib] -name = "alpha_id" -path = "src/lib.rs" - -[[bin]] -name = "alpha-id" -path = "src/bin/alpha_id.rs" - -[dependencies] -clap = { version = "4", features = ["derive"] } -serde = { version = "1", features = ["derive"] } -serde_json = "1" -toml = "0.8" -quick-xml = "0.36" -tantivy = "0.22" -reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } -sha2 = "0.10" -rand = "0.8" -thiserror = "1" - -[dev-dependencies] -tiny_http = "0.12" -tempfile = "3" -``` - -- [ ] **Step 2: Create `src/lib.rs`** - -```rust -pub mod model; -pub mod corpus; -pub mod claml; -pub mod normalize; -pub mod segment; -pub mod lexical; -pub mod tags; -pub mod fusion; -pub mod ionos; -pub mod embed; -pub mod vector; -pub mod rerank; -pub mod pipeline; -pub mod eval; -``` - -- [ ] **Step 3: Create `config/default.toml`** - -```toml -ionos_base_url = "https://openai.inference.de-txl.ionos.com/v1" -token_path = "~/.ionos_token" -embed_model = "BAAI/bge-m3" -rerank_model = "Qwen/Qwen3-VL-Reranker-8B" -alpha_id_path = "data/icd10gm2026_alphaidse_edvtxt_20250926.txt" -claml_path = "icd-claml/Klassifikationsdateien/icd10gm2026syst_claml_20250912.xml" -index_dir = "index" -pool_size = 60 -top_k = 10 -``` - -- [ ] **Step 4: Create placeholder `src/bin/alpha_id.rs`** - -```rust -fn main() { - println!("alpha-id CLI — see `alpha-id --help`"); -} -``` - -- [ ] **Step 5: Stub the remaining modules so `lib.rs` compiles** - -Create each of `src/model.rs`, `src/corpus.rs`, `src/claml.rs`, `src/normalize.rs`, `src/segment.rs`, `src/lexical.rs`, `src/tags.rs`, `src/fusion.rs`, `src/ionos.rs`, `src/embed.rs`, `src/vector.rs`, `src/rerank.rs`, `src/pipeline.rs`, `src/eval.rs` containing exactly one line: - -```rust -// implemented in a later task -``` - -- [ ] **Step 6: Verify it builds** - -Run: `cargo build` -Expected: compiles with warnings about empty modules, no errors. - -- [ ] **Step 7: Commit** - -```bash -git add Cargo.toml Cargo.lock src/ config/ -git commit -m "chore: scaffold alpha-id crate skeleton" -``` - ---- - -## Phase C — Lexical baseline + data layer + eval (fully working, no IONOS) - -### Task 2: Core types - -**Files:** -- Modify: `src/model.rs` -- Test: `tests/model_tests.rs` - -- [ ] **Step 1: Write the failing test** — `tests/model_tests.rs` - -```rust -use alpha_id::model::{Filter, Tags, Mode}; - -#[test] -fn filter_billable_only_rejects_non_primary() { - let tags = Tags { billable: false, valid: true, chapter: "I".into(), exotic: false }; - let f = Filter { billable_only: true, valid_only: false, chapters: None, exclude_exotic: false }; - assert!(!f.accepts(&tags)); -} - -#[test] -fn filter_default_accepts_everything() { - let tags = Tags { billable: false, valid: false, chapter: "II".into(), exotic: true }; - assert!(Filter::default().accepts(&tags)); -} - -#[test] -fn mode_parses_from_str() { - assert_eq!("a".parse::().unwrap(), Mode::A); - assert_eq!("c".parse::().unwrap(), Mode::C); -} -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test --test model_tests` -Expected: FAIL — `model` items not found. - -- [ ] **Step 3: Write minimal implementation** — `src/model.rs` - -```rust -use serde::{Deserialize, Serialize}; -use std::str::FromStr; - -#[derive(Debug, Clone)] -pub struct AlphaIdEntry { - pub alpha_id: String, - pub valid: bool, - pub icd_primary: String, // Field 3, normalized below - pub icd_star: String, // Field 4 - pub icd_addon: String, // Field 5 ("!") — NOT ICD exclusion - pub icd_primary2: String, // Field 6 - pub orpha: String, // Field 7 - pub text: String, // Field 8 -} - -#[derive(Debug, Clone, Default)] -pub struct IcdMeta { - pub code: String, // normalized, no + * ! - pub description: String, - pub chapter: String, - pub group: String, - pub para295: String, // P/O/Z/V - pub para301: String, - pub exotic: bool, - pub ifsg: bool, - pub content: bool, -} - -#[derive(Debug, Clone, Serialize)] -pub struct Tags { - pub billable: bool, // para295 == "P" - pub valid: bool, // Alpha-ID field 1 - pub chapter: String, - pub exotic: bool, -} - -#[derive(Debug, Clone)] -pub struct Filter { - pub billable_only: bool, - pub valid_only: bool, - pub chapters: Option>, - pub exclude_exotic: bool, -} - -impl Default for Filter { - fn default() -> Self { - Filter { billable_only: false, valid_only: false, chapters: None, exclude_exotic: false } - } -} - -impl Filter { - pub fn accepts(&self, t: &Tags) -> bool { - if self.billable_only && !t.billable { return false; } - if self.valid_only && !t.valid { return false; } - if self.exclude_exotic && t.exotic { return false; } - if let Some(ch) = &self.chapters { - if !ch.iter().any(|c| c == &t.chapter) { return false; } - } - true - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Mode { C, A } - -impl FromStr for Mode { - type Err = String; - fn from_str(s: &str) -> Result { - match s.to_ascii_lowercase().as_str() { - "c" => Ok(Mode::C), - "a" => Ok(Mode::A), - other => Err(format!("unknown mode: {other}")), - } - } -} - -#[derive(Debug, Clone)] -pub struct Candidate { - pub icd_code: String, - pub alpha_text: String, - pub segment_idx: usize, - pub lexical: Option, - pub semantic: Option, - pub rerank: Option, -} - -#[derive(Debug, Clone, Serialize)] -pub struct Suggestion { - pub icd_code: String, - pub description: String, - pub score: f32, - pub matched_phrase: String, - pub source_segments: Vec, - pub tags: Tags, -} - -#[derive(Debug, Clone, Serialize, Default)] -pub struct Diagnostics { - pub mode: String, - pub degraded: bool, - pub segment_count: usize, - pub ionos_calls: usize, - pub millis: u128, -} - -#[derive(Debug, Serialize)] -pub struct SuggestResult { - pub suggestions: Vec, - pub diagnostics: Diagnostics, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct Config { - pub ionos_base_url: String, - pub token_path: String, - pub embed_model: String, - pub rerank_model: String, - pub alpha_id_path: String, - pub claml_path: String, - pub index_dir: String, - pub pool_size: usize, - pub top_k: usize, -} - -impl Config { - pub fn load(path: &str) -> Result { - let s = std::fs::read_to_string(path) - .map_err(|e| AppError::Io(format!("config {path}: {e}")))?; - toml::from_str(&s).map_err(|e| AppError::Parse(format!("config: {e}"))) - } -} - -#[derive(Debug, thiserror::Error)] -pub enum AppError { - #[error("io error: {0}")] - Io(String), - #[error("parse error: {0}")] - Parse(String), - #[error("ionos error: {0}")] - Ionos(String), - #[error("config error: {0}")] - Config(String), -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cargo test --test model_tests` -Expected: PASS (3 tests). - -- [ ] **Step 5: Commit** - -```bash -git add src/model.rs tests/model_tests.rs -git commit -m "feat: core domain types and Filter predicate" -``` - ---- - -### Task 3: Alpha-ID corpus parser - -**Files:** -- Modify: `src/corpus.rs` -- Test: `tests/corpus_tests.rs` - -- [ ] **Step 1: Write the failing test** — `tests/corpus_tests.rs` - -```rust -use alpha_id::corpus; - -#[test] -fn parses_pipe_line_into_entry() { - let line = "1|I69812|A01.0+|I39.8*|||99745|Typhus-Endokarditis"; - let e = corpus::parse_line(line).expect("should parse"); - assert_eq!(e.alpha_id, "I69812"); - assert!(e.valid); - assert_eq!(e.icd_primary, "A01.0"); // normalized: no '+' - assert_eq!(e.icd_star, "I39.8"); // normalized: no '*' - assert_eq!(e.orpha, "99745"); - assert_eq!(e.text, "Typhus-Endokarditis"); -} - -#[test] -fn invalid_flag_is_false() { - let line = "0|I97837|||U81!|||Bakterien mit Multiresistenz"; - let e = corpus::parse_line(line).unwrap(); - assert!(!e.valid); - assert_eq!(e.icd_addon, "U81"); // field 5, '!' stripped - assert_eq!(e.icd_primary, ""); -} - -#[test] -fn loads_full_real_corpus() { - let entries = corpus::load("data/icd10gm2026_alphaidse_edvtxt_20250926.txt").unwrap(); - assert_eq!(entries.len(), 90_898); - assert!(entries.iter().any(|e| e.text.contains("Diabetes mellitus Typ 2"))); -} -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test --test corpus_tests` -Expected: FAIL — `corpus::parse_line` not found. - -- [ ] **Step 3: Write minimal implementation** — `src/corpus.rs` - -```rust -use crate::model::{AlphaIdEntry, AppError}; - -/// Normalize an ICD code field by removing classification marks. -pub fn normalize_code(raw: &str) -> String { - raw.trim().trim_end_matches(['+', '*', '!']).trim().to_string() -} - -pub fn parse_line(line: &str) -> Option { - let f: Vec<&str> = line.split('|').collect(); - if f.len() < 8 { return None; } - Some(AlphaIdEntry { - valid: f[0].trim() == "1", - alpha_id: f[1].trim().to_string(), - icd_primary: normalize_code(f[2]), - icd_star: normalize_code(f[3]), - icd_addon: normalize_code(f[4]), - icd_primary2: normalize_code(f[5]), - orpha: f[6].trim().to_string(), - text: f[7].trim().to_string(), - }) -} - -pub fn load(path: &str) -> Result, AppError> { - let raw = std::fs::read_to_string(path) - .map_err(|e| AppError::Io(format!("{path}: {e}")))?; - let entries: Vec = raw - .lines() - .filter(|l| !l.trim().is_empty()) - .filter_map(parse_line) - .collect(); - if entries.is_empty() { - return Err(AppError::Parse(format!("no entries parsed from {path}"))); - } - Ok(entries) -} - -/// The primary ICD code a matched entry contributes (falls back to star/addon). -pub fn primary_code(e: &AlphaIdEntry) -> Option<&str> { - for c in [&e.icd_primary, &e.icd_primary2, &e.icd_star, &e.icd_addon] { - if !c.is_empty() { return Some(c); } - } - None -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cargo test --test corpus_tests` -Expected: PASS (3 tests). The full-corpus test reads the real gitignored file. - -- [ ] **Step 5: Commit** - -```bash -git add src/corpus.rs tests/corpus_tests.rs -git commit -m "feat: Alpha-ID-SE corpus parser with code normalization" -``` - ---- - -### Task 4: ClaML metadata parser - -**Files:** -- Modify: `src/claml.rs` -- Test: `tests/claml_tests.rs` - -- [ ] **Step 1: Write the failing test** — `tests/claml_tests.rs` - -```rust -use alpha_id::claml; - -#[test] -fn parses_real_claml_meta_and_titles() { - let map = claml::load("icd-claml/Klassifikationsdateien/icd10gm2026syst_claml_20250912.xml").unwrap(); - let a010 = map.get("A01.0").expect("A01.0 present"); - assert_eq!(a010.para295, "P"); // billable primary - assert_eq!(a010.chapter, "01"); - assert!(a010.description.contains("Typhus abdominalis")); - let a01 = map.get("A01").expect("A01 present"); - assert_eq!(a01.para295, "V"); // 3-digit non-codable - assert!(map.len() > 10_000); -} -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test --test claml_tests` -Expected: FAIL — `claml::load` not found. - -- [ ] **Step 3: Write minimal implementation** — `src/claml.rs` - -```rust -use crate::model::{AppError, IcdMeta}; -use quick_xml::events::Event; -use quick_xml::reader::Reader; -use std::collections::HashMap; - -/// Stream-parse the ClaML XML. We only need : -/// its code attr, children, owning chapter, and the -/// preferred