Add scripts to build anatomy vocabulary

2026-04-16 17:34:38 +02:00
parent 8a394ec8c0
commit 3da5a36dc8
3 changed files with 165 additions and 0 deletions
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+#
+# Crawl the German Wikipedia "Kategorie:Anatomie" (recursively into
+# subcategories) and print each page title on stdout, one per line,
+# sorted and deduplicated. Progress info goes to stderr.
+#
+# Dependencies: curl, jq.
+# Rate limit: ~10 requests/sec (sleep 0.1 between requests).
+# Depth limit: MAX_DEPTH — prevents infinite recursion via cross-links.
+#
+# Usage: ./fetch-wiki-anatomy.sh > raw/anatomy-wiki.txt
+
+set -euo pipefail
+
+USER_AGENT="doctate-vocab-builder (mould73@gmail.com)"
+API="https://de.wikipedia.org/w/api.php"
+ROOT_CATEGORY="Kategorie:Anatomie"
+MAX_DEPTH=4
+
+for tool in curl jq; do
+    if ! command -v "$tool" >/dev/null 2>&1; then
+        echo "Error: $tool is required but not installed." >&2
+        exit 1
+    fi
+done
+
+PAGES=$(mktemp)
+SEEN=$(mktemp)
+trap 'rm -f "$PAGES" "$SEEN"' EXIT
+
+# Fetch all pages under one category, recursing into subcategories.
+# Args: category_title depth
+fetch_category() {
+    local cat="$1"
+    local depth="$2"
+
+    if [[ $depth -gt $MAX_DEPTH ]]; then
+        return
+    fi
+    if grep -qFx "$cat" "$SEEN"; then
+        return
+    fi
+    echo "$cat" >> "$SEEN"
+
+    echo "[depth=$depth] $cat" >&2
+
+    local cmcontinue=""
+    while true; do
+        local response
+        if [[ -n "$cmcontinue" ]]; then
+            response=$(curl -sS -A "$USER_AGENT" -G "$API" \
+                --data-urlencode "action=query" \
+                --data-urlencode "list=categorymembers" \
+                --data-urlencode "cmtitle=$cat" \
+                --data-urlencode "cmlimit=500" \
+                --data-urlencode "cmtype=page|subcat" \
+                --data-urlencode "format=json" \
+                --data-urlencode "cmcontinue=$cmcontinue")
+        else
+            response=$(curl -sS -A "$USER_AGENT" -G "$API" \
+                --data-urlencode "action=query" \
+                --data-urlencode "list=categorymembers" \
+                --data-urlencode "cmtitle=$cat" \
+                --data-urlencode "cmlimit=500" \
+                --data-urlencode "cmtype=page|subcat" \
+                --data-urlencode "format=json")
+        fi
+
+        # Collect page titles (ns=0 is "Main" namespace, i.e. articles).
+        echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 0) | .title' >> "$PAGES"
+
+        # Recurse into subcategories (ns=14).
+        local subcats
+        subcats=$(echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 14) | .title')
+        while IFS= read -r subcat; do
+            [[ -z "$subcat" ]] && continue
+            fetch_category "$subcat" $((depth + 1))
+        done <<< "$subcats"
+
+        cmcontinue=$(echo "$response" | jq -r '.continue.cmcontinue // empty')
+        if [[ -z "$cmcontinue" ]]; then
+            break
+        fi
+
+        sleep 0.1
+    done
+}
+
+fetch_category "$ROOT_CATEGORY" 0
+
+echo "Collected $(wc -l < "$PAGES") raw titles, emitting sorted-unique..." >&2
+sort -u "$PAGES"