#!/usr/bin/env bash
#
# Crawl the German Wikipedia "Kategorie:Anatomie" (recursively into
# subcategories) and print each page title on stdout, one per line,
# sorted and deduplicated. Progress info goes to stderr.
#
# Dependencies: curl, jq.
# Rate limit: ~10 requests/sec (sleep 0.1 between requests).
# Depth limit: MAX_DEPTH — prevents infinite recursion via cross-links.
#
# Usage: ./fetch-wiki-anatomy.sh > raw/anatomy-wiki.txt

set -euo pipefail

USER_AGENT="doctate-vocab-builder (mould73@gmail.com)"
API="https://de.wikipedia.org/w/api.php"
ROOT_CATEGORY="Kategorie:Anatomie"
MAX_DEPTH=4

for tool in curl jq; do
    if ! command -v "$tool" >/dev/null 2>&1; then
        echo "Error: $tool is required but not installed." >&2
        exit 1
    fi
done

PAGES=$(mktemp)
SEEN=$(mktemp)
trap 'rm -f "$PAGES" "$SEEN"' EXIT

# Fetch all pages under one category, recursing into subcategories.
# Args: category_title depth
fetch_category() {
    local cat="$1"
    local depth="$2"

    if [[ $depth -gt $MAX_DEPTH ]]; then
        return
    fi
    if grep -qFx "$cat" "$SEEN"; then
        return
    fi
    echo "$cat" >> "$SEEN"

    echo "[depth=$depth] $cat" >&2

    local cmcontinue=""
    while true; do
        local response
        if [[ -n "$cmcontinue" ]]; then
            response=$(curl -sS -A "$USER_AGENT" -G "$API" \
                --data-urlencode "action=query" \
                --data-urlencode "list=categorymembers" \
                --data-urlencode "cmtitle=$cat" \
                --data-urlencode "cmlimit=500" \
                --data-urlencode "cmtype=page|subcat" \
                --data-urlencode "format=json" \
                --data-urlencode "cmcontinue=$cmcontinue")
        else
            response=$(curl -sS -A "$USER_AGENT" -G "$API" \
                --data-urlencode "action=query" \
                --data-urlencode "list=categorymembers" \
                --data-urlencode "cmtitle=$cat" \
                --data-urlencode "cmlimit=500" \
                --data-urlencode "cmtype=page|subcat" \
                --data-urlencode "format=json")
        fi

        # Collect page titles (ns=0 is "Main" namespace, i.e. articles).
        echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 0) | .title' >> "$PAGES"

        # Recurse into subcategories (ns=14).
        local subcats
        subcats=$(echo "$response" | jq -r '.query.categorymembers[] | select(.ns == 14) | .title')
        while IFS= read -r subcat; do
            [[ -z "$subcat" ]] && continue
            fetch_category "$subcat" $((depth + 1))
        done <<< "$subcats"

        cmcontinue=$(echo "$response" | jq -r '.continue.cmcontinue // empty')
        if [[ -z "$cmcontinue" ]]; then
            break
        fi

        sleep 0.1
    done
}

fetch_category "$ROOT_CATEGORY" 0

echo "Collected $(wc -l < "$PAGES") raw titles, emitting sorted-unique..." >&2
sort -u "$PAGES"