Sketch: admin rate-limits dashboard architecture (CF Access + Tunnel)

backend/app/dependencies.py

@@ -1,9 +1,35 @@
 """Shared FastAPI dependencies."""
 
-from fastapi import Query, Request
+import os
+
+from fastapi import HTTPException, Query, Request
 from slowapi.util import get_remote_address
 
 
+def require_admin(request: Request) -> str:
+    """Reject requests missing a valid `X-Admin-Token` header.
+
+    Used as a FastAPI dependency on every `/api/admin/*` route. The
+    token comes from the `ADMIN_TOKEN` env var (sourced from
+    1Password). If `ADMIN_TOKEN` isn't set, the entire admin surface
+    fails closed (503) — safer than silently allowing through.
+
+    Defense in depth: in production the admin endpoints are also
+    gated by Cloudflare Access OAuth at the edge (see
+    `docker-compose.admin.yml` + `playbooks/admin-install.yml`).
+    Token check is the second layer in case CF Access is ever
+    bypassed or misconfigured. Returns the constant string "ok"
+    so the dependency can be used with `Depends(require_admin)`.
+    """
+    expected = os.environ.get("ADMIN_TOKEN", "").strip()
+    if not expected:
+        raise HTTPException(status_code=503, detail="Admin disabled")
+    presented = request.headers.get("x-admin-token", "")
+    if presented != expected:
+        raise HTTPException(status_code=401, detail="Bad admin token")
+    return "ok"
+
+
 def client_ip(request: Request) -> str:
     """Resolve the real visitor IP behind Cloudflare → nginx → uvicorn.
 

backend/app/routers/admin_api_keys.py

@@ -0,0 +1,83 @@
+"""API key management — issue, list, rotate, revoke.
+
+Backed by `services/api_keys_store.py`. The store handles hashing
+and Mongo persistence; this router is the admin surface.
+
+## Endpoints
+
+  GET    /api/admin/api-keys             — list (without plaintext)
+  POST   /api/admin/api-keys             — create — returns plaintext ONCE
+  POST   /api/admin/api-keys/{id}/rotate — issue new plaintext, invalidate old
+  DELETE /api/admin/api-keys/{id}        — soft revoke (audit trail preserved)
+
+## Single-show semantics
+
+Creation and rotation are the only times the operator sees the
+plaintext key. The response body contains it once with explicit
+"this won't be shown again" language; the UI surfaces it as a
+copy-button + warning banner. Hash is what's stored.
+
+## What keys unlock
+
+Once this PR lands AND the consumer-side wiring lands:
+
+- Service accounts (e.g. Spire Compendium / Overwolf desktop app)
+  authenticate as themselves rather than per-user-IP. Higher rate
+  limits, revocable identity, accurate attribution.
+- Power users opt into a "claimed" identity that survives across
+  IP changes (so a user with VPN-rotation behaviour can still get
+  consistent rate limits + run attribution).
+- Third-party widget embedders get analytics on who's using the
+  public API; we can revoke a single abuser without IP-banning.
+
+## Future: scope strings
+
+`scopes: ["runs:submit", "runs:read", "guides:read", "admin:*"]` —
+intentionally not enforced in this sketch. The first PR ships with
+all keys having full read+submit access; scope checking lands as a
+follow-up once a real use case for narrower access shows up.
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+
+from ..dependencies import require_admin
+
+router = APIRouter(
+    prefix="/api/admin/api-keys",
+    tags=["Admin"],
+    dependencies=[Depends(require_admin)],
+)
+
+
+@router.get("")
+async def list_keys(request: Request):
+    """List every key (hashes + metadata, never plaintext). Query:
+    ?include_revoked=false (default)."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("")
+async def create_key(request: Request):
+    """Body: `{"owner": "spire-compendium", "owner_kind": "service",
+              "scopes": ["runs:submit"], "rate_limit_override": null}`.
+
+    Response includes plaintext key — show to operator immediately;
+    it's not recoverable after this response is closed."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("/{key_id}/rotate")
+async def rotate_key(key_id: str, request: Request):
+    """Issue new plaintext for an existing key_id. Old plaintext stops
+    working immediately. Response shape same as create."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.delete("/{key_id}")
+async def revoke_key(key_id: str, request: Request):
+    """Soft revoke — sets `revoked=true` + `revoked_at` + `revoked_by`.
+    Key doc stays for audit trail; subsequent presentations are
+    rejected by the lookup hot path."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")

backend/app/routers/admin_audit.py

@@ -0,0 +1,50 @@
+"""Audit log — read-only view of every admin action.
+
+Records are written by `services/audit_log.py::record()` from each
+admin write endpoint. This router is read-only — it never writes,
+and there's deliberately no DELETE endpoint. An append-only log
+that can be edited isn't an audit log.
+
+## Endpoints
+
+  GET /api/admin/audit
+    Last 100 entries, newest first.
+    Query: ?limit=100, ?since=ISO, ?actor=..., ?action=...
+
+  GET /api/admin/audit/by-target/{target}
+    Every entry referencing a specific run hash, slug, etc. — useful
+    for "show me everything that's ever been done to run X."
+
+## Retention
+
+Append-only, no TTL on the application side. Mongo collection has a
+date-based partial index for fast `?since=...` queries. For
+long-term retention, run `playbooks/backup.yml` (existing) which
+already snapshots Mongo dumps to B2.
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+
+from ..dependencies import require_admin
+
+router = APIRouter(
+    prefix="/api/admin/audit",
+    tags=["Admin"],
+    dependencies=[Depends(require_admin)],
+)
+
+
+@router.get("")
+async def list_audit(request: Request):
+    """Most recent entries, newest first.
+    Query: ?limit=100, ?since=2026-05-20T00:00:00, ?actor=..., ?action=..."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.get("/by-target/{target}")
+async def by_target(target: str, request: Request):
+    """All audit entries referencing a specific target (run hash,
+    guide slug, username, rate-limit slug, etc.)."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")

backend/app/routers/admin_bulk.py

@@ -0,0 +1,101 @@
+"""Bulk operations — the "I'd rather click than re-write a script"
+endpoints.
+
+These are operator power tools: ops you'd otherwise do via a
+one-shot Python script in tools/. Each is a long-running job that
+returns a job id immediately and lets you poll status separately.
+
+## Job pattern
+
+Every bulk op writes a `bulk_jobs` doc:
+
+  {
+    "_id": "job_<uuid>",
+    "kind": "rehash_runs",          # operation slug
+    "params": {...},                # original request body
+    "status": "queued|running|done|failed",
+    "started_at": ..., "finished_at": ...,
+    "processed": 0, "total": null,  # progress counters
+    "error": null,
+    "actor": "peter",
+  }
+
+Endpoint kicks off the job in a background thread (FastAPI's
+BackgroundTasks for the simple case, or a separate worker if jobs
+ever get bigger). GET /jobs/{id} returns current status.
+
+## Initial set of bulk ops
+
+- `POST /api/admin/bulk/rehash-runs` — recompute run_hash for runs
+  matching a filter (after a hash-formula change)
+- `POST /api/admin/bulk/dedupe-runs` — find duplicates by
+  (seed, character, start, run_time) and hide all but the oldest
+- `POST /api/admin/bulk/reattach-files` — for runs missing JSON
+  files on disk, attempt sibling-copy or synthesize from Mongo doc
+- `POST /api/admin/bulk/import-beta-version` — parse a fresh beta
+  PCK + DLL extraction into a versioned data-beta/ dir
+- `POST /api/admin/bulk/recompute-scores` — full rebuild of
+  spire_codex_entity_scores (Codex Score) — same as /ops/refresh-
+  entity-scores but as a tracked job for visibility
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+
+from ..dependencies import require_admin
+
+router = APIRouter(
+    prefix="/api/admin/bulk",
+    tags=["Admin"],
+    dependencies=[Depends(require_admin)],
+)
+
+
+@router.get("/jobs")
+async def list_jobs(request: Request):
+    """Last N jobs across all kinds. Query: `?limit=50&kind=...&status=...`."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.get("/jobs/{job_id}")
+async def get_job(job_id: str, request: Request):
+    """Poll a specific job's status + progress."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("/rehash-runs")
+async def rehash_runs(request: Request):
+    """Body: `{"filter": {...}}`. Recompute run_hash on matching docs."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("/dedupe-runs")
+async def dedupe_runs(request: Request):
+    """Find duplicates by (seed, character, start_time, run_time) and
+    hide all but the oldest. Returns a job id; results in audit log."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("/reattach-files")
+async def reattach_files(request: Request):
+    """For runs missing JSON files on disk: try sibling-copy first,
+    fall back to synthesizing a minimal blob from Mongo. Same logic
+    as `/api/runs/shared/{hash}` does on-demand, batched."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("/import-beta-version")
+async def import_beta_version(request: Request):
+    """Body: `{"version": "v0.106.0"}`. Parse a fresh beta extraction
+    (assumes extraction/beta/raw + decompiled are populated) into
+    data-beta/<version>/."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("/recompute-scores")
+async def recompute_scores(request: Request):
+    """Full Codex Score rebuild as a tracked job. Same logic as the
+    startup pre-warm and `/ops/refresh-entity-scores`, but progress
+    is visible in /bulk/jobs."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")

backend/app/routers/admin_integrations.py

@@ -0,0 +1,121 @@
+"""Outbound integration health + test-fire endpoints.
+
+Every external dependency this app calls — Discord webhooks, Resend,
+Sentry, GitHub App, Cloudflare API, IndexNow — can fail silently
+when a token rotates or an upstream API changes shape. The admin
+dashboard surfaces last-success + last-error timestamps + a button
+to fire a test request, so credential rotation breakage is found in
+seconds, not days.
+
+## Health endpoint shape
+
+  GET /api/admin/integrations
+  →
+  {
+    "discord_feedback":  {"last_ok": ISO, "last_error": ISO|null, "last_error_msg": "..."},
+    "discord_guide":     {...},
+    "resend":            {...},
+    "sentry":            {...},
+    "github_app":        {"last_ok": ISO, "token_expires_at": ISO|null, ...},
+    "cloudflare":        {...},
+    "indexnow":          {...},
+  }
+
+Status comes from a `integration_health` Mongo collection that each
+existing outbound call writes to on success/failure (single
+`upsert_one` per call, fire-and-forget).
+
+## Test endpoints
+
+  POST /api/admin/integrations/discord-feedback/test  → fires a
+       "[test from admin dashboard at <ts>]" message
+  POST /api/admin/integrations/resend/test            → sends a test
+       email to UNINSTALL_FORWARD_TO
+  POST /api/admin/integrations/github-app/test        → calls GitHub
+       API /repos/<repo> to verify the JWT signs valid
+  POST /api/admin/integrations/cloudflare/test        → calls
+       /zones/<id> to verify the API token still authenticates
+  POST /api/admin/integrations/indexnow/test          → pings one
+       URL via IndexNow
+
+All test endpoints write the result to the same integration_health
+collection so the GET endpoint reflects the test outcome too.
+
+## Why this matters
+
+The breakage modes for these aren't loud — Discord webhooks return
+404 silently, Resend's quota errors are 429s buried in logs, GitHub
+App tokens expire after a year. The "things broke and nobody told
+me" half-life is usually weeks, sometimes months. A one-click test
++ a one-glance panel collapses that to minutes.
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+
+from ..dependencies import require_admin
+
+router = APIRouter(
+    prefix="/api/admin/integrations",
+    tags=["Admin"],
+    dependencies=[Depends(require_admin)],
+)
+
+
+@router.get("")
+async def integration_health(request: Request):
+    """Latest success + failure timestamps for every external dep,
+    plus token expiry where applicable. One Mongo find, sub-ms."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("/discord-feedback/test")
+async def test_discord_feedback(request: Request):
+    """Fire a test message at the FEEDBACK_WEBHOOK_URL. Body:
+    `{"message": "..."}` (optional override — default is a
+    timestamped probe)."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("/discord-guide/test")
+async def test_discord_guide(request: Request):
+    """Same as above for the GUIDE_WEBHOOK_URL."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("/resend/test")
+async def test_resend(request: Request):
+    """Send a test email via Resend to UNINSTALL_FORWARD_TO. Confirms
+    the API key + the from-address are still valid."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("/sentry/test")
+async def test_sentry(request: Request):
+    """Capture a synthetic exception via the Sentry SDK to verify
+    SENTRY_DSN is still valid and events flow."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("/github-app/test")
+async def test_github_app(request: Request):
+    """Call GitHub API /repos/<GITHUB_APP_REPO> with the App JWT.
+    Confirms knowledge-demon.private-key.pem is still installed,
+    valid, and authorized on the configured repo."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("/cloudflare/test")
+async def test_cloudflare(request: Request):
+    """Call CF /zones/{zone_id} with the stored API token. Verifies
+    the token still authenticates and still has scope on the zone
+    (rotation breaks this silently otherwise)."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")
+
+
+@router.post("/indexnow/test")
+async def test_indexnow(request: Request):
+    """Ping IndexNow for one URL (e.g. the home page). Confirms
+    api.indexnow.org is accepting our key."""
+    raise HTTPException(status_code=501, detail="Not implemented yet")