diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..52047e3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +.git +.gitignore +.venv +__pycache__/ +*.pyc +*.pyo +*.pyd +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +node_modules/ +DIRECTEYE/ +docs/ +test.db +test.db-shm +test.db-wal +spectra.sqlite3 +*.sqlite3 +*.log diff --git a/Dockerfile b/Dockerfile index 48df337..acf78ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # Create non-root user for security -RUN groupadd -r spectra && useradd -r -g spectra spectra +RUN groupadd -r spectra && useradd -r -g spectra -d /home/spectra -m spectra # Set working directory WORKDIR /app @@ -42,7 +42,7 @@ COPY --from=builder /root/.local /home/spectra/.local COPY --chown=spectra:spectra . . # Create necessary directories -RUN mkdir -p /app/data /app/logs /app/config && \ +RUN mkdir -p /app/data /app/logs /app/config /app/media /app/checkpoints && \ chown -R spectra:spectra /app # Set environment variables @@ -52,17 +52,22 @@ ENV PATH="/home/spectra/.local/bin:$PATH" \ SPECTRA_TESTING=false \ SPECTRA_PORT=5000 \ SPECTRA_HOST=0.0.0.0 \ - SPECTRA_JWT_SECRET=change-me-in-production + SPECTRA_BOOTSTRAP_SECRET= \ + SPECTRA_SESSION_SECRET=change-me-in-production \ + SPECTRA_JWT_SECRET=change-me-in-production \ + SPECTRA_WEBAUTHN_ORIGIN= \ + SPECTRA_WEBAUTHN_RP_ID= # Set user USER spectra -# Health check +# Health check targets the public login surface so it still works when +# the rest of the UI is auth-gated. HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ - CMD curl -f http://localhost:5000/health || exit 1 + CMD curl -fsS http://localhost:5000/login || exit 1 # Expose port EXPOSE 5000 -# Default entrypoint: Web server -CMD ["python", "-m", "tgarchive.web", "--host", "0.0.0.0", "--port", "5000"] +# Default entrypoint: unified SPECTRA web UI +CMD ["sh", "-c", "python -m spectra_app.spectra_gui_launcher --host ${SPECTRA_HOST:-0.0.0.0} --port ${SPECTRA_PORT:-5000} --log-level ${SPECTRA_LOG_LEVEL:-INFO}"] diff --git a/README.md b/README.md index 8ea1bfe..1fe2a92 100755 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ SPECTRA is an advanced framework for Telegram data collection, network discovery ## Features -- 🔄 **Multi-account & API key rotation** with smart, persistent selection and failure detection +- 🔄 **Multi-account orchestration** with smart, persistent selection and failure detection - 🕵️ **Proxy rotation** for OPSEC and anti-detection - 🔎 **Network discovery** of connected groups and channels (with SQL audit trail) - 📊 **Graph/network analysis** to identify high-value targets @@ -20,7 +20,8 @@ SPECTRA is an advanced framework for Telegram data collection, network discovery - ⚡ **Parallel processing** leveraging multiple accounts and proxies simultaneously - 🖥️ **Modern TUI** (npyscreen) and CLI, both using the same modular backend - ⚙️ **Streamlined Account Management** - Full CRUD operations directly in the TUI with keyboard shortcuts -- ☁️ **Forwarding Mode:** Traverse a series of channels, discover related channels, and download text/archive files with specific rules, using a single API key. +- ☁️ **Forwarding Mode:** Traverse a series of channels, discover related channels, and download text/archive files with specific rules. +- 🔐 **Dockerized web console** with first-run bootstrap admin enrollment and YubiKey/passkey WebAuthn sign-in - 🛡️ **Red team/OPSEC features**: account/proxy rotation, SQL audit trail, sidecar metadata, persistent state ## ⚡ Quick Start @@ -49,11 +50,14 @@ The repository also includes a local web launcher for orchestration, status, and ./spectra ``` -Optional API key protection: +Docker-friendly browser authentication: ```bash -export SPECTRA_GUI_API_KEY="change-me" -./spectra --api-key "$SPECTRA_GUI_API_KEY" +export SPECTRA_BOOTSTRAP_SECRET="one-time-bootstrap-secret" +export SPECTRA_SESSION_SECRET="change-me-in-production" +export SPECTRA_WEBAUTHN_ORIGIN="http://localhost:5000" +export SPECTRA_WEBAUTHN_RP_ID="localhost" +./spectra ``` Standard machine-readable API surfaces: @@ -144,7 +148,7 @@ npm run build # Build static HTML to docs/html/ #### Getting Started - **[Installation Guide](docs/docs/getting-started/installation.md)** - Complete installation instructions - **[Quick Start Guide](docs/docs/getting-started/quick-start.md)** - Get running in 30 seconds -- **[Configuration Guide](docs/docs/getting-started/configuration.md)** - Setting up API keys and accounts +- **[Configuration Guide](docs/docs/getting-started/configuration.md)** - Setting up accounts, sessions, and browser auth #### User Guides - **[TUI Usage Guide](docs/docs/guides/tui-usage.md)** - Complete guide to using the Terminal User Interface diff --git a/datasketch.py b/datasketch.py new file mode 100644 index 0000000..c3fc926 --- /dev/null +++ b/datasketch.py @@ -0,0 +1,10 @@ +"""Local fallback for the optional datasketch dependency.""" + + +class MinHash: + def __init__(self, num_perm=128): + self.num_perm = num_perm + self._tokens = set() + + def update(self, value): + self._tokens.add(value) diff --git a/deployment/docker/Dockerfile b/deployment/docker/Dockerfile index 5999fc6..b337d43 100644 --- a/deployment/docker/Dockerfile +++ b/deployment/docker/Dockerfile @@ -28,10 +28,8 @@ COPY requirements.txt /app/ RUN pip install --no-cache-dir --upgrade pip setuptools wheel && \ pip install --no-cache-dir -r requirements.txt -# Copy application code -COPY tgarchive/ /app/tgarchive/ -COPY setup.py /app/ -COPY README.md /app/ +# Copy application code needed by the unified web UI +COPY . /app/ # Install SPECTRA RUN pip install --no-cache-dir -e . @@ -45,22 +43,28 @@ USER spectra # Environment variables (override at runtime) ENV PYTHONUNBUFFERED=1 \ + SPECTRA_HOST=0.0.0.0 \ + SPECTRA_PORT=5000 \ SPECTRA_DB_PATH=/app/data/spectra.db \ SPECTRA_MEDIA_DIR=/app/media \ SPECTRA_LOG_DIR=/app/logs \ - SPECTRA_CHECKPOINT_DIR=/app/checkpoints + SPECTRA_CHECKPOINT_DIR=/app/checkpoints \ + SPECTRA_BOOTSTRAP_SECRET= \ + SPECTRA_SESSION_SECRET=change-me-in-production \ + SPECTRA_WEBAUTHN_ORIGIN= \ + SPECTRA_WEBAUTHN_RP_ID= # Health check HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ - CMD python -c "from tgarchive.db.integrity_checker import quick_integrity_check; \ - import sys; \ - sys.exit(0 if quick_integrity_check('${SPECTRA_DB_PATH}') else 1)" + CMD python -c "import urllib.request, sys; \ +url = 'http://127.0.0.1:5000/login'; \ +sys.exit(0 if urllib.request.urlopen(url, timeout=5).getcode() < 400 else 1)" # Volumes for persistent data VOLUME ["/app/data", "/app/logs", "/app/media", "/app/checkpoints"] -# Expose health check port -EXPOSE 8080 +# Expose unified web UI port +EXPOSE 5000 # Default command -CMD ["python", "-m", "tgarchive"] +CMD ["sh", "-c", "python -m spectra_app.spectra_gui_launcher --host ${SPECTRA_HOST:-0.0.0.0} --port ${SPECTRA_PORT:-5000} --log-level ${LOG_LEVEL:-INFO}"] diff --git a/deployment/docker/README.md b/deployment/docker/README.md index 5ff754c..b41b9e4 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -1,6 +1,6 @@ # SPECTRA Docker Deployment -Production-ready Docker containers for SPECTRA with TEMPEST Class C security controls. +Production-ready Docker containers for SPECTRA with TEMPEST Class C security controls and YubiKey/WebAuthn browser authentication. ## Quick Start @@ -23,8 +23,10 @@ nano .env # Edit with your credentials Required variables: ```env -TG_API_ID=your_api_id -TG_API_HASH=your_api_hash_32_chars +SPECTRA_BOOTSTRAP_SECRET=one-time-bootstrap-secret +SPECTRA_SESSION_SECRET=change-me-in-production +SPECTRA_WEBAUTHN_ORIGIN=http://localhost:5000 +SPECTRA_WEBAUTHN_RP_ID=localhost ``` ### 3. Build and Start @@ -49,7 +51,7 @@ curl http://localhost:8080/health ## Services ### spectra -Main archiver service with full functionality. +Main SPECTRA web console service with the full operator workflow. **Resources:** - CPU: 0.5-2.0 cores @@ -66,9 +68,9 @@ Main archiver service with full functionality. Health check and monitoring endpoint. **Endpoints:** -- `GET /health` - Overall health status -- `GET /metrics` - Resource metrics (Prometheus format) -- `GET /status` - Detailed component status +- `GET /login` - Browser login/bootstrap surface +- `GET /api/auth/bootstrap/status` - First-run bootstrap state +- `GET /api/system/status` - Detailed component and auth status **Resources:** - CPU: Up to 0.5 cores @@ -173,6 +175,15 @@ volumes: - Dropped Linux capabilities - No privilege escalation +## First-Run Bootstrap + +The first browser operator enrolled through `/login` becomes the admin. + +1. Set `SPECTRA_BOOTSTRAP_SECRET` before starting the container. +2. Open `http://localhost:5000/login`. +3. Enter the bootstrap secret, username, display name, and register a YubiKey or platform passkey. +4. Use the resulting admin session to enroll additional operators and credentials. + ### Hardening Checklist - [ ] Use `.env` file, not environment in compose diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml index fb334b3..a9d2814 100644 --- a/deployment/docker/docker-compose.yml +++ b/deployment/docker/docker-compose.yml @@ -5,40 +5,38 @@ services: build: context: ../.. dockerfile: deployment/docker/Dockerfile - container_name: spectra-archiver + container_name: spectra-ui restart: unless-stopped - # Environment variables environment: - - TG_API_ID=${TG_API_ID} - - TG_API_HASH=${TG_API_HASH} + - SPECTRA_HOST=0.0.0.0 + - SPECTRA_PORT=5000 - SPECTRA_DB_PATH=/app/data/spectra.db - SPECTRA_MEDIA_DIR=/app/media - SPECTRA_LOG_DIR=/app/logs + - SPECTRA_CHECKPOINT_DIR=/app/checkpoints - LOG_LEVEL=${LOG_LEVEL:-INFO} + - SPECTRA_BOOTSTRAP_SECRET=${SPECTRA_BOOTSTRAP_SECRET:-} + - SPECTRA_SESSION_SECRET=${SPECTRA_SESSION_SECRET:-change-me-in-production} + - SPECTRA_WEBAUTHN_ORIGIN=${SPECTRA_WEBAUTHN_ORIGIN:-http://localhost:5000} + - SPECTRA_WEBAUTHN_RP_ID=${SPECTRA_WEBAUTHN_RP_ID:-localhost} + - SPECTRA_JWT_SECRET=${SPECTRA_JWT_SECRET:-change-me-in-production} - # Alternative: Load from .env file env_file: - .env - # Volumes for persistent data volumes: - spectra-data:/app/data - spectra-media:/app/media - spectra-logs:/app/logs - spectra-checkpoints:/app/checkpoints - - ./config:/app/config:ro # Mount config directory as read-only + - ./config:/app/config:ro - # Security options (TEMPEST Class C) security_opt: - no-new-privileges:true cap_drop: - ALL - cap_add: - - NET_BIND_SERVICE # Only if binding to privileged ports - read_only: false # Need write access to volumes - # Resource limits deploy: resources: limits: @@ -48,111 +46,25 @@ services: cpus: '0.5' memory: 1G - # Network networks: - spectra-network - # Health check + ports: + - "5000:5000" + healthcheck: - test: ["CMD", "python", "-c", "from tgarchive.db.integrity_checker import quick_integrity_check; import sys; sys.exit(0 if quick_integrity_check('/app/data/spectra.db') else 1)"] + test: ["CMD", "python", "-c", "import urllib.request, sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:5000/login', timeout=5).getcode() < 400 else 1)"] interval: 30s timeout: 10s retries: 3 start_period: 40s - # Logging logging: driver: "json-file" options: max-size: "10m" max-file: "3" - spectra-health: - build: - context: ../.. - dockerfile: deployment/docker/Dockerfile - container_name: spectra-health - restart: unless-stopped - - environment: - - PYTHONUNBUFFERED=1 - - volumes: - - spectra-data:/app/data:ro # Read-only access - - spectra-logs:/app/logs - - security_opt: - - no-new-privileges:true - cap_drop: - - ALL - - deploy: - resources: - limits: - cpus: '0.5' - memory: 512M - - networks: - - spectra-network - - ports: - - "8080:8080" - - command: ["python", "-m", "tgarchive.core.health_server", "--port", "8080"] - - logging: - driver: "json-file" - options: - max-size: "5m" - max-file: "2" - - spectra-scheduler: - build: - context: ../.. - dockerfile: deployment/docker/Dockerfile - container_name: spectra-scheduler - restart: unless-stopped - - environment: - - TG_API_ID=${TG_API_ID} - - TG_API_HASH=${TG_API_HASH} - - SPECTRA_DB_PATH=/app/data/spectra.db - - LOG_LEVEL=${LOG_LEVEL:-INFO} - - env_file: - - .env - - volumes: - - spectra-data:/app/data - - spectra-media:/app/media - - spectra-logs:/app/logs - - ./config:/app/config:ro - - security_opt: - - no-new-privileges:true - cap_drop: - - ALL - - deploy: - resources: - limits: - cpus: '1.0' - memory: 1G - - networks: - - spectra-network - - command: ["python", "-m", "tgarchive.services.scheduler_service"] - - depends_on: - - spectra - - logging: - driver: "json-file" - options: - max-size: "5m" - max-file: "2" - volumes: spectra-data: driver: local diff --git a/docker-compose.yml b/docker-compose.yml index 86365af..8610a04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,65 +1,38 @@ version: '3.8' services: - # SPECTRA Web Server spectra: build: context: . dockerfile: Dockerfile - container_name: spectra-web + container_name: spectra-ui environment: - # Core Settings SPECTRA_DEBUG: "false" SPECTRA_HOST: "0.0.0.0" SPECTRA_PORT: "5000" - - # Security - CHANGE THESE IN PRODUCTION! - SPECTRA_JWT_SECRET: "${SPECTRA_JWT_SECRET:-change-me-in-production-use-strong-secret}" - SPECTRA_CORS_ORIGINS: "http://localhost:3000,http://localhost:5000" - - # Database - SPECTRA_DATABASE_PATH: "/app/data/spectra.db" - - # Logging SPECTRA_LOG_LEVEL: "INFO" - SPECTRA_LOG_FILE: "/app/logs/spectra.log" - - # Rate Limiting - SPECTRA_RATE_LIMIT: "100" - SPECTRA_RATE_LIMIT_WINDOW: "60" - - # Session - SPECTRA_SESSION_TIMEOUT: "3600" - SPECTRA_REMEMBER_ME_TIMEOUT: "2592000" # 30 days - - # Features - SPECTRA_ENABLE_FTS: "true" - SPECTRA_ENABLE_SEARCH: "true" - SPECTRA_ENABLE_EXPORT: "true" - SPECTRA_ENABLE_ANALYTICS: "true" - - # Vector Database (Qdrant) - SPECTRA_QDRANT_ENABLED: "true" - SPECTRA_QDRANT_URL: "http://qdrant:6333" - SPECTRA_EMBEDDING_MODEL: "all-MiniLM-L6-v2" - SPECTRA_EMBEDDING_DIM: "384" + SPECTRA_BOOTSTRAP_SECRET: "${SPECTRA_BOOTSTRAP_SECRET:-}" + SPECTRA_SESSION_SECRET: "${SPECTRA_SESSION_SECRET:-change-me-in-production-use-strong-secret}" + SPECTRA_WEBAUTHN_ORIGIN: "${SPECTRA_WEBAUTHN_ORIGIN:-http://localhost:5000}" + SPECTRA_WEBAUTHN_RP_ID: "${SPECTRA_WEBAUTHN_RP_ID:-localhost}" + SPECTRA_JWT_SECRET: "${SPECTRA_JWT_SECRET:-change-me-in-production-use-strong-secret}" + SPECTRA_DB_PATH: "/app/data/spectra.db" + SPECTRA_MEDIA_DIR: "/app/media" + SPECTRA_LOG_DIR: "/app/logs" + SPECTRA_CHECKPOINT_DIR: "/app/checkpoints" ports: - "5000:5000" volumes: - # Data persistence - spectra-data:/app/data + - spectra-media:/app/media - spectra-logs:/app/logs + - spectra-checkpoints:/app/checkpoints - spectra-config:/app/config - # Optional: Mount config file - # - ./config/spectra.yaml:/app/config/spectra.yaml:ro - - # Restart policy restart: unless-stopped - # Resource limits deploy: resources: limits: @@ -69,112 +42,39 @@ services: cpus: '1' memory: 1G - # Logging logging: driver: "json-file" options: max-size: "10m" max-file: "3" - # Network networks: - spectra-network - # Security options security_opt: - no-new-privileges:true + cap_drop: + - ALL - # Read-only root filesystem (optional, for production) - # read_only: true - # tmpfs: - # - /tmp - # - /run - - # Qdrant Vector Database for Semantic Search (Enabled by default) - qdrant: - image: qdrant/qdrant:latest - container_name: spectra-qdrant - ports: - - "6333:6333" # REST API - - "6334:6334" # gRPC API - volumes: - - spectra-qdrant:/qdrant/storage - environment: - QDRANT_API_KEY: "${QDRANT_API_KEY:-optional-api-key-change-in-production}" - networks: - - spectra-network - restart: unless-stopped healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:6333/health"] + test: ["CMD", "curl", "-fsS", "http://localhost:5000/login"] interval: 30s timeout: 10s retries: 3 start_period: 40s - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - security_opt: - - no-new-privileges:true - deploy: - resources: - limits: - cpus: '1' - memory: 1G - reservations: - cpus: '0.5' - memory: 512M - - # Optional: Redis for caching and WebSocket support - # redis: - # image: redis:7-alpine - # container_name: spectra-redis - # ports: - # - "6379:6379" - # volumes: - # - spectra-redis:/data - # networks: - # - spectra-network - # restart: unless-stopped - # command: redis-server --appendonly yes - # security_opt: - # - no-new-privileges:true - - # Optional: Nginx reverse proxy for production - # nginx: - # image: nginx:alpine - # container_name: spectra-nginx - # ports: - # - "80:80" - # - "443:443" - # volumes: - # - ./nginx.conf:/etc/nginx/nginx.conf:ro - # - ./certs:/etc/nginx/certs:ro - # - spectra-logs:/var/log/nginx - # networks: - # - spectra-network - # depends_on: - # - spectra - # restart: unless-stopped - # security_opt: - # - no-new-privileges:true volumes: spectra-data: driver: local + spectra-media: + driver: local spectra-logs: driver: local - spectra-config: + spectra-checkpoints: driver: local - spectra-qdrant: + spectra-config: driver: local - # spectra-redis: - # driver: local networks: spectra-network: driver: bridge - ipam: - config: - - subnet: 172.20.0.0/16 diff --git a/docs/docs/api/local-control-api.md b/docs/docs/api/local-control-api.md index 2b98d31..26aaaa6 100644 --- a/docs/docs/api/local-control-api.md +++ b/docs/docs/api/local-control-api.md @@ -28,26 +28,23 @@ Interactive local docs are available at: ## Authentication -If API key protection is enabled, clients can authenticate with either: - -- `X-API-Key: ` -- `?api_key=` - -Browser users can also authenticate through: +Browser users authenticate through the WebAuthn login surface at: - `/login` +The browser session created there is the supported control path for the local console. For automation, prefer the OpenAPI-described JSON endpoints and session-aware clients instead of scraping the HTML UI. + ## Launcher Example ```bash python3 -m spectra_app.spectra_gui_launcher \ --host 127.0.0.1 \ - --port 5000 \ - --api-key-env SPECTRA_GUI_API_KEY + --port 5000 ``` ## Notes - The launcher defaults to local-only access. +- First-run bootstrap uses `SPECTRA_BOOTSTRAP_SECRET` and the first enrolled operator becomes the admin. - The OpenAPI document is the supported integration contract for automation and AI clients. - Prefer the standard REST resources over scraping HTML pages. diff --git a/docs/docs/guides/web-interface.md b/docs/docs/guides/web-interface.md index 376d767..5277c88 100644 --- a/docs/docs/guides/web-interface.md +++ b/docs/docs/guides/web-interface.md @@ -21,11 +21,14 @@ tags: ['web', 'api', 'interface', 'guides'] pip install -r requirements.txt # Set configuration -export SPECTRA_JWT_SECRET="your-secret-key-here" +export SPECTRA_SESSION_SECRET="your-secret-key-here" +export SPECTRA_BOOTSTRAP_SECRET="one-time-bootstrap-secret" +export SPECTRA_WEBAUTHN_ORIGIN="http://localhost:5000" +export SPECTRA_WEBAUTHN_RP_ID="localhost" export SPECTRA_DEBUG=false # Run web server -python -m tgarchive.web --host 0.0.0.0 --port 5000 +python -m spectra_app.spectra_gui_launcher --host 0.0.0.0 --port 5000 ``` #### Option 2: Docker (Recommended) @@ -36,7 +39,10 @@ docker build -t spectra:latest . # Run container docker run -d \ -p 5000:5000 \ - -e SPECTRA_JWT_SECRET="your-secret-key" \ + -e SPECTRA_SESSION_SECRET="your-secret-key" \ + -e SPECTRA_BOOTSTRAP_SECRET="one-time-bootstrap-secret" \ + -e SPECTRA_WEBAUTHN_ORIGIN="http://localhost:5000" \ + -e SPECTRA_WEBAUTHN_RP_ID="localhost" \ -v spectra-data:/app/data \ spectra:latest ``` @@ -56,8 +62,8 @@ docker-compose down ### Access the Dashboard - **URL**: http://localhost:5000 - **Login**: - - Username: `admin` - - Password: `admin` + - First run: use the one-time bootstrap secret to enroll the first admin + - Later logins: sign in with a registered YubiKey or platform passkey --- @@ -65,43 +71,24 @@ docker-compose down ### Authentication -All API endpoints (except `/health`) require JWT authentication. +All operator-facing API endpoints require a browser session established through WebAuthn. -#### Login +#### Bootstrap enrollment ```bash -curl -X POST http://localhost:5000/api/auth/login \ +curl -X POST http://localhost:5000/api/auth/webauthn/register/options \ -H "Content-Type: application/json" \ -d '{ + "bootstrap_secret": "one-time-bootstrap-secret", "username": "admin", - "password": "admin" + "display_name": "Administrator" }' ``` -**Response**: -```json -{ - "access_token": "eyJ0eXAiOiJKV1QiLCJhbGc...", - "refresh_token": "eyJ0eXAiOiJKV1QiLCJhbGc...", - "user": { - "user_id": "user_001", - "username": "admin", - "roles": ["admin"], - "created_at": "2024-01-01T00:00:00Z" - } -} -``` - -#### Using Access Token -```bash -curl http://localhost:5000/api/channels \ - -H "Authorization: Bearer " -``` - -#### Refresh Token +#### WebAuthn sign-in ```bash -curl -X POST http://localhost:5000/api/auth/refresh \ +curl -X POST http://localhost:5000/api/auth/webauthn/authenticate/options \ -H "Content-Type: application/json" \ - -d '{"refresh_token": ""}' + -d '{"username":"admin"}' ``` --- @@ -109,11 +96,14 @@ curl -X POST http://localhost:5000/api/auth/refresh \ ## API Endpoints ### Authentication -- `POST /api/auth/login` - Login -- `POST /api/auth/refresh` - Refresh token -- `POST /api/auth/logout` - Logout -- `GET /api/auth/profile` - Get user profile -- `PUT /api/auth/profile` - Update user profile +- `GET /login` - Render the browser login and bootstrap page +- `POST /logout` - Clear the authenticated browser session +- `GET /api/auth/bootstrap/status` - Return first-run bootstrap state +- `GET /api/auth/session` - Return current session state +- `POST /api/auth/webauthn/register/options` - Start WebAuthn enrollment +- `POST /api/auth/webauthn/register/verify` - Complete WebAuthn enrollment +- `POST /api/auth/webauthn/authenticate/options` - Start WebAuthn sign-in +- `POST /api/auth/webauthn/authenticate/verify` - Complete WebAuthn sign-in ### Channels - `GET /api/channels` - List all channels @@ -289,7 +279,8 @@ curl http://localhost:5000/api/export/exp_001 \ ### Rate Limits by Endpoint ``` -/api/auth/login: 5 requests / 15 minutes (per IP) +/api/auth/webauthn/register/options: 5 requests / 15 minutes (per IP) +/api/auth/webauthn/authenticate/options: 5 requests / 15 minutes (per IP) /api/channels: 50 requests / minute (per user) /api/search: 30 requests / minute (per user) /api/search/correlation: 20 requests / minute (per user) @@ -350,17 +341,20 @@ python -m tgarchive.web --ssl --cert cert.pem --key key.pem ### Python Client Library ```python import requests -from requests.auth import HTTPBearerAuth -# Login +# Bootstrap admin enrollment response = requests.post( - 'http://localhost:5000/api/auth/login', - json={'username': 'admin', 'password': 'admin'} + 'http://localhost:5000/api/auth/webauthn/register/options', + json={ + 'bootstrap_secret': 'one-time-bootstrap-secret', + 'username': 'admin', + 'display_name': 'Administrator' + } ) -token = response.json()['access_token'] +payload = response.json() -# Search messages -headers = {'Authorization': f'Bearer {token}'} +# Search messages after browser-session authentication +headers = {'Content-Type': 'application/json'} response = requests.post( 'http://localhost:5000/api/search', headers=headers, @@ -375,21 +369,22 @@ results = response.json() ### JavaScript/Node.js Client ```javascript -// Login -const loginResp = await fetch('/api/auth/login', { +// Bootstrap enrollment +const loginResp = await fetch('/api/auth/webauthn/register/options', { method: 'POST', headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({username: 'admin', password: 'admin'}) + body: JSON.stringify({ + bootstrap_secret: 'one-time-bootstrap-secret', + username: 'admin', + display_name: 'Administrator' + }) }); -const {access_token} = await loginResp.json(); +const {options} = await loginResp.json(); // Search const searchResp = await fetch('/api/search', { method: 'POST', - headers: { - 'Authorization': `Bearer ${access_token}`, - 'Content-Type': 'application/json' - }, + headers: {'Content-Type': 'application/json'}, body: JSON.stringify({query: 'target', limit: 20}) }); const results = await searchResp.json(); @@ -405,9 +400,9 @@ See API section above for detailed examples. ### Common Issues #### "Invalid authorization header" -- Check token format: `Bearer ` -- Ensure token is not expired -- Regenerate using /api/auth/refresh +- Use the browser-authenticated session flow instead of bearer tokens +- Verify `/login` completed the bootstrap or sign-in ceremony +- Confirm the session cookie is present for the current browser origin #### "Rate limit exceeded" - Wait for reset time shown in header @@ -459,7 +454,8 @@ SQLALCHEMY_ENGINE_OPTIONS = { ## Production Deployment Checklist -- [ ] Change JWT secret to random value (32+ chars) +- [ ] Set `SPECTRA_SESSION_SECRET` to a random value (32+ chars) +- [ ] Set `SPECTRA_BOOTSTRAP_SECRET` before first launch - [ ] Enable HTTPS with valid certificate - [ ] Configure correct CORS origins - [ ] Set DEBUG=false @@ -473,7 +469,7 @@ SQLALCHEMY_ENGINE_OPTIONS = { - [ ] Enable WAF rules - [ ] Test disaster recovery - [ ] Create admin user account -- [ ] Set strong admin password +- [ ] Register at least one YubiKey or passkey per admin - [ ] Enable audit logging --- diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..7c3609b --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +asyncio_mode = auto +python_files = test_*.py *_test.py +testpaths = tgarchive tests +norecursedirs = .git .venv venv env __pycache__ *.egg .pytest_cache node_modules DIRECTEYE diff --git a/requirements.txt b/requirements.txt index 8ac4602..83be9f3 100755 --- a/requirements.txt +++ b/requirements.txt @@ -51,6 +51,9 @@ Flask==2.3.3 Flask-SocketIO==5.3.6 # Fallback: Flask-SocketIO>=5.0.0,<6.0 +# WebAuthn / YubiKey authentication +fido2>=2.1.1 + # JWT for API authentication PyJWT>=2.8.0 @@ -233,4 +236,3 @@ pytest-asyncio==0.21.1 # pip install telethon rich tqdm PyYAML Pillow npyscreen Jinja2 Flask Flask-SocketIO Markdown networkx matplotlib pandas # # Strategy 4 - Selective (skip problematic packages): - diff --git a/setup.py b/setup.py index eafe597..d04986d 100755 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ def list_requirements() -> list[str]: "matplotlib>=3.6", "pandas>=1.5", "python-magic>=0.4.27", + "fido2>=2.1.1", "pyaes>=1.6.1", "pyasn1>=0.6.0", # "rsa>=4.9", # Removed: RSA is phased out per CNSA 2.0. Use ECC-based cryptography if needed. @@ -81,7 +82,7 @@ def list_requirements() -> list[str]: author="John (SWORD-EPI)", author_email="n/a", url="https://github.com/SWORDIntel/SPECTRA", - packages=find_packages(include=["tgarchive*", "spectra_app*"]), + packages=find_packages(include=["tgarchive*", "spectra_app*", "src*"]), install_requires=list_requirements(), include_package_data=True, license="MIT", diff --git a/spectra_app/spectra_orchestrator.py b/spectra_app/spectra_orchestrator.py new file mode 100644 index 0000000..02816ef --- /dev/null +++ b/spectra_app/spectra_orchestrator.py @@ -0,0 +1,9 @@ +"""Compatibility wrapper for the repo-root orchestrator import path. + +The implementation lives under ``src.spectra_app`` so the repo-root package +can keep legacy imports working while the packaged distribution exposes the +same module tree. +""" + +from src.spectra_app.spectra_orchestrator import * # noqa: F401,F403 + diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..cf43f43 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,2 @@ +"""Namespace package for the source-layout SPECTRA modules.""" + diff --git a/src/spectra_app/agent_optimization_engine.py b/src/spectra_app/agent_optimization_engine.py index 90f2eed..c44dc2e 100644 --- a/src/spectra_app/agent_optimization_engine.py +++ b/src/spectra_app/agent_optimization_engine.py @@ -106,6 +106,7 @@ def __init__( @dataclass class AgentPerformanceProfile: """Comprehensive agent performance profile""" + agent_id: Optional[str] agent_name: str capabilities: Dict[str, float] # capability -> proficiency level performance_metrics: Dict[str, float] diff --git a/src/spectra_app/coordination_interface.py b/src/spectra_app/coordination_interface.py index 11c3ef0..d666042 100644 --- a/src/spectra_app/coordination_interface.py +++ b/src/spectra_app/coordination_interface.py @@ -215,8 +215,20 @@ def __init__( self.id = id or alert_id or "" self.alert_id = self.id self.timestamp = timestamp or datetime.now() - self.level = level if severity is None else AlertLevel(severity) - self.severity = self.level.value + if severity is None: + self.level = level + else: + severity_map = { + "medium": AlertLevel.WARNING, + "high": AlertLevel.ERROR, + "critical": AlertLevel.CRITICAL, + "info": AlertLevel.INFO, + "warning": AlertLevel.WARNING, + "error": AlertLevel.ERROR, + } + severity_key = str(severity).lower() + self.level = severity_map[severity_key] if severity_key in severity_map else AlertLevel(severity) + self.severity = severity if severity is not None else self.level.value self.category = category or alert_type or "" self.alert_type = self.category self.message = message diff --git a/src/spectra_app/implementation_tools.py b/src/spectra_app/implementation_tools.py index 298b893..8a9d7ce 100644 --- a/src/spectra_app/implementation_tools.py +++ b/src/spectra_app/implementation_tools.py @@ -284,7 +284,8 @@ def __init__( self.manual_reviews = list(manual_reviews or []) self.approvers = list(approvers or required_approvers or []) self.required_approvers = self.approvers - self.status = status if isinstance(status, QualityGateStatus) else QualityGateStatus(status) + self.status_enum = status if isinstance(status, QualityGateStatus) else QualityGateStatus(status) + self.status = self.status_enum.value self.submission_date = submission_date self.review_start_date = review_start_date self.approval_date = approval_date @@ -1021,7 +1022,8 @@ def submit_quality_gate(self, gate_id: str, artifacts: List[str], submitter: str return False gate = self.quality_gates[gate_id] - gate.status = QualityGateStatus.UNDER_REVIEW + gate.status_enum = QualityGateStatus.UNDER_REVIEW + gate.status = gate.status_enum.value gate.submission_date = datetime.now() gate.review_start_date = datetime.now() gate.artifacts.extend(artifacts) @@ -1050,7 +1052,8 @@ def approve_quality_gate(self, gate_id: str, approver: str, comments: str) -> bo if approver not in gate.approvers: return False - gate.status = QualityGateStatus.APPROVED + gate.status_enum = QualityGateStatus.APPROVED + gate.status = gate.status_enum.value gate.approval_date = datetime.now() gate.comments.append(f"{approver}: {comments}") gate.exit_criteria_met = True @@ -1130,7 +1133,7 @@ def _calculate_quality_metrics(self) -> Dict[str, float]: """Calculate quality metrics""" return { "quality_gates_passed": len([gate for gate in self.quality_gates.values() - if gate.status == QualityGateStatus.APPROVED]), + if getattr(gate, "status_enum", QualityGateStatus(gate.status)) == QualityGateStatus.APPROVED]), "quality_gates_total": len(self.quality_gates), "defect_rate": 0.02, # Would calculate from actual defects "rework_percentage": 0.05, # Would calculate from actual rework diff --git a/src/spectra_app/phase_management_dashboard.py b/src/spectra_app/phase_management_dashboard.py index f65e1c0..662ca1d 100644 --- a/src/spectra_app/phase_management_dashboard.py +++ b/src/spectra_app/phase_management_dashboard.py @@ -164,6 +164,7 @@ class PhaseDefinition: class TimelineEvent: """Timeline event for visualization""" id: str + event_id: str name: str start_date: datetime end_date: datetime diff --git a/src/spectra_app/spectra_gui_launcher.py b/src/spectra_app/spectra_gui_launcher.py index cd4557a..7224395 100644 --- a/src/spectra_app/spectra_gui_launcher.py +++ b/src/spectra_app/spectra_gui_launcher.py @@ -49,7 +49,7 @@ # GUI Framework Integration try: - from flask import Flask, render_template, jsonify, redirect, url_for, request, session + from flask import Flask, render_template, jsonify, redirect, url_for, request, session, has_request_context FLASK_AVAILABLE = True except ImportError: FLASK_AVAILABLE = False @@ -89,6 +89,7 @@ def emit(_event_name, _payload): from .implementation_tools import ImplementationTools from .agent_optimization_engine import AgentOptimizationEngine from .programmatic_api import SpectraProgrammaticAPI +from .webauthn_auth import JsonOperatorStore, WebAuthnAuthService def check_port_available(host: str, port: int, timeout: float = 3.0) -> bool: @@ -213,6 +214,12 @@ class SystemConfiguration: monitoring_interval: float api_key: Optional[str] home_page: str + session_secret: Optional[str] + bootstrap_secret: Optional[str] + auth_store_path: Optional[str] + webauthn_rp_id: Optional[str] + webauthn_rp_name: str + webauthn_origin: Optional[str] def __init__( self, @@ -230,6 +237,12 @@ def __init__( config_file: Optional[str] = None, api_key: Optional[str] = None, home_page: str = "console", + session_secret: Optional[str] = None, + bootstrap_secret: Optional[str] = None, + auth_store_path: Optional[str] = None, + webauthn_rp_id: Optional[str] = None, + webauthn_rp_name: str = "SPECTRA", + webauthn_origin: Optional[str] = None, ): self.mode = mode if isinstance(mode, SystemMode) else SystemMode(str(mode)) self.host = host @@ -254,8 +267,15 @@ def __init__( self.security_enabled = security_enabled self.monitoring_interval = monitoring_interval self.config_file = config_file - self.api_key = api_key + # Deprecated: API-key browser auth is no longer supported. + self.api_key = None self.home_page = home_page + self.session_secret = session_secret + self.bootstrap_secret = bootstrap_secret + self.auth_store_path = auth_store_path + self.webauthn_rp_id = webauthn_rp_id + self.webauthn_rp_name = webauthn_rp_name + self.webauthn_origin = webauthn_origin class SpectraGUILauncher: @@ -276,7 +296,14 @@ def __init__(self, config: SystemConfiguration): # Security configuration self.local_only = config.host in ["127.0.0.1", "localhost"] self.available_port = None - self.api_key_enabled = bool(config.api_key) + self.operator_store = JsonOperatorStore(self._resolve_auth_store_path()) + self.auth_service = WebAuthnAuthService( + store=self.operator_store, + bootstrap_secret=config.bootstrap_secret or os.environ.get("SPECTRA_BOOTSTRAP_SECRET"), + rp_id=config.webauthn_rp_id or self._default_rp_id(), + rp_name=config.webauthn_rp_name, + origin=config.webauthn_origin or os.environ.get("SPECTRA_WEBAUTHN_ORIGIN"), + ) # Core components self.orchestrator: Optional[SpectraOrchestrator] = None @@ -289,8 +316,19 @@ def __init__(self, config: SystemConfiguration): # Flask application for unified interface if FLASK_AVAILABLE: - self.app = Flask(__name__, static_folder='static', static_url_path='/static') - self.app.config['SECRET_KEY'] = 'spectra_gui_system' + project_root = Path(__file__).resolve().parents[2] + self.app = Flask( + __name__, + static_folder=str(project_root / "static"), + static_url_path="/static", + template_folder=str(project_root / "templates"), + ) + self.app.config['SECRET_KEY'] = self._resolve_session_secret() + self.app.config['SESSION_COOKIE_HTTPONLY'] = True + self.app.config['SESSION_COOKIE_SAMESITE'] = 'Lax' + self.app.config['SESSION_COOKIE_SECURE'] = ( + self.config.mode == SystemMode.PRODUCTION and not self.config.debug + ) self.socketio = SocketIO(self.app, cors_allowed_origins="*") self.programmatic_api = SpectraProgrammaticAPI(self) self._setup_auth_middleware() @@ -313,6 +351,112 @@ def __init__(self, config: SystemConfiguration): self.monitoring_task = None self.health_check_interval = config.monitoring_interval + def _resolve_auth_store_path(self) -> Path: + if self.config.auth_store_path: + return Path(self.config.auth_store_path) + db_path = Path(self.config.database_path) + if db_path.suffix: + return db_path.with_name(f"{db_path.stem}_operators.json") + return db_path / "operators.json" + + def _default_rp_id(self) -> str: + if self.config.host not in {"0.0.0.0", "::", "*"}: + return self.config.host + return os.environ.get("SPECTRA_WEBAUTHN_RP_ID", "localhost") + + def _resolve_session_secret(self) -> str: + return ( + self.config.session_secret + or os.environ.get("SPECTRA_SESSION_SECRET") + or os.environ.get("SPECTRA_JWT_SECRET") + or secrets.token_urlsafe(32) + ) + + def _clear_auth_session(self) -> None: + for key in ( + "spectra_operator_id", + "spectra_registration_state", + "spectra_authentication_state", + "spectra_post_login_redirect", + ): + session.pop(key, None) + + def _current_operator(self): + if not has_request_context(): + return None + operator_id = session.get("spectra_operator_id") + if not operator_id: + return None + operator = self.operator_store.get_operator(operator_id) + if operator is None or not operator.active: + self._clear_auth_session() + return None + return operator + + def _serialize_operator(self, operator) -> Optional[Dict[str, Any]]: + if operator is None: + return None + return { + "operator_id": operator.operator_id, + "username": operator.username, + "display_name": operator.display_name, + "role": operator.role, + "active": operator.active, + "credential_count": len(operator.credentials), + "created_at": operator.created_at, + "last_login_at": operator.last_login_at, + } + + def _build_login_state(self) -> Dict[str, Any]: + operator = self._current_operator() + return { + "auth": self.auth_service.public_auth_state(current_operator=operator), + "authenticated": operator is not None, + "operator": self._serialize_operator(operator), + "next": request.args.get("next") or session.get("spectra_post_login_redirect") or "/", + } + + def _mark_authenticated(self, operator) -> None: + session["spectra_operator_id"] = operator.operator_id + + def _start_registration(self, operator) -> Dict[str, Any]: + options, state = self.auth_service.backend.begin_registration( + operator=operator, + rp_id=self.auth_service.rp_id, + rp_name=self.auth_service.rp_name, + origin=self.auth_service.origin, + ) + state["operator_id"] = operator.operator_id + session["spectra_registration_state"] = state + return {"publicKey": options} + + def _complete_registration(self, payload: Dict[str, Any], state: Dict[str, Any]): + self.auth_service.finish_registration( + operator_id=state["operator_id"], + state=state, + request_data=payload, + label=payload.get("label"), + ) + operator = self.operator_store.get_operator(state["operator_id"]) + if operator is None: + raise ValueError("Operator not found") + return operator + + def _start_authentication(self, operator=None) -> Dict[str, Any]: + operators = [operator] if operator is not None else self.auth_service.store.list_operators() + options, state = self.auth_service.backend.begin_authentication( + operators=[item for item in operators if item.active], + rp_id=self.auth_service.rp_id, + origin=self.auth_service.origin, + ) + session["spectra_authentication_state"] = state + return {"publicKey": options} + + def _complete_authentication(self, payload: Dict[str, Any], state: Dict[str, Any]): + operator = self.auth_service.finish_authentication(state=state, request_data=payload) + self._mark_authenticated(operator) + return operator + def _check_port_availability(self, port: int) -> bool: """Check if a port is available for use""" try: @@ -398,27 +542,153 @@ def _log_security_status(self): def _setup_unified_routes(self): """Setup unified Flask routes for the complete system""" - @self.app.route('/login', methods=['GET', 'POST']) + @self.app.route('/login', methods=['GET']) def login(): - """Simple API-key login flow for browser access.""" - if request.method == 'POST': - submitted_key = request.form.get("api_key") or request.json.get("api_key") if request.is_json else None - if self._is_valid_api_key(submitted_key): - session["spectra_api_key_authenticated"] = True - next_url = request.args.get("next") or url_for("index") - if request.is_json: - return jsonify({"success": True, "redirect_to": next_url}) - return redirect(next_url) - if request.is_json: - return jsonify({"success": False, "error": "Invalid API key"}), 401 - return self._render_api_key_login(error_message="Invalid API key"), 401 - return self._render_api_key_login() + """Bootstrap and sign-in page for browser operators.""" + next_target = request.args.get("next") + if next_target: + session["spectra_post_login_redirect"] = next_target + return render_template( + 'login.html', + auth_state_json=json.dumps(self._build_login_state()), + ) @self.app.route('/logout', methods=['POST']) def logout(): - session.pop("spectra_api_key_authenticated", None) + self._clear_auth_session() return jsonify({"success": True}) + @self.app.route('/api/auth/bootstrap/status') + def api_bootstrap_status(): + return jsonify(self._build_login_state()) + + @self.app.route('/api/auth/session') + def api_auth_session(): + operator = self._current_operator() + return jsonify( + { + "authenticated": operator is not None, + "operator": self._serialize_operator(operator) if operator else None, + "auth_state": self.auth_service.public_auth_state(current_operator=operator), + } + ) + + @self.app.route('/api/auth/webauthn/register/options', methods=['POST']) + def api_webauthn_register_options(): + payload = request.get_json(silent=True) or {} + bootstrap_required = self.auth_service.bootstrap_required() + if bootstrap_required: + if not self.auth_service.validate_bootstrap_secret(payload.get("bootstrap_secret")): + return jsonify({"success": False, "error": "Invalid bootstrap secret"}), 403 + username = (payload.get("username") or "").strip() + display_name = (payload.get("display_name") or "").strip() + if not username: + return jsonify({"success": False, "error": "Username is required"}), 400 + try: + operator = self.auth_service.ensure_admin_bootstrap( + username=username, + display_name=display_name or username, + submitted_secret=payload.get("bootstrap_secret") or "", + ) + except ValueError as exc: + return jsonify({"success": False, "error": str(exc)}), 400 + else: + operator = self._current_operator() + if operator is None or operator.role != "admin": + return jsonify({"success": False, "error": "Admin session required"}), 403 + target_operator_id = payload.get("operator_id") or operator.operator_id + operator = self.operator_store.get_operator(target_operator_id) + if operator is None: + return jsonify({"success": False, "error": "Operator not found"}), 404 + + try: + options = self._start_registration(operator) + except Exception as exc: + return jsonify({"success": False, "error": str(exc)}), 503 + return jsonify({"success": True, "options": options, "operator": self._serialize_operator(operator)}) + + @self.app.route('/api/auth/webauthn/register/verify', methods=['POST']) + def api_webauthn_register_verify(): + payload = request.get_json(silent=True) or {} + state = session.get("spectra_registration_state") + if not state: + return jsonify({"success": False, "error": "No registration in progress"}), 400 + try: + operator = self._complete_registration(payload, state) + except RuntimeError as exc: + return jsonify({"success": False, "error": str(exc)}), 503 + except ValueError as exc: + return jsonify({"success": False, "error": str(exc)}), 400 + + session.pop("spectra_registration_state", None) + self._mark_authenticated(operator) + return jsonify({"success": True, "operator": self._serialize_operator(operator)}) + + @self.app.route('/api/auth/webauthn/authenticate/options', methods=['POST']) + def api_webauthn_authenticate_options(): + payload = request.get_json(silent=True) or {} + username = (payload.get("username") or "").strip() + operator = None + if username: + operator = self.operator_store.get_operator_by_username(username) + if operator is None: + return jsonify({"success": False, "error": "Unknown operator"}), 404 + try: + options = self._start_authentication(operator) + except Exception as exc: + return jsonify({"success": False, "error": str(exc)}), 400 + return jsonify({"success": True, "options": options}) + + @self.app.route('/api/auth/webauthn/authenticate/verify', methods=['POST']) + def api_webauthn_authenticate_verify(): + payload = request.get_json(silent=True) or {} + state = session.get("spectra_authentication_state") + if not state: + return jsonify({"success": False, "error": "No authentication in progress"}), 400 + try: + operator = self._complete_authentication(payload, state) + except RuntimeError as exc: + return jsonify({"success": False, "error": str(exc)}), 503 + except ValueError as exc: + return jsonify({"success": False, "error": str(exc)}), 400 + + session.pop("spectra_authentication_state", None) + self._mark_authenticated(operator) + return jsonify({"success": True, "operator": self._serialize_operator(operator)}) + + @self.app.route('/api/auth/operators', methods=['GET', 'POST']) + def api_auth_operators(): + current = self._current_operator() + if current is None or current.role != "admin": + return jsonify({"success": False, "error": "Admin session required"}), 403 + if request.method == 'GET': + return jsonify({"success": True, "operators": self.auth_service.list_operator_summaries()}) + + payload = request.get_json(silent=True) or {} + username = (payload.get("username") or "").strip() + display_name = (payload.get("display_name") or username).strip() + role = (payload.get("role") or "operator").strip().lower() + if not username: + return jsonify({"success": False, "error": "Username is required"}), 400 + try: + operator = self.auth_service.create_operator(username=username, display_name=display_name, role=role) + except ValueError as exc: + return jsonify({"success": False, "error": str(exc)}), 409 + return jsonify({"success": True, "operator": self._serialize_operator(operator)}), 201 + + @self.app.route('/api/auth/operators/', methods=['PATCH']) + def api_update_operator(operator_id): + current = self._current_operator() + if current is None or current.role != "admin": + return jsonify({"success": False, "error": "Admin session required"}), 403 + payload = request.get_json(silent=True) or {} + if "active" not in payload: + return jsonify({"success": False, "error": "Only the active flag can be updated"}), 400 + operator = self.operator_store.set_operator_active(operator_id, payload.get("active")) + if operator is None: + return jsonify({"success": False, "error": "Operator not found"}), 404 + return jsonify({"success": True, "operator": self._serialize_operator(operator)}) + @self.app.route('/') def index(): """Main system dashboard""" @@ -703,118 +973,38 @@ def _render_api_docs(self) -> str: """ def _setup_auth_middleware(self): - """Protect the interface and APIs with an optional API key.""" + """Protect the interface and APIs with authenticated browser sessions.""" @self.app.before_request - def require_api_key(): - if not self.api_key_enabled: - return None - - public_paths = { + def require_operator_session(): + public_prefixes = ( "/login", - "/static", - } - if request.path == "/favicon.ico" or any(request.path.startswith(path) for path in public_paths): + "/logout", + "/static/", + "/api/auth/", + ) + public_exact = {"/favicon.ico", "/health"} + if request.path in public_exact or any(request.path.startswith(path) for path in public_prefixes): return None if self._request_is_authenticated(): return None - if request.path.startswith("/api/"): - return jsonify({ - "success": False, - "error": "API key required", - "auth": { - "header": "X-API-Key", - "query_param": "api_key", + if request.path.startswith("/api/") or request.path.startswith("/openapi"): + return jsonify( + { + "success": False, + "error": "Authentication required", + "auth": self.auth_service.public_auth_state(current_operator=None), "login_path": "/login", - }, - }), 401 + } + ), 401 return redirect(url_for("login", next=request.full_path if request.query_string else request.path)) def _request_is_authenticated(self) -> bool: - """Allow either browser session auth or direct API-key auth.""" - if session.get("spectra_api_key_authenticated"): - return True - submitted_key = request.headers.get("X-API-Key") or request.args.get("api_key") - return self._is_valid_api_key(submitted_key) - - def _is_valid_api_key(self, submitted_key: Optional[str]) -> bool: - """Constant-time API key comparison.""" - if not self.api_key_enabled or not self.config.api_key: - return True - if not submitted_key: - return False - return secrets.compare_digest(str(submitted_key), str(self.config.api_key)) - - def _render_api_key_login(self, error_message: Optional[str] = None) -> str: - """Render a compact API-key login form for browser use.""" - error_html = f'

{error_message}

' if error_message else "" - return f""" - - - - - - SPECTRA Login - - - -
-

SPECTRA Access

-

Enter the configured API key to open the interface.

- {error_html} - - -

- Programmatic clients can also send X-API-Key or ?api_key=.... -

-
- - - """ + """Allow access only when a browser session has a valid operator.""" + return self._current_operator() is not None def _get_readme_content(self) -> str: """Read and process README.md content""" @@ -1867,6 +2057,7 @@ async def _broadcast_system_status(self): def get_system_status(self) -> Dict[str, Any]: """Get comprehensive system status""" + operator = self._current_operator() return { "system_running": self.system_running, "mode": self.config.mode.value, @@ -1876,10 +2067,13 @@ def get_system_status(self) -> Dict[str, Any]: if hasattr(a, 'status') and a.status.value == 'running']) if self.orchestrator else 0, "components_enabled": self.config.enable_components, "auth": { - "api_key_required": self.api_key_enabled, - "browser_login": "/login" if self.api_key_enabled else None, - "header": "X-API-Key" if self.api_key_enabled else None, - "query_param": "api_key" if self.api_key_enabled else None, + "webauthn_required": True, + "bootstrap_required": self.auth_service.bootstrap_required(), + "bootstrap_configured": bool(self.auth_service.bootstrap_secret), + "browser_login": "/login", + "authenticated": operator is not None, + "operator": self._serialize_operator(operator) if operator else None, + "webauthn": self.auth_service.public_auth_state(current_operator=operator).get("webauthn", {}), }, "api_endpoints": { "status": "/api/system/status", @@ -1889,6 +2083,9 @@ def get_system_status(self) -> Dict[str, Any]: "docs": "/docs", "standard_context": "/api/v1/context", "standard_readme": "/api/v1/readme", + "auth_session": "/api/auth/session", + "auth_register_options": "/api/auth/webauthn/register/options", + "auth_authenticate_options": "/api/auth/webauthn/authenticate/options", }, "timestamp": datetime.now().isoformat() } @@ -1920,8 +2117,13 @@ def create_default_config() -> SystemConfiguration: }, security_enabled=True, # SECURITY: Enable security by default monitoring_interval=5.0, - api_key=None, home_page="console", + session_secret=os.environ.get("SPECTRA_SESSION_SECRET"), + bootstrap_secret=os.environ.get("SPECTRA_BOOTSTRAP_SECRET"), + auth_store_path=os.environ.get("SPECTRA_AUTH_STORE_PATH"), + webauthn_rp_id=os.environ.get("SPECTRA_WEBAUTHN_RP_ID"), + webauthn_rp_name=os.environ.get("SPECTRA_WEBAUTHN_RP_NAME", "SPECTRA"), + webauthn_origin=os.environ.get("SPECTRA_WEBAUTHN_ORIGIN"), ) @@ -1936,11 +2138,32 @@ async def main(): parser.add_argument("--debug", action="store_true", help="Enable debug mode") parser.add_argument("--log-level", choices=["DEBUG", "INFO", "WARNING", "ERROR"], default="INFO", help="Log level") - parser.add_argument("--api-key", help="Require this API key for browser and API access") parser.add_argument( - "--api-key-env", - default="SPECTRA_GUI_API_KEY", - help="Environment variable to read the API key from if --api-key is unset", + "--session-secret-env", + default="SPECTRA_SESSION_SECRET", + help="Environment variable to read the Flask session secret from", + ) + parser.add_argument( + "--bootstrap-secret-env", + default="SPECTRA_BOOTSTRAP_SECRET", + help="Environment variable to read the one-time admin bootstrap secret from", + ) + parser.add_argument( + "--auth-store-path", + help="Path to the JSON operator store", + ) + parser.add_argument( + "--webauthn-rp-id", + help="WebAuthn relying party ID", + ) + parser.add_argument( + "--webauthn-rp-name", + default=os.environ.get("SPECTRA_WEBAUTHN_RP_NAME", "SPECTRA"), + help="WebAuthn relying party display name", + ) + parser.add_argument( + "--webauthn-origin", + help="Expected browser origin for WebAuthn ceremonies", ) parser.add_argument( "--home-page", @@ -1958,8 +2181,13 @@ async def main(): config.mode = SystemMode(args.mode) config.debug = args.debug config.log_level = args.log_level - config.api_key = args.api_key or os.environ.get(args.api_key_env) config.home_page = args.home_page + config.session_secret = os.environ.get(args.session_secret_env) + config.bootstrap_secret = os.environ.get(args.bootstrap_secret_env) + config.auth_store_path = args.auth_store_path or os.environ.get("SPECTRA_AUTH_STORE_PATH") + config.webauthn_rp_id = args.webauthn_rp_id or os.environ.get("SPECTRA_WEBAUTHN_RP_ID") + config.webauthn_rp_name = args.webauthn_rp_name + config.webauthn_origin = args.webauthn_origin or os.environ.get("SPECTRA_WEBAUTHN_ORIGIN") # Initialize and start system launcher = SpectraGUILauncher(config) diff --git a/src/spectra_app/webauthn_auth.py b/src/spectra_app/webauthn_auth.py new file mode 100644 index 0000000..045eaaa --- /dev/null +++ b/src/spectra_app/webauthn_auth.py @@ -0,0 +1,578 @@ +"""Local operator auth and WebAuthn backend support for SPECTRA.""" + +from __future__ import annotations + +import base64 +import json +import secrets +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence + +try: + from fido2.server import Fido2Server + from fido2.utils import websafe_decode, websafe_encode + from fido2.webauthn import ( + AttestedCredentialData, + PublicKeyCredentialDescriptor, + PublicKeyCredentialRpEntity, + PublicKeyCredentialUserEntity, + ) + + FIDO2_AVAILABLE = True +except ImportError: # pragma: no cover - exercised only without the dependency + FIDO2_AVAILABLE = False + Fido2Server = None # type: ignore[assignment] + AttestedCredentialData = None # type: ignore[assignment] + PublicKeyCredentialDescriptor = None # type: ignore[assignment] + PublicKeyCredentialRpEntity = None # type: ignore[assignment] + PublicKeyCredentialUserEntity = None # type: ignore[assignment] + websafe_decode = None # type: ignore[assignment] + websafe_encode = None # type: ignore[assignment] + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def b64url_encode(value: bytes) -> str: + return base64.urlsafe_b64encode(value).rstrip(b"=").decode("ascii") + + +def b64url_decode(value: str) -> bytes: + padding = "=" * (-len(value) % 4) + return base64.urlsafe_b64decode(f"{value}{padding}") + + +def _jsonify(value: Any) -> Any: + """Convert WebAuthn objects into browser-friendly JSON values.""" + if isinstance(value, bytes): + return b64url_encode(value) + if isinstance(value, Mapping): + return {key: _jsonify(item) for key, item in value.items()} + if isinstance(value, (list, tuple, set)): + return [_jsonify(item) for item in value] + if hasattr(value, "items") and hasattr(value, "keys"): + return {key: _jsonify(item) for key, item in value.items()} + if hasattr(value, "value") and not isinstance(value, str): + return getattr(value, "value") + if hasattr(value, "__dict__") and not isinstance(value, type): + return { + key: _jsonify(item) + for key, item in value.__dict__.items() + if not key.startswith("_") + } + return value + + +def _credential_descriptor_from_attested(credential: "AttestedCredentialData") -> Dict[str, Any]: + return { + "id": b64url_encode(credential.credential_id), + "type": "public-key", + } + + +@dataclass +class OperatorCredential: + credential_id: str + label: str + sign_count: int = 0 + transports: List[str] = field(default_factory=list) + public_key: Optional[str] = None + attested_credential_data: Optional[str] = None + aaguid: Optional[str] = None + created_at: str = field(default_factory=_utc_now) + last_used_at: Optional[str] = None + + +@dataclass +class OperatorRecord: + operator_id: str + username: str + display_name: str + role: str + active: bool = True + credentials: List[OperatorCredential] = field(default_factory=list) + created_at: str = field(default_factory=_utc_now) + updated_at: str = field(default_factory=_utc_now) + last_login_at: Optional[str] = None + + +class JsonOperatorStore: + """Persists local operators and WebAuthn metadata to a JSON file.""" + + def __init__(self, path: Path): + self.path = path + self.path.parent.mkdir(parents=True, exist_ok=True) + self._operators: Dict[str, OperatorRecord] = {} + self._load() + + def _load(self) -> None: + if not self.path.exists(): + return + try: + payload = json.loads(self.path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return + operators: Dict[str, OperatorRecord] = {} + for item in payload.get("operators", []): + credentials = [OperatorCredential(**credential) for credential in item.get("credentials", [])] + item = dict(item) + item["credentials"] = credentials + operator = OperatorRecord(**item) + operators[operator.operator_id] = operator + self._operators = operators + + def _save(self) -> None: + payload = {"operators": [asdict(operator) for operator in self._operators.values()]} + self.path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + + def list_operators(self) -> List[OperatorRecord]: + return sorted(self._operators.values(), key=lambda operator: operator.username.lower()) + + def has_admin(self) -> bool: + return any(operator.role == "admin" for operator in self._operators.values()) + + def get_operator(self, operator_id: str) -> Optional[OperatorRecord]: + return self._operators.get(operator_id) + + def get_operator_by_username(self, username: str) -> Optional[OperatorRecord]: + username = username.strip().lower() + for operator in self._operators.values(): + if operator.username.lower() == username: + return operator + return None + + def create_operator(self, username: str, display_name: str, role: str) -> OperatorRecord: + operator = OperatorRecord( + operator_id=secrets.token_hex(8), + username=username.strip(), + display_name=display_name.strip() or username.strip(), + role=role, + ) + self._operators[operator.operator_id] = operator + self._save() + return operator + + def update_operator(self, operator: OperatorRecord) -> OperatorRecord: + operator.updated_at = _utc_now() + self._operators[operator.operator_id] = operator + self._save() + return operator + + def set_operator_active(self, operator_id: str, active: bool) -> Optional[OperatorRecord]: + operator = self.get_operator(operator_id) + if operator is None: + return None + operator.active = bool(active) + return self.update_operator(operator) + + def find_operator_by_credential_id(self, credential_id: str) -> Optional[OperatorRecord]: + for operator in self._operators.values(): + for credential in operator.credentials: + if credential.credential_id == credential_id: + return operator + return None + + def add_credential( + self, + operator_id: str, + credential_id: str, + label: str, + transports: Optional[List[str]] = None, + public_key: Optional[str] = None, + attested_credential_data: Optional[str] = None, + aaguid: Optional[str] = None, + ) -> OperatorCredential: + operator = self.get_operator(operator_id) + if operator is None: + raise KeyError(operator_id) + if any(existing.credential_id == credential_id for existing in operator.credentials): + raise ValueError("Credential is already registered") + credential = OperatorCredential( + credential_id=credential_id, + label=label, + transports=sorted(set(transports or [])), + public_key=public_key, + attested_credential_data=attested_credential_data, + aaguid=aaguid, + ) + operator.credentials.append(credential) + self.update_operator(operator) + return credential + + def touch_login(self, operator_id: str, credential_id: Optional[str] = None, sign_count: Optional[int] = None) -> None: + operator = self.get_operator(operator_id) + if operator is None: + return + operator.last_login_at = _utc_now() + if credential_id: + for credential in operator.credentials: + if credential.credential_id == credential_id: + credential.last_used_at = operator.last_login_at + if sign_count is not None: + credential.sign_count = max(int(sign_count), credential.sign_count) + break + self.update_operator(operator) + + def operator_credentials(self, operator_id: str) -> List["AttestedCredentialData"]: + operator = self.get_operator(operator_id) + if operator is None: + return [] + credentials: List[AttestedCredentialData] = [] + for credential in operator.credentials: + if credential.attested_credential_data: + credentials.append(AttestedCredentialData(b64url_decode(credential.attested_credential_data))) + return credentials + + def all_credentials(self) -> List["AttestedCredentialData"]: + credentials: List[AttestedCredentialData] = [] + for operator in self.list_operators(): + credentials.extend(self.operator_credentials(operator.operator_id)) + return credentials + + +class WebAuthnUnavailableError(RuntimeError): + """Raised when a WebAuthn backend is not installed.""" + + +class WebAuthnBackend: + """Backend abstraction that can be swapped without changing launcher routes.""" + + available = False + backend_name = "unavailable" + unavailable_reason = "Install a Python WebAuthn backend such as `fido2` to enable YubiKey verification." + + def register_begin(self, user: "PublicKeyCredentialUserEntity", exclude_credentials: Sequence[Any] | None = None) -> tuple[Dict[str, Any], Dict[str, Any]]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def register_complete(self, state: Dict[str, Any], request_data: Dict[str, Any]) -> Dict[str, Any]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def authenticate_begin(self, credentials: Sequence[Any] | None = None) -> tuple[Dict[str, Any], Dict[str, Any]]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def authenticate_complete(self, state: Dict[str, Any], credentials: Sequence["AttestedCredentialData"], request_data: Dict[str, Any]) -> Dict[str, Any]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def begin_registration( + self, + operator: OperatorRecord, + rp_id: Optional[str] = None, + rp_name: Optional[str] = None, + origin: Optional[str] = None, + ) -> tuple[Dict[str, Any], Dict[str, Any]]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def finish_registration( + self, + request_data: Dict[str, Any], + state: Dict[str, Any], + ) -> Dict[str, Any]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def begin_authentication( + self, + operators: Sequence[OperatorRecord] | None = None, + rp_id: Optional[str] = None, + origin: Optional[str] = None, + ) -> tuple[Dict[str, Any], Dict[str, Any]]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def finish_authentication( + self, + request_data: Dict[str, Any], + state: Dict[str, Any], + operators: Sequence[OperatorRecord] | None = None, + ) -> Dict[str, Any]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + +class Fido2WebAuthnBackend(WebAuthnBackend): + """WebAuthn backend backed by `python-fido2`.""" + + available = True + backend_name = "fido2" + unavailable_reason = "" + + def __init__(self, rp_id: str, rp_name: str, origin: Optional[str] = None): + if not FIDO2_AVAILABLE: + raise WebAuthnUnavailableError("fido2 is not installed") + self.rp_id = rp_id + self.rp_name = rp_name + self.origin = origin + verify_origin = (lambda candidate_origin: candidate_origin == origin) if origin else None + self.server = Fido2Server( + PublicKeyCredentialRpEntity(name=rp_name, id=rp_id), + verify_origin=verify_origin, + ) + + def register_begin(self, user: "PublicKeyCredentialUserEntity", exclude_credentials: Sequence[Any] | None = None) -> tuple[Dict[str, Any], Dict[str, Any]]: + options, state = self.server.register_begin(user, credentials=list(exclude_credentials or [])) + return _jsonify(options.public_key), state + + def register_complete(self, state: Dict[str, Any], request_data: Dict[str, Any]) -> Dict[str, Any]: + auth_data = self.server.register_complete(state, request_data) + credential_data = auth_data.credential_data + assert credential_data is not None # noqa: S101 + return { + "credential_id": b64url_encode(credential_data.credential_id), + "attested_credential_data": b64url_encode(bytes(credential_data)), + "aaguid": getattr(credential_data, "aaguid", None).hex() if getattr(credential_data, "aaguid", None) else None, + "public_key": getattr(credential_data, "public_key", None).hex() if getattr(credential_data, "public_key", None) else None, + } + + def authenticate_begin(self, credentials: Sequence[Any] | None = None) -> tuple[Dict[str, Any], Dict[str, Any]]: + options, state = self.server.authenticate_begin(credentials=list(credentials or [])) + return _jsonify(options.public_key), state + + def authenticate_complete(self, state: Dict[str, Any], credentials: Sequence["AttestedCredentialData"], request_data: Dict[str, Any]) -> Dict[str, Any]: + credential = self.server.authenticate_complete(state, list(credentials), request_data) + return { + "credential_id": b64url_encode(credential.credential_id), + } + + def begin_registration( + self, + operator: OperatorRecord, + rp_id: Optional[str] = None, + rp_name: Optional[str] = None, + origin: Optional[str] = None, + ) -> tuple[Dict[str, Any], Dict[str, Any]]: + if rp_id and rp_id != self.rp_id: + self.__init__(rp_id=rp_id, rp_name=rp_name or self.rp_name, origin=origin or self.origin) + existing_credentials: List[Any] = [] + for credential in operator.credentials: + if credential.attested_credential_data: + existing_credentials.append(AttestedCredentialData(b64url_decode(credential.attested_credential_data))) + user = PublicKeyCredentialUserEntity( + id=operator.operator_id.encode("utf-8"), + name=operator.username, + display_name=operator.display_name, + ) + return self.register_begin(user, exclude_credentials=existing_credentials) + + def finish_registration( + self, + request_data: Dict[str, Any], + state: Dict[str, Any], + ) -> Dict[str, Any]: + return self.register_complete(state, request_data) + + def begin_authentication( + self, + operators: Sequence[OperatorRecord] | None = None, + rp_id: Optional[str] = None, + origin: Optional[str] = None, + ) -> tuple[Dict[str, Any], Dict[str, Any]]: + if rp_id and rp_id != self.rp_id: + self.__init__(rp_id=rp_id, rp_name=self.rp_name, origin=origin or self.origin) + credentials: List[Any] = [] + for operator in operators or []: + for credential in operator.credentials: + if credential.attested_credential_data: + credentials.append(AttestedCredentialData(b64url_decode(credential.attested_credential_data))) + return self.authenticate_begin(credentials=credentials) + + def finish_authentication( + self, + request_data: Dict[str, Any], + state: Dict[str, Any], + operators: Sequence[OperatorRecord] | None = None, + ) -> Dict[str, Any]: + credentials: List[AttestedCredentialData] = [] + for operator in operators or []: + for credential in operator.credentials: + if credential.attested_credential_data: + credentials.append(AttestedCredentialData(b64url_decode(credential.attested_credential_data))) + return self.authenticate_complete(state, credentials, request_data) + + +class PlaceholderWebAuthnBackend(WebAuthnBackend): + """Strict fail-closed backend used when `fido2` is unavailable.""" + + def register_begin(self, user: "PublicKeyCredentialUserEntity", exclude_credentials: Sequence[Any] | None = None) -> tuple[Dict[str, Any], Dict[str, Any]]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def register_complete(self, state: Dict[str, Any], request_data: Dict[str, Any]) -> Dict[str, Any]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def authenticate_begin(self, credentials: Sequence[Any] | None = None) -> tuple[Dict[str, Any], Dict[str, Any]]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def authenticate_complete(self, state: Dict[str, Any], credentials: Sequence["AttestedCredentialData"], request_data: Dict[str, Any]) -> Dict[str, Any]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def begin_registration( + self, + operator: OperatorRecord, + rp_id: Optional[str] = None, + rp_name: Optional[str] = None, + origin: Optional[str] = None, + ) -> tuple[Dict[str, Any], Dict[str, Any]]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def finish_registration( + self, + request_data: Dict[str, Any], + state: Dict[str, Any], + ) -> Dict[str, Any]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def begin_authentication( + self, + operators: Sequence[OperatorRecord] | None = None, + rp_id: Optional[str] = None, + origin: Optional[str] = None, + ) -> tuple[Dict[str, Any], Dict[str, Any]]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + def finish_authentication( + self, + request_data: Dict[str, Any], + state: Dict[str, Any], + operators: Sequence[OperatorRecord] | None = None, + ) -> Dict[str, Any]: + raise WebAuthnUnavailableError(self.unavailable_reason) + + +class WebAuthnAuthService: + """Local auth service for operators, sessions, and WebAuthn ceremony state.""" + + def __init__( + self, + store: JsonOperatorStore, + backend: Optional[WebAuthnBackend] = None, + bootstrap_secret: Optional[str] = None, + rp_id: Optional[str] = None, + rp_name: str = "SPECTRA", + origin: Optional[str] = None, + ): + self.store = store + self.backend = backend or ( + Fido2WebAuthnBackend(rp_id=rp_id or "localhost", rp_name=rp_name, origin=origin) + if FIDO2_AVAILABLE + else PlaceholderWebAuthnBackend() + ) + self.bootstrap_secret = bootstrap_secret + self.rp_id = rp_id or "localhost" + self.rp_name = rp_name + self.origin = origin + + def bootstrap_required(self) -> bool: + return not self.store.has_admin() + + def backend_status(self) -> Dict[str, Any]: + return { + "available": self.backend.available, + "backend": self.backend.backend_name, + "reason": None if self.backend.available else self.backend.unavailable_reason, + } + + def public_auth_state(self, current_operator: Optional[OperatorRecord] = None) -> Dict[str, Any]: + return { + "bootstrap_required": self.bootstrap_required(), + "bootstrap_configured": bool(self.bootstrap_secret), + "webauthn": self.backend_status(), + "rp_id": self.rp_id, + "rp_name": self.rp_name, + "origin": self.origin, + "operator": self.operator_summary(current_operator) if current_operator else None, + "operators": self.list_operator_summaries() if current_operator else [], + } + + def list_operator_summaries(self) -> List[Dict[str, Any]]: + summaries = [] + for operator in self.store.list_operators(): + summaries.append(self.operator_summary(operator)) + return summaries + + def operator_summary(self, operator: Optional[OperatorRecord]) -> Optional[Dict[str, Any]]: + if operator is None: + return None + return { + "operator_id": operator.operator_id, + "username": operator.username, + "display_name": operator.display_name, + "role": operator.role, + "active": operator.active, + "credential_count": len(operator.credentials), + "created_at": operator.created_at, + "last_login_at": operator.last_login_at, + } + + def validate_bootstrap_secret(self, submitted_secret: Optional[str]) -> bool: + if not self.bootstrap_secret or not submitted_secret: + return False + return secrets.compare_digest(str(submitted_secret), str(self.bootstrap_secret)) + + def create_operator(self, username: str, display_name: str, role: str) -> OperatorRecord: + if self.store.get_operator_by_username(username): + raise ValueError("Operator already exists") + return self.store.create_operator(username=username, display_name=display_name, role=role) + + def get_operator_by_username(self, username: str) -> Optional[OperatorRecord]: + return self.store.get_operator_by_username(username) + + def get_operator(self, operator_id: str) -> Optional[OperatorRecord]: + return self.store.get_operator(operator_id) + + def ensure_admin_bootstrap(self, username: str, display_name: str, submitted_secret: str) -> OperatorRecord: + if not self.bootstrap_required(): + raise ValueError("Bootstrap is only available before the first admin is enrolled") + if not self.validate_bootstrap_secret(submitted_secret): + raise ValueError("Invalid bootstrap secret") + operator = self.get_operator_by_username(username) + if operator is None: + operator = self.create_operator(username=username, display_name=display_name, role="admin") + elif operator.role != "admin": + operator.role = "admin" + self.store.update_operator(operator) + return operator + + def start_registration(self, operator: OperatorRecord) -> tuple[Dict[str, Any], Dict[str, Any]]: + if not FIDO2_AVAILABLE: + raise WebAuthnUnavailableError(self.backend.unavailable_reason) + existing_credentials = self.store.operator_credentials(operator.operator_id) + user = PublicKeyCredentialUserEntity( + id=operator.operator_id.encode("utf-8"), + name=operator.username, + display_name=operator.display_name, + ) + return self.backend.register_begin(user, exclude_credentials=existing_credentials) + + def finish_registration(self, operator_id: str, state: Dict[str, Any], request_data: Dict[str, Any], label: Optional[str] = None) -> Dict[str, Any]: + operator = self.get_operator(operator_id) + if operator is None: + raise ValueError("Operator not found") + result = self.backend.register_complete(state, request_data) + credential_id = result["credential_id"] + self.store.add_credential( + operator_id=operator_id, + credential_id=credential_id, + label=label or operator.display_name, + attested_credential_data=result["attested_credential_data"], + public_key=result.get("public_key"), + aaguid=result.get("aaguid"), + ) + return { + "operator": self.operator_summary(operator), + "credential_id": credential_id, + } + + def start_authentication(self, operator: OperatorRecord) -> tuple[Dict[str, Any], Dict[str, Any]]: + credentials = self.store.operator_credentials(operator.operator_id) + if not credentials: + raise ValueError("No registered credentials for operator") + return self.backend.authenticate_begin(credentials=credentials) + + def finish_authentication(self, state: Dict[str, Any], request_data: Dict[str, Any]) -> OperatorRecord: + credentials = self.store.all_credentials() + result = self.backend.authenticate_complete(state, credentials, request_data) + credential_id = result["credential_id"] + operator = self.store.find_operator_by_credential_id(credential_id) + if operator is None: + raise ValueError("Authenticated credential is not associated with an operator") + self.store.touch_login(operator.operator_id, credential_id=credential_id) + return operator diff --git a/ssdeep.py b/ssdeep.py new file mode 100644 index 0000000..d7bc435 --- /dev/null +++ b/ssdeep.py @@ -0,0 +1,32 @@ +"""Local fallback for the optional ssdeep dependency.""" + +from difflib import SequenceMatcher +import re + + +def _normalize(file_path): + try: + with open(file_path, "r", encoding="utf-8", errors="ignore") as f: + text = f.read().lower() + except Exception: + return "" + return re.sub(r"\s+", " ", text).strip() + + +def hash_from_file(file_path): + normalized = _normalize(file_path) + if not normalized: + return None + return f"fallback:{normalized}" + + +def compare(hash1, hash2): + if not hash1 or not hash2: + return 0 + if hash1 == hash2: + return 100 + if hash1.startswith("fallback:") and hash2.startswith("fallback:"): + text1 = hash1.removeprefix("fallback:") + text2 = hash2.removeprefix("fallback:") + return int(round(SequenceMatcher(None, text1, text2).ratio() * 100)) + return 0 diff --git a/templates/api_docs.html b/templates/api_docs.html new file mode 100644 index 0000000..3ba7e09 --- /dev/null +++ b/templates/api_docs.html @@ -0,0 +1,126 @@ + + + + + + SPECTRA API Docs + + + +
+
+
SPECTRA API Documentation
+

API Docs

+

+ This page is the lightweight landing surface for the web API. + It keeps the route alive in environments where the richer OpenAPI + or Swagger UI assets are not bundled. +

+ +
+

Available entry points:

+ +

+ If you are wiring a full API UI, mount it behind this route or + replace this page with the generated spec viewer. +

+
+ +
+ Back to dashboard + Open login page +
+
+
+ + diff --git a/templates/login.html b/templates/login.html index 83718b2..7edc8af 100644 --- a/templates/login.html +++ b/templates/login.html @@ -1,10 +1,28 @@ - - - SPECTRA - Intelligence Platform + + + SPECTRA Access Console -
-
-

◆ SPECTRA ◆

-

Intelligence Gathering Platform

+ + +
+
+
+

SPECTRA

+

Secure operator console

+
+
+
WebAuthn
+
Checking access policy
+
+
+ +
+
+

Bootstrap admin enrollment

+

+ First-run enrollment requires a bootstrap secret. The first operator to enroll becomes the admin. + Use a YubiKey or passkey-ready security key to register. +

+
+ Required only once. After the first admin exists, this path is disabled. +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ +
+

Sign in with security key

+

+ Authenticate with your registered YubiKey or passkey. Usernames are optional if the key is already + registered. +

+
+
+ + +
+
+ + +
+
+ +
+ Session active. You are already authenticated. Use the dashboard shortcut below. +
+
+ + +
+
+ +
+

Access status

+
+
Relying party: unknown
+
Origin: unknown
+
+
+
Operator: not signed in
+
Role: n/a
+
+
+
+
+
-
+
YubiKey-first access for containerized SPECTRA deployments
+
-
-
- - -
+ diff --git a/templates/readme.html b/templates/readme.html index 0f351cf..3fae98c 100644 --- a/templates/readme.html +++ b/templates/readme.html @@ -191,6 +191,7 @@
Documentation Launcher

SPECTRA Docs

Operator-facing documentation inside the same NSO-inspired shell as the rest of the web interface.

+

LOCAL ONLY by default: the launcher binds to 127.0.0.1 unless you explicitly reconfigure it.

Web Console diff --git a/tests/test_enhanced_gui_security.py b/tests/test_enhanced_gui_security.py index f7e9e63..bb54608 100644 --- a/tests/test_enhanced_gui_security.py +++ b/tests/test_enhanced_gui_security.py @@ -18,6 +18,7 @@ import socket import sys import time +import tempfile from pathlib import Path ROOT_DIR = Path(__file__).resolve().parents[1] if str(ROOT_DIR) not in sys.path: @@ -30,6 +31,24 @@ from spectra_app.spectra_coordination_gui import SpectraCoordinationGUI +def _reserve_free_port() -> tuple[socket.socket, int]: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", 0)) + return sock, sock.getsockname()[1] + + +def _build_auth_launcher(bootstrap_secret: str = "bootstrap-admin-key", host: str = "127.0.0.1", port: int = 5000): + config = create_default_config() + config.host = host + config.port = port + config.home_page = "console" + config.bootstrap_secret = bootstrap_secret + auth_dir = Path(tempfile.mkdtemp(prefix="spectra-auth-")) + config.auth_store_path = str(auth_dir / "operators.json") + return SpectraGUILauncher(config) + + class MockOrchestrator: """Mock orchestrator for testing""" def __init__(self): @@ -54,10 +73,11 @@ def test_default_security_configuration(): # Verify localhost-only default assert config.host == "127.0.0.1", f"Expected localhost, got {config.host}" + assert config.port == 5000, f"Expected default port 5000, got {config.port}" assert config.security_enabled == True, "Security should be enabled by default" + assert config.home_page == "console", "Default home page should remain console" print("✅ Default configuration is secure (localhost only)") - return True def test_port_availability_checking(): @@ -67,21 +87,16 @@ def test_port_availability_checking(): config = create_default_config() launcher = SpectraGUILauncher(config) - # Test with a port that should be available - available_port = 9999 - if launcher._check_port_availability(available_port): - print(f"✅ Port {available_port} correctly detected as available") - else: - print(f"⚠️ Port {available_port} detected as unavailable (may be in use)") + # Test with a port that is guaranteed to be available. + probe_socket, available_port = _reserve_free_port() + probe_socket.close() + assert launcher._check_port_availability(available_port), f"Port {available_port} should be available" + print(f"✅ Port {available_port} correctly detected as available") # Test port finding functionality - found_port = launcher._find_available_port(9990, 5) - if found_port: - print(f"✅ Found available port: {found_port}") - else: - print("⚠️ No available ports found in test range") - - return True + found_port = launcher._find_available_port(available_port, 5) + assert found_port is not None, "Expected to find an available port" + print(f"✅ Found available port: {found_port}") def test_security_warnings(): @@ -108,8 +123,6 @@ def test_security_warnings(): assert has_critical_warning, "Should have critical security warning" print("✅ Insecure configuration generates critical warnings") - return True - def test_coordination_gui_defaults(): """Test coordination GUI security defaults""" @@ -118,16 +131,21 @@ def test_coordination_gui_defaults(): mock_orchestrator = MockOrchestrator() # Test default localhost configuration - gui = SpectraCoordinationGUI( - orchestrator=mock_orchestrator - # Using default parameters - should be localhost - ) + try: + gui = SpectraCoordinationGUI( + orchestrator=mock_orchestrator + # Using default parameters - should be localhost + ) + except Exception as e: + if "Flask is required" in str(e): + print("⚠️ Coordination GUI skipped: Flask is not installed in this environment") + return + raise assert gui.host == "127.0.0.1", f"Expected localhost, got {gui.host}" assert gui.local_only == True, "Should be configured for local only access" print("✅ Coordination GUI defaults to secure localhost configuration") - return True def test_security_status_logging(): @@ -143,9 +161,62 @@ def test_security_status_logging(): print("✅ Security status logging completed without errors") except Exception as e: print(f"❌ Security status logging failed: {e}") - return False + raise + + +def test_bootstrap_login_logout_session_flow(): + """Test browser login, session auth, and logout for the bootstrap admin flow.""" + print("🔍 Testing bootstrap login/logout session flow...") + + launcher = _build_auth_launcher() + client = launcher.app.test_client() - return True + unauthenticated = client.get("/", follow_redirects=False) + assert unauthenticated.status_code in (301, 302) + assert "/login" in unauthenticated.headers["Location"] + + login_page = client.get("/login") + login_html = login_page.get_data(as_text=True) + assert login_page.status_code == 200 + assert "Bootstrap admin enrollment" in login_html + assert "YubiKey" in login_html + assert "passkey" in login_html.lower() + + bootstrap_status = client.get("/api/auth/bootstrap/status") + assert bootstrap_status.status_code == 200 + bootstrap_json = bootstrap_status.get_json() + assert bootstrap_json["auth"]["bootstrap_required"] is True + assert bootstrap_json["auth"]["bootstrap_configured"] is True + + operator = launcher.auth_service.ensure_admin_bootstrap( + username="admin", + display_name="Admin", + submitted_secret="bootstrap-admin-key", + ) + with client.session_transaction() as sess: + sess["spectra_operator_id"] = operator.operator_id + + dashboard = client.get("/") + assert dashboard.status_code == 200 + assert "SPECTRA Web Console" in dashboard.get_data(as_text=True) + + status_response = client.get("/api/system/status") + assert status_response.status_code == 200 + status_json = status_response.get_json() + assert status_json["auth"]["webauthn_required"] is True + assert status_json["auth"]["authenticated"] is True + assert status_json["auth"]["browser_login"] == "/login" + + logout = client.post("/logout") + assert logout.status_code == 200 + assert logout.get_json()["success"] is True + + with client.session_transaction() as sess: + assert not sess.get("spectra_operator_id") + + post_logout = client.get("/", follow_redirects=False) + assert post_logout.status_code in (301, 302) + assert "/login" in post_logout.headers["Location"] def simulate_port_conflict(): @@ -153,12 +224,9 @@ def simulate_port_conflict(): print("🔍 Testing port conflict handling...") # Create a socket to occupy a port - test_port = 9998 - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket, test_port = _reserve_free_port() try: - server_socket.bind(("127.0.0.1", test_port)) server_socket.listen(1) print(f"🔒 Occupied port {test_port} for testing") @@ -168,39 +236,29 @@ def simulate_port_conflict(): launcher = SpectraGUILauncher(config) is_available = launcher._check_port_availability(test_port) - if not is_available: - print("✅ Port conflict correctly detected") - - # Test that it finds an alternative - alternative = launcher._find_available_port(test_port + 1, 5) - if alternative: - print(f"✅ Found alternative port: {alternative}") - else: - print("⚠️ No alternative port found (may be expected)") - else: - print("❌ Port conflict not detected (unexpected)") - return False + assert not is_available, "Port conflict was not detected" + print("✅ Port conflict correctly detected") + + # Test that it finds an alternative + alternative = launcher._find_available_port(test_port + 1, 5) + assert alternative is not None, "Expected an alternative port" + print(f"✅ Found alternative port: {alternative}") except Exception as e: print(f"⚠️ Port conflict test had issues: {e}") - return False + raise finally: server_socket.close() - return True - -async def test_initialization_with_port_conflict(): +def test_initialization_with_port_conflict(): """Test system initialization with port conflict""" print("🔍 Testing system initialization with port conflict...") # Create a socket to occupy the default port - test_port = 5099 # Use a different port to avoid conflicts - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket, test_port = _reserve_free_port() try: - server_socket.bind(("127.0.0.1", test_port)) server_socket.listen(1) # Configure launcher to use the occupied port @@ -210,16 +268,15 @@ async def test_initialization_with_port_conflict(): # Test initialization - should find alternative port # Note: We're not actually starting the server, just testing the port logic + assert launcher._check_port_availability(test_port) is False, "Port conflict should be detected" print("✅ Port conflict handling during initialization works") except Exception as e: print(f"⚠️ Initialization test had issues: {e}") - return False + raise finally: server_socket.close() - return True - def run_all_tests(): """Run all security enhancement tests""" @@ -244,11 +301,9 @@ def run_all_tests(): print("-" * 50) try: - if test_func(): - passed += 1 - print(f"✅ {test_name} PASSED") - else: - print(f"❌ {test_name} FAILED") + test_func() + passed += 1 + print(f"✅ {test_name} PASSED") except Exception as e: print(f"❌ {test_name} FAILED with exception: {e}") @@ -256,11 +311,9 @@ def run_all_tests(): print(f"\n🧪 System Initialization with Port Conflict") print("-" * 50) try: - if asyncio.run(test_initialization_with_port_conflict()): - passed += 1 - print("✅ System Initialization with Port Conflict PASSED") - else: - print("❌ System Initialization with Port Conflict FAILED") + test_initialization_with_port_conflict() + passed += 1 + print("✅ System Initialization with Port Conflict PASSED") except Exception as e: print(f"❌ System Initialization with Port Conflict FAILED with exception: {e}") diff --git a/tests/test_local_only_gui.py b/tests/test_local_only_gui.py index 641f2db..f5cc114 100755 --- a/tests/test_local_only_gui.py +++ b/tests/test_local_only_gui.py @@ -20,7 +20,6 @@ def test_port_checking(): print("1. Testing port availability checking...") # Import the functions - sys.path.insert(0, str(Path(__file__).parent)) from spectra_app.spectra_gui_launcher import check_port_available, find_available_port, get_security_level # Test port checking @@ -31,8 +30,6 @@ def test_port_checking(): available_port, is_preferred = find_available_port("127.0.0.1", 5000) print(f" Found available port: {available_port} (preferred: {is_preferred})") - return True - def test_security_levels(): """Test security level detection""" print("2. Testing security level detection...") @@ -49,8 +46,6 @@ def test_security_levels(): print(f" 0.0.0.0: {level} - {desc}") assert level == "CRITICAL", f"Expected CRITICAL security for 0.0.0.0, got {level}" - return True - def test_default_config(): """Test default configuration is localhost""" print("3. Testing default configuration...") @@ -62,34 +57,41 @@ def test_default_config(): print(f" Default port: {config.port}") assert config.host == "127.0.0.1", f"Expected localhost default, got {config.host}" + assert config.port == 5000, f"Expected default port 5000, got {config.port}" print(" ✅ Default configuration is LOCAL ONLY") - return True +def test_login_template_copy(): + """Test that login UI copy reflects the first-run bootstrap flow""" + print("4. Testing login template bootstrap messaging...") + + login_template = ROOT_DIR / "templates" / "login.html" + if login_template.exists(): + content = login_template.read_text() + assert "Bootstrap admin enrollment" in content, "Login template missing bootstrap messaging" + assert "YubiKey" in content, "Login template missing YubiKey copy" + assert "passkey" in content.lower(), "Login template missing passkey copy" + print(" ✅ Login template contains first-run bootstrap messaging") + else: + raise AssertionError("Login template not found") def test_template_content(): """Test that templates contain LOCAL ONLY messaging""" - print("4. Testing template LOCAL ONLY messaging...") + print("5. Testing template LOCAL ONLY messaging...") # Check README template - readme_template = Path("templates/readme.html") + readme_template = ROOT_DIR / "templates" / "readme.html" if readme_template.exists(): content = readme_template.read_text() - if "LOCAL ONLY" in content: - print(" ✅ README template contains LOCAL ONLY messaging") - else: - print(" ❌ README template missing LOCAL ONLY messaging") - return False + assert "LOCAL ONLY" in content, "README template missing LOCAL ONLY messaging" + print(" ✅ README template contains LOCAL ONLY messaging") else: - print(" ❌ README template not found") - return False - - return True + raise AssertionError("README template not found") def test_security_api_routes(): """Test that security API routes are present in the launcher""" - print("5. Testing security API routes...") + print("6. Testing security API routes...") - launcher_file = Path("spectra_app/spectra_gui_launcher.py") + launcher_file = ROOT_DIR / "spectra_app" / "spectra_gui_launcher.py" content = launcher_file.read_text() required_routes = [ @@ -105,7 +107,7 @@ def test_security_api_routes(): print(f" ❌ Missing route: {route}") all_present = False - return all_present + assert all_present, "Missing security API routes" def main(): """Run all tests""" @@ -116,6 +118,7 @@ def main(): test_port_checking, test_security_levels, test_default_config, + test_login_template_copy, test_template_content, test_security_api_routes ] @@ -125,11 +128,9 @@ def main(): for test_func in tests: try: - if test_func(): - passed += 1 - print(" ✅ PASSED\n") - else: - print(" ❌ FAILED\n") + test_func() + passed += 1 + print(" ✅ PASSED\n") except Exception as e: print(f" ❌ ERROR: {e}\n") @@ -150,7 +151,6 @@ def main(): print("python3 examples/demo_readme_gui.py") print("Then visit: http://127.0.0.1:5000/readme") - return True else: print("⚠️ Some tests failed. Check the issues above.") return False diff --git a/tests/test_readme_gui.py b/tests/test_readme_gui.py index 0237b60..06a61eb 100755 --- a/tests/test_readme_gui.py +++ b/tests/test_readme_gui.py @@ -101,15 +101,27 @@ def test_readme_integration(): except: print(" ❌ Could not check requirements.txt") + # Test 7: Check login template copy for first-run auth flow + total_tests += 1 + print("7. Checking login template first-run copy...") + login_template = Path("templates/login.html") + if login_template.exists(): + content = login_template.read_text() + if "Bootstrap admin enrollment" in content and "YubiKey" in content and "passkey" in content: + print(" ✅ Login template contains first-run bootstrap copy") + tests_passed += 1 + else: + print(" ❌ Login template missing first-run bootstrap copy") + else: + print(" ❌ Login template file not found") + # Summary print(f"\n📊 Test Results: {tests_passed}/{total_tests} tests passed") if tests_passed == total_tests: print("🎉 All tests passed! README integration is ready.") - return True else: - print("⚠️ Some tests failed. Check the issues above.") - return False + raise AssertionError("Some README integration checks failed") def create_demo_launcher(): """Create a simple demo launcher for testing""" @@ -177,15 +189,14 @@ async def main(): print("SPECTRA GUI README Integration Test") print("==================================\\n") - success = test_readme_integration() - - if success: + try: + test_readme_integration() print("\\n🚀 Creating demo launcher...") create_demo_launcher() print("\\n✅ README integration is ready!") print("\\nTo test the GUI:") print("1. python3 examples/demo_readme_gui.py") print("2. Open http://localhost:5000/readme") - else: + except Exception: print("\\n❌ Please fix the issues above before testing.") sys.exit(1) diff --git a/tests/test_readme_integration.py b/tests/test_readme_integration.py index 943bd24..818164d 100644 --- a/tests/test_readme_integration.py +++ b/tests/test_readme_integration.py @@ -6,7 +6,9 @@ import sys import asyncio +import json import logging +import tempfile from pathlib import Path ROOT_DIR = Path(__file__).resolve().parents[1] if str(ROOT_DIR) not in sys.path: @@ -15,38 +17,65 @@ # Import the main launcher from spectra_app.spectra_gui_launcher import SpectraGUILauncher, create_default_config, SystemMode + +class _StubWebAuthnBackend: + available = True + backend_name = "stub" + unavailable_reason = "" + + def begin_registration(self, operator, rp_id=None, rp_name=None, origin=None): + return ({"challenge": "register-challenge", "rpId": rp_id or "localhost"}, {"operator_id": operator.operator_id}) + + def register_complete(self, state, request_data): + return { + "credential_id": "cred-1", + "attested_credential_data": "Y3JlZC1kYXRh", + "aaguid": None, + "public_key": None, + } + + def begin_authentication(self, operators=None, rp_id=None, origin=None): + return ({"challenge": "auth-challenge", "rpId": rp_id or "localhost"}, {"operator_id": operators[0].operator_id if operators else None}) + + def authenticate_complete(self, state, credentials, request_data): + return {"credential_id": "cred-1"} + def test_readme_integration(): """Test the README integration functionality""" print("🧪 Testing README Integration...") + failures = [] # Create test configuration config = create_default_config() config.mode = SystemMode.DEMO config.debug = True config.port = 5555 # Use different port for testing + config.bootstrap_secret = "bootstrap-admin-key" + auth_dir = Path(tempfile.mkdtemp(prefix="spectra-auth-")) + config.auth_store_path = str(auth_dir / "operators.json") # Initialize launcher launcher = SpectraGUILauncher(config) + launcher.auth_service.backend = _StubWebAuthnBackend() + client = launcher.app.test_client() # Test README content processing print("\n📄 Testing README content processing...") try: readme_content = launcher._get_readme_content() - if readme_content: - print(f"✅ README content loaded successfully ({len(readme_content)} characters)") + assert readme_content, "No README content loaded" + print(f"✅ README content loaded successfully ({len(readme_content)} characters)") - # Check for key sections - if "SPECTRA" in readme_content: - print("✅ Main title found") - if "Installation" in readme_content: - print("✅ Installation section found") - if "Features" in readme_content: - print("✅ Features section found") - - else: - print("❌ No README content loaded") + # Check for key sections + assert "SPECTRA" in readme_content, "Main title missing from README" + print("✅ Main title found") + assert "Installation" in readme_content, "Installation section missing from README" + print("✅ Installation section found") + assert "Features" in readme_content, "Features section missing from README" + print("✅ Features section found") except Exception as e: + failures.append(f"README content processing: {e}") print(f"❌ Error loading README content: {e}") # Test markdown fallback @@ -68,84 +97,118 @@ def test_readme_integration(): """ fallback_html = launcher._markdown_to_html_fallback(test_markdown) - if "

Test Header

" in fallback_html: - print("✅ Header conversion working") - if "

Subheader

" in fallback_html: - print("✅ Subheader conversion working") - if "
" in fallback_html:
-            print("✅ Code block conversion working")
-        if 'Link test' in fallback_html:
-            print("✅ Link conversion working")
+        assert "
Test Header
" in fallback_html, "Header conversion failed"
+        print("✅ Header conversion working")
+        assert "
Subheader
" in fallback_html, "Subheader conversion failed"
+        print("✅ Subheader conversion working")
+        assert "
" in fallback_html, "Code block conversion failed"
+        print("✅ Code block conversion working")
+        assert 'Link test' in fallback_html, "Link conversion failed"
+        print("✅ Link conversion working")
 
     except Exception as e:
+        failures.append(f"Markdown fallback: {e}")
         print(f"❌ Error testing markdown fallback: {e}")
 
     # Test Flask app setup
     print("\n🌐 Testing Flask app setup...")
     try:
         app = launcher.app
-        if app:
-            print("✅ Flask app initialized")
-
-            # Check routes
-            routes = [rule.rule for rule in app.url_map.iter_rules()]
-
-            if '/readme' in routes:
-                print("✅ README route registered")
-            if '/help' in routes:
-                print("✅ Help route registered")
-            if '/documentation' in routes:
-                print("✅ Documentation route registered")
-            if '/api/system/status' in routes:
-                print("✅ API routes registered")
-
-        else:
-            print("❌ Flask app not initialized")
+        assert app, "Flask app not initialized"
+        print("✅ Flask app initialized")
+
+        # Check routes
+        routes = [rule.rule for rule in app.url_map.iter_rules()]
+        assert '/readme' in routes, "README route not registered"
+        print("✅ README route registered")
+        assert '/help' in routes, "Help route not registered"
+        print("✅ Help route registered")
+        assert '/documentation' in routes, "Documentation route not registered"
+        print("✅ Documentation route registered")
+        assert '/api/system/status' in routes, "API routes not registered"
+        print("✅ API routes registered")
 
     except Exception as e:
+        failures.append(f"Flask app setup: {e}")
         print(f"❌ Error testing Flask app: {e}")
 
     # Test template existence
     print("\n📋 Testing template files...")
     try:
-        templates_dir = Path("templates")
-        if templates_dir.exists():
-            print("✅ Templates directory exists")
-
-            readme_template = templates_dir / "readme.html"
-            if readme_template.exists():
-                print("✅ README template exists")
-
-                # Check template content
-                template_content = readme_template.read_text()
-                if "{{ readme_content|safe }}" in template_content:
-                    print("✅ Template has proper Jinja2 integration")
-                if "system_status" in template_content:
-                    print("✅ Template has system status integration")
-
-            else:
-                print("❌ README template not found")
-        else:
-            print("❌ Templates directory not found")
+        templates_dir = ROOT_DIR / "templates"
+        assert templates_dir.exists(), "Templates directory not found"
+        print("✅ Templates directory exists")
+
+        readme_template = templates_dir / "readme.html"
+        assert readme_template.exists(), "README template not found"
+        print("✅ README template exists")
+
+        # Check template content
+        template_content = readme_template.read_text()
+        assert "{{ readme_content|safe }}" in template_content, "Jinja2 content placeholder missing"
+        print("✅ Template has proper Jinja2 integration")
+        assert "system_status" in template_content, "System status integration missing"
+        print("✅ Template has system status integration")
 
     except Exception as e:
+        failures.append(f"Template files: {e}")
         print(f"❌ Error testing templates: {e}")
 
     # Test system status integration
     print("\n📊 Testing system status integration...")
     try:
-        status = launcher.get_system_status()
-        if status:
-            print("✅ System status available")
-            print(f"   Mode: {status.get('mode', 'Unknown')}")
-            print(f"   Running: {status.get('system_running', False)}")
-            print(f"   Agents: {status.get('total_agents', 0)}")
-        else:
-            print("❌ No system status available")
+        with launcher.app.test_request_context("/"):
+            status = launcher.get_system_status()
+        assert status, "No system status available"
+        print("✅ System status available")
+        print(f"   Mode: {status.get('mode', 'Unknown')}")
+        print(f"   Running: {status.get('system_running', False)}")
+        print(f"   Agents: {status.get('total_agents', 0)}")
 
     except Exception as e:
+        failures.append(f"System status: {e}")
         print(f"❌ Error getting system status: {e}")
 
+    print("\n🔐 Testing browser auth gates...")
+    try:
+        redirect_response = client.get("/readme", follow_redirects=False)
+        assert redirect_response.status_code in (301, 302), "Unauthenticated README access should redirect"
+        assert "/login" in redirect_response.headers["Location"], "Redirect should point to /login"
+
+        login_page = client.get("/login")
+        login_html = login_page.get_data(as_text=True)
+        assert "Bootstrap admin enrollment" in login_html, "Bootstrap login copy missing"
+        assert "YubiKey" in login_html, "Passkey-first copy missing"
+
+        bootstrap_status = client.get("/api/auth/bootstrap/status")
+        assert bootstrap_status.status_code == 200
+
+        register_response = client.post(
+            "/api/auth/webauthn/register/options",
+            json={"bootstrap_secret": "bootstrap-admin-key", "username": "admin", "display_name": "Admin"},
+        )
+        assert register_response.status_code == 200, "Bootstrap registration should succeed"
+
+        verify_response = client.post(
+            "/api/auth/webauthn/register/verify",
+            json={"id": "cred-1", "type": "public-key", "response": {}},
+        )
+        assert verify_response.status_code == 200, "Bootstrap verification should succeed"
+
+        with client.session_transaction() as sess:
+            assert sess.get("spectra_operator_id") is not None
+
+        readme_response = client.get("/readme")
+        assert readme_response.status_code == 200, "Authenticated README access should succeed"
+        assert "SPECTRA Documentation" in readme_response.get_data(as_text=True)
+
+        logout_response = client.post("/logout")
+        assert logout_response.status_code == 200, "Logout should succeed"
+        assert logout_response.get_json()["success"] is True
+    except Exception as e:
+        failures.append(f"Auth gate flow: {e}")
+        print(f"❌ Error testing auth gates: {e}")
+
     print("\n🎯 README Integration Test Summary:")
     print("=" * 50)
     print("✅ All core functionality tested")
@@ -153,6 +216,9 @@ def test_readme_integration():
     print("✅ Template integration ready")
     print("✅ Markdown processing available")
     print("✅ System status integration working")
+    if failures:
+        raise AssertionError("; ".join(failures))
+
     print("\n🚀 Ready for deployment!")
 
 if __name__ == "__main__":
@@ -161,6 +227,12 @@ def test_readme_integration():
 
     try:
         test_readme_integration()
+        print("\n🚀 Creating demo launcher...")
+        create_demo_launcher()
+        print("\n✅ README integration is ready!")
+        print("\nTo test the GUI:")
+        print("1. python3 examples/demo_readme_gui.py")
+        print("2. Open http://localhost:5000/readme")
     except KeyboardInterrupt:
         print("\n🛑 Test interrupted by user")
     except Exception as e:
diff --git a/tests/test_spectra_gui_simplified.py b/tests/test_spectra_gui_simplified.py
index 04930a9..2fc3831 100644
--- a/tests/test_spectra_gui_simplified.py
+++ b/tests/test_spectra_gui_simplified.py
@@ -13,6 +13,8 @@
 import asyncio
 import json
 import sys
+import tempfile
+from pathlib import Path
 from datetime import datetime, timedelta
 from typing import Dict, List, Any, Optional
 
@@ -26,44 +28,42 @@ def test_imports():
         print("✓ SpectraCoordinationGUI imported successfully")
     except ImportError as e:
         print(f"✗ SpectraCoordinationGUI import failed: {e}")
-        return False
+        raise
 
     try:
         from spectra_app.agent_optimization_engine import AgentOptimizationEngine, AgentPerformanceProfile
         print("✓ Agent optimization components imported successfully")
     except ImportError as e:
         print(f"✗ Agent optimization import failed: {e}")
-        return False
+        raise
 
     try:
         from spectra_app.phase_management_dashboard import PhaseManagementDashboard, TimelineEvent, MilestoneStatus
         print("✓ Phase management components imported successfully")
     except ImportError as e:
         print(f"✗ Phase management import failed: {e}")
-        return False
+        raise
 
     try:
         from spectra_app.coordination_interface import CoordinationInterface, AgentHealthMetrics
         print("✓ Coordination interface components imported successfully")
     except ImportError as e:
         print(f"✗ Coordination interface import failed: {e}")
-        return False
+        raise
 
     try:
         from spectra_app.implementation_tools import ImplementationTools, WorkBreakdownItem
         print("✓ Implementation tools components imported successfully")
     except ImportError as e:
         print(f"✗ Implementation tools import failed: {e}")
-        return False
+        raise
 
     try:
         from spectra_app.spectra_gui_launcher import SpectraGUILauncher
         print("✓ GUI launcher imported successfully")
     except ImportError as e:
         print(f"✗ GUI launcher import failed: {e}")
-        return False
-
-    return True
+        raise
 
 
 def test_data_structures():
@@ -95,7 +95,7 @@ def test_data_structures():
 
     except Exception as e:
         print(f"✗ AgentPerformanceProfile creation failed: {e}")
-        return False
+        raise
 
     try:
         from spectra_app.phase_management_dashboard import TimelineEvent, MilestoneStatus
@@ -121,7 +121,7 @@ def test_data_structures():
 
     except Exception as e:
         print(f"✗ TimelineEvent creation failed: {e}")
-        return False
+        raise
 
     try:
         from spectra_app.coordination_interface import AgentHealthMetrics
@@ -145,9 +145,7 @@ def test_data_structures():
 
     except Exception as e:
         print(f"✗ AgentHealthMetrics creation failed: {e}")
-        return False
-
-    return True
+        raise
 
 
 def test_orchestrator_integration():
@@ -184,7 +182,7 @@ async def get_system_metrics(self):
 
     except Exception as e:
         print(f"✗ PhaseManagementDashboard integration failed: {e}")
-        return False
+        raise
 
     try:
         from spectra_app.coordination_interface import CoordinationInterface
@@ -198,12 +196,10 @@ async def get_system_metrics(self):
 
     except Exception as e:
         print(f"✗ CoordinationInterface integration failed: {e}")
-        return False
+        raise
 
-    return True
 
-
-async def test_async_functionality():
+async def _test_async_functionality():
     """Test asynchronous functionality"""
     print("\n=== Testing Async Functionality ===")
 
@@ -222,11 +218,16 @@ async def get_available_agents(self):
         assert isinstance(agents, dict)
         assert "agent_001" in agents
         print("✓ Async functionality working")
-        return True
+        return None
 
     except Exception as e:
         print(f"✗ Async functionality failed: {e}")
-        return False
+        raise
+
+
+def test_async_functionality():
+    """Pytest-visible wrapper for async functionality."""
+    asyncio.run(_test_async_functionality())
 
 
 def test_json_serialization():
@@ -251,11 +252,11 @@ def test_json_serialization():
         assert restored_data["agent_id"] == "agent_001"
         assert restored_data["capabilities"]["security"] == 0.9
         print("✓ JSON serialization working")
-        return True
+        return None
 
     except Exception as e:
         print(f"✗ JSON serialization failed: {e}")
-        return False
+        raise
 
 
 def test_gui_launcher_basic():
@@ -278,11 +279,52 @@ def test_gui_launcher_basic():
         assert config.port == 5001
         assert config.mode == "demo"
         print("✓ SystemConfiguration creation successful")
-        return True
+        return None
 
     except Exception as e:
         print(f"✗ GUI launcher basic test failed: {e}")
-        return False
+        raise
+
+
+def test_gui_launcher_auth_surface():
+    """Test login page copy and auth metadata for the browser UI."""
+    print("\n=== Testing GUI Launcher Auth Surface ===")
+
+    try:
+        from spectra_app.spectra_gui_launcher import SpectraGUILauncher, SystemConfiguration
+
+        auth_dir = Path(tempfile.mkdtemp(prefix="spectra-auth-"))
+        config = SystemConfiguration(
+            host="127.0.0.1",
+            port=5001,
+            mode="demo",
+            debug=True,
+            home_page="console",
+            auth_store_path=str(auth_dir / "operators.json"),
+        )
+        config.bootstrap_secret = "bootstrap-admin-key"
+
+        launcher = SpectraGUILauncher(config)
+        client = launcher.app.test_client()
+
+        assert launcher.config.host == "127.0.0.1"
+        assert launcher.config.port == 5001
+
+        login_page = client.get("/login")
+        login_html = login_page.get_data(as_text=True)
+        assert login_page.status_code == 200
+        assert "Bootstrap admin enrollment" in login_html
+        assert "YubiKey" in login_html
+        assert "passkey" in login_html.lower()
+
+        status = launcher.get_system_status()
+        assert status["auth"]["webauthn_required"] is True
+        assert status["auth"]["browser_login"] == "/login"
+        print("✓ GUI launcher auth surface verified")
+        return None
+    except Exception as e:
+        print(f"✗ GUI launcher auth surface test failed: {e}")
+        raise
 
 
 def run_all_tests():
@@ -297,6 +339,7 @@ def run_all_tests():
         ("Orchestrator Integration", test_orchestrator_integration),
         ("JSON Serialization", test_json_serialization),
         ("GUI Launcher Basic", test_gui_launcher_basic),
+        ("GUI Launcher Auth Surface", test_gui_launcher_auth_surface),
     ]
 
     # Run sync tests
@@ -304,8 +347,8 @@ def run_all_tests():
     for test_name, test_func in tests:
         print(f"\n🔄 Running {test_name} test...")
         try:
-            result = test_func()
-            results.append((test_name, result))
+            test_func()
+            results.append((test_name, True))
         except Exception as e:
             print(f"✗ {test_name} test crashed: {e}")
             results.append((test_name, False))
@@ -313,8 +356,8 @@ def run_all_tests():
     # Run async test
     print(f"\n🔄 Running Async Functionality test...")
     try:
-        async_result = asyncio.run(test_async_functionality())
-        results.append(("Async Functionality", async_result))
+        test_async_functionality()
+        results.append(("Async Functionality", True))
     except Exception as e:
         print(f"✗ Async Functionality test crashed: {e}")
         results.append(("Async Functionality", False))
@@ -354,4 +397,4 @@ def run_all_tests():
 
 if __name__ == "__main__":
     success = run_all_tests()
-    sys.exit(0 if success else 1)
\ No newline at end of file
+    sys.exit(0 if success else 1)
diff --git a/tests/test_spectra_gui_system.py b/tests/test_spectra_gui_system.py
index 14b2b36..3368189 100644
--- a/tests/test_spectra_gui_system.py
+++ b/tests/test_spectra_gui_system.py
@@ -15,7 +15,9 @@
 import unittest
 import json
 import sys
+import tempfile
 from datetime import datetime, timedelta
+from pathlib import Path
 from typing import Dict, List, Any, Optional
 from dataclasses import asdict
 
@@ -39,6 +41,29 @@
 )
 
 
+class _StubWebAuthnBackend:
+    available = True
+    backend_name = "stub"
+    unavailable_reason = ""
+
+    def begin_registration(self, operator, rp_id=None, rp_name=None, origin=None):
+        return ({"challenge": "register-challenge", "rpId": rp_id or "localhost"}, {"operator_id": operator.operator_id})
+
+    def register_complete(self, state, request_data):
+        return {
+            "credential_id": "cred-1",
+            "attested_credential_data": "Y3JlZC1kYXRh",
+            "aaguid": None,
+            "public_key": None,
+        }
+
+    def begin_authentication(self, operators=None, rp_id=None, origin=None):
+        return ({"challenge": "auth-challenge", "rpId": rp_id or "localhost"}, {"operator_id": operators[0].operator_id if operators else None})
+
+    def authenticate_complete(self, state, credentials, request_data):
+        return {"credential_id": "cred-1"}
+
+
 class MockSpectraOrchestrator:
     """Mock orchestrator for testing purposes"""
 
@@ -304,8 +329,75 @@ def test_implementation_tools(self):
 
         print("✓ Implementation tools validation successful")
 
-    async def test_async_functionality(self):
+    def test_gui_launcher_auth_surface(self):
+        """Test browser auth flow and bootstrap login copy."""
+        print("\n=== Testing GUI Launcher Auth Surface ===")
+
+        from spectra_app.spectra_gui_launcher import SpectraGUILauncher, create_default_config
+
+        config = create_default_config()
+        config.bootstrap_secret = "bootstrap-admin-key"
+        auth_dir = Path(tempfile.mkdtemp(prefix="spectra-auth-"))
+        config.auth_store_path = str(auth_dir / "operators.json")
+        launcher = SpectraGUILauncher(config)
+        launcher.auth_service.backend = _StubWebAuthnBackend()
+        client = launcher.app.test_client()
+
+        self.assertEqual(config.host, "127.0.0.1")
+        self.assertEqual(config.port, 5000)
+
+        login_response = client.get("/login")
+        login_html = login_response.get_data(as_text=True)
+        self.assertEqual(login_response.status_code, 200)
+        self.assertIn("Bootstrap admin enrollment", login_html)
+        self.assertIn("YubiKey", login_html)
+        self.assertIn("passkey", login_html.lower())
+
+        unauthenticated = client.get("/", follow_redirects=False)
+        self.assertIn(unauthenticated.status_code, (301, 302))
+        self.assertIn("/login", unauthenticated.headers["Location"])
+
+        bootstrap_status = client.get("/api/auth/bootstrap/status")
+        self.assertEqual(bootstrap_status.status_code, 200)
+        self.assertTrue(bootstrap_status.get_json()["auth"]["bootstrap_required"])
+
+        register_response = client.post(
+            "/api/auth/webauthn/register/options",
+            json={"bootstrap_secret": "bootstrap-admin-key", "username": "admin", "display_name": "Admin"},
+        )
+        self.assertEqual(register_response.status_code, 200)
+        self.assertTrue(register_response.get_json()["success"])
+
+        verify_response = client.post(
+            "/api/auth/webauthn/register/verify",
+            json={"id": "cred-1", "type": "public-key", "response": {}},
+        )
+        self.assertEqual(verify_response.status_code, 200)
+        self.assertTrue(verify_response.get_json()["success"])
+
+        with client.session_transaction() as sess:
+            self.assertTrue(sess.get("spectra_operator_id"))
+
+        status_response = client.get("/api/system/status")
+        self.assertEqual(status_response.status_code, 200)
+        status_json = status_response.get_json()
+        self.assertTrue(status_json["auth"]["webauthn_required"])
+        self.assertEqual(status_json["auth"]["browser_login"], "/login")
+
+        logout_response = client.post("/logout")
+        self.assertEqual(logout_response.status_code, 200)
+        self.assertTrue(logout_response.get_json()["success"])
+
+        with client.session_transaction() as sess:
+            self.assertFalse(sess.get("spectra_operator_id"))
+
+        print("✓ GUI launcher auth surface validation successful")
+
+    def test_async_functionality(self):
         """Test asynchronous functionality of components"""
+        asyncio.run(self._test_async_functionality())
+
+    async def _test_async_functionality(self):
         print("\n=== Testing Async Functionality ===")
 
         # Test coordination interface async methods
@@ -448,6 +540,14 @@ def run_comprehensive_tests():
         print(f"✗ Implementation tools test failed: {e}")
         tests_run += 1
 
+    try:
+        test_instance.test_gui_launcher_auth_surface()
+        tests_run += 1
+        tests_passed += 1
+    except Exception as e:
+        print(f"✗ GUI launcher auth surface test failed: {e}")
+        tests_run += 1
+
     try:
         test_instance.test_data_serialization()
         tests_run += 1
@@ -491,6 +591,7 @@ def run_comprehensive_tests():
         "Phase Management Dashboard",
         "Coordination Interface",
         "Implementation Tools",
+        "GUI Launcher Auth Surface",
         "Data Serialization",
         "Async Functionality"
     ]
@@ -519,4 +620,4 @@ def run_comprehensive_tests():
     result = run_comprehensive_tests()
 
     # Exit with appropriate code
-    sys.exit(0 if result == "PASSED" else 1)
\ No newline at end of file
+    sys.exit(0 if result == "PASSED" else 1)
diff --git a/tgarchive/__main__.py b/tgarchive/__main__.py
index 6640aca..3cc9a9f 100644
--- a/tgarchive/__main__.py
+++ b/tgarchive/__main__.py
@@ -1167,22 +1167,38 @@ async def handle_migrate(args: argparse.Namespace) -> int:
     """Handle migrate command"""
     cfg = Config(Path(args.config))
     db = SpectraDB(cfg.data.get("db", {}).get("path", "spectra.db"))
-    # Placeholder for a proper client factory
     from telethon import TelegramClient
-    client = TelegramClient('anon', 12345, 'hash') # Replace with actual session logic
-    manager = MassMigrationManager(cfg, db, client)
-    await manager.one_time_migration(args.source, args.destination, args.dry_run, args.parallel)
-    return 0
+    account = cfg.auto_select_account()
+    if not account:
+        logger.error("No account available for this operation.")
+        return 1
+
+    client = TelegramClient(account['session_name'], account['api_id'], account['api_hash'])
+    await client.connect()
+    try:
+        manager = MassMigrationManager(cfg, db, client)
+        await manager.one_time_migration(args.source, args.destination, args.dry_run, args.parallel)
+        return 0
+    finally:
+        await client.disconnect()
 
 async def handle_rollback(args: argparse.Namespace) -> int:
     """Handle rollback command"""
     cfg = Config(Path(args.config))
     db = SpectraDB(cfg.data.get("db", {}).get("path", "spectra.db"))
     from telethon import TelegramClient
-    client = TelegramClient('anon', 12345, 'hash')
-    manager = MassMigrationManager(cfg, db, client)
-    manager.rollback_migration(args.migration_id)
-    return 0
+    account = cfg.auto_select_account()
+    if not account:
+        logger.error("No account available for this operation.")
+        return 1
+
+    client = TelegramClient(account['session_name'], account['api_id'], account['api_hash'])
+    await client.connect()
+    try:
+        manager = MassMigrationManager(cfg, db, client)
+        return 0 if manager.rollback_migration(args.migration_id) else 1
+    finally:
+        await client.disconnect()
 
 async def handle_migrate_report(args: argparse.Namespace) -> int:
     """Handle migrate-report command"""
@@ -1200,7 +1216,7 @@ async def handle_download_users(args: argparse.Namespace) -> int:
     cfg = Config(Path(args.config))
     if args.import_accounts:
         cfg = enhance_config_with_gen_accounts(cfg)
-    from .user_operations import get_server_users
+    from .utils.user_operations import get_server_users
     from telethon import TelegramClient
     account = cfg.auto_select_account()
     if not account:
diff --git a/tgarchive/api/__init__.py b/tgarchive/api/__init__.py
index f71cccd..054be10 100644
--- a/tgarchive/api/__init__.py
+++ b/tgarchive/api/__init__.py
@@ -36,13 +36,18 @@ def create_app(config=None):
     app = Flask(__name__, template_folder='../../templates', static_folder='../../static')
 
     # Configuration
-    if config:
+    config_obj = config
+    if isinstance(config, dict):
         app.config.from_mapping(config)
-    else:
-        # Load from environment
-        app.config['JWT_SECRET'] = app.config.get('SPECTRA_JWT_SECRET', 'dev-secret-key')
-        app.config['DEBUG'] = app.config.get('SPECTRA_DEBUG', False)
-        app.config['TESTING'] = app.config.get('SPECTRA_TESTING', False)
+        config_obj = Config(Path('spectra_config.json'))
+        config_obj.data.update(config)
+    elif config:
+        app.config.from_mapping(getattr(config, "data", config))
+
+    # Load defaults from environment or safe fallbacks.
+    app.config.setdefault('JWT_SECRET', app.config.get('SPECTRA_JWT_SECRET', 'dev-secret-key'))
+    app.config.setdefault('DEBUG', app.config.get('SPECTRA_DEBUG', False))
+    app.config.setdefault('TESTING', app.config.get('SPECTRA_TESTING', False))
 
     # Initialize security
     SecurityHeaders.init_app(app)
@@ -70,12 +75,12 @@ def create_app(config=None):
     task_manager = TaskManager()
     
     # Load config if not provided
-    if not config:
+    if config_obj is None:
         config_path = Path('spectra_config.json')
         if config_path.exists():
-            config = Config(config_path)
+            config_obj = Config(config_path)
         else:
-            config = Config(config_path)  # Will use defaults
+            config_obj = Config(config_path)  # Will use defaults
     
     # Register blueprints
     app.register_blueprint(auth_bp, url_prefix='/api/auth')
@@ -83,20 +88,20 @@ def create_app(config=None):
     # Initialize and register route modules
     try:
         from .routes import core, accounts, forwarding, threat, analytics, ml, crypto, database, osint, services, channels, messages
-        
+
         # Initialize routes with dependencies
-        core.init_core_routes(app, config, task_manager)
-        accounts.init_accounts_routes(app, config)
-        forwarding.init_forwarding_routes(app, config, task_manager)
+        core.init_core_routes(app, config_obj, task_manager)
+        accounts.init_accounts_routes(app, config_obj)
+        forwarding.init_forwarding_routes(app, config_obj, task_manager)
         threat.init_threat_routes(app)
         analytics.init_analytics_routes(app)
         ml.init_ml_routes(app)
         crypto.init_crypto_routes(app)
-        database.init_database_routes(app, config)
-        osint.init_osint_routes(app, config)
-        services.init_services_routes(app, config)
-        channels.init_channels_routes(app, config)
-        messages.init_messages_routes(app, config)
+        database.init_database_routes(app, config_obj)
+        osint.init_osint_routes(app, config_obj)
+        services.init_services_routes(app, config_obj)
+        channels.init_channels_routes(app, config_obj)
+        messages.init_messages_routes(app, config_obj)
     except Exception as e:
         logger.warning(f"Some route modules failed to initialize: {e}")
     
diff --git a/tgarchive/api/routes/__init__.py b/tgarchive/api/routes/__init__.py
index 3a6c208..a4ab2be 100644
--- a/tgarchive/api/routes/__init__.py
+++ b/tgarchive/api/routes/__init__.py
@@ -61,6 +61,7 @@
 
 # Import route handlers
 from .auth import *
+from .core import *
 from .channels import *
 from .messages import *
 from .search import *
diff --git a/tgarchive/api/services/analytics_service.py b/tgarchive/api/services/analytics_service.py
index 72f7d4d..349bcc3 100644
--- a/tgarchive/api/services/analytics_service.py
+++ b/tgarchive/api/services/analytics_service.py
@@ -9,9 +9,9 @@
 from typing import Dict, Any, Optional, List
 
 try:
-    from ..analytics.forecasting import ForecastingEngine
-    from ..analytics.time_series_analyzer import TimeSeriesAnalyzer
-    from ..analytics.predictive_engine import PredictiveEngine
+    from ...analytics.forecasting import ForecastingEngine
+    from ...analytics.time_series_analyzer import TimeSeriesAnalyzer
+    from ...analytics.predictive_engine import PredictiveEngine
     ANALYTICS_AVAILABLE = True
 except ImportError:
     ANALYTICS_AVAILABLE = False
diff --git a/tgarchive/api/services/archive_service.py b/tgarchive/api/services/archive_service.py
index 137e156..ca7d0a7 100644
--- a/tgarchive/api/services/archive_service.py
+++ b/tgarchive/api/services/archive_service.py
@@ -10,9 +10,9 @@
 from pathlib import Path
 from typing import Dict, Any, Optional, List
 
-from ..core.config_models import Config
-from ..core.sync import runner, archive_channel, ProxyCycler
-from ..db import SpectraDB
+from ...core.config_models import Config
+from ...core.sync import runner, archive_channel, ProxyCycler
+from ...db import SpectraDB
 from .tasks import TaskManager, TaskStatus
 
 logger = logging.getLogger(__name__)
diff --git a/tgarchive/api/services/crypto_service.py b/tgarchive/api/services/crypto_service.py
index b683faf..c50938f 100644
--- a/tgarchive/api/services/crypto_service.py
+++ b/tgarchive/api/services/crypto_service.py
@@ -10,7 +10,7 @@
 import base64
 
 try:
-    from ..crypto.pqc import CNSA20CryptoManager, CNSAKeyPair
+    from ...crypto.pqc import CNSA20CryptoManager, CNSAKeyPair
     CRYPTO_AVAILABLE = True
 except ImportError:
     CRYPTO_AVAILABLE = False
@@ -263,7 +263,7 @@ async def decrypt_data(
             return {"error": "Crypto manager not available"}
         
         try:
-            from ..crypto.pqc import EncryptedPackage
+            from ...crypto.pqc import EncryptedPackage
             
             encrypted = EncryptedPackage(
                 kem_ciphertext=base64.b64decode(encrypted_package["kem_ciphertext"]),
diff --git a/tgarchive/api/services/database_service.py b/tgarchive/api/services/database_service.py
index 105be67..c4c3f13 100644
--- a/tgarchive/api/services/database_service.py
+++ b/tgarchive/api/services/database_service.py
@@ -9,8 +9,8 @@
 from pathlib import Path
 from typing import Dict, Any, Optional, List
 
-from ..db import SpectraDB
-from ..core.config_models import Config
+from ...db import SpectraDB
+from ...core.config_models import Config
 
 logger = logging.getLogger(__name__)
 
diff --git a/tgarchive/api/services/discovery_service.py b/tgarchive/api/services/discovery_service.py
index 95c807d..fbb296d 100644
--- a/tgarchive/api/services/discovery_service.py
+++ b/tgarchive/api/services/discovery_service.py
@@ -10,9 +10,9 @@
 from pathlib import Path
 from typing import Dict, Any, Optional, List
 
-from ..core.config_models import Config
-from ..utils.discovery import SpectraCrawlerManager
-from ..db import SpectraDB
+from ...core.config_models import Config
+from ...utils.discovery import SpectraCrawlerManager
+from ...db import SpectraDB
 from .tasks import TaskManager, TaskStatus
 
 logger = logging.getLogger(__name__)
diff --git a/tgarchive/api/services/ml_service.py b/tgarchive/api/services/ml_service.py
index 7b5bb03..cd89a40 100644
--- a/tgarchive/api/services/ml_service.py
+++ b/tgarchive/api/services/ml_service.py
@@ -9,11 +9,11 @@
 from typing import Dict, Any, Optional, List
 
 try:
-    from ..ml.pattern_detector import PatternDetector
-    from ..ml.correlation_engine import CorrelationEngine
-    from ..ml.continuous_learner import ContinuousLearner
-    from ..ai.entity_extraction import EntityExtractor
-    from ..ai.semantic_search import SemanticSearcher
+    from ...ml.pattern_detector import PatternDetector
+    from ...ml.correlation_engine import CorrelationEngine
+    from ...ml.continuous_learner import ContinuousLearner
+    from ...ai.entity_extraction import EntityExtractor
+    from ...ai.semantic_search import SemanticSearcher
     ML_AVAILABLE = True
 except ImportError:
     ML_AVAILABLE = False
diff --git a/tgarchive/api/services/threat_service.py b/tgarchive/api/services/threat_service.py
index 43af9ec..67eb235 100644
--- a/tgarchive/api/services/threat_service.py
+++ b/tgarchive/api/services/threat_service.py
@@ -8,12 +8,12 @@
 import logging
 from typing import Dict, Any, Optional, List
 
-from ..threat.attribution import AttributionEngine
-from ..threat.temporal import TemporalAnalyzer
-from ..threat.network import ThreatNetworkAnalyzer
-from ..threat.scoring import ThreatScorer
-from ..threat.indicators import ThreatIndicatorExtractor
-from ..threat.visualization import ThreatVisualizer
+from ...threat.attribution import AttributionEngine
+from ...threat.temporal import TemporalAnalyzer
+from ...threat.network import ThreatNetworkTracker
+from ...threat.scoring import ThreatScorer
+from ...threat.indicators import ThreatIndicatorDetector, ThreatIndicator
+from ...threat.visualization import MermaidGenerator, ThreatReportGenerator
 
 logger = logging.getLogger(__name__)
 
@@ -24,10 +24,11 @@ class ThreatService:
     def __init__(self):
         self.attribution_engine = AttributionEngine()
         self.temporal_analyzer = TemporalAnalyzer()
-        self.network_analyzer = ThreatNetworkAnalyzer()
+        self.network_tracker = ThreatNetworkTracker()
         self.threat_scorer = ThreatScorer()
-        self.indicator_extractor = ThreatIndicatorExtractor()
-        self.visualizer = ThreatVisualizer()
+        self.indicator_detector = ThreatIndicatorDetector()
+        self.visualizer = MermaidGenerator()
+        self.report_generator = ThreatReportGenerator()
     
     async def analyze_attribution(
         self,
@@ -107,17 +108,19 @@ async def analyze_temporal_patterns(
             Temporal analysis results
         """
         try:
-            analysis = self.temporal_analyzer.analyze_activity_patterns(
+            analysis = self.temporal_analyzer.analyze_activity_patterns(messages)
+            prediction = self.temporal_analyzer.predict_next_activity(
                 messages,
-                time_window_days=options.get("time_window_days", 30) if options else 30
+                forecast_hours=options.get("prediction_window_hours", 24) if options else 24,
             )
             
             return {
-                "timezone_inference": analysis.get("timezone_inference"),
+                "timezone_inference": analysis.get("inferred_timezone"),
                 "peak_hours": analysis.get("peak_hours"),
-                "activity_bursts": analysis.get("activity_bursts"),
+                "activity_bursts": analysis.get("burst_periods"),
                 "regularity_score": analysis.get("regularity_score"),
-                "entropy": analysis.get("entropy")
+                "entropy": analysis.get("hour_distribution"),
+                "prediction": prediction,
             }
         except Exception as e:
             logger.error(f"Temporal analysis failed: {e}", exc_info=True)
@@ -169,12 +172,32 @@ async def analyze_threat_network(
             Network analysis results
         """
         try:
-            # This would use ThreatNetworkAnalyzer
+            if not entities:
+                return {
+                    "nodes": [],
+                    "edges": [],
+                    "centrality_metrics": {},
+                    "communities": [],
+                }
+
+            for entity in entities:
+                source_id = int(entity.get("source_id", entity.get("user_id", 0)))
+                target_id = int(entity.get("target_id", entity.get("related_user_id", 0)))
+                if source_id and target_id and source_id != target_id:
+                    self.network_tracker.add_interaction(
+                        source_id=source_id,
+                        target_id=target_id,
+                        interaction_type=entity.get("interaction_type", "same_channel"),
+                        timestamp=entity.get("timestamp"),
+                        message_id=entity.get("message_id"),
+                        channel_id=entity.get("channel_id"),
+                    )
+
             return {
-                "nodes": [],
-                "edges": [],
+                "nodes": [i.to_dict() for i in self.network_tracker.interactions],
+                "edges": [r.to_dict() for r in self.network_tracker.relationships.values()],
                 "centrality_metrics": {},
-                "communities": []
+                "communities": self.network_tracker.detect_communities(),
             }
         except Exception as e:
             logger.error(f"Threat network analysis failed: {e}", exc_info=True)
@@ -196,15 +219,25 @@ async def calculate_threat_score(
             Threat score and details
         """
         try:
-            # This would use ThreatScorer
-            score = self.threat_scorer.calculate_score(entity_id, indicators or [])
+            total_severity = 0.0
+            normalized_indicators: List[ThreatIndicator] = []
+
+            for indicator in indicators or []:
+                if isinstance(indicator, ThreatIndicator):
+                    normalized_indicators.append(indicator)
+                    total_severity += float(indicator.severity)
+                elif isinstance(indicator, dict):
+                    severity = float(indicator.get("severity", 0.0))
+                    total_severity += severity
+
+            score = min(10.0, total_severity)
             
             return {
                 "entity_id": entity_id,
-                "score": score.get("score", 0.0),
-                "severity": score.get("severity", "low"),
-                "indicators": score.get("indicators", []),
-                "reasoning": score.get("reasoning", "")
+                "score": score,
+                "severity": "high" if score >= 7 else "medium" if score >= 4 else "low",
+                "indicators": [i.to_dict() for i in normalized_indicators] if normalized_indicators else indicators or [],
+                "reasoning": "Derived from indicator severities",
             }
         except Exception as e:
             logger.error(f"Threat scoring failed: {e}", exc_info=True)
@@ -224,9 +257,15 @@ async def get_threat_indicators(
             Threat indicators
         """
         try:
-            # This would use ThreatIndicatorExtractor
+            if entity_id is None:
+                return {
+                    "indicators": [],
+                    "entity_id": entity_id
+                }
+
+            indicators = self.indicator_detector.detect_indicators(entity_id)
             return {
-                "indicators": [],
+                "indicators": [indicator.to_dict() for indicator in indicators],
                 "entity_id": entity_id
             }
         except Exception as e:
@@ -249,10 +288,14 @@ async def get_threat_visualization(
             Visualization data
         """
         try:
-            # This would use ThreatVisualizer
+            if visualization_type == "report":
+                data = self.report_generator.generate_executive_report([], self.network_tracker)
+            else:
+                data = self.visualizer.generate_network_graph([], self.network_tracker)
+
             return {
                 "type": visualization_type,
-                "data": {},
+                "data": data,
                 "entity_id": entity_id
             }
         except Exception as e:
diff --git a/tgarchive/config_models.py b/tgarchive/config_models.py
new file mode 100644
index 0000000..156edbd
--- /dev/null
+++ b/tgarchive/config_models.py
@@ -0,0 +1,11 @@
+"""Compatibility wrapper for legacy config imports.
+
+The canonical configuration model lives in `tgarchive.core.config_models`,
+but a number of integration scripts still import `tgarchive.config_models`.
+"""
+
+from __future__ import annotations
+
+from tgarchive.core.config_models import Config, DEFAULT_CFG
+
+__all__ = ["Config", "DEFAULT_CFG"]
diff --git a/tgarchive/core/deduplication.py b/tgarchive/core/deduplication.py
index 99739dc..70c84fe 100644
--- a/tgarchive/core/deduplication.py
+++ b/tgarchive/core/deduplication.py
@@ -6,6 +6,8 @@
 """
 
 import hashlib
+import re
+from difflib import SequenceMatcher
 import imagehash
 try:
     import ssdeep
@@ -47,7 +49,18 @@ def get_fuzzy_hash(file_path):
     Generates a fuzzy hash for a file.
     """
     if not HAS_SSDEEP:
-        return None
+        try:
+            with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+                text = f.read().lower()
+        except Exception:
+            return None
+
+        normalized = re.sub(r"\s+", " ", text).strip()
+        if not normalized:
+            return None
+
+        # Deterministic fallback signature for environments without ssdeep.
+        return "fallback:" + normalized
     try:
         return ssdeep.hash_from_file(file_path)
     except Exception:
@@ -57,6 +70,19 @@ def compare_fuzzy_hashes(hash1, hash2):
     """
     Compares two fuzzy hashes and returns a similarity percentage.
     """
+    if not hash1 or not hash2:
+        return 0
+
+    if hash1 == hash2:
+        return 100
+
+    if hash1.startswith("fallback:") and hash2.startswith("fallback:"):
+        text1 = hash1.removeprefix("fallback:")
+        text2 = hash2.removeprefix("fallback:")
+        if not text1 or not text2:
+            return 0
+        return int(round(SequenceMatcher(None, text1, text2).ratio() * 100))
+
     if not HAS_SSDEEP:
         return 0
     try:
diff --git a/tgarchive/db/db_base.py b/tgarchive/db/db_base.py
index a72d9e0..82f7c9b 100644
--- a/tgarchive/db/db_base.py
+++ b/tgarchive/db/db_base.py
@@ -7,9 +7,10 @@
 import math
 import sqlite3
 import time
-from contextlib import AbstractContextManager
+from contextlib import AbstractContextManager, asynccontextmanager
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import AsyncIterator
 
 import pytz  # type: ignore
 
@@ -23,6 +24,40 @@ def _page(n: int, multiple: int) -> int:
     return math.ceil(n / multiple) if n > 0 else 1
 
 
+class _AsyncCursor:
+    """Minimal async wrapper around a sqlite3 cursor."""
+
+    def __init__(self, cursor: sqlite3.Cursor) -> None:
+        self._cursor = cursor
+
+    async def fetchone(self):
+        return self._cursor.fetchone()
+
+    async def fetchall(self):
+        return self._cursor.fetchall()
+
+    async def fetchmany(self, size: int | None = None):
+        if size is None:
+            return self._cursor.fetchmany()
+        return self._cursor.fetchmany(size)
+
+
+class _AsyncConnection:
+    """Minimal async wrapper around the shared sqlite3 connection."""
+
+    def __init__(self, conn: sqlite3.Connection) -> None:
+        self._conn = conn
+
+    async def execute(self, sql: str, params: tuple = ()) -> _AsyncCursor:
+        return _AsyncCursor(self._conn.execute(sql, params))
+
+    async def commit(self) -> None:
+        self._conn.commit()
+
+    async def rollback(self) -> None:
+        self._conn.rollback()
+
+
 class BaseDB(AbstractContextManager):
     """Base SQLite wrapper providing WAL mode, foreign keys, and retry logic."""
 
@@ -67,6 +102,17 @@ def __exit__(self, exc_type, exc, tb):
         logger.info("Connection closed")
         return False
 
+    @asynccontextmanager
+    async def connect(self) -> AsyncIterator[_AsyncConnection]:
+        """Provide an async-compatible view over the live SQLite connection."""
+        wrapper = _AsyncConnection(self.conn)
+        try:
+            yield wrapper
+            self.conn.commit()
+        except Exception:
+            self.conn.rollback()
+            raise
+
     def _exec_retry(self, sql: str, params: tuple = ()) -> None:
         """Execute SQL with exponential backoff on lock errors."""
         backoff = 1.0
diff --git a/tgarchive/db/schema.py b/tgarchive/db/schema.py
index 931b112..82f704c 100644
--- a/tgarchive/db/schema.py
+++ b/tgarchive/db/schema.py
@@ -41,6 +41,26 @@
 CREATE INDEX IF NOT EXISTS idx_messages_date ON messages(date);
 CREATE INDEX IF NOT EXISTS idx_messages_user ON messages(user_id);
 
+CREATE TABLE IF NOT EXISTS osint_targets (
+    user_id       INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
+    username      TEXT NOT NULL UNIQUE,
+    notes         TEXT DEFAULT '',
+    created_at    TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_osint_targets_username ON osint_targets(username);
+
+CREATE TABLE IF NOT EXISTS osint_interactions (
+    id               INTEGER PRIMARY KEY AUTOINCREMENT,
+    source_user_id   INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    target_user_id   INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    interaction_type TEXT NOT NULL,
+    channel_id       BIGINT NOT NULL,
+    message_id       INTEGER NOT NULL,
+    timestamp        TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_osint_interactions_source ON osint_interactions(source_user_id);
+CREATE INDEX IF NOT EXISTS idx_osint_interactions_target ON osint_interactions(target_user_id);
+
 CREATE TABLE IF NOT EXISTS checkpoints (
     id               INTEGER PRIMARY KEY AUTOINCREMENT,
     last_message_id  INTEGER,
diff --git a/tgarchive/db/spectra_db.py b/tgarchive/db/spectra_db.py
index 5d653b7..044ae8b 100644
--- a/tgarchive/db/spectra_db.py
+++ b/tgarchive/db/spectra_db.py
@@ -4,7 +4,8 @@
 Main SpectraDB class combining all operation modules.
 """
 from pathlib import Path
-from typing import Optional, Dict, Any
+from datetime import datetime
+from typing import Optional, Dict, Any, List, Tuple
 
 from .core_operations import CoreOperations
 from .db_base import BaseDB
@@ -169,5 +170,47 @@ def update_mirror_progress(self, *args, **kwargs):
     def get_mirror_progress(self, *args, **kwargs):
         return self.mirror.get_mirror_progress(*args, **kwargs)
 
+    def find_messages_by_timestamp_range(
+        self,
+        start_time: int,
+        end_time: int,
+        channel_id: Optional[int] = None,
+        cache_manager=None,
+    ) -> List[int]:
+        """Compatibility search helper used by integration tests."""
+        query = "SELECT id, date FROM messages"
+        params: tuple = ()
+        if channel_id is not None:
+            query += " WHERE channel_id = ?"
+            params = (channel_id,)
+        query += " ORDER BY date"
+
+        rows = self.cur.execute(query, params).fetchall()
+        results: List[int] = []
+        for message_id, date_value in rows:
+            if isinstance(date_value, str):
+                dt = datetime.fromisoformat(date_value.replace("Z", "+00:00"))
+            else:
+                dt = date_value
+            ts = int(dt.timestamp())
+            if start_time <= ts <= end_time:
+                results.append(message_id)
+        return results
+
+    def find_message_by_id_fast(self, message_id: int, channel_id: Optional[int] = None) -> Optional[dict]:
+        """Compatibility fast lookup helper used by integration tests."""
+        query = "SELECT * FROM messages WHERE id = ?"
+        params: tuple = (message_id,)
+        if channel_id is not None:
+            query += " AND channel_id = ?"
+            params = (message_id, channel_id)
+
+        row = self.cur.execute(query, params).fetchone()
+        if not row:
+            return None
+
+        columns = [desc[0] for desc in self.cur.description]
+        return dict(zip(columns, row))
+
 
 __all__ = ["SpectraDB"]
diff --git a/tgarchive/deduplication.py b/tgarchive/deduplication.py
new file mode 100644
index 0000000..e973965
--- /dev/null
+++ b/tgarchive/deduplication.py
@@ -0,0 +1,6 @@
+"""Compatibility alias for deduplication helpers."""
+
+from .core import deduplication as _deduplication
+import sys as _sys
+
+_sys.modules[__name__] = _deduplication
diff --git a/tgarchive/forwarding/forwarder.py b/tgarchive/forwarding/forwarder.py
index 8fe9751..11e50ac 100644
--- a/tgarchive/forwarding/forwarder.py
+++ b/tgarchive/forwarding/forwarder.py
@@ -5,6 +5,8 @@
 
 import asyncio
 import logging
+import os
+import tempfile
 from typing import List, Optional, Sequence, Tuple
 
 try:
@@ -53,11 +55,24 @@ class Chat:
 
 from tgarchive.utils.attribution import AttributionFormatter
 from tgarchive.core.config_models import Config
+from tgarchive.core.deduplication import (
+    compare_fuzzy_hashes,
+    get_fuzzy_hash,
+    get_perceptual_hash,
+    get_sha256_hash,
+)
 try:
     from tgarchive.db import SpectraDB
 except ImportError:  # pragma: no cover - optional for unit tests that do not touch the DB
     SpectraDB = object
 
+try:
+    import imagehash
+    IMAGEHASH_AVAILABLE = True
+except ImportError:  # pragma: no cover - optional dependency in some environments
+    imagehash = None
+    IMAGEHASH_AVAILABLE = False
+
 from .client import ClientManager
 from .deduplication import Deduplicator
 from .grouping import MessageGrouper
@@ -72,6 +87,7 @@ def __init__(
         self,
         config: Config,
         db: Optional[SpectraDB] = None,
+        deduplicator: Optional[Deduplicator] = None,
         forward_to_all_saved_messages: bool = False,
         destination_topic_id: Optional[int] = None,
         source_topic_id: Optional[int] = None,
@@ -88,7 +104,7 @@ def __init__(
         self.db = db
         self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
         self.client_manager = ClientManager(config)
-        self.deduplicator = Deduplicator(db, enable_deduplication)
+        self.deduplicator = deduplicator or Deduplicator(db, enable_deduplication)
         self.grouper = MessageGrouper(grouping_strategy, grouping_time_window_seconds)
         self.attribution_formatter = AttributionFormatter(self.config.data)
 
@@ -143,6 +159,80 @@ def _message_is_forwardable(self, message) -> bool:
             return True
         return self.include_text_messages and bool(self._get_message_text(message))
 
+    async def _get_client(self, account_identifier: Optional[str] = None):
+        return await self.client_manager.get_client(account_identifier)
+
+    async def _is_duplicate(self, message_group: List[TLMessage], client: TelegramClient) -> bool:
+        """
+        Check if any file in a message group is a duplicate or near-duplicate.
+        """
+        if not self.deduplicator.enable_deduplication or not message_group:
+            return False
+
+        if await self.deduplicator.is_duplicate(message_group, client):
+            return True
+
+        if not self.config.data.get("deduplication", {}).get("enable_near_duplicates"):
+            return False
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            for message in message_group:
+                if not message.file:
+                    continue
+
+                file_path = os.path.join(tmpdir, message.file.name or str(message.file.id))
+                try:
+                    await client.download_media(message.media, file=file_path)
+                except Exception as e:
+                    self.logger.error(f"Failed to download file for hashing (Msg ID: {message.id}): {e}", exc_info=True)
+                    continue
+
+                if not os.path.exists(file_path) or os.path.getsize(file_path) == 0:
+                    self.logger.warning(
+                        f"File for hashing (Msg ID: {message.id}) was not downloaded correctly or is empty. "
+                        "Skipping duplicate check for this file."
+                    )
+                    continue
+
+                if self.db and IMAGEHASH_AVAILABLE:
+                    import mimetypes
+
+                    mime_type, _ = mimetypes.guess_type(file_path)
+                    if mime_type and mime_type.startswith("image/"):
+                        perceptual_hash = get_perceptual_hash(file_path)
+                        if perceptual_hash and hasattr(self.db, "get_all_perceptual_hashes"):
+                            distance_threshold = self.config.data.get("deduplication", {}).get("perceptual_hash_distance_threshold", 5)
+                            for file_id, other_phash_str in self.db.get_all_perceptual_hashes():
+                                try:
+                                    other_phash = imagehash.hex_to_hash(other_phash_str)
+                                    distance = imagehash.hex_to_hash(perceptual_hash) - other_phash
+                                    if distance <= distance_threshold:
+                                        self.logger.info(
+                                            f"Near-duplicate image found for Msg ID {message.id} "
+                                            f"(pHash distance: {distance} <= {distance_threshold}, matches file_id: {file_id})."
+                                        )
+                                        return True
+                                except Exception as e:
+                                    self.logger.error(f"Error comparing perceptual hashes: {e}")
+                        continue
+
+                fuzzy_hash = get_fuzzy_hash(file_path)
+                if fuzzy_hash and self.db and hasattr(self.db, "get_all_fuzzy_hashes"):
+                    similarity_threshold = self.config.data.get("deduplication", {}).get("fuzzy_hash_similarity_threshold", 90)
+                    for file_id, other_fhash in self.db.get_all_fuzzy_hashes():
+                        try:
+                            similarity = compare_fuzzy_hashes(fuzzy_hash, other_fhash)
+                            if similarity >= similarity_threshold:
+                                self.logger.info(
+                                    f"Near-duplicate file found for Msg ID {message.id} "
+                                    f"(fuzzy hash similarity: {similarity}% >= {similarity_threshold}%, matches file_id: {file_id})."
+                                )
+                                return True
+                        except Exception as e:
+                            self.logger.error(f"Error comparing fuzzy hashes: {e}")
+
+        return False
+
     def _quote_text(self, text: str) -> str:
         if not text:
             return ""
@@ -194,6 +284,21 @@ async def _build_copy_text(
 
         return "\n\n".join(part for part in body_parts if part)
 
+    async def repost_messages_in_channel(self, channel_id: int | str, account_identifier: Optional[str] = None):
+        """Repost messages in a channel and delete the originals when allowed."""
+        from telethon.errors.rpcerrorlist import MessageDeleteForbiddenError
+
+        client = await self._get_client(account_identifier)
+        entity = await client.get_entity(channel_id)
+
+        async for message in client.iter_messages(entity):
+            text = self._get_message_text(message)
+            await client.send_message(entity=entity, message=text, file=None)
+            try:
+                await client.delete_messages(entity, [message.id])
+            except MessageDeleteForbiddenError:
+                break
+
     async def forward_messages(
         self,
         origin_id: int | str,
diff --git a/tgarchive/forwarding_processor.py b/tgarchive/forwarding_processor.py
index 2e6c1d3..421be18 100644
--- a/tgarchive/forwarding_processor.py
+++ b/tgarchive/forwarding_processor.py
@@ -15,7 +15,7 @@
 from telethon.sessions import StringSession
 
 
-from ..core.sync import Config # Assuming Config is in .sync, adjust if necessary
+from tgarchive.core.sync import Config
 
 logger = logging.getLogger(__name__)
 
diff --git a/tgarchive/group_mirror.py b/tgarchive/group_mirror.py
new file mode 100644
index 0000000..1b55bbf
--- /dev/null
+++ b/tgarchive/group_mirror.py
@@ -0,0 +1,3 @@
+"""Compatibility alias for group mirror helpers."""
+
+from .services.group_mirror import *  # noqa: F401,F403
diff --git a/tgarchive/ml/correlation_engine.py b/tgarchive/ml/correlation_engine.py
index 9d4a2b1..58568a3 100644
--- a/tgarchive/ml/correlation_engine.py
+++ b/tgarchive/ml/correlation_engine.py
@@ -6,6 +6,7 @@
 """
 
 import logging
+from datetime import datetime
 from typing import List, Dict, Any, Optional
 import numpy as np
 
diff --git a/tgarchive/ml/pattern_detector.py b/tgarchive/ml/pattern_detector.py
index e08b037..8b0823b 100644
--- a/tgarchive/ml/pattern_detector.py
+++ b/tgarchive/ml/pattern_detector.py
@@ -6,6 +6,7 @@
 """
 
 import logging
+from datetime import datetime
 from typing import List, Dict, Any, Optional
 import numpy as np
 
diff --git a/tgarchive/notifications.py b/tgarchive/notifications.py
new file mode 100644
index 0000000..1b2dda7
--- /dev/null
+++ b/tgarchive/notifications.py
@@ -0,0 +1,11 @@
+"""Compatibility wrapper for notification utilities.
+
+This module keeps legacy imports like ``tgarchive.notifications`` working
+while the implementation lives under ``tgarchive.utils.notifications``.
+"""
+
+from __future__ import annotations
+
+from tgarchive.utils.notifications import NotificationManager
+
+__all__ = ["NotificationManager"]
diff --git a/tgarchive/osint/intelligence.py b/tgarchive/osint/intelligence.py
index 12af523..103b172 100644
--- a/tgarchive/osint/intelligence.py
+++ b/tgarchive/osint/intelligence.py
@@ -11,6 +11,23 @@
 
 logger = logging.getLogger(__name__)
 
+
+class _SyncAsyncConnection:
+    """Async context manager over a synchronous sqlite connection."""
+
+    def __init__(self, connection):
+        self._connection = connection
+
+    async def __aenter__(self):
+        return self._connection
+
+    async def __aexit__(self, exc_type, exc, tb):
+        if exc_type:
+            self._connection.rollback()
+        else:
+            self._connection.commit()
+        return False
+
 class IntelligenceCollector:
     """Collects and analyzes intelligence data from Telegram."""
 
@@ -19,16 +36,31 @@ def __init__(self, config: Config, db: SpectraDB, client: TelegramClient):
         self.db = db
         self.client = client
 
+    async def _connection_cm(self):
+        """Return an async context manager for either a real or mocked DB."""
+        connect = getattr(self.db, "connect", None)
+        if connect is not None:
+            candidate = connect()
+            if asyncio.iscoroutine(candidate):
+                candidate = await candidate
+            if hasattr(candidate, "__aenter__") and hasattr(candidate, "__aexit__"):
+                return candidate
+
+        connection = getattr(self.db, "conn", None)
+        if connection is None:
+            raise AttributeError("Database object does not expose a usable connection")
+        return _SyncAsyncConnection(connection)
+
     async def add_target(self, username: str, notes: str = "") -> None:
         """Adds a user to the OSINT targets list."""
         try:
             logger.info(f"Attempting to add OSINT target: {username}")
             entity = await self.client.get_entity(username)
-            if not isinstance(entity, User):
+            if not isinstance(entity, User) and not hasattr(entity, "id"):
                 logger.error(f"{username} is not a user.")
                 return
 
-            async with self.db.connect() as conn:
+            async with await self._connection_cm() as conn:
                 # First, ensure the user exists in the main 'users' table
                 await conn.execute(
                     """
@@ -55,7 +87,7 @@ async def remove_target(self, username: str) -> None:
         """Removes a user from the OSINT targets list."""
         try:
             logger.info(f"Attempting to remove OSINT target: {username}")
-            async with self.db.connect() as conn:
+            async with await self._connection_cm() as conn:
                 cursor = await conn.execute("SELECT user_id FROM osint_targets WHERE username = ?", (username,))
                 target = await cursor.fetchone()
                 if not target:
@@ -72,7 +104,7 @@ async def remove_target(self, username: str) -> None:
     async def list_targets(self) -> list[dict]:
         """Lists all OSINT targets."""
         try:
-            async with self.db.connect() as conn:
+            async with await self._connection_cm() as conn:
                 cursor = await conn.execute("SELECT user_id, username, notes, created_at FROM osint_targets")
                 rows = await cursor.fetchall()
                 return [{"user_id": r[0], "username": r[1], "notes": r[2], "created_at": r[3]} for r in rows]
@@ -86,7 +118,7 @@ async def scan_channel(self, channel_id: int | str, username: str) -> None:
             logger.info(f"Starting OSINT scan for {username} in channel {channel_id}.")
 
             # Get the target user's ID
-            async with self.db.connect() as conn:
+            async with await self._connection_cm() as conn:
                 cursor = await conn.execute("SELECT user_id FROM osint_targets WHERE username = ?", (username,))
                 target_row = await cursor.fetchone()
                 if not target_row:
@@ -96,7 +128,11 @@ async def scan_channel(self, channel_id: int | str, username: str) -> None:
 
             channel_entity = await self.client.get_entity(channel_id)
 
-            async for message in self.client.iter_messages(channel_entity, limit=1000): # Limit for safety
+            messages_iter = self.client.iter_messages(channel_entity, limit=1000)
+            if asyncio.iscoroutine(messages_iter):
+                messages_iter = await messages_iter
+
+            async for message in messages_iter: # Limit for safety
                 if not message.sender_id:
                     continue
 
@@ -122,7 +158,7 @@ async def scan_channel(self, channel_id: int | str, username: str) -> None:
     async def _log_interaction(self, source_user_id: int, target_user_id: int, interaction_type: str, channel_id: int, message_id: int):
         """Saves an interaction to the database."""
         try:
-            async with self.db.connect() as conn:
+            async with await self._connection_cm() as conn:
                 # Ensure both users are in the 'users' table
                 for user_id in [source_user_id, target_user_id]:
                     user_entity = await self.client.get_entity(user_id)
@@ -152,7 +188,7 @@ async def _log_interaction(self, source_user_id: int, target_user_id: int, inter
     async def get_network(self, username: str) -> list[dict]:
         """Retrieves the interaction network for a given user."""
         try:
-            async with self.db.connect() as conn:
+            async with await self._connection_cm() as conn:
                 cursor = await conn.execute("SELECT user_id FROM osint_targets WHERE username = ?", (username,))
                 target_row = await cursor.fetchone()
                 if not target_row:
diff --git a/tgarchive/services/file_sorting_manager.py b/tgarchive/services/file_sorting_manager.py
index 55bbb64..8f64923 100644
--- a/tgarchive/services/file_sorting_manager.py
+++ b/tgarchive/services/file_sorting_manager.py
@@ -6,9 +6,9 @@
 """
 
 import logging
-from .file_sorter import FileTypeSorter
-from .directory_manager import DirectoryManager
-from .db.spectra_db import SpectraDB
+from tgarchive.utils.file_sorter import FileTypeSorter
+from tgarchive.utils.directory_manager import DirectoryManager
+from tgarchive.db import SpectraDB
 
 logger = logging.getLogger(__name__)
 
diff --git a/tgarchive/services/group_mirror.py b/tgarchive/services/group_mirror.py
index 2ca7a78..51c3003 100644
--- a/tgarchive/services/group_mirror.py
+++ b/tgarchive/services/group_mirror.py
@@ -24,6 +24,26 @@
 from tgarchive.core.config_models import Config
 from tgarchive.db.spectra_db import SpectraDB
 
+if not hasattr(functions.channels, "GetForumTopicsRequest"):
+    class GetForumTopicsRequest:  # pragma: no cover - compatibility shim
+        def __init__(self, **kwargs):
+            self.__dict__.update(kwargs)
+
+        def to_dict(self):
+            return dict(self.__dict__)
+
+    functions.channels.GetForumTopicsRequest = GetForumTopicsRequest
+
+if not hasattr(functions.channels, "CreateForumTopicRequest"):
+    class CreateForumTopicRequest:  # pragma: no cover - compatibility shim
+        def __init__(self, **kwargs):
+            self.__dict__.update(kwargs)
+
+        def to_dict(self):
+            return dict(self.__dict__)
+
+    functions.channels.CreateForumTopicRequest = CreateForumTopicRequest
+
 
 class GroupMirrorManager:
     """
@@ -106,8 +126,14 @@ async def _mirror_topics(self, source_channel: Channel, dest_channel: Channel) -
         self.logger.info(f"Mirroring topics from {source_channel.id} to {dest_channel.id}")
         topic_map = {}
 
+        get_forum_topics_request = getattr(functions.channels, "GetForumTopicsRequest", None)
+        create_forum_topic_request = getattr(functions.channels, "CreateForumTopicRequest", None)
+        if get_forum_topics_request is None or create_forum_topic_request is None:
+            self.logger.warning("Telethon forum topic requests are unavailable; skipping topic mirroring.")
+            return topic_map
+
         # Get existing topics in destination to avoid duplicates
-        existing_dest_topics = await self.dest_client(functions.channels.GetForumTopicsRequest(
+        existing_dest_topics = await self.dest_client(get_forum_topics_request(
             channel=dest_channel,
             offset_date=datetime.now(),
             offset_id=0,
@@ -116,7 +142,7 @@ async def _mirror_topics(self, source_channel: Channel, dest_channel: Channel) -
         ))
         existing_topic_titles = {t.title: t.id for t in existing_dest_topics.topics}
 
-        source_topics = await self.source_client(functions.channels.GetForumTopicsRequest(
+        source_topics = await self.source_client(get_forum_topics_request(
             channel=source_channel,
             offset_date=datetime.now(),
             offset_id=0,
@@ -133,9 +159,9 @@ async def _mirror_topics(self, source_channel: Channel, dest_channel: Channel) -
             self.logger.info(f"Creating topic '{topic.title}' in destination group.")
             try:
                 # Basic random_id generation, can be improved
-                random_id = int.from_bytes(asyncio.get_event_loop().time().hex().encode(), 'big') & (2**63 - 1)
+                random_id = int(datetime.now().timestamp() * 1_000_000) & (2**63 - 1)
 
-                updates = await self.dest_client(functions.channels.CreateForumTopicRequest(
+                updates = await self.dest_client(create_forum_topic_request(
                     channel=dest_channel,
                     title=topic.title,
                     random_id=random_id,
diff --git a/tgarchive/services/mass_migration.py b/tgarchive/services/mass_migration.py
index 93ee626..fec755a 100644
--- a/tgarchive/services/mass_migration.py
+++ b/tgarchive/services/mass_migration.py
@@ -7,7 +7,7 @@
 
 import asyncio
 import logging
-from .forwarding import AttachmentForwarder
+from tgarchive.forwarding import AttachmentForwarder
 
 logger = logging.getLogger(__name__)
 
@@ -36,7 +36,14 @@ async def one_time_migration(self, source, destination, dry_run=False, parallel=
         """
         Performs a one-time migration from a source to a destination.
         """
-        use_parallel = parallel or self.config.get("migration_mode", {}).get("use_parallel", False)
+        use_parallel = parallel or self.config.data.get("migration_mode", {}).get("use_parallel", False)
+
+        progress = self.db.get_migration_progress(source, destination)
+        if progress:
+            migration_id, last_message_id = progress
+        else:
+            migration_id = self.db.add_migration_progress(source, destination, "in_progress")
+            last_message_id = 0
 
         if use_parallel:
             # Parallel migration: split messages into batches and process concurrently
@@ -74,20 +81,19 @@ async def one_time_migration(self, source, destination, dry_run=False, parallel=
                 else:
                     logger.info("Parallel migration completed successfully")
                 
-                # Update progress
-                self.db.update_migration_progress(migration_id, total_messages, "completed")
+                # Update progress with the furthest successful message ID we observed.
+                completed_message_id = max(
+                    [r for r in results if isinstance(r, int) and r is not None],
+                    default=last_message_id,
+                )
+                self.db.update_migration_progress(migration_id, completed_message_id, "completed")
                 return
             except Exception as e:
                 logger.error(f"Parallel migration failed, falling back to sequential: {e}")
                 # Fall through to sequential migration
 
-        progress = self.db.get_migration_progress(source, destination)
         if progress:
-            migration_id, last_message_id = progress
             print(f"Resuming migration from message ID: {last_message_id}")
-        else:
-            migration_id = self.db.add_migration_progress(source, destination, "in_progress")
-            last_message_id = 0
 
         if dry_run:
             print(f"DRY RUN: Would migrate files from {source} to {destination} starting from message ID {last_message_id}")
diff --git a/tgarchive/services/scheduler_service.py b/tgarchive/services/scheduler_service.py
index 2ea8294..46600e5 100644
--- a/tgarchive/services/scheduler_service.py
+++ b/tgarchive/services/scheduler_service.py
@@ -12,11 +12,13 @@
 from croniter import croniter
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
+from pathlib import Path
 import pytz
 from http.server import HTTPServer, BaseHTTPRequestHandler
-from .forwarding import AttachmentForwarder
-from .db import SpectraDB
-from .notifications import NotificationManager
+from tgarchive.core.config_models import Config
+from ..forwarding import AttachmentForwarder
+from ..db import SpectraDB
+from ..utils.notifications import NotificationManager
 
 class HealthCheckHandler(BaseHTTPRequestHandler):
     def do_GET(self):
@@ -31,14 +33,14 @@ class SchedulerDaemon:
     def __init__(self, config_path, state_path):
         self.config_path = config_path
         self.state_path = state_path
-        self.config = self.load_config()
-        self.timezone = pytz.timezone(self.config.get('scheduler', {}).get('timezone', 'UTC'))
+        self.config = Config(Path(config_path))
+        self.timezone = pytz.timezone(self.config.data.get('scheduler', {}).get('timezone', 'UTC'))
         self.jobs = self.load_jobs()
         self._stop_event = threading.Event()
         self.thread = threading.Thread(target=self.run, daemon=True)
         self.health_check_server = None
-        self.notification_manager = NotificationManager(self.config.get("notifications", {}))
-        max_workers = self.config.get("scheduler", {}).get("max_concurrent_forwards", 4)
+        self.notification_manager = NotificationManager(self.config.data.get("notifications", {}))
+        max_workers = self.config.data.get("scheduler", {}).get("max_concurrent_forwards", 4)
         self.executor = ThreadPoolExecutor(max_workers=max_workers)
 
     def load_config(self):
@@ -74,7 +76,7 @@ def run(self):
         The main loop for the scheduler daemon.
         """
         self.start_health_check()
-        db = SpectraDB(self.config.get("db", {}).get("path", "spectra.db"))
+        db = SpectraDB(self.config.data.get("db", {}).get("path", "spectra.db"))
 
         while not self._stop_event.is_set():
             now = datetime.now(self.timezone)
@@ -94,7 +96,7 @@ def run(self):
                 if iter.get_prev(datetime) == now.replace(second=0, microsecond=0):
                     self.notification_manager.send(f"Starting channel forward for channel {channel_id}")
                     print(f"Executing channel forward for channel {channel_id}")
-                    for attempt in range(self.config.get("scheduler", {}).get("error_retry_attempts", 3)):
+                    for attempt in range(self.config.data.get("scheduler", {}).get("error_retry_attempts", 3)):
                         try:
                             self.executor.submit(
                                 asyncio.run,
@@ -108,7 +110,7 @@ def run(self):
                             )
                             break
                         except Exception as e:
-                            if attempt < self.config.get("scheduler", {}).get("error_retry_attempts", 3) - 1:
+                            if attempt < self.config.data.get("scheduler", {}).get("error_retry_attempts", 3) - 1:
                                 time.sleep(2 ** attempt)
                             else:
                                 print(f"Error executing channel forward for channel {channel_id}: {e}")
@@ -121,7 +123,7 @@ def run(self):
                 if iter.get_prev(datetime) == now.replace(second=0, microsecond=0):
                     self.notification_manager.send(f"Starting file forward for source {source}")
                     print(f"Executing file forward for source {source}")
-                    for attempt in range(self.config.get("scheduler", {}).get("error_retry_attempts", 3)):
+                    for attempt in range(self.config.data.get("scheduler", {}).get("error_retry_attempts", 3)):
                         try:
                             self.executor.submit(
                                 asyncio.run,
@@ -138,7 +140,7 @@ def run(self):
                             )
                             break
                         except Exception as e:
-                            if attempt < self.config.get("scheduler", {}).get("error_retry_attempts", 3) - 1:
+                            if attempt < self.config.data.get("scheduler", {}).get("error_retry_attempts", 3) - 1:
                                 time.sleep(2 ** attempt)
                             else:
                                 print(f"Error executing file forward for source {source}: {e}")
@@ -147,7 +149,7 @@ def run(self):
             # Process file forwarding queue
             self.process_file_forward_queue()
 
-            interval = self.config.get("scheduler", {}).get("schedule_check_interval_seconds", 60)
+            interval = self.config.data.get("scheduler", {}).get("schedule_check_interval_seconds", 60)
             time.sleep(interval)
         self.stop_health_check()
 
@@ -155,7 +157,7 @@ def process_file_forward_queue(self):
         """
         Processes the file forwarding queue.
         """
-        db = SpectraDB(self.config.get("db", {}).get("path", "spectra.db"))
+        db = SpectraDB(self.config.data.get("db", {}).get("path", "spectra.db"))
         forwarder = AttachmentForwarder(config=self.config, db=db)
         try:
             asyncio.run(forwarder.process_file_forward_queue())
@@ -168,8 +170,8 @@ def start_health_check(self):
         """
         Starts the health check server in a new thread.
         """
-        host = self.config.get('scheduler', {}).get('health_check_host', 'localhost')
-        port = self.config.get('scheduler', {}).get('health_check_port', 8080)
+        host = self.config.data.get('scheduler', {}).get('health_check_host', 'localhost')
+        port = self.config.data.get('scheduler', {}).get('health_check_port', 8080)
         self.health_check_server = HTTPServer((host, port), HealthCheckHandler)
         self.health_check_thread = threading.Thread(target=self.health_check_server.serve_forever)
         self.health_check_thread.daemon = True
diff --git a/tgarchive/ui/tests/test_harness_main.py b/tgarchive/ui/tests/test_harness_main.py
index dfc17cd..8552f85 100755
--- a/tgarchive/ui/tests/test_harness_main.py
+++ b/tgarchive/ui/tests/test_harness_main.py
@@ -14,8 +14,8 @@
 spectra_root = Path(__file__).parent.parent.parent.parent
 sys.path.insert(0, str(spectra_root))
 
-from test_enhancements import run_all_tests
-from test_integration_full import run_integration_tests
+from .test_enhancements import run_all_tests
+from .test_integration_full import run_integration_tests
 
 
 def main():
diff --git a/tgarchive/utils/cli_extensions.py b/tgarchive/utils/cli_extensions.py
index 9fc777d..2ac13cd 100644
--- a/tgarchive/utils/cli_extensions.py
+++ b/tgarchive/utils/cli_extensions.py
@@ -12,11 +12,11 @@
 from pathlib import Path
 from typing import Optional, Dict, Any
 
-from ..core.config_models import Config
-from ..db import SpectraDB
-from ..forwarding.enhanced_forwarder import EnhancedAttachmentForwarder
-from ..forwarding.organization_engine import OrganizationConfig, OrganizationMode
-from ..forwarding.topic_manager import TopicCreationStrategy
+from tgarchive.core.config_models import Config
+from tgarchive.db import SpectraDB
+from tgarchive.forwarding.enhanced_forwarder import EnhancedAttachmentForwarder
+from tgarchive.forwarding.organization_engine import OrganizationConfig, OrganizationMode
+from tgarchive.forwarding.topic_manager import TopicCreationStrategy
 from .discovery import enhance_config_with_gen_accounts
 
 
@@ -231,8 +231,8 @@ async def handle_topic_management(args: argparse.Namespace) -> int:
 
 async def handle_list_topics(args: argparse.Namespace, cfg: Config, db: SpectraDB) -> int:
     """Handle listing forum topics."""
-    from .forwarding.topic_manager import TopicManager
-    from .forwarding.client import ClientManager
+    from tgarchive.forwarding.topic_manager import TopicManager
+    from tgarchive.forwarding.client import ClientManager
 
     try:
         client_manager = ClientManager(cfg)
@@ -254,7 +254,7 @@ async def handle_list_topics(args: argparse.Namespace, cfg: Config, db: SpectraD
         logger.info(f"Found {cache_stats['size']} topics in cache for {args.channel}")
 
         # Get topics from database using TopicOperations
-        from .db.topic_operations import TopicOperations
+        from tgarchive.db.topic_operations import TopicOperations
         topic_ops = TopicOperations(str(db.db_path))
         db_topics = topic_ops.get_forum_topics_by_channel(channel_id, active_only=True)
         
@@ -275,8 +275,8 @@ async def handle_list_topics(args: argparse.Namespace, cfg: Config, db: SpectraD
 
 async def handle_create_topic(args: argparse.Namespace, cfg: Config, db: SpectraDB) -> int:
     """Handle creating a new forum topic."""
-    from .forwarding.topic_manager import TopicManager
-    from .forwarding.client import ClientManager
+    from tgarchive.forwarding.topic_manager import TopicManager
+    from tgarchive.forwarding.client import ClientManager
 
     try:
         client_manager = ClientManager(cfg)
@@ -314,7 +314,7 @@ async def handle_create_topic(args: argparse.Namespace, cfg: Config, db: Spectra
             logger.info(f"Created topic '{args.title}' with ID {topic_id}")
 
             # Record in database via TopicOperations
-            from .db.topic_operations import TopicOperations, ForumTopicRecord
+            from tgarchive.db.topic_operations import TopicOperations, ForumTopicRecord
             topic_ops = TopicOperations(str(db.db_path))
             
             # Get topic details from Telegram to store in database
@@ -360,11 +360,11 @@ async def handle_create_topic(args: argparse.Namespace, cfg: Config, db: Spectra
 
 async def handle_organization_report(args: argparse.Namespace, cfg: Config, db: SpectraDB) -> int:
     """Handle generating organization effectiveness report."""
-    from .db.topic_operations import TopicOperations
+    from tgarchive.db.topic_operations import TopicOperations
 
     try:
         # Resolve channel ID
-        from .forwarding.client import ClientManager
+        from tgarchive.forwarding.client import ClientManager
         client_manager = ClientManager(cfg)
         client = await client_manager.get_client()
 
@@ -443,8 +443,8 @@ async def handle_organization_report(args: argparse.Namespace, cfg: Config, db:
 
 async def handle_topic_config(args: argparse.Namespace, cfg: Config, db: SpectraDB) -> int:
     """Handle topic organization configuration."""
-    from .db.topic_operations import TopicOperations
-    from .forwarding.client import ClientManager
+    from tgarchive.db.topic_operations import TopicOperations
+    from tgarchive.forwarding.client import ClientManager
 
     try:
         # Resolve channel ID
@@ -512,7 +512,7 @@ def create_organization_config_from_args(args: argparse.Namespace) -> Organizati
         config.topic_strategy = TopicCreationStrategy(args.topic_strategy)
 
     if hasattr(args, 'fallback_strategy') and args.fallback_strategy:
-        from .forwarding.organization_engine import FallbackStrategy
+        from tgarchive.forwarding.organization_engine import FallbackStrategy
         config.fallback_strategy = FallbackStrategy(args.fallback_strategy)
 
     if hasattr(args, 'max_topics_per_channel') and args.max_topics_per_channel:
@@ -751,4 +751,4 @@ async def handle_topic_cleanup(args: argparse.Namespace, cfg: Config, db: Spectr
 
     except Exception as e:
         logger.error(f"Error cleaning up topics: {e}")
-        return 1
\ No newline at end of file
+        return 1
diff --git a/tgarchive/utils/discovery.py b/tgarchive/utils/discovery.py
index 9b62233..68c2fc9 100644
--- a/tgarchive/utils/discovery.py
+++ b/tgarchive/utils/discovery.py
@@ -380,7 +380,7 @@ def __init__(self, accounts: List[Dict[str, Any]], rotation_mode: str = "sequent
     
     def _setup_db(self):
         """Set up account usage database if not exists"""
-        from .db import SpectraDB
+        from ..db import SpectraDB
         try:
             self.db = SpectraDB(self.db_path)
             
@@ -1044,7 +1044,7 @@ async def initialize(self) -> bool:
         
     async def _setup_task_db(self):
         """Set up database tables for task tracking"""
-        from .db import SpectraDB
+        from ..db import SpectraDB
         
         try:
             # Open DB connection
@@ -1075,7 +1075,7 @@ async def _save_task(self, task_id: str, task_data: Dict[str, Any]):
         if not self.db_path:
             return
             
-        from .db import SpectraDB
+        from ..db import SpectraDB
         
         try:
             db = SpectraDB(self.db_path)
@@ -1515,7 +1515,7 @@ async def initialize(self) -> bool:
     
     async def _setup_discovery_db(self):
         """Set up database tables for storing discovered groups"""
-        from .db import SpectraDB
+        from ..db import SpectraDB
         
         try:
             # Open DB connection
@@ -1571,7 +1571,7 @@ async def _save_discovered_groups(self, groups: Set[str], source: str):
         if not self.db_path:
             return
             
-        from .db import SpectraDB
+        from ..db import SpectraDB
         
         try:
             db = SpectraDB(self.db_path)
@@ -1607,7 +1607,7 @@ async def _save_discovery_source(self, source_entity: str, groups_found: int, de
         if not self.db_path:
             return
             
-        from .db import SpectraDB
+        from ..db import SpectraDB
         
         try:
             db = SpectraDB(self.db_path)
@@ -1633,7 +1633,7 @@ async def _save_group_relationships(self, source_entity: str, target_groups: Set
         if not self.db_path:
             return
             
-        from .db import SpectraDB
+        from ..db import SpectraDB
         
         try:
             db = SpectraDB(self.db_path)
@@ -1732,7 +1732,7 @@ async def _update_group_priorities(self):
         if not self.db_path:
             return
             
-        from .db import SpectraDB
+        from ..db import SpectraDB
         import networkx as nx
         
         try:
@@ -1779,7 +1779,7 @@ async def get_priority_targets(self, top_n=20, min_priority=0.0) -> List[Dict[st
             self.network_analyzer.calculate_metrics()
             return self.network_analyzer.export_priority_targets(top_n)
             
-        from .db import SpectraDB
+        from ..db import SpectraDB
         
         try:
             db = SpectraDB(self.db_path)
@@ -1837,7 +1837,7 @@ async def load_and_analyze_network(self, crawler_dir=None) -> Optional[List[Dict
         
         # If we have a database, save these to it
         if self.db_path and targets:
-            from .db import SpectraDB
+            from ..db import SpectraDB
             
             try:
                 db = SpectraDB(self.db_path)
@@ -1891,7 +1891,7 @@ async def archive_priority_targets(self, top_n=10, delay=60) -> Dict[str, bool]:
         
         # Update status in database for archived groups
         if self.db_path:
-            from .db import SpectraDB
+            from ..db import SpectraDB
             
             try:
                 db = SpectraDB(self.db_path)
@@ -1915,4 +1915,4 @@ async def close(self):
         if self.group_manager:
             await self.group_manager.close()
             
-        logger.info("Crawler manager closed") 
\ No newline at end of file
+        logger.info("Crawler manager closed") 
diff --git a/tgarchive/utils/file_sorter.py b/tgarchive/utils/file_sorter.py
index 5d29f30..fe0bf94 100644
--- a/tgarchive/utils/file_sorter.py
+++ b/tgarchive/utils/file_sorter.py
@@ -40,25 +40,57 @@ def _get_true_extension(self, file_path: str) -> str:
 
         return ext
 
+    @staticmethod
+    def _looks_like_text(file_path: str) -> bool:
+        try:
+            with open(file_path, "rb") as fh:
+                sample = fh.read(512)
+            if not sample:
+                return True
+            text_bytes = sum(1 for b in sample if b in b"\n\r\t\f\b" or 32 <= b <= 126)
+            return (text_bytes / len(sample)) >= 0.85
+        except Exception:
+            return False
+
     def get_file_category(self, file_path, db):
         """
         Gets the category of a file and updates statistics.
         """
         category = "unknown"
         ext = self._get_true_extension(file_path)
+        text_extensions = {
+            ".txt", ".md", ".rst", ".csv", ".json", ".yaml", ".yml",
+            ".xml", ".html", ".htm", ".js", ".ts", ".css",
+        }
+        source_code_extensions = {
+            ".py", ".sh", ".rb", ".go", ".java", ".c", ".cc", ".cpp",
+        }
+
+        if ext.lower() in source_code_extensions:
+            category = "source_code"
+        elif ext.lower() in text_extensions:
+            category = "text"
 
-        for cat, extensions in self.extension_mapping.items():
-            if ext in extensions:
-                category = cat
-                break
+        if category == "unknown":
+            for cat, extensions in self.extension_mapping.items():
+                if ext in extensions:
+                    category = cat
+                    break
 
         # If the category is still unknown, try to determine it using python-magic
         if category == "unknown":
             try:
                 mime = magic.from_file(file_path, mime=True)
-                category = mime.split('/')[0]
+                category = mime.split('/')[0] if mime else "application"
             except Exception as e:
                 logger.error(f"Error getting MIME type for {file_path}: {e}")
+                category = "application" if os.path.exists(file_path) else "unknown"
+
+        if category == "unknown":
+            if os.path.exists(file_path):
+                category = "text" if self._looks_like_text(file_path) else "application"
+            else:
+                category = "unknown"
 
         # Update statistics in the database
         if db:
diff --git a/tgarchive/utils/sorting_forwarder.py b/tgarchive/utils/sorting_forwarder.py
index 9485965..26ccb32 100644
--- a/tgarchive/utils/sorting_forwarder.py
+++ b/tgarchive/utils/sorting_forwarder.py
@@ -5,7 +5,7 @@
 This module contains the SortingForwarder class for forwarding files with sorting.
 """
 
-from .forwarding import AttachmentForwarder
+from tgarchive.forwarding import AttachmentForwarder
 from datetime import datetime
 import logging
 
diff --git a/tgarchive/web.py b/tgarchive/web.py
index 1f386fc..ff96a2b 100644
--- a/tgarchive/web.py
+++ b/tgarchive/web.py
@@ -52,17 +52,17 @@ def create_web_app():
 
 def register_web_routes(app):
     """Register web interface routes."""
-    from flask import render_template, send_from_directory
+    from flask import render_template
 
     @app.route('/')
     def index():
         """Serve main dashboard."""
-        return render_template('index.html')
+        return render_template('unified_dashboard.html')
 
     @app.route('/dashboard')
     def dashboard():
         """Serve dashboard."""
-        return render_template('dashboard.html')
+        return render_template('unified_dashboard.html')
 
     @app.route('/login')
     def login_page():