diff --git a/.circleci/config.yml b/.circleci/config.yml index 349012d..c793148 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,4 +48,4 @@ workflows: - test: matrix: parameters: - py-version: ['3.11', '3.12', '3.13', '3.14'] + py-version: ['3.11', '3.12', '3.13'] diff --git a/.env.example b/.env.example index b6280ae..cb6cd0b 100644 --- a/.env.example +++ b/.env.example @@ -38,6 +38,19 @@ MAM_ID=your-mam-session-cookie-value # Optional: known safe torrent ID for manual download integration testing # MAM_TEST_TID=1234567 +# ============================================================================= +# AUDIBLE SETTINGS (OPTIONAL AUTHENTICATED SEARCH) +# ============================================================================= + +# Optional override for the encrypted Audible auth file consumed by mkb79/Audible +# AUDIBLE_AUTH_FILE=secrets/audible-auth.json + +# Password used to decrypt the Audible auth file (not your Audible/Amazon account password) +# AUDIBLE_AUTH_FILE_PASSWORD=your-audible-auth-file-password + +# NOTE: The repo install path adds mkb79/Audible from GitHub. The auth file and +# decrypt password are required only when the authenticated Audible backend is enabled. + # ============================================================================= # QBITTORRENT SETTINGS # ============================================================================= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89a3cac..f2fdbed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] + python-version: ["3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v6 @@ -56,6 +56,47 @@ jobs: fail_ci_if_error: false token: ${{ secrets.CODECOV_TOKEN }} + audible-install: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v6 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Verify no-deps Audible install path + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + make install-audible + python - <<'PY' + import asyncio + from pathlib import Path + + from src.audible_client import AudibleClientProvider + + auth_file = Path("tmp-audible-auth.json") + auth_file.write_text("{}") + + async def main() -> None: + provider = AudibleClientProvider(auth_file=str(auth_file), auth_file_password="test-password") + client = await provider.get_client("us") + assert client is None + await provider.aclose() + + try: + asyncio.run(main()) + finally: + auth_file.unlink(missing_ok=True) + PY + security: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index 07b48e5..a793553 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,15 @@ # Makefile for audiobook-dev project -.PHONY: help install install-dev test test-fast test-integration lint lint-fix format format-check type-check clean run pre-commit ci +.PHONY: help install install-dev install-audible test test-fast test-integration lint lint-fix format format-check type-check clean run pre-commit ci + +AUDIBLE_GIT_REF := 458131b4702cca48a8a6eb68c19c21b91b276d37 +AUDIBLE_PIP_SPEC := git+https://github.com/mkb79/Audible.git@$(AUDIBLE_GIT_REF) help: @echo "Available commands:" @echo " make install - Install production dependencies" @echo " make install-dev - Install development dependencies" + @echo " make install-audible - Install mkb79/Audible from GitHub" @echo " make test - Run tests with coverage" @echo " make test-fast - Run tests without coverage" @echo " make test-integration - Run integration tests only" @@ -19,12 +23,17 @@ help: install: pip install -r requirements.txt + $(MAKE) install-audible install-dev: pip install -r requirements.txt + $(MAKE) install-audible pip install -e ".[dev]" pre-commit install +install-audible: + pip install --force-reinstall --no-deps "$(AUDIBLE_PIP_SPEC)" + test: pytest --cov=src --cov-branch --cov-report=term-missing --cov-report=html --cov-fail-under=50 -v diff --git a/README.md b/README.md index 3700ff2..f15a982 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ A modern, secure, and delightfully over-engineered FastAPI microservice for auto ## ✨ Features - **πŸ”’ Secure Webhook Endpoint** - Token-validated integration with Autobrr/MAM -- **πŸ“– Metadata Enrichment** - Audnex API and Audible scraping for rich book data +- **πŸ“– Metadata Enrichment** - Audnex API and authenticated Audible lookups for rich book data - **πŸ’Ύ Persistent Storage** - SQLite database with comprehensive audit trails - **⏰ Time-Limited Tokens** - Cryptographically secure, single-use approval tokens - **πŸ“± Multi-Platform Notifications** - Pushover, Discord, Gotify, and Ntfy support @@ -66,7 +66,7 @@ python -m venv .venv source .venv/bin/activate # Windows: .venv\Scripts\activate # Install dependencies -pip install -r requirements.txt +make install-dev # Configure the system cp config/config.yaml.example config/config.yaml @@ -87,7 +87,7 @@ For detailed setup instructions, see the [Getting Started Guide](docs/user-guide ## πŸ—οΈ Project Structure -``` +```text audiobook_dev/ β”œβ”€β”€ docs/ # πŸ“š Comprehensive documentation β”‚ β”œβ”€β”€ user-guide/ # User documentation and guides @@ -145,9 +145,11 @@ audiobook_dev/ 3. **Install dependencies** ```bash - pip install -r requirements.txt + make install-dev ``` + This installs the upstream `mkb79/Audible` package from GitHub for the authenticated Audible backend. + 4. **Copy and edit config** - Edit `config/config.yaml` for your environment (API URLs, notification settings, etc). - Create a `.env` file with your secrets (see `.env.example`). @@ -179,7 +181,7 @@ Configure each in `config/config.yaml` and `.env`. ## Metadata - Uses Audnex API for fast, reliable metadata. -- Falls back to Audible scraping if needed. +- Uses `mkb79/Audible` with an encrypted auth file for Audible-backed search. - Cleans and normalizes author, narrator, series, and description fields. - Caches lookups with LRU cache for efficiency. diff --git a/config/config.yaml.example b/config/config.yaml.example index 65eba4f..3c0e405 100644 --- a/config/config.yaml.example +++ b/config/config.yaml.example @@ -27,6 +27,7 @@ metadata: audible: base_url: "https://api.audible.com" search_endpoint: "/1.0/catalog/products" + auth_file: "secrets/audible-auth.json" # Optional mkb79/Audible auth file for authenticated fallback searches notifications: diff --git a/docs/PR1_REVIEW_FIXES.md b/docs/PR1_REVIEW_FIXES.md index 58d7a5e..72f3c5d 100644 --- a/docs/PR1_REVIEW_FIXES.md +++ b/docs/PR1_REVIEW_FIXES.md @@ -143,7 +143,7 @@ Most Round 3 review comments re-flagged issues already resolved in Round 2: - **File**: [src/audible_scraper.py](https://github.com/H2OKing89/audiobook_dev/pull/1#discussion_r2642782765) - **Lines**: 260, 272 - **Issue**: Catching broad `Exception` instead of specific exceptions -- **Note**: Deferred - requires deeper analysis of Audible scraping error modes +- **Note**: Deferred - requires deeper analysis of authenticated Audible backend error modes - **Status**: ⏳ Future Enhancement ### 8. βœ… Missing Cleanup - `alpine-components.js` Loading Screen (FIXED) diff --git a/docs/SYSTEM_COMPLETION_SUMMARY.md b/docs/SYSTEM_COMPLETION_SUMMARY.md index 4944019..9597aae 100644 --- a/docs/SYSTEM_COMPLETION_SUMMARY.md +++ b/docs/SYSTEM_COMPLETION_SUMMARY.md @@ -9,7 +9,7 @@ - **Modular Architecture**: Refactored metadata workflow into separate, focused modules - `mam_api/` - MAM JSON API client, models, and metadata adapter - `audnex_metadata.py` - Comprehensive metadata cleaning and enrichment - - `audible_scraper.py` - Audible fallback scraping + - `src/audible_scraper.py` - Authenticated Audible metadata backend - `metadata_coordinator.py` - Orchestrates the entire workflow ### ⚑ **Async & Concurrency** diff --git a/docs/api/config-reference.md b/docs/api/config-reference.md index ce2d695..ecaf216 100644 --- a/docs/api/config-reference.md +++ b/docs/api/config-reference.md @@ -354,7 +354,7 @@ logging: - **Type:** Boolean - **Default:** `true` -- **Description:** Enable Audible scraping for metadata +- **Description:** Enable authenticated Audible lookups for metadata ### `metadata.cache_expiry_hours` @@ -520,7 +520,7 @@ The system validates configuration on startup: Configuration errors are reported clearly: -``` +```text Configuration Error: notifications.discord.webhook_url is required when Discord is enabled Configuration Error: server.port must be between 1 and 65535 Configuration Error: security.token_expiry_hours cannot exceed 168 (1 week) diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 068c35f..5362300 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -59,7 +59,9 @@ metadata: timeout_seconds: 10 audible: enabled: true - timeout_seconds: 15 + base_url: "https://api.audible.com" + search_endpoint: "/1.0/catalog/products" + auth_file: "secrets/audible-auth.json" # Optional - required only for authenticated Audible lookups ``` ### Notifications @@ -94,6 +96,10 @@ NTFY_URL=https://ntfy.sh/your-topic # MAM API auth (optional, required for MAM metadata lookups) MAM_ID=your-mam-session-cookie-value + +# Authenticated Audible backend +# AUDIBLE_AUTH_FILE=secrets/audible-auth.json +# AUDIBLE_AUTH_FILE_PASSWORD=your-audible-auth-file-password ``` ## πŸ” MAM API Configuration (Optional) @@ -108,6 +114,16 @@ MAM_ID=your-mam-session-cookie-value Security note: `MAM_ID` is a session token. Keep it only in `.env`, never commit it, and rotate it if it is shared or exposed. +## 🎧 Authenticated Audible Integration + +The Audible backend now uses `mkb79/Audible`. Configure `AUDIBLE_AUTH_FILE` and `AUDIBLE_AUTH_FILE_PASSWORD` only when you want authenticated Audible lookups so the app can decrypt the stored auth JSON and authenticate requests. + +The encrypted auth file format used by `Authenticator.from_file(...)` matches the `salt` / `iv` / `ciphertext` JSON envelope already used by this project. + +`AUDIBLE_AUTH_FILE_PASSWORD` is the decryption password for the auth file. It is not your Audible or Amazon login password. + +Installation note: this repo installs `mkb79/Audible` directly from GitHub because the PyPI release is behind upstream. The supported Python range for the Audible install path is 3.11-3.13. + ## 🎯 Configuration Examples ### Development/Testing @@ -137,7 +153,7 @@ Test your configuration: ```bash # Test main config -python -c "from src.config import load_config; print('βœ… Config valid')" +python -c "from src.config import load_config; load_config(); print('βœ… Config valid')" # Test MAM API auth (if configured) pytest tests/test_mam_api.py -k Integration --no-cov @@ -177,6 +193,7 @@ cp config/config.yaml.example config/config.yaml - [ ] `config/config.yaml` created and configured - [ ] `.env` file created with required tokens - [ ] `MAM_ID` set in `.env` (if using MAM) +- [ ] `AUDIBLE_AUTH_FILE` and `AUDIBLE_AUTH_FILE_PASSWORD` set if using authenticated Audible lookups - [ ] Configuration validated with test scripts - [ ] Notification services tested (if enabled) - [ ] Rate limiting configured appropriately diff --git a/docs/vendor/audible/.env.example b/docs/vendor/audible/.env.example new file mode 100644 index 0000000..be4c77e --- /dev/null +++ b/docs/vendor/audible/.env.example @@ -0,0 +1,128 @@ +# Audiobook Approval System - Environment Variables Template +# Copy this file to .env and fill in your actual values +# NEVER commit .env with real values to version control + +# ============================================================================= +# SECURITY SETTINGS +# ============================================================================= + +# Force HTTPS redirects (set to 'true' in production) +FORCE_HTTPS=false + +# API Key for admin endpoints (generate a strong random key) +# API_KEY=your-secure-api-key-here + +# ============================================================================= +# DATABASE +# ============================================================================= + +# Path to SQLite database file +# DB_PATH=/var/lib/audiobook/db.sqlite + +# ============================================================================= +# AUTOBRR INTEGRATION +# ============================================================================= + +# Token for webhook authentication (set in autobrr) +AUTOBRR_TOKEN=your-autobrr-webhook-token + +# ============================================================================= +# MAM (MYANONAMOUSE) SETTINGS +# ============================================================================= + +# MAM session cookie value for API access +# Get this from your browser after logging into MAM (cookie name: mam_id) +# SECURITY: Never share this value - it grants full access to your MAM account +MAM_ID=your-mam-session-cookie-value + +# Optional: known safe torrent ID for manual download integration testing +# MAM_TEST_TID=1234567 + +# ============================================================================= +# AUDIBLE SETTINGS (OPTIONAL AUTHENTICATED SEARCH) +# ============================================================================= + +# Optional override for the encrypted Audible auth file consumed by mkb79/Audible +# AUDIBLE_AUTH_FILE=secrets/audible-auth.json + +# Password used to decrypt the Audible auth file (not your Audible/Amazon account password) +# AUDIBLE_AUTH_FILE_PASSWORD=your-audible-auth-file-password + +# NOTE: The repo install path adds mkb79/Audible from GitHub. The auth file and +# decrypt password are required for the Audible backend to return results. + +# ============================================================================= +# QBITTORRENT SETTINGS +# ============================================================================= + +# qBittorrent connection details +QB_HOST=http://localhost:8080 +QB_USERNAME=admin +QB_PASSWORD=your-qbittorrent-password + +# ============================================================================= +# NOTIFICATION SERVICES +# ============================================================================= + +# Discord Webhook +DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/your/webhook/url + +# Gotify Notifications +GOTIFY_SERVER_URL=https://gotify.example.com +GOTIFY_APP_TOKEN=your-gotify-app-token + +# Ntfy Notifications +NTFY_TOPIC=audiobook_requests +NTFY_TOKEN=optional-ntfy-bearer-token + +# Pushover Notifications +PUSHOVER_USER_KEY=your-pushover-user-key +PUSHOVER_API_TOKEN=your-pushover-api-token + +# ============================================================================= +# APPLICATION SETTINGS +# ============================================================================= + +# Environment (development, staging, production) +ENVIRONMENT=development + +# Base URL for the application (used for redirects and notifications) +BASE_URL=https://audiobook-requests.example.com + +# Server host and port +HOST=0.0.0.0 +PORT=8000 + +# ============================================================================= +# LOGGING +# ============================================================================= + +# Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) +LOG_LEVEL=INFO + +# Log file path +LOG_FILE=logs/audiobook_requests.log + +# ============================================================================= +# SECURITY NOTES +# ============================================================================= + +# 1. Generate strong, unique passwords for all services +# 2. Use HTTPS in production (set FORCE_HTTPS=true) +# 3. Keep this file secure and never commit it to version control +# 4. Rotate secrets regularly +# 5. Use environment-specific configurations +# 6. Monitor logs for security events + +# ============================================================================= +# PRODUCTION CHECKLIST +# ============================================================================= + +# [ ] FORCE_HTTPS=true +# [ ] Strong API_KEY set +# [ ] All webhook tokens configured +# [ ] Database path secured +# [ ] Log rotation configured +# [ ] File permissions secured (600 for .env, 640 for logs, or 600 where feasible) using a dedicated service user and log group for intentional group-read access +# [ ] Verify deployed log files are not world-readable (for example: `stat -c '%a %n' logs/*`) +# [ ] Regular security updates scheduled diff --git a/docs/vendor/audible/README.md b/docs/vendor/audible/README.md new file mode 100644 index 0000000..6d560c8 --- /dev/null +++ b/docs/vendor/audible/README.md @@ -0,0 +1,229 @@ +# 🎧 Audiobook Automation System + +A modern, secure, and delightfully over-engineered FastAPI microservice for automated audiobook approval workflows. Built by Quentin with maximum automation and minimum manual intervention in mind. + +## Security and CI + +[![CI](https://github.com/H2OKing89/audiobook_dev/actions/workflows/ci.yml/badge.svg)](https://github.com/H2OKing89/audiobook_dev/actions/workflows/ci.yml) +[![Dependency Review](https://github.com/H2OKing89/audiobook_dev/actions/workflows/dependency-review.yml/badge.svg)](https://github.com/H2OKing89/audiobook_dev/actions/workflows/dependency-review.yml) + +Current security and test status is tracked in GitHub Actions. + +--- + +## ✨ Features + +- **πŸ”’ Secure Webhook Endpoint** - Token-validated integration with Autobrr/MAM +- **πŸ“– Metadata Enrichment** - Audnex API and authenticated Audible lookups for rich book data +- **πŸ’Ύ Persistent Storage** - SQLite database with comprehensive audit trails +- **⏰ Time-Limited Tokens** - Cryptographically secure, single-use approval tokens +- **πŸ“± Multi-Platform Notifications** - Pushover, Discord, Gotify, and Ntfy support +- **🎨 Beautiful Web Interface** - Modern, responsive UI with cyberpunk/anime aesthetics +- **🌐 Social Media Ready** - Dynamic OG/Twitter meta tags for all pages +- **βš™οΈ qBittorrent Integration** - Automated torrent handling with MAM cookie support +- **πŸš€ Async Performance** - Threadpool handling for optimal responsiveness +- **πŸ“Š Comprehensive Logging** - Centralized, rotating logs with detailed audit trails +- **β™Ώ Accessibility First** - WCAG 2.1 AA compliance with ARIA labels and keyboard navigation +- **πŸ§ͺ Test Coverage** - Comprehensive unit and integration test suite + +--- + +## πŸ“š Documentation + +Complete documentation is available in the [`docs/`](docs/) directory: + +### 🎯 For Users + +- **[πŸ“– Getting Started](docs/user-guide/getting-started.md)** - Installation and setup guide +- **[βš™οΈ Configuration](docs/user-guide/configuration.md)** - Configuration options and examples +- **[🌐 Web Interface](docs/user-guide/web-interface.md)** - Using the web UI +- **[πŸ“± Notifications](docs/user-guide/notifications.md)** - Setting up notification services +- **[πŸ”§ Troubleshooting](docs/user-guide/troubleshooting.md)** - Common issues and solutions + +### πŸ› οΈ For Developers + +- **[πŸ—οΈ Architecture](docs/development/architecture.md)** - System design and component overview +- **[πŸ” Security](docs/development/SECURITY.md)** - Security implementation details +- **[🎨 Interactive Fixes](docs/development/INTERACTIVE_FIXES.md)** - UI/UX improvements +- **[πŸ“‹ Logging](docs/development/LOGGING_IMPROVEMENTS.md)** - Enhanced logging system +- **[πŸ§ͺ Testing](docs/development/testing.md)** - Testing strategies and guidelines + +### πŸ”Œ API Reference + +- **[🌐 REST API](docs/api/rest-api.md)** - Complete API documentation +- **[πŸ”— Webhooks](docs/api/webhooks.md)** - Webhook configuration and payloads +- **[πŸ’Ύ Database](docs/api/database.md)** - Database schema and queries +- **[πŸ“‹ Configuration](docs/api/config-reference.md)** - Complete configuration reference + +--- + +## πŸš€ Quick Start + +```bash +# Clone the repository +git clone https://github.com/H2OKing89/audiobook_dev.git +cd audiobook_dev + +# Set up virtual environment +python -m venv .venv +source .venv/bin/activate # Windows: .venv\Scripts\activate + +# Install dependencies +make install-dev + +# Configure the system +cp config/config.yaml.example config/config.yaml +# Edit config/config.yaml with your settings + +# Initialize database +python src/db.py + +# Start the application +python src/main.py +``` + +Visit `http://localhost:8000` to access the beautiful web interface! + +For detailed setup instructions, see the [Getting Started Guide](docs/user-guide/getting-started.md). + +--- + +## πŸ—οΈ Project Structure + +```text +audiobook_dev/ +β”œβ”€β”€ docs/ # πŸ“š Comprehensive documentation +β”‚ β”œβ”€β”€ user-guide/ # User documentation and guides +β”‚ β”œβ”€β”€ development/ # Developer and architecture docs +β”‚ └── api/ # API reference and webhooks +β”œβ”€β”€ src/ # 🐍 Python source code +β”‚ β”œβ”€β”€ main.py # FastAPI application entry point +β”‚ β”œβ”€β”€ webui.py # Web interface and routes +β”‚ β”œβ”€β”€ metadata.py # Audiobook metadata handling +β”‚ β”œβ”€β”€ token_gen.py # Secure token generation/validation +β”‚ β”œβ”€β”€ notify/ # πŸ“± Notification service modules +β”‚ β”‚ β”œβ”€β”€ pushover.py # Pushover notifications +β”‚ β”‚ β”œβ”€β”€ gotify.py # Gotify notifications +β”‚ β”‚ β”œβ”€β”€ discord.py # Discord notifications +β”‚ β”‚ └── ntfy.py # Ntfy notifications +β”‚ β”œβ”€β”€ qbittorrent.py # qBittorrent integration +β”‚ β”œβ”€β”€ db.py # SQLite database operations +β”‚ β”œβ”€β”€ config.py # Configuration management +β”‚ β”œβ”€β”€ html.py # Jinja2 template utilities +β”‚ └── utils.py # Shared utility functions +β”œβ”€β”€ templates/ # 🎨 Jinja2 HTML templates +β”‚ β”œβ”€β”€ base.html # Base template with common elements +β”‚ β”œβ”€β”€ index.html # Enhanced home page +β”‚ β”œβ”€β”€ approval.html # Approval workflow page +β”‚ β”œβ”€β”€ rejection.html # Witty rejection page +β”‚ └── *.html # Additional UI templates +β”œβ”€β”€ static/ # 🌐 Static web assets +β”‚ β”œβ”€β”€ css/style.css # Enhanced cyberpunk styling +β”‚ └── js/app.js # Interactive JavaScript features +β”œβ”€β”€ tests/ # πŸ§ͺ Comprehensive test suite +β”œβ”€β”€ config/ # βš™οΈ Configuration files +β”‚ └── config.yaml # Main application configuration +β”œβ”€β”€ logs/ # πŸ“‹ Application logs +└── db.sqlite # πŸ’Ύ SQLite database +``` + +--- + +## Setup + +1. **Clone the repo** + + ```bash + git clone + cd audiobook_dev + ``` + +2. **Create and activate a virtualenv** + + ```bash + python3 -m venv .venv + source .venv/bin/activate + ``` + +3. **Install dependencies** + + ```bash + make install-dev + ``` + + This installs the upstream `mkb79/Audible` package from GitHub for the authenticated Audible backend. + +4. **Copy and edit config** + - Edit `config/config.yaml` for your environment (API URLs, notification settings, etc). + - Create a `.env` file with your secrets (see `.env.example`). + +--- + +## Running + +```bash +uvicorn src.main:app --host 0.0.0.0 --port 8000 --reload +``` + +- The webhook endpoint is set in `config.yaml` (default: `/webhook/audiobook-requests`). +- The web UI is available at `/`. + +--- + +## Notifications + +- **Pushover**: Rich HTML, cover image, approval link. +- **Discord**: Embed with cover, links, and markdown. +- **Gotify**: Markdown, cover image, action links. +- **ntfy**: Markdown, cover image, action links. + +Configure each in `config/config.yaml` and `.env`. + +--- + +## Metadata + +- Uses Audnex API for fast, reliable metadata. +- Uses `mkb79/Audible` with an encrypted auth file for Audible-backed search. +- Cleans and normalizes author, narrator, series, and description fields. +- Caches lookups with LRU cache for efficiency. + +--- + +## Testing + +- Run all tests: + + ```bash + pytest -vv + ``` + +- Tests cover: + - Metadata cleaning and validation + - Notification formatting + - Web UI endpoints + - Error cases +- Fixtures in `tests/conftest.py` for isolation. + +--- + +## Development + +- Code style: Black, isort, flake8 recommended. +- Logging is configurable in `config.yaml`. +- All user input is sanitized before rendering or sending to notification services. +- For async/production, consider running with Gunicorn/Uvicorn workers. + +--- + +## Security + +- Webhook endpoints require a token (set in `.env`). +- Never commit `.env` or real secrets. +- All user input is sanitized. + +--- + +## License + +MIT License. See `LICENSE` for details. diff --git a/docs/vendor/audible/authentication.md b/docs/vendor/audible/authentication.md new file mode 100644 index 0000000..79bc65b --- /dev/null +++ b/docs/vendor/audible/authentication.md @@ -0,0 +1,48 @@ +# Audible Authentication + +Source URLs: + +- +- + +## Supported Modes + +The docs describe two API authentication modes: + +- sign request +- bearer token + +## Sign Request + +This is the preferred mode because it provides unrestricted API access. + +It relies on device registration data such as: + +- RSA private key +- `adp_token` + +The package applies this automatically when the `Authenticator` has the required data. + +## Bearer Mode + +Bearer mode is more limited. + +The docs specifically note that some calls, including content license requests, do not work with bearer-only auth. + +Headers look like: + +```text +Authorization: Bearer Atna|... +client-id: 0 +``` + +## Website Cookies + +The `Authenticator` also exposes website cookies that can be used with `httpx.Client` for web endpoints that are not part of the external Audible API. + +## Practical Guidance For This Repo + +- do not store Audible usernames or passwords in repo files +- prefer an external auth file loaded with `Authenticator.from_file(...)` +- keep auth material outside committed files +- treat Audible auth as optional and secondary to ABS plus Audnex during early development diff --git a/docs/vendor/audible/config/config.yaml.example b/docs/vendor/audible/config/config.yaml.example new file mode 100644 index 0000000..2bd88dd --- /dev/null +++ b/docs/vendor/audible/config/config.yaml.example @@ -0,0 +1,138 @@ +audnex: + api_url: "https://api.audnex.us/books" +payload: + # Regex to extract ASIN from torrent name or payload fields + asin_regex: "B[0-9A-Z]{9}" + required_keys: + - name + - url + - download_url + +######################################################### +# Configuration for Audiobook metadata and sources +# This section defines how to fetch and handle audiobook metadata +# This includes API endpoints, rate limits, and other settings +######################################################### +metadata: + rate_limit_seconds: 10 # seconds between API calls + mam: + base_url: "https://www.myanonamouse.net" + audnex: + base_url: "https://api.audnex.us" + rate_limit_seconds: 0.15 # 150ms between requests + # Try multiple regions if one fails - ordered by priority/success rate + regions: ["us", "uk", "es", "ca", "au", "de", "fr", "it", "jp", "in"] + try_all_regions_on_error: true # If a region returns error, try others + max_regions_to_try: 9 # Maximum number of regions to try (prevents excessive API calls) + audible: + base_url: "https://api.audible.com" + search_endpoint: "/1.0/catalog/products" + auth_file: "secrets/audible-auth.json" # Optional mkb79/Audible auth file for authenticated fallback searches + + +notifications: + pushover: + enabled: true + sound: roxy_waterball + html: 1 + priority: 0 + ntfy: + enabled: false # Enable ntfy notifications + topic: "audiobook-requests" # Set your ntfy.sh topic here + url: "https://ntfy.sh" # Base ntfy URL (or use your own ntfy server) + icon_url: "https://ptpimg.me/4larvz.jpg" # Icon for ntfy notifications + discord: + icon_url: "https://ptpimg.me/44pi19.png" + author_url: "https://example.com/audiobookshelf/" # Replace with your Audiobookshelf URL + footer_icon_url: "https://ptpimg.me/44pi19.png" + footer_text: "Powered by Autobrr" + + +server: + host: "0.0.0.0" + port: 8000 + reload: true + base_url: "https://your-domain.com" # Replace with your domain + autobrr_webhook_endpoint: "/webhook/audiobook-requests" # autobrr webhook token location .env "AUTOBRR_TOKEN" + reply_token_ttl: 3600 # 1 hour in seconds + # approve_success_autoclose: 10 # seconds to auto-close success page + # reject_autoclose: 10 # seconds to auto-close rejection page + # token_expired_autoclose: 10 # seconds to auto-close token expired page + +logging: + level: "DEBUG" # Options: DEBUG, INFO, WARNING, ERROR, CRITICAL + format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + file: "logs/audiobook_requests.log" + max_size: 10 # MB + max_files: 5 # Number of log files to keep (used with size-based rotation) + backup_count: 5 # Number of backup log files to keep (alternative to max_files) + rotation: "midnight" # Rotate logs at midnight (time-based rotation) + compression: "zip" # Compress rotated logs + retention: "7 days" # How long to keep rotated logs (time-based cleanup) + + +qbittorrent: + enabled: true # disable for testing or if not using qBittorrent + host: "http://localhost:8080" # Replace with your qBittorrent host + qbittorrent_ui: "https://your-qbittorrent-url.com/" # Link to UI for notifications + use_auto_torrent_management: true # Use qBittorrent's auto management + content_layout: "Subfolder" # Subfolder or Original + category: "audiobooks" # Default category if not specified in webhook + tags: ["myanonamouse"] # Default tags + paused: true + +security: + # HTTPS enforcement + force_https: false # Set to true in production to redirect HTTP to HTTPS + + # Rate limiting + rate_limit_window: 3600 # Time window in seconds (1 hour) + max_tokens_per_window: 10 # Maximum tokens per IP per window + max_failed_logins: 5 # Max failed logins before temporary lockout + lockout_time: 300 # Seconds (5 minutes) + + # API security + api_key_enabled: true # Enable API key requirement for admin endpoints + api_key: "your-strong-api-key-here" # CHANGE THIS! Strong API key for admin endpoints + # WARNING: Runtime validation will reject default placeholder values. Generate a strong random key before deployment. + + # Endpoint protection + endpoint_protection_enabled: true # Enable endpoint authorization checks + require_auth_for_unknown_endpoints: false # Require auth for endpoints not explicitly listed + protected_endpoints: # Additional endpoints that require authentication + - "/admin" + - "/api/admin" + - "/config" + - "/logs" + - "/stats" + - "/health/detailed" + - "/debug" + public_endpoints: # Additional endpoints that are publicly accessible + - "/" + - "/static" + - "/approve" + - "/reject" + - "/health" + - "/favicon.ico" + + # CSRF protection + csrf_protection: true # Enable CSRF protection for forms + + # JavaScript security + use_external_js: true # Use external JS files for stricter CSP (recommended) + + # Note: For production environments, consider tightening brute-force settings: + # - Lower max_failed_logins to 3 + # - Increase lockout_time to 900 (15 minutes) + + # Input validation + max_payload_size: 1048576 # 1MB max payload size + sanitize_inputs: true # Sanitize inputs to prevent XSS + + # Content Security + allowed_image_domains: + - "your-image-host.com" # Replace with your image hosting domain + - "ptpimg.me" + - "i.imgur.com" + - "audnex.us" + - "m.media-amazon.com" diff --git a/docs/vendor/audible/docs/README.md b/docs/vendor/audible/docs/README.md new file mode 100644 index 0000000..59bfbb2 --- /dev/null +++ b/docs/vendor/audible/docs/README.md @@ -0,0 +1,110 @@ +# πŸ“š Audiobook Automation Documentation + +Welcome to the comprehensive documentation for the Audiobook Automation System! This documentation is organized to help both users and developers understand, use, and contribute to the system. + +## Quick Start + +New to the system? Start here: + +1. [Getting Started](user-guide/getting-started.md) - Installation and basic setup +2. [Configuration](user-guide/configuration.md) - Configure the system for your needs +3. [Web Interface](user-guide/web-interface.md) - Using the web UI + +## Documentation Structure + +### 🎯 User Guide (`user-guide/`) + +Documentation for end users who want to use the audiobook automation system. + +- **[Getting Started](user-guide/getting-started.md)** - Installation, setup, and first run +- **[Configuration](user-guide/configuration.md)** - Configuration options and examples +- **[Web Interface](user-guide/web-interface.md)** - Using the web UI for approvals and monitoring +- **[Notifications](user-guide/notifications.md)** - Setting up Discord, Pushover, etc. +- **[Troubleshooting](user-guide/troubleshooting.md)** - Common issues and solutions + +### πŸ› οΈ Development (`development/`) + +Documentation for developers who want to understand, modify, or contribute to the codebase. + +- **[Architecture](development/architecture.md)** - System architecture and design overview +- **[Security](development/SECURITY.md)** - Security considerations and best practices +- **[Testing](development/testing.md)** - Testing strategies, test suite, and guidelines +- **[Contributing](development/contributing.md)** - How to contribute to the project + +### πŸ”Œ API Reference (`api/`) + +Technical API documentation for integrations and advanced usage. + +- **[REST API](api/rest-api.md)** - HTTP API endpoints and examples +- **[Configuration Reference](api/config-reference.md)** - Complete configuration options + +## 🎯 Key Features Documented + +### βœ… Core System + +- **Audiobook Request Processing** - Automated processing of audiobook requests +- **MAM Integration** - MyAnonaMouse scraping and ASIN extraction +- **Metadata Enrichment** - Audnex and Audible metadata fetching +- **Web Interface** - Modern approval/rejection interface +- **Security** - CSRF protection, rate limiting, input validation + +### βœ… Advanced Features + +- **Rate Limiting** - Configurable API rate limiting (30s test, 120s production) +- **Fallback Systems** - Multiple metadata sources with intelligent fallbacks +- **Notification Systems** - Discord, Pushover, Gotify, NTFY support +- **Webhook Integration** - Autobrr and other webhook sources + +## πŸ“Š System Status + +- βœ… **Production Ready** - All core features tested and working +- βœ… **Security Audited** - Comprehensive security testing completed +- βœ… **Well Tested** - Full test suite with real data validation +- βœ… **Documented** - Complete documentation for users and developers + +## πŸ”— External Resources + +- **MyAnonaMouse** - Primary torrent source for audiobooks +- **Audnex API** - Rich audiobook metadata and chapter information +- **Audible API** - Fallback metadata source + +## πŸ“ Archive + +Historical development documentation and implementation logs are stored in `archive/` for reference but are not part of the current documentation. + +--- + +**Last Updated**: May 6, 2026 +**System Version**: Production v1.0 + +- [Database Schema](api/database.md) - Database structure +- [Configuration Reference](api/config-reference.md) - Complete configuration options + +## πŸš€ Quick Links + +- **[Installation Guide](user-guide/getting-started.md#installation)** - Get up and running quickly +- **[Configuration Examples](user-guide/configuration.md#examples)** - Common configuration scenarios +- **[API Documentation](api/rest-api.md)** - For developers integrating with the system +- **[Troubleshooting](user-guide/troubleshooting.md)** - When things go wrong + +## πŸ€– About This System + +This audiobook automation system was built by Quentin with the philosophy of "maximum automation, minimum manual intervention." It handles: + +- **Automated Approval Workflows** - Smart request processing +- **Multi-platform Notifications** - Discord, Gotify, Ntfy, Pushover +- **Security-First Design** - Token-based authentication, CSP compliance +- **Modern Web Interface** - Beautiful, responsive UI with personality +- **Comprehensive Logging** - Detailed audit trails and debugging + +## πŸ“ Documentation Standards + +All documentation in this project follows these standards: + +- **Clear Structure** - Organized sections with logical flow +- **Practical Examples** - Real-world usage scenarios +- **Up-to-date** - Regularly maintained and current +- **Accessible** - Written for both beginners and experts +- **Searchable** - Well-indexed with consistent terminology + +**Need help?** Check the [troubleshooting guide](user-guide/troubleshooting.md) or open an issue on GitHub! diff --git a/docs/vendor/audible/docs/SYSTEM_COMPLETION_SUMMARY.md b/docs/vendor/audible/docs/SYSTEM_COMPLETION_SUMMARY.md new file mode 100644 index 0000000..6fb6ee3 --- /dev/null +++ b/docs/vendor/audible/docs/SYSTEM_COMPLETION_SUMMARY.md @@ -0,0 +1,153 @@ +# πŸš€ AUDIOBOOK SYSTEM COMPLETION SUMMARY + +## Date: June 18, 2025 + +## βœ… COMPLETED FEATURES + +### πŸ”§ **Core System Refactoring** + +- **Modular Architecture**: Refactored metadata workflow into separate, focused modules + - `mam_api/` - MAM JSON API client, models, and metadata adapter + - `audnex_metadata.py` - Comprehensive metadata cleaning and enrichment + - `audible_scraper.py` - Authenticated Audible metadata backend + - `metadata_coordinator.py` - Orchestrates the entire workflow + +### ⚑ **Async & Concurrency** + +- **Global Queue System**: Implemented `asyncio.Queue` for safe, sequential webhook processing +- **Background Worker**: Persistent worker thread processes requests without blocking +- **Rate Limiting**: Global rate limiting across all metadata sources +- **MAM API Client**: Uses MAM's JSON API through httpx instead of browser automation + +### πŸ“Š **Monitoring & Health** + +- **Health Endpoint**: `/health` - Public health check for monitoring +- **Queue Status**: `/queue/status` - Internal queue monitoring (IP-restricted) +- **Security Documentation**: Comprehensive reverse proxy security guide +- **Logging**: Enhanced logging with request IDs and structured output + +### πŸ“§ **Metadata & Notifications** + +- **Complete Field Passthrough**: All webhook and metadata fields preserved +- **Robust Field Extraction**: `get_notification_fields()` handles all metadata formats +- **Narrator & Series Support**: Comprehensive extraction from multiple field formats +- **HTML Cleaning**: Sanitizes descriptions for notifications +- **Size Formatting**: Human-readable file size display + +### 🎨 **UI/UX Enhancements** + +- **Automatic Light/Dark Mode**: Both approval AND rejection pages adapt to browser's `prefers-color-scheme` +- **CSS Variables**: Complete variable system for consistent theming across all pages +- **Cyberpunk Aesthetic**: Maintained dark theme with light mode compatibility for both approval and rejection +- **Responsive Design**: Works across different screen sizes +- **CSS Test Pages**: Development endpoints `/css-test` and `/rejection-css-test` for theme validation + +### πŸ”’ **Security** + +- **Endpoint Protection**: Queue status restricted to local IPs +- **API Key Support**: Optional additional security layer +- **Rate Limiting**: Protection against abuse +- **CSP Headers**: Content Security Policy implementation + +### πŸ§ͺ **Testing & Validation** + +- **Comprehensive Test Suite**: Full system validation script +- **Integration Tests**: End-to-end workflow testing +- **CSS Test Page**: `/css-test` endpoint for theme validation +- **Metadata Tests**: Validates field extraction and formatting + +## πŸ“ **KEY FILES MODIFIED/CREATED** + +### Core Application + +- `src/main.py` - FastAPI app with queue system and endpoints +- `src/metadata_coordinator.py` - Async metadata orchestration +- `src/mam_api/` - MAM JSON API integration +- `src/audnex_metadata.py` - Comprehensive metadata cleaning +- `src/utils.py` - Enhanced notification field extraction + +### UI/Styling + +- `static/css/pages/approval.css` - Light/dark mode with CSS variables +- `static/css/pages/rejection.css` - Light/dark mode with CSS variables (NEW) +- `templates/approval.html` - Approval page template +- `templates/rejection.html` - Rejection page template +- `templates/css_test.html` - CSS testing page + +### Configuration & Documentation + +- `config/config.yaml` - Rate limits and service configuration +- `docs/security/REVERSE_PROXY_SECURITY.md` - Nginx security guide +- `test_system_validation.py` - Comprehensive system tests + +### Testing + +- Multiple test scripts for metadata, queues, and integration testing +- Real webhook payload testing +- MAM login validation scripts + +## 🎯 **PRODUCTION READINESS** + +### βœ… Validated Features + +1. **Health Monitoring**: Service responds to health checks +2. **Queue Processing**: Sequential webhook processing with rate limiting +3. **Metadata Extraction**: All fields (narrators, series, etc.) properly extracted +4. **Theme Adaptation**: Automatic light/dark mode switching on BOTH approval and rejection pages +5. **Security**: Proper endpoint protection and access control +6. **CSS Test Pages**: Development endpoints for both approval and rejection themes + +### πŸ”§ **System Status** + +- **Service Running**: βœ… Active on port 8000 +- **Queue Empty**: βœ… 0 pending requests +- **All Tests Passing**: βœ… 5/5 validation tests successful +- **Endpoints Active**: βœ… Health, queue status, webhook, CSS test endpoints all functional +- **CSS Variables**: βœ… Complete light/dark mode support on BOTH approval and rejection pages + +## 🚦 **NEXT STEPS (Optional)** + +### High Priority + +- **Production Deployment**: Configure reverse proxy (Nginx/SWAG) with security settings +- **Monitoring Setup**: Configure log rotation and monitoring alerts +- **Rate Limit Tuning**: Adjust rate limits based on actual usage patterns + +### Medium Priority + +- **Additional Notification Channels**: Extend notification system if needed +- **Metadata Caching**: Add caching layer for frequently requested metadata +- **UI Polish**: Minor CSS refinements based on user feedback + +### Low Priority + +- **Advanced Analytics**: Queue performance metrics and statistics +- **Admin Interface**: Web-based configuration and monitoring panel +- **API Documentation**: Swagger/OpenAPI documentation generation + +## πŸ“ˆ **PERFORMANCE CHARACTERISTICS** + +- **Queue Capacity**: 50 concurrent requests +- **Rate Limiting**: 120 seconds between metadata API calls +- **Memory Usage**: Minimal due to async design +- **Response Times**: + - Health check: ~1ms + - Queue status: ~5ms + - Webhook processing: Async (no blocking) + +## πŸŽ‰ **CONCLUSION** + +The audiobook approval system has been successfully refactored into a robust, production-ready application with: + +- **Modular, maintainable code architecture** +- **Async processing with proper concurrency controls** +- **Comprehensive metadata handling with full field passthrough** +- **Modern, adaptive UI supporting both light and dark themes** +- **Proper security controls and monitoring capabilities** +- **Complete test coverage and validation** + +The system is now **READY FOR PRODUCTION** and can handle real-world webhook traffic with proper rate limiting, error handling, and metadata processing. + +--- + +*Generated on June 18, 2025 - All systems operational* ✨ diff --git a/docs/vendor/audible/docs/api/config-reference.md b/docs/vendor/audible/docs/api/config-reference.md new file mode 100644 index 0000000..ecaf216 --- /dev/null +++ b/docs/vendor/audible/docs/api/config-reference.md @@ -0,0 +1,548 @@ +# πŸ“‹ Configuration Reference + +Complete reference for all configuration options in the Audiobook Automation System. + +## πŸ“ Configuration File Structure + +The main configuration is stored in `config/config.yaml` using YAML format. + +```yaml +# Example complete configuration +server: + host: "0.0.0.0" + port: 8000 + debug: false + workers: 1 + +database: + path: "db.sqlite" + backup_enabled: true + backup_interval_hours: 24 + +security: + token_expiry_hours: 24 + max_requests_per_hour: 10 + allowed_hosts: [] + cors_enabled: false + +notifications: + enabled: true + discord: + enabled: false + webhook_url: "" + username: "Audiobook Bot" + color: 0xFF69B4 + + gotify: + enabled: false + server_url: "" + app_token: "" + priority: 5 + + ntfy: + enabled: false + server_url: "https://ntfy.sh" + topic: "" + priority: "default" + + pushover: + enabled: false + user_key: "" + api_token: "" + priority: 0 + +qbittorrent: + enabled: false + host: "localhost" + port: 8080 + username: "" + password: "" + download_path: "/downloads" + category: "audiobooks" + +metadata: + audnex_enabled: true + audible_enabled: true + cache_expiry_hours: 168 + +logging: + level: "INFO" + file_enabled: true + file_path: "logs/audiobook_requests.log" + max_file_size_mb: 10 + backup_count: 5 + console_enabled: true +``` + +## 🌐 Server Configuration + +### `server.host` + +- **Type:** String +- **Default:** `"0.0.0.0"` +- **Description:** Host address to bind the server to +- **Examples:** + - `"localhost"` - Local access only + - `"0.0.0.0"` - All interfaces + - `"192.168.1.100"` - Specific IP address + +### `server.port` + +- **Type:** Integer +- **Default:** `8000` +- **Range:** 1-65535 +- **Description:** Port number for the web server + +### `server.debug` + +- **Type:** Boolean +- **Default:** `false` +- **Description:** Enable debug mode with enhanced logging and error details +- **⚠️ Warning:** Never enable in production + +### `server.workers` + +- **Type:** Integer +- **Default:** `1` +- **Description:** Number of worker processes (for production deployment) + +--- + +## πŸ’Ύ Database Configuration + +### `database.path` + +- **Type:** String +- **Default:** `"db.sqlite"` +- **Description:** Path to SQLite database file +- **Examples:** + - `"db.sqlite"` - Relative path + - `"/var/lib/audiobook/db.sqlite"` - Absolute path + - `":memory:"` - In-memory database (testing only) + +### `database.backup_enabled` + +- **Type:** Boolean +- **Default:** `true` +- **Description:** Enable automatic database backups + +### `database.backup_interval_hours` + +- **Type:** Integer +- **Default:** `24` +- **Description:** Hours between automatic backups + +--- + +## πŸ” Security Configuration + +### `security.token_expiry_hours` + +- **Type:** Integer +- **Default:** `24` +- **Range:** 1-168 (1 week max) +- **Description:** Hours before approval/rejection tokens expire + +### `security.max_requests_per_hour` + +- **Type:** Integer +- **Default:** `10` +- **Description:** Maximum requests per IP address per hour + +### `security.allowed_hosts` + +- **Type:** Array of Strings +- **Default:** `[]` (all hosts allowed) +- **Description:** Restrict access to specific hostnames +- **Example:** + + ```yaml + allowed_hosts: + - "audiobooks.example.com" + - "localhost" + ``` + +### `security.cors_enabled` + +- **Type:** Boolean +- **Default:** `false` +- **Description:** Enable Cross-Origin Resource Sharing + +--- + +## πŸ“± Notification Configuration + +### Global Notification Settings + +#### `notifications.enabled` + +- **Type:** Boolean +- **Default:** `true` +- **Description:** Enable/disable all notifications + +### Discord Notifications + +#### `notifications.discord.enabled` + +- **Type:** Boolean +- **Default:** `false` +- **Description:** Enable Discord notifications + +#### `notifications.discord.webhook_url` + +- **Type:** String +- **Required:** Yes (if Discord enabled) +- **Description:** Discord webhook URL +- **Example:** `"https://discord.com/api/webhooks/123456789/abcdef..."` + +#### `notifications.discord.username` + +- **Type:** String +- **Default:** `"Audiobook Bot"` +- **Description:** Bot username for Discord messages + +#### `notifications.discord.color` + +- **Type:** Integer (Hex) +- **Default:** `0xFF69B4` +- **Description:** Embed color for Discord messages +- **Examples:** + - `0xFF69B4` - Hot pink + - `0x00FF00` - Green + - `0x0099FF` - Blue + +### Gotify Notifications + +#### `notifications.gotify.enabled` + +- **Type:** Boolean +- **Default:** `false` +- **Description:** Enable Gotify notifications + +#### `notifications.gotify.server_url` + +- **Type:** String +- **Required:** Yes (if Gotify enabled) +- **Description:** Gotify server URL +- **Example:** `"https://gotify.example.com"` + +#### `notifications.gotify.app_token` + +- **Type:** String +- **Required:** Yes (if Gotify enabled) +- **Description:** Gotify application token + +#### `notifications.gotify.priority` + +- **Type:** Integer +- **Default:** `5` +- **Range:** 0-10 +- **Description:** Message priority level + +### Ntfy Notifications + +#### `notifications.ntfy.enabled` + +- **Type:** Boolean +- **Default:** `false` +- **Description:** Enable Ntfy notifications + +#### `notifications.ntfy.server_url` + +- **Type:** String +- **Default:** `"https://ntfy.sh"` +- **Description:** Ntfy server URL + +#### `notifications.ntfy.topic` + +- **Type:** String +- **Required:** Yes (if Ntfy enabled) +- **Description:** Ntfy topic name +- **Example:** `"audiobook_requests"` + +#### `notifications.ntfy.priority` + +- **Type:** String +- **Default:** `"default"` +- **Options:** `"max"`, `"high"`, `"default"`, `"low"`, `"min"` +- **Description:** Message priority + +### Pushover Notifications + +#### `notifications.pushover.enabled` + +- **Type:** Boolean +- **Default:** `false` +- **Description:** Enable Pushover notifications + +#### `notifications.pushover.user_key` + +- **Type:** String +- **Required:** Yes (if Pushover enabled) +- **Description:** Pushover user key + +#### `notifications.pushover.api_token` + +- **Type:** String +- **Required:** Yes (if Pushover enabled) +- **Description:** Pushover API token + +#### `notifications.pushover.priority` + +- **Type:** Integer +- **Default:** `0` +- **Range:** -2 to 2 +- **Description:** Message priority (-2=lowest, 2=emergency) + +--- + +## βš™οΈ qBittorrent Configuration + +### `qbittorrent.enabled` + +- **Type:** Boolean +- **Default:** `false` +- **Description:** Enable qBittorrent integration + +### `qbittorrent.host` + +- **Type:** String +- **Default:** `"localhost"` +- **Description:** qBittorrent server hostname/IP + +### `qbittorrent.port` + +- **Type:** Integer +- **Default:** `8080` +- **Description:** qBittorrent web UI port + +### `qbittorrent.username` + +- **Type:** String +- **Required:** Yes (if qBittorrent enabled) +- **Description:** qBittorrent web UI username + +### `qbittorrent.password` + +- **Type:** String +- **Required:** Yes (if qBittorrent enabled) +- **Description:** qBittorrent web UI password + +### `qbittorrent.download_path` + +- **Type:** String +- **Default:** `"/downloads"` +- **Description:** Download directory path + +### `qbittorrent.category` + +- **Type:** String +- **Default:** `"audiobooks"` +- **Description:** Category for audiobook torrents + +--- + +## πŸ“– Metadata Configuration + +### `metadata.audnex_enabled` + +- **Type:** Boolean +- **Default:** `true` +- **Description:** Enable Audnex API for metadata + +### `metadata.audible_enabled` + +- **Type:** Boolean +- **Default:** `true` +- **Description:** Enable authenticated Audible lookups for metadata + +### `metadata.cache_expiry_hours` + +- **Type:** Integer +- **Default:** `168` (1 week) +- **Description:** Hours to cache metadata responses + +--- + +## πŸ“‹ Logging Configuration + +### `logging.level` + +- **Type:** String +- **Default:** `"INFO"` +- **Options:** `"DEBUG"`, `"INFO"`, `"WARNING"`, `"ERROR"`, `"CRITICAL"` +- **Description:** Minimum log level to record + +### `logging.file_enabled` + +- **Type:** Boolean +- **Default:** `true` +- **Description:** Enable logging to file + +### `logging.file_path` + +- **Type:** String +- **Default:** `"logs/audiobook_requests.log"` +- **Description:** Log file path + +### `logging.max_file_size_mb` + +- **Type:** Integer +- **Default:** `10` +- **Description:** Maximum log file size before rotation + +### `logging.backup_count` + +- **Type:** Integer +- **Default:** `5` +- **Description:** Number of backup log files to keep + +### `logging.console_enabled` + +- **Type:** Boolean +- **Default:** `true` +- **Description:** Enable logging to console/stdout + +--- + +## 🌍 Environment Variables + +Sensitive configuration can be provided via environment variables: + +```bash +# Database +export DB_PATH="/var/lib/audiobook/db.sqlite" + +# Discord +export DISCORD_WEBHOOK_URL="https://discord.com/api/webhooks/..." + +# Gotify +export GOTIFY_SERVER_URL="https://gotify.example.com" +export GOTIFY_APP_TOKEN="AbCdEf123456" + +# Ntfy +export NTFY_TOPIC="audiobook_requests" + +# Pushover +export PUSHOVER_USER_KEY="abc123..." +export PUSHOVER_API_TOKEN="def456..." + +# qBittorrent +export QB_HOST="qbittorrent.local" +export QB_USERNAME="admin" +export QB_PASSWORD="password123" +``` + +Environment variables take precedence over YAML configuration. + +--- + +## πŸ“‹ Configuration Examples + +### Minimal Configuration + +```yaml +server: + port: 8000 + +notifications: + enabled: false + +qbittorrent: + enabled: false +``` + +### Production Configuration + +```yaml +server: + host: "0.0.0.0" + port: 8000 + debug: false + workers: 4 + +database: + path: "/var/lib/audiobook/db.sqlite" + backup_enabled: true + +security: + token_expiry_hours: 12 + max_requests_per_hour: 20 + allowed_hosts: + - "audiobooks.company.com" + +notifications: + enabled: true + discord: + enabled: true + webhook_url: "${DISCORD_WEBHOOK_URL}" + username: "Audiobook Bot" + +logging: + level: "INFO" + file_path: "/var/log/audiobook/requests.log" + max_file_size_mb: 50 + backup_count: 10 +``` + +### Development Configuration + +```yaml +server: + host: "localhost" + port: 8001 + debug: true + +database: + path: "dev_db.sqlite" + +notifications: + enabled: false + +logging: + level: "DEBUG" + console_enabled: true +``` + +--- + +## βœ… Configuration Validation + +The system validates configuration on startup: + +- **Required fields** - Ensures all mandatory settings are present +- **Type checking** - Validates data types (string, integer, boolean) +- **Range validation** - Checks numeric values are within acceptable ranges +- **Format validation** - Validates URLs, file paths, etc. +- **Dependency checking** - Ensures required settings for enabled features + +### Validation Errors + +Configuration errors are reported clearly: + +```text +Configuration Error: notifications.discord.webhook_url is required when Discord is enabled +Configuration Error: server.port must be between 1 and 65535 +Configuration Error: security.token_expiry_hours cannot exceed 168 (1 week) +``` + +--- + +## πŸ”„ Dynamic Configuration + +Some settings can be updated without restarting: + +- **Notification settings** - Webhook URLs, priorities +- **Logging levels** - Change verbosity on the fly +- **Rate limits** - Adjust request limits +- **Metadata cache** - Clear or update cache settings + +Send a `SIGHUP` signal to reload configuration: + +```bash +kill -HUP $(pgrep -f "python.*main.py") +``` + +--- + +**Need help with configuration?** Check the [Getting Started Guide](../user-guide/getting-started.md) or [Troubleshooting Guide](../user-guide/troubleshooting.md)! diff --git a/docs/vendor/audible/docs/user-guide/configuration.md b/docs/vendor/audible/docs/user-guide/configuration.md new file mode 100644 index 0000000..e65273b --- /dev/null +++ b/docs/vendor/audible/docs/user-guide/configuration.md @@ -0,0 +1,199 @@ +# βš™οΈ Configuration Guide + +This guide covers all configuration options for the Audiobook Automation System. + +## πŸ“ Configuration Files + +All configuration files are located in the `config/` directory: + +```text +config/ +β”œβ”€β”€ config.yaml # Main application configuration +└── config.yaml.example # Template for main config +``` + +## πŸ”§ Main Configuration (`config.yaml`) + +### Server Settings + +```yaml +server: + host: "127.0.0.1" + port: 8080 + debug: false +``` + +### Database + +```yaml +database: + path: "db.sqlite" + backup_enabled: true + backup_interval_hours: 24 +``` + +### Security + +```yaml +security: + csrf_enabled: true + token_length: 32 + rate_limit: + enabled: true + max_requests: 10 + window_hours: 1 +``` + +### Metadata Workflow + +```yaml +metadata: + rate_limit_seconds: 120 # Production: 120s, Testing: 30s + sources: + mam: + enabled: true + timeout_seconds: 30 + audnex: + enabled: true + base_url: "https://api.audnex.us" + timeout_seconds: 10 + audible: + enabled: true + base_url: "https://api.audible.com" + search_endpoint: "/1.0/catalog/products" + auth_file: "secrets/audible-auth.json" # Optional encrypted auth file for mkb79/Audible +``` + +### Notifications + +```yaml +notifications: + discord: + enabled: false + webhook_url: "" # Set in .env as DISCORD_WEBHOOK_URL + + pushover: + enabled: false + user_key: "" # Set in .env as PUSHOVER_USER_KEY + api_token: "" # Set in .env as PUSHOVER_API_TOKEN +``` + +## πŸ” Environment Variables (`.env`) + +Create a `.env` file for sensitive configuration: + +```bash +# Required for webhook authentication +AUTOBRR_TOKEN=your-autobrr-webhook-token + +# Notification services (optional) +DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/your/webhook/url +PUSHOVER_USER_KEY=your-pushover-user-key +PUSHOVER_API_TOKEN=your-pushover-api-token +GOTIFY_URL=https://gotify.example.com +GOTIFY_TOKEN=your-gotify-token +NTFY_URL=https://ntfy.sh/your-topic + +# MAM API auth (optional, required for MAM metadata lookups) +MAM_ID=your-mam-session-cookie-value + +# Authenticated Audible backend +# AUDIBLE_AUTH_FILE=secrets/audible-auth.json +# AUDIBLE_AUTH_FILE_PASSWORD=your-audible-auth-file-password +``` + +## πŸ” MAM API Configuration (Optional) + +For full MAM integration with ASIN extraction, set `MAM_ID` in `.env` to the value of your MAM `mam_id` browser cookie. The application uses MAM's JSON API directly and does not log in through the website. + +### Find the Cookie Value + +```bash +MAM_ID=your-mam-session-cookie-value +``` + +Security note: `MAM_ID` is a session token. Keep it only in `.env`, never commit it, and rotate it if it is shared or exposed. + +## 🎧 Authenticated Audible Integration + +The Audible backend now uses `mkb79/Audible`. Configure both `AUDIBLE_AUTH_FILE` and `AUDIBLE_AUTH_FILE_PASSWORD` so the app can decrypt the stored auth JSON and authenticate requests. + +The encrypted auth file format used by `Authenticator.from_file(...)` matches the `salt` / `iv` / `ciphertext` JSON envelope already used by this project. + +`AUDIBLE_AUTH_FILE_PASSWORD` is the decryption password for the auth file. It is not your Audible or Amazon login password. + +Installation note: this repo installs `mkb79/Audible` directly from GitHub because the PyPI release is behind upstream. The supported Python range for the Audible install path is 3.11-3.13. + +## 🎯 Configuration Examples + +### Development/Testing + +```yaml +metadata: + rate_limit_seconds: 30 # Faster testing +server: + debug: true # Enable debug mode +``` + +### Production + +```yaml +metadata: + rate_limit_seconds: 120 # Respectful API usage +server: + debug: false # Disable debug mode +security: + rate_limit: + max_requests: 5 # Stricter rate limiting +``` + +## βœ… Configuration Validation + +Test your configuration: + +```bash +# Test main config +python -c "from src.config import load_config; load_config(); print('βœ… Config valid')" + +# Test MAM API auth (if configured) +pytest tests/test_mam_api.py -k Integration --no-cov + +# Test metadata workflow +python tests/test_metadata_workflow.py +``` + +## πŸ”§ Troubleshooting + +### Common Issues + +**Config file not found:** + +```bash +cp config/config.yaml.example config/config.yaml +``` + +**MAM API auth fails:** + +- Verify `MAM_ID` in `.env` is the current `mam_id` cookie value +- Log in to MAM in your browser and refresh the cookie value if the API reports authentication failure +- Make sure the value is not URL-encoded twice or surrounded by quotes + +**Rate limiting too slow:** + +- Adjust `metadata.rate_limit_seconds` in config.yaml +- Use 30s for testing, 120s for production + +**Webhook authentication fails:** + +- Verify `AUTOBRR_TOKEN` in `.env` file +- Check autobrr webhook configuration + +## πŸ“‹ Configuration Checklist + +- [ ] `config/config.yaml` created and configured +- [ ] `.env` file created with required tokens +- [ ] `MAM_ID` set in `.env` (if using MAM) +- [ ] `AUDIBLE_AUTH_FILE` and `AUDIBLE_AUTH_FILE_PASSWORD` set +- [ ] Configuration validated with test scripts +- [ ] Notification services tested (if enabled) +- [ ] Rate limiting configured appropriately diff --git a/docs/vendor/audible/docs/user-guide/troubleshooting.md b/docs/vendor/audible/docs/user-guide/troubleshooting.md new file mode 100644 index 0000000..ff37e67 --- /dev/null +++ b/docs/vendor/audible/docs/user-guide/troubleshooting.md @@ -0,0 +1,490 @@ +# πŸ”§ Troubleshooting Guide + +Common issues and solutions for the Audiobook Automation System. + +## 🚨 System Won't Start + +### Port Already in Use + +**Error:** `Address already in use: 8080` + +**Solution:** + +```bash +# Find process using port 8080 +sudo lsof -i :8080 + +# Kill the process (replace PID) +sudo kill -9 + +# Or use a different port in config.yaml +server: + port: 8081 +``` + +### Missing Configuration + +**Error:** `Config file not found` + +**Solution:** + +```bash +# Copy example config +cp config/config.yaml.example config/config.yaml + +# Create .env file +cp .env.example .env +# Edit .env with your tokens +``` + +### Python Dependencies + +**Error:** `ModuleNotFoundError: No module named 'xyz'` + +**Solution:** + +```bash +# Install dependencies +pip install -r requirements.txt + +# Or use virtual environment +python -m venv .venv +source .venv/bin/activate # Linux/Mac +# .venv\Scripts\activate # Windows +pip install -r requirements.txt +``` + +## πŸ” Authentication Issues + +### Webhook Token Mismatch + +**Error:** `401 Unauthorized` on webhook requests + +**Solution:** + +1. Check `AUTOBRR_TOKEN` in `.env` file +2. Verify autobrr webhook configuration +3. Test token manually: + +```bash +read -rsp "AUTOBRR_TOKEN: " AUTOBRR_TOKEN && echo +curl -H "X-Autobrr-Token: $AUTOBRR_TOKEN" http://localhost:8080/webhook/test +``` + +Run the command after exporting or reading `AUTOBRR_TOKEN` so you can verify the request without printing the token. + +### CSRF Token Issues + +**Error:** `CSRF token mismatch` + +**Solution:** + +1. Clear browser cache and cookies +2. Refresh the page +3. Check if `csrf_enabled: true` in config.yaml +4. Verify browser accepts cookies + +## 🌐 Web Interface Problems + +### Page Not Loading + +**Symptoms:** Blank page or 404 errors + +**Solutions:** + +1. **Check server status:** + +```bash +# Verify server is running +ps aux | grep python +``` + +1. **Check logs:** + +```bash +tail -f logs/audiobook_requests.log +``` + +1. **Test direct access:** + +```bash +curl http://localhost:8080 +``` + +### JavaScript Errors + +**Symptoms:** Buttons not working, keyboard shortcuts broken + +**Solutions:** + +1. **Open browser console** (F12) +2. **Clear browser cache** +3. **Check for JavaScript errors:** + - Look for red errors in console + - Verify static files are loading +4. **Test in incognito mode** + +### Mobile Interface Issues + +**Symptoms:** Interface not responsive on mobile + +**Solutions:** + +1. **Clear mobile browser cache** +2. **Test in different mobile browsers** +3. **Check viewport meta tag** in templates +4. **Verify CSS media queries** are working + +## πŸ“Š Metadata Workflow Issues + +### MAM API Auth Failed + +**Error:** `MAM API authentication failed; update MAM_ID` + +**Solutions:** + +1. **Check API cookie:** + +```bash +# Confirm MAM_ID is present without printing the value +test -n "$MAM_ID" && echo "MAM_ID is set" + +pytest tests/test_mam_api.py -k Integration --no-cov +``` + +1. **Refresh the cookie value:** + +Verify the account is active, log in to MAM in your browser, copy the current `mam_id` cookie value into `.env` as `MAM_ID`, and restart the app so the environment reloads. + +### ASIN Not Found + +**Error:** `No ASIN found on MAM page` + +**This is normal behavior:** + +- Not all MAM torrents have ASINs +- System will fallback to Audible search +- Check logs for fallback success + +### Audnex API Timeout + +**Error:** `Audnex API timeout` or `Connection failed` + +**Solutions:** + +1. **Check Audnex status:** + +```bash +curl https://api.audnex.us/books/health +``` + +1. **Increase timeout:** + +```yaml +# In config.yaml +metadata: + sources: + audnex: + timeout_seconds: 30 # Increase from 10 +``` + +1. **Check network connectivity:** + +```bash +ping api.audnex.us +``` + +### Rate Limiting Too Slow + +**Issue:** Metadata workflow takes too long + +**Solutions:** + +1. **Adjust rate limit for testing:** + +```yaml +# In config.yaml (testing only) +metadata: + rate_limit_seconds: 30 # Instead of 120 +``` + +1. **Check last API call time:** + +```bash +# View coordinator logs +tail -f logs/metadata_coordinator.log +``` + +## πŸ”” Notification Issues + +### Discord Webhook Not Working + +**Error:** Discord notifications not received + +**Solutions:** + +1. **Test webhook URL:** + +```bash +curl -X POST -H "Content-Type: application/json" \ + -d '{"content":"Test message"}' \ + "$DISCORD_WEBHOOK_URL" +``` + +Do not paste or share your real Discord webhook URL in public issues or support channels. + +1. **Check webhook permissions:** + - Verify webhook has send message permissions + - Check channel permissions + +1. **Verify configuration:** + +```bash +# Check .env file +if grep -q '^DISCORD_WEBHOOK_URL=' .env; then echo 'DISCORD_WEBHOOK_URL set'; else echo 'DISCORD_WEBHOOK_URL unset'; fi + +# Test notification system +python -c "import os; print('DISCORD_WEBHOOK_URL set' if os.environ.get('DISCORD_WEBHOOK_URL') else 'DISCORD_WEBHOOK_URL unset')" +``` + +### Pushover Not Working + +**Error:** Pushover notifications not received + +**Solutions:** + +1. **Verify credentials:** + - Check User Key and API Token + - Test on Pushover website + +2. **Check device registration:** + - Install Pushover app + - Register device with account + +3. **Test API:** + +```bash +curl -s -F "token=YOUR_API_TOKEN" \ + -F "user=YOUR_USER_KEY" \ + -F "message=Test message" \ + https://api.pushover.net/1/messages.json +``` + +## πŸ’Ύ Database Issues + +### Database Locked + +**Error:** `Database is locked` + +**Solutions:** + +1. **Check for hung processes:** + +```bash +# Find processes using database +lsof db.sqlite +``` + +1. **Restart application:** + +```bash +# Stop only the audiobook service process +pkill -f "src/main.py" + +# Start fresh +python src/main.py +``` + +If `pkill` is too broad for your environment, use the specific PID you identified earlier with `kill ` or stop the relevant service/container instead. + +1. **Backup and recreate:** + +```bash +# Backup database +cp db.sqlite db.sqlite.backup + +# Remove lock files +rm -f db.sqlite-shm db.sqlite-wal +``` + +### Database Corruption + +**Error:** `Database disk image is malformed` + +**Solutions:** + +1. **Check database integrity:** + +```bash +sqlite3 db.sqlite "PRAGMA integrity_check;" +``` + +1. **Restore from backup:** + +```bash +# Find latest backup +ls -la db.sqlite* + +# Restore +cp db.sqlite.backup db.sqlite +``` + +1. **Recreate database:** + +```bash +# Last resort: recreate (loses data) +rm db.sqlite +python src/db.py # Recreates tables +``` + +## 🐳 Docker Issues + +### Container Won't Start + +**Error:** Docker container exits immediately + +**Solutions:** + +1. **Check container logs:** + +```bash +docker logs audiobook-automation +``` + +1. **Verify volume mounts:** + +```bash +# Check config files exist +ls -la config/ +``` + +1. **Test without Docker:** + +```bash +# Run directly to see errors +python src/main.py +``` + +### Port Mapping Issues + +**Error:** Cannot access web interface + +**Solutions:** + +1. **Check port mapping:** + +```bash +docker ps # Verify ports are mapped +``` + +1. **Test container networking:** + +```bash +# Access from within container +docker exec -it audiobook-automation curl localhost:8080 +``` + +## πŸ“ Log Analysis + +### Enable Debug Logging + +```yaml +# In config.yaml +server: + debug: true + +logging: + level: DEBUG +``` + +### Key Log Files + +```bash +# Main application log +tail -f logs/audiobook_requests.log + +# Metadata workflow +tail -f logs/metadata_coordinator.log + +# MAM API client +tail -f logs/audiobook_requests.log + +# Notifications +tail -f logs/notifications.log +``` + +### Log Patterns to Look For + +```bash +# Errors +grep -i "error" logs/*.log + +# Authentication issues +grep -i "auth\|token" logs/*.log + +# Rate limiting +grep -i "rate" logs/*.log + +# Database issues +grep -i "database\|sqlite" logs/*.log +``` + +## πŸ†˜ Getting Help + +### Information to Gather + +Before seeking help, collect: + +1. **System Information:** + +```bash +# Python version +python --version + +# OS information +uname -a + +# Package versions +pip freeze | grep -E "(fastapi|httpx|pydantic)" +``` + +1. **Configuration (sanitized):** + +```bash +# Remove sensitive data before sharing +cp config/config.yaml config/config-debug.yaml +# Edit config-debug.yaml to remove secrets +``` + +1. **Log Excerpts:** + +```bash +# Last 50 lines of relevant logs +tail -50 logs/audiobook_requests.log +tail -50 logs/metadata_coordinator.log +``` + +1. **Error Messages:** + - Full error text + - Steps to reproduce + - Expected vs actual behavior + +### Support Channels + +- Check documentation first +- Search existing issues +- Create detailed bug reports +- Include system information and logs + +## πŸ“‹ Troubleshooting Checklist + +- [ ] System requirements met +- [ ] Configuration files exist and valid +- [ ] Environment variables set +- [ ] Dependencies installed +- [ ] Ports available +- [ ] Network connectivity working +- [ ] Authentication tokens valid +- [ ] Database accessible +- [ ] Log files readable +- [ ] Error messages documented diff --git a/docs/vendor/audible/examples.md b/docs/vendor/audible/examples.md new file mode 100644 index 0000000..575809c --- /dev/null +++ b/docs/vendor/audible/examples.md @@ -0,0 +1,35 @@ +# Audible Examples + +Source URL: + +- + +## Marketplace Iteration Example + +```python +import audible + +filename = "path/to/credentials.json" +auth = audible.Authenticator.from_file(filename) +client = audible.Client(auth) +country_codes = ["de", "us", "ca", "uk", "au", "fr", "jp", "it", "in"] + +for country in country_codes: + client.switch_marketplace(country) + library = client.get("library", num_results=1000) + asins = [book["asin"] for book in library["items"]] + print(f"Country: {client.marketplace.upper()} | Number of books: {len(asins)}") +``` + +## Why This Matters Here + +This is useful for the importer because marketplace differences affect: + +- ASIN availability +- title variants like `Philosopher's` versus `Sorcerer's` +- narrator and edition differences +- region-unavailable results that may still exist in the local library + +## Stats Example + +The docs also show that `client.get(...)` can target other endpoints such as `1.0/stats/aggregates`, which confirms the client is a generic API wrapper rather than a library-only helper. diff --git a/docs/vendor/audible/external-api.md b/docs/vendor/audible/external-api.md new file mode 100644 index 0000000..c3e08e6 --- /dev/null +++ b/docs/vendor/audible/external-api.md @@ -0,0 +1,85 @@ +# Audible External API Reference + +Source URL: + +- + +## Important Note + +The Audible API is not publicly documented by Audible. The `audible` package docs provide community-maintained endpoint notes. + +Responses often return minimal data unless `response_groups` are requested. + +## Endpoints Relevant To This Importer + +### Library + +`GET /1.0/library` + +Useful query parameters: + +- `num_results` up to `1000` +- `page` +- `title` +- `author` +- `sort_by` +- `response_groups` + +Relevant response groups include: + +- `contributors` +- `media` +- `product_attrs` +- `product_desc` +- `product_details` +- `product_extended_attrs` +- `series` +- `relationships` +- `origin_asin` +- `pdf_url` + +`GET /1.0/library/{asin}` + +Use this for richer per-book library data when a title is already known. + +### Catalog + +`GET /1.0/catalog/products/{asin}` + +This is the key product lookup endpoint for the importer. + +Useful response groups: + +- `contributors` +- `media` +- `product_attrs` +- `product_desc` +- `product_details` +- `product_extended_attrs` +- `series` +- `relationships` +- `rating` +- `customer_rights` + +`GET /1.0/catalog/products` + +Supports batch lookup using `asins`. + +### Content + +`GET /1.0/content/{asin}/metadata` + +Potentially useful for content reference and chapter-related metadata. + +`POST /1.0/content/{asin}/licenserequest` + +Not needed for importer naming. This is relevant for download and DRM workflows, which should stay out of scope for now. + +## Recommended Usage For Naming + +For importer metadata, the best starting sequence is: + +1. enumerate owned titles with `GET /1.0/library` +2. enrich a specific ASIN with `GET /1.0/catalog/products/{asin}` +3. compare Audible fields against Audnex, ABS, and local filenames +4. use Audible as a tie-breaker and ownership-aware source, not the only source of truth diff --git a/docs/vendor/audible/getting-started.md b/docs/vendor/audible/getting-started.md new file mode 100644 index 0000000..9a99302 --- /dev/null +++ b/docs/vendor/audible/getting-started.md @@ -0,0 +1,53 @@ +# Audible Getting Started + +Source URL: + +- + +## Device Registration + +Before using the Audible API, you authorize against Amazon or Audible and register a virtual device. + +Reference pattern: + +```python +import audible + +auth = audible.Authenticator.from_login( + USERNAME, + PASSWORD, + locale=COUNTRY_CODE, + with_username=False, +) +auth.to_file(FILENAME) +``` + +Notes: + +- every device registration appears in the Amazon devices list +- the docs recommend registering once and reusing the saved auth file +- two-factor auth can be handled by appending the current OTP to the password in some cases + +## First Library Call + +Reference pattern: + +```python +with audible.Client(auth=auth) as client: + library = client.get( + "1.0/library", + num_results=1000, + response_groups="product_desc, product_attrs", + sort_by="-PurchaseDate", + ) +``` + +## Reusing Credentials + +Reference pattern: + +```python +auth = audible.Authenticator.from_file(FILENAME) +``` + +For this importer, file-based auth reuse is the right default. Interactive login should stay outside the normal import path. diff --git a/docs/vendor/audible/overview.md b/docs/vendor/audible/overview.md new file mode 100644 index 0000000..64a94c0 --- /dev/null +++ b/docs/vendor/audible/overview.md @@ -0,0 +1,53 @@ +# Audible Overview + +Source URLs: + +- +- + +## Workspace Notes + +- Installed package version in this workspace: `audible 0.8.2` +- ReadTheDocs pages currently describe `0.10.0` +- The core runtime surface we verified is present in `0.8.2`: + - `audible.Authenticator.from_file(...)` + - `audible.Authenticator.from_login(...)` + - `audible.Client` + - `audible.AsyncClient` + - `client.get/post/put/delete/switch_marketplace` + +## What The Package Is + +`audible` is a Python client for Audible's non-public API. It provides: + +- device registration and credential handling +- automatic request authentication +- synchronous and asynchronous clients +- access to library, catalog, content, and account endpoints + +## Why It Matters For This Importer + +Audible support gives the importer a second strong metadata source next to Audnex. + +Use it for: + +- direct library enumeration from the user's Audible account +- product metadata lookups by ASIN +- richer response groups than Audnex in some cases +- validating ASINs and edition variants against the user's owned library + +Do not use it as the only metadata source. It should complement: + +- Audiobookshelf item metadata +- Audnex product and chapter data +- local tags and filenames + +## Key Runtime Pattern + +```python +import audible + +auth = audible.Authenticator.from_file("credentials.json", password="...") +with audible.Client(auth=auth) as client: + library = client.get("1.0/library", num_results=1000) +``` diff --git a/docs/vendor/audible/raw/README.md b/docs/vendor/audible/raw/README.md new file mode 100644 index 0000000..972d462 --- /dev/null +++ b/docs/vendor/audible/raw/README.md @@ -0,0 +1,11 @@ +# Audible Raw Sources + +This directory mirrors the raw `View page source` text files from the Audible ReadTheDocs site. + +Use [manifest.json](manifest.json) to see the synced source URLs and local file paths. + +Refresh the mirror with: + +```bash +python scripts/sync_audible_raw_docs.py +``` diff --git a/docs/vendor/audible/raw/auth/authentication.rst.txt b/docs/vendor/audible/raw/auth/authentication.rst.txt new file mode 100644 index 0000000..62ec70f --- /dev/null +++ b/docs/vendor/audible/raw/auth/authentication.rst.txt @@ -0,0 +1,94 @@ +============== +Authentication +============== + +API Authentication +================== + +Audible uses the `sign request` or the `bearer` method to authenticate the +requests to the Audible API. + +The authentication is done automatically when using the +:class:`audible.Authenticator`. Simply use the ``Authenticator`` with +the :class:`audible.Client` or :class:`audible.AsyncClient` like so:: + + auth = audible.Authenticator.from_file(...) + client = audible.Client(auth=auth) + +The Authenticator will try to use the sign request method if available. +Otherwise the Authenticator will try the bearer method. If no method is +available an exception is raised. + +Sign request method +------------------- + +With the sign request method you gain unrestricted access to the Audible API. +To use this method, you need the RSA private key and the adp_token from a +*device registration*. This method is used by the Audible apps for iOS and +Android too. A device registration is done automatically with +:meth:`audible.Authenticator.from_login` or +:meth:`audible.Authenticator.from_login_external` + +Request signing is fairly straight-forward and uses a signed SHA256 digest. +Headers look like:: + + x-adp-alg: SHA256withRSA:1.0 + x-adp-signature: AAAAAAAA...:2019-02-16T00:00:01.000000000Z, + x-adp-token: {enc:...} + +Bearer method +------------- + +API requests with the bearer method have some restrictions. Some API call, like +the :http:post:`/1.0/content/(string:asin)/licenserequest`, doesn't work. To use +the bearer method you need an access token and a client id. You receive the +token after a device registration. Which values are valid for the client-id +is unknown but 0 does work. An access token expires after 60 minutes. It +can be renewed with a refresh token. A refresh token is obtained by a device +registration only. Headers for the bearer method look like:: + + Authorization: Bearer Atna|... + client-id: 0 + +Website Authentication +====================== + +To authenticate website requests you need the website cookies received from an +authorization or device registration. + +You can use the website cookies from an ``Authenticator`` with a +:class:`httpx.Client` or :class:`httpx.AsyncClient` like so:: + + auth = audible.Authenticator.from_file(...) + with httpx.Client(cookies=auth.website_cookies) as client: + resp = client.get("https://www.amazon.com/cpe/yourpayments/wallet?ref_=ya_d_c_pmt_mpo") + resp = client.get("https://www.audible.com") + +.. note:: + + Website cookies are limited to the scope of a top level domain + (e.g. com, de, ...). To set website cookies for another top level domain + scope, you can call ``auth.set_website_cookies_for_country(COUNTRY_CODE)``. + +.. warning:: + + Set website cookies for another country will override the old ones. If you + want to keep the new cookies, please make sure to save your authentication data. + +Using Postman for authentication +================================ + +`Postman `_ is a helpful utility to test API's. + +To use Postman with the Audible API, every request needs to be authenticated. +You can use the bearer method (with his limitions) with Postman out of the box. + +Using the sign request method with Postman is possible, but needs some extra work. + +HOWTO: + +1. Install the `postman_util_lib `_ +2. Copy the content from the :download:`pre-request-script <../../../utils/postman/pm_pre_request.js>` + into the `Pre-request Scripts` Tab for the Collection or request +3. Create an Environment and define the variables `adp-token` and `private key` + with the counterparts from the authentication data file diff --git a/docs/vendor/audible/raw/auth/authorization.rst.txt b/docs/vendor/audible/raw/auth/authorization.rst.txt new file mode 100644 index 0000000..6975a0c --- /dev/null +++ b/docs/vendor/audible/raw/auth/authorization.rst.txt @@ -0,0 +1,186 @@ +===================== +Authorization (Login) +===================== + +Information +=========== + +Clients are authorized using OpenID in Authorization Code Flow with PKCE. +Once a client has successfully authorized to Amazon, they receive an +`authorization code` for device registration to Audible/Amazon. + +.. _authorization: + +Authorization +============= + +For an example on authorization, please take a look at :ref:`hello_library`. + +CAPTCHA +------- + +.. versionadded:: v0.5.2 + + Init cookies added to login function to prevent CAPTCHAs in most cases. + +Authorization requires answering a CAPTCHA in some cases. By default Pillow is used +to show captcha and a user prompt will be provided to enter your answer, which +looks like:: + + Answer for CAPTCHA: + +A custom callback can be provided (for example submitting the CAPTCHA to an +external service), like so:: + + def custom_captcha_callback(captcha_url): + + # Do some things with the captcha_url ... + # maybe you can call webbrowser.open(captcha_url) + # or simply print out the captcha_url + + return "My answer for CAPTCHA" + + auth = audible.Authenticator.from_login( + ... + captcha_callback=custom_captcha_callback + ) + +2FA (OTP Code) +-------------- + +If two-factor authentication (2FA) is activated, a user prompt will be provided +using `input` to enter your one time password (OTP), which looks like:: + + "OTP Code: " + +A custom callback can be provided, like so:: + + def custom_otp_callback(): + + # Do some things to insert otp code + + return "My answer for otp code" + + auth = audible.Authenticator.from_login( + ... + otp_callback=custom_otp_callback + ) + +If you have to enter an OTP often and don't care about security, you can use +the `pyotp `_ package with a custom callback +like so:: + + from pyotp.totp import TOTP + + def otp_callback(): + secret = "YOUR-AMAZON-OTP-SECRET" + secret = secret.replace(" ", "") + otp = TOTP(secret) + return str(otp.now()) + +Another approach is to append the current OTP to the password. + +CVF Code +-------- + +If 2FA is deactivated and Amazon detects some security risks (too many logins +in short times, etc.) you will be asked for a verify code (CVF). In that case, +amazon sends you an email or SMS with a code, which you enter here:: + + "CVF Code: " + +A custom callback can be provided, like so:: + + def custom_cvf_callback(): + + # Do some things to insert cvf code + + return "My answer for cvf code" + + auth = audible.Authenticator.from_login( + ... + cvf_callback=custom_cvf_callback + ) + +Approval Alert +-------------- + +Some users report that trying to authorize with audible gives them an approval alert and an email from amazon. +Since audible v0.5 you will get a user prompt which looks like:: + + "Approval alert detected! Amazon sends you a mail." + "Please press enter when you approve the notification." + +Please approve the email/SMS, and press any key to continue. + +.. versionadded:: 0.5.1 + + Provide a custom callback with ``approval_callback`` + +A custom callback can be provided, like so:: + + def custom_approval_callback(): + + # You can let python check for the received Amazon mail and + # open the approval link. The login function waits until + # the callback function is executed. The returned value will be + # ignored by the login function. + + + auth = audible.Authenticator.from_login( + ... + approval_callback=custom_approval_callback + ) + +Authorization with external browser or program logic +==================================================== + +.. versionadded:: v0.5.1 + + Login with external browser or program logic + +To handle the login with a external browser or program logic you can do the following:: + + import audible + + auth = audible.Authenticator.from_login_external(locale=COUNTRY_CODE) + +By default, this code prints out the login url for the selected country code. +Paste this url into a web browser or use it programmatically to authorize yourself. +You have to enter your credentials two times (because of missing init cookies). +First time, the password can be a random one. +Second time, you have to solve a captcha before you can submit the login form and +you must use your correct password. +After logging in, you will end in an error page (not found). This is correct. +Copy the url from the address bar from your browser and paste the url into the input +field of the python code. It will look something like +"https://www.amazon.{domain}/ap/maplanding?...&openid.oa2.authorization_code=..." + +.. note:: + If you have `playwright `_ installed and + use the default ``login_url_callback``, a new browser is opened, where you can + authorize to your account. + +.. note:: + + If you are using macOS and have trouble inserting the login result url, simply import the + readline module in your script. See + `#34 `_. + +Custom callback +--------------- + +A custom callback can be provided (for example open the url in a web browser directly), like so:: + + def custom_login_url_callback(login_url): + + # Do some things with the login_url ... + # maybe you can call webbrowser.open(login_url) + # or simply print out the login_url + + return "The postlogin url" + + auth = audible.Authenticator.from_login_external( + ... + login_url_callback=custom_login_url_callback + ) diff --git a/docs/vendor/audible/raw/auth/register.rst.txt b/docs/vendor/audible/raw/auth/register.rst.txt new file mode 100644 index 0000000..686612c --- /dev/null +++ b/docs/vendor/audible/raw/auth/register.rst.txt @@ -0,0 +1,46 @@ +========================= +Register a Audible device +========================= + +Register +======== + +Clients are obtaining additional authentication data and information after +registration a "virtual" Audible device. + +To authorize and register a new device you can do:: + + auth = audible.Authenticator.from_login( + username, + password, + locale=country_code, + with_username=False, + ) + +This will authorize you to your account and register an Audible device. + +.. important:: + + Every device registration will be shown on the Amazon devices list. So only + register once and save your credentials or deregister the device before you + close your session. + +Deregister +========== + +Authentication data obtained by a device registration are valid until +deregister. Call ``auth.deregister_device()`` to deregister the current used +device. + +Call ``auth.deregister_device(deregister_all=True)`` to deregister **ALL** +Audible devices. This function is helpful to remove hanging slots. This can +happens if you registered a device and forgot to store the given authentication +data or to deregister. This also deregister all other devices such as an +Audible app on mobile devices. If you only want to remove one registration you +can also open the amazon devices list on the the amazon website. + +.. important:: + + Deregister needs an valid access token. The authentication data from a + device registration contains a refresh token. With these token, an access + token can be renewed with ``auth.refresh_access_token()``. diff --git a/docs/vendor/audible/raw/index.rst.txt b/docs/vendor/audible/raw/index.rst.txt new file mode 100644 index 0000000..4d6ab2e --- /dev/null +++ b/docs/vendor/audible/raw/index.rst.txt @@ -0,0 +1,75 @@ +================================ +Audible |version| documentation! +================================ + +.. image:: https://img.shields.io/pypi/v/audible.svg + :target: https://pypi.org/project/audible/ + +.. image:: https://img.shields.io/pypi/l/audible.svg + :target: https://pypi.org/project/audible + +.. image:: https://img.shields.io/pypi/pyversions/audible.svg + :target: https://pypi.org/project/audible + +.. image:: https://img.shields.io/pypi/status/audible.svg + :target: https://pypi.org/project/audible + +.. image:: https://img.shields.io/pypi/wheel/audible.svg + :target: https://pypi.org/project/audible + +.. image:: https://img.shields.io/travis/mkb79/audible/master.svg?logo=travis + :target: https://travis-ci.org/mkb79/audible + +.. image:: https://www.codefactor.io/repository/github/mkb79/audible/badge + :target: https://www.codefactor.io/repository/github/mkb79/audible + +.. image:: https://img.shields.io/pypi/dm/audible.svg + :target: https://pypi.org/project/audible + +------------------- + +**Audible** is a Python low-level interface to communicate with the non-publicly +`Audible `_ API. +It enables Python developers to create there own Audible services. Asynchronous +communication with the Audible API is supported. + +.. note:: + + For a basic command line interface take a look at my + `audible-cli `_ package. + This package supports: + + - downloading audiobooks (aax/aaxc), cover, PDF and chapter file + - export library to `csv `_ + files + - get activation bytes + - add own plugin commands + +| + +.. toctree:: + :maxdepth: 1 + :caption: Table of Contents + + intro/install + intro/getting_started + marketplaces/marketplaces + auth/authorization + auth/authentication + auth/register + misc/load_save + misc/async + misc/advanced + misc/logging + misc/external_api + misc/examples + misc/changelog + modules/audible + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/vendor/audible/raw/intro/getting_started.rst.txt b/docs/vendor/audible/raw/intro/getting_started.rst.txt new file mode 100644 index 0000000..5ab647d --- /dev/null +++ b/docs/vendor/audible/raw/intro/getting_started.rst.txt @@ -0,0 +1,94 @@ +=============== +Getting started +=============== + +Introduction +============ + +If you are new to Audible, this is the place to begin. The goal of this tutorial +is to get you set-up and rolling with Audible. I won't go into too much detail +here, just some important basics. + +First Audible device +==================== + +Before you can communicate with the non-public Audible Api, you need to +authorize (login) yourself to Amazon (or Audible) and register a new "virtual" +Audible device. Please make sure to select the correct Audible marketplace. +An overview about all known Audible marketplaces and associated country codes +can be found at :ref:`country_codes`. + +.. code-block:: + + import audible + + # Authorize and register in one step + auth = audible.Authenticator.from_login( + USERNAME, + PASSWORD, + locale=COUNTRY_CODE, + with_username=False + ) + + # Save credentials to file + auth.to_file(FILENAME) + +.. important:: + + Every device registration will be shown on the Amazon devices list. So only + register once and reuse your authentication data or deregister the device + with ``auth.deregister_device()`` before you close your session. + +.. note:: + + If you have activated 2-factor-authentication for your Amazon account, you + can append the current OTP to your password. This eliminates the need for a + new OTP prompt. + +.. note:: + + Set `with_username=True` to login with your pre-Amazon account (for US, UK or + DE marketplace only). + +.. note:: + + For security reasons in some cases you have to solve a Captcha and complete + some extra steps. Please take a look at the :ref:`authorization` section for + more information. + +.. _hello_library: + +Hello Library +============= + +After the device creation was successfully completed, you are ready to make +your first API call. To fetch and print out all books from your Audible library +(sorted by purchase date in descending order) you can do:: + + with audible.Client(auth=auth) as client: + library = client.get( + "1.0/library", + num_results=1000, + response_groups="product_desc, product_attrs", + sort_by="-PurchaseDate" + ) + for book in library["items"]: + print(book) + +.. note:: + + The information returned by the API depends on the requested `response_groups`. + The response for the example above is minimal. Please take a look at + :http:get:`/1.0/library` for all known `response_groups` and other parameter + for the library endpoint. + +Reuse authentication data +========================= + +You can store your authentication data after an device registration with:: + + auth.to_file(FILENAME) + +And load the data from file to reuse it with:: + + auth = audible.Authenticator.from_file(FILENAME) diff --git a/docs/vendor/audible/raw/intro/install.rst.txt b/docs/vendor/audible/raw/intro/install.rst.txt new file mode 100644 index 0000000..7e2dd96 --- /dev/null +++ b/docs/vendor/audible/raw/intro/install.rst.txt @@ -0,0 +1,200 @@ +================== +Installation Guide +================== + +Requirements / Dependencies +=========================== + +Audible needs at least *Python 3.10*. + +It depends on the following packages: + +* beautifulsoup4 +* httpx +* pbkdf2 +* Pillow +* pyaes +* rsa + +Optional Dependencies +===================== + +**Available in audible >= 0.11.0** + +For significantly improved performance, you can optionally install +high-performance cryptographic backends: + +* **cryptography** - Modern, Rust-accelerated library (recommended) +* **pycryptodome** - Mature, C-based cryptographic library + +The library automatically selects the best available provider: + +1. ``cryptography`` (preferred, Rust-accelerated) +2. ``pycryptodome`` (C-based) +3. ``legacy`` fallback (pure Python: ``pyaes``, ``rsa``, ``pbkdf2``) + +Performance improvements with optimized backends: + +* **5-10x faster** AES encryption/decryption +* **10-20x faster** RSA signing operations +* **3-5x faster** PBKDF2 key derivation +* **5-10x faster** SHA-256 and SHA-1 hashing + +These benefits are most noticeable when you: + +* Make frequent API requests (RSA signing on each request) +* Handle authentication workflows often (PBKDF2 key derivation) +* Encrypt or decrypt larger payloads + +For significantly improved JSON serialization performance, you can optionally +install high-performance JSON backends: + +* **orjson** - Rust-based library (recommended for compact JSON) +* **ujson** - C-based library (supports pretty-printing with indent=4) +* **rapidjson** - C++ based library + +The library automatically selects the best available provider: + +1. ``orjson`` (preferred for compact JSON, Rust-based) +2. ``ujson`` (C-based, supports indent=4) +3. ``rapidjson`` (C++ based) +4. ``json`` (standard library fallback) + +Performance improvements with optimized JSON backends: + +* **4-5x faster** compact JSON serialization (orjson) +* **2-3x faster** pretty-printed JSON with indent=4 (ujson/rapidjson) +* Smart fallback: orjson automatically uses ujson/rapidjson for indent=4 + +These benefits are most noticeable when you: + +* Load/save encrypted authentication credentials frequently +* Parse large API responses +* Work with JSON-heavy authentication flows + +Installation +============ + +Standard Installation +--------------------- + +The easiest way to install the latest version from PyPI is by using pip:: + + pip install audible + +Using uv (faster alternative to pip):: + + uv pip install audible + +Recommended: With Performance Optimizations +-------------------------------------------- + +**Available in audible >= 0.11.0** + +Install with optional extras to enable high-performance crypto and JSON providers. + +Using pip (choose one or more extras):: + + # Crypto providers + pip install audible[cryptography] # Rust-accelerated crypto (recommended) + pip install audible[pycryptodome] # C-based crypto + + # JSON providers + pip install audible[json-full] # Complete JSON coverage: orjson + ujson (recommended) + pip install audible[json-fast] # Fast compact JSON only: orjson + pip install audible[orjson] # Just orjson + pip install audible[ujson] # Just ujson + pip install audible[rapidjson] # Just rapidjson + + # Combined (recommended for best performance) + pip install audible[cryptography,json-full] + + # All performance optimizations + pip install audible[cryptography,pycryptodome,json-full] + +Using uv:: + + uv pip install audible[cryptography,json-full] + uv pip install audible[cryptography,pycryptodome,json-full] + +Or run with extras inside the project:: + + uv run --extra cryptography --extra json-full your_script.py + +Provider Overrides (advanced) +----------------------------- + +**Crypto Providers** + +Most use cases do not need direct access to the crypto registry. Prefer wiring +providers through high-level APIs such as ``Authenticator``:: + + from audible import Authenticator + from audible.crypto_provider import CryptographyProvider, set_default_crypto_provider + + auth = Authenticator.from_file( + "auth.json", + password="secret", + crypto_provider=CryptographyProvider, + ) + + # Optional: set and later reset a process-wide default provider + set_default_crypto_provider(CryptographyProvider) + ... + set_default_crypto_provider() + +``get_crypto_providers()`` is considered an internal helper and is not intended +for general external use. + +**JSON Providers** + +The library automatically selects the best available JSON provider. For explicit +control, use ``set_default_json_provider()``:: + + from audible.json_provider import set_default_json_provider + + # Use orjson explicitly (if installed) + set_default_json_provider("orjson") + + # Use ujson explicitly (if installed) + set_default_json_provider("ujson") + + # Reset to auto-detection + set_default_json_provider() + +The JSON provider system is fully automatic. Explicit configuration is rarely +needed and mainly useful for testing or performance tuning specific use cases. + +Development Installation +------------------------ + +You can also use Git to clone the repository from GitHub to install the latest +development version:: + + git clone https://github.com/mkb79/audible.git + cd audible + pip install . + +With optional dependencies:: + + pip install .[cryptography] + pip install .[pycryptodome] + pip install .[json-full] + pip install .[cryptography,pycryptodome,json-full] + +Using uv:: + + uv pip install -e .[cryptography,pycryptodome,json-full] + +Or when working in the project:: + + uv sync --extra cryptography --extra pycryptodome --extra json-full + +Alternatively, install it directly from the GitHub repository:: + + pip install git+https://github.com/mkb79/audible.git + +With optional dependencies:: + + pip install "audible[cryptography,json-full] @ git+https://github.com/mkb79/audible.git" + pip install "audible[cryptography,pycryptodome,json-full] @ git+https://github.com/mkb79/audible.git" diff --git a/docs/vendor/audible/raw/manifest.json b/docs/vendor/audible/raw/manifest.json new file mode 100644 index 0000000..f922a29 --- /dev/null +++ b/docs/vendor/audible/raw/manifest.json @@ -0,0 +1,81 @@ +{ + "generated_at": "2026-03-27T00:45:50.799815+00:00", + "count": 15, + "entries": [ + { + "url": "https://audible.readthedocs.io/en/latest/_sources/index.rst.txt", + "path": "docs/vendor/audible/raw/index.rst.txt", + "bytes": 2097 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/intro/install.rst.txt", + "path": "docs/vendor/audible/raw/intro/install.rst.txt", + "bytes": 5972 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/intro/getting_started.rst.txt", + "path": "docs/vendor/audible/raw/intro/getting_started.rst.txt", + "bytes": 2762 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/marketplaces/marketplaces.rst.txt", + "path": "docs/vendor/audible/raw/marketplaces/marketplaces.rst.txt", + "bytes": 2115 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/auth/authorization.rst.txt", + "path": "docs/vendor/audible/raw/auth/authorization.rst.txt", + "bytes": 5493 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/auth/authentication.rst.txt", + "path": "docs/vendor/audible/raw/auth/authentication.rst.txt", + "bytes": 3644 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/auth/register.rst.txt", + "path": "docs/vendor/audible/raw/auth/register.rst.txt", + "bytes": 1536 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/misc/load_save.rst.txt", + "path": "docs/vendor/audible/raw/misc/load_save.rst.txt", + "bytes": 4456 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/misc/async.rst.txt", + "path": "docs/vendor/audible/raw/misc/async.rst.txt", + "bytes": 295 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/misc/advanced.rst.txt", + "path": "docs/vendor/audible/raw/misc/advanced.rst.txt", + "bytes": 7215 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/misc/logging.rst.txt", + "path": "docs/vendor/audible/raw/misc/logging.rst.txt", + "bytes": 1039 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/misc/external_api.rst.txt", + "path": "docs/vendor/audible/raw/misc/external_api.rst.txt", + "bytes": 20890 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/misc/examples.rst.txt", + "path": "docs/vendor/audible/raw/misc/examples.rst.txt", + "bytes": 1163 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/misc/changelog.md.txt", + "path": "docs/vendor/audible/raw/misc/changelog.md.txt", + "bytes": 68 + }, + { + "url": "https://audible.readthedocs.io/en/latest/_sources/modules/audible.rst.txt", + "path": "docs/vendor/audible/raw/modules/audible.rst.txt", + "bytes": 4308 + } + ] +} diff --git a/docs/vendor/audible/raw/marketplaces/marketplaces.rst.txt b/docs/vendor/audible/raw/marketplaces/marketplaces.rst.txt new file mode 100644 index 0000000..848ed57 --- /dev/null +++ b/docs/vendor/audible/raw/marketplaces/marketplaces.rst.txt @@ -0,0 +1,87 @@ +============ +Marketplaces +============ + +General Information +=================== + +Audible offers its service on 11 different marketplaces. You can read more about +marketplaces +`here `_. + +.. note:: + + Except website cookies, authentication data from device registration are valid + for all marketplaces, no matter which marketplace is used. + +.. note:: + + The Brazilian marketplace was added in mid-2023. + +.. _country_codes: + +Country Codes +============= + +This app supports all marketplaces provided by Audible. For every marketplace a +country code is associated. + +.. note:: + + The country code of the selected marketplace is stored to file, when you + save your authentication data. So, after loading this data from file, the + stored country code is used by default. + +.. list-table:: Marketplaces with country codes + :widths: 20 50 15 + :header-rows: 1 + + * - Marketplace + - Supported Countries + - Country Code + * - Audible.com + - US and all other countries not listed + - us + * - Audible.ca + - Canada + - ca + * - Audible.co.uk + - UK and Ireland + - uk + * - Audible.com.au + - Australia and New Zealand + - au + * - Audible.fr + - France, Belgium, Switzerland + - fr + * - Audible.de + - Germany, Austria, Switzerland + - de + * - Audible.co.jp + - Japan + - jp + * - Audible.it + - Italy + - it + * - Audible.co.in + - India + - in + * - Audible.es + - Spain + - es + * - Audible.com.br + - Brazil + - br + +The locale argument +=================== + +The locale argument has the same meaning as the country code argument. For +backward compatibility we have not renamed the locale argument yet, so if you +are asked for a `locale` then provide a country code from above. + +.. note:: + + The country code for the Brazilian marketplace needs Audible > 0.8.2. + To use this marketplace with an earlier Audible version, see + `this comment `_. diff --git a/docs/vendor/audible/raw/misc/advanced.rst.txt b/docs/vendor/audible/raw/misc/advanced.rst.txt new file mode 100644 index 0000000..d0ff82d --- /dev/null +++ b/docs/vendor/audible/raw/misc/advanced.rst.txt @@ -0,0 +1,253 @@ +============== +Advanced Usage +============== + +Client classes +============== + +Here are some information about the ``Client`` and the ``AsyncClient`` classes. + +Instantiate a client +-------------------- + +A client needs at least an :class:`audible.Authenticator` at instantiation. The +following args and kwargs can be passed to the client instantiation: + +* country_code (overrides the country code set in :class:`audible.Authenticator`) +* headers (will be bypassed to the underlying httpx client) +* timeout (will be bypassed to the underlying httpx client) +* response_callback (custom response preparation - read more below) +* all other kwargs (will be bypassed to the underlying httpx client) + +Make API requests +----------------- + +Both client classes have the following methods to send requests +to the external API: + +- get +- post +- delete +- put + +The external Audible API offers currently two API versions, `0.0` and +`1.0`. The Client use the `1.0` by default. So both terms are equal:: + + resp = client.get("library") + resp = client.get("1.0/library") + +Each query parameter can be written as a separate keyword argument or you can +merge them as a dict to the `params` keyword. So both terms are equal:: + + resp = client.get("library", response_groups="...", num_results=20) + resp = client.get( + "library", + params={ + "response_groups": "...", + "num_results": 20 + } + ) + +The external Audible API awaits a request body in JSON format. You have to +provide the body as a dict to the Client. The Client converts and sends them +in JSON style to the API. You can send them like so:: + + resp = client.post( + "wishlist", + body={"asin": ASIN_OF_BOOK_TO_ADD} + ) + +The Audible API responses are in JSON format. The client converts API JSON +responses into Python dicts and returns them. + +.. note:: + + For all known API endpoints take a look at :ref:`api_endpoints`. + +Client responses +---------------- + +.. versionadded:: v0.8.0 + + The ``response_callback`` kwarg to client __init__, get, post, delete and put methods. + +By default requesting the API with the client get, post, delete and put methods +will call :func:`audible.client.raise_for_status` and try to convert +the response with :func:`audible.client.convert_response_content` to a Python dict, +which is finally returned. + +If you want to implement your own response preparation, you can do:: + + def own_response_callback(resp): + return resp + + client = audible.Client(auth=..., response_callback=own_response_callback) + +This will return the unprepared response (include headers). + +Show/Change Marketplace +----------------------- + +The currently selected marketplace can be shown with:: + + client.marketplace + +The marketplace can be changed with:: + + client.switch_marketplace(COUNTRY_CODE) + +Username/Userprofile +-------------------- + +To get the profile for the user, which authentication data are used you +can do this:: + + user_profile = client.get_user_profile() + + # or from an Authenticator instance + auth.refresh_access_token() + user_profile = auth.user_profile() + +To get the username only:: + + user_name = client.user_name + +Switch User +----------- + +If you work with multiple users you can do this:: + + # instantiate 1st user + auth = audible.Authenticator.from_file(FILENAME) + + # instantiate 2nd user + auth2 = audible.Authenticator.from_file(FILENAME2) + + # instantiate client with 1st user + client = audible.AudibleAPI(auth) + print(client.user_name) + + # now change user with auth2 + client.switch_user(auth2) + print(client.user_name) + + # optional set default marketplace from 2nd user + client.switch_user(auth2, switch_to_default_marketplace=True) + +Misc +---- + +The underlying Authenticator can be accessed via the `auth` attribute. + +Authenticator classes +===================== + +.. deprecated:: v0.5.0 + + The ``LoginAuthenticator`` and the ``FileAuthenticator`` + +.. versionchanged:: v0.6.0 + +The ``LoginAuthenticator`` and the ``FileAuthenticator`` are removed from the +Audible package. + +.. versionadded:: v0.5.0 + + The :class:`Authenticator` with the classmethods ``from_file`` and + ``from_login`` + +The :meth:`Authenticator.from_login` classmethod is used to authorize +an user and then authenticate requests with the received data. The +:meth:`Authenticator.from_file` classmethod is used to load +previous saved authentication data. + +With an Authenticator you can: + +- Save credentials to file with ``auth.to_file()`` +- Deregister a previously registered device with ``auth.deregister_device()``. +- Refresh an access token from a previously registered device with + ``auth.refresh_access_token()``. +- Get user profile with ``auth.user_profile()``. Needs a valid access token. + +To check if a access token is expired you can call:: + + auth.access_token_expired + +Or to check the time left before token expires:: + + auth.access_token_expires + +Activation Bytes +================ + +.. versionadded:: v0.4.0 + + Get activation bytes + +.. versionadded:: v0.5.0 + + the ``extract`` param + +To retrieve activation bytes an authentication :class:`Authenticator` is needed. + +The Activation bytes can be obtained like so:: + + activation_bytes = auth.get_activation_bytes() + + # the whole activation blob can fetched with + auth.get_activation_bytes(extract=False) + +The activation blob can be saved to file too:: + + activation_bytes = auth.get_activation_bytes(FILENAME) + +.. attention:: + + Please only use this for gaining full access to your own audiobooks for + archiving / conversion / convenience. DeDRMed audiobooks should not be uploaded + to open servers, torrents, or other methods of mass distribution. No help + will be given to people doing such things. Authors, retailers, and + publishers all need to make a living, so that they can continue to produce + audiobooks for us to hear, and enjoy. Don't be a parasite. + +PDF Url +======= + +PDF urls received by the Audible API don't work anymore. Authentication data +are missing in the provided link. As a workaround you can do:: + + import audible + import httpx + + asin = ASIN_FROM_BOOK + auth = audible.Authenticator.from_file(...) # or Authenticator.from_login + tld = auth.locale.domain + + with httpx.Client(auth=auth) as client: + resp = client.head( + f"https://www.audible.{tld}/companion-file/{asin}" + ) + url = resp.url + +Decrypting license +================== + +Responses from the :http:post:`/1.0/content/(string:asin)/licenserequest` +endpoint contains the encrypted license (voucher). + +To decrypt the license response you can do:: + + from audible.aescipher import decrypt_voucher_from_licenserequest + + auth = YOUR_AUTH_INSTANCE + lr = RESPONSE_FROM_LICENSEREQUEST_ENDPOINT + dlr = decrypt_voucher_from_licenserequest(auth, lr) + +.. attention:: + + Please only use this for gaining full access to your own audiobooks for + archiving / conversion / convenience. DeDRMed audiobooks should not be uploaded + to open servers, torrents, or other methods of mass distribution. No help + will be given to people doing such things. Authors, retailers, and + publishers all need to make a living, so that they can continue to produce + audiobooks for us to hear, and enjoy. Don't be a parasite. diff --git a/docs/vendor/audible/raw/misc/async.rst.txt b/docs/vendor/audible/raw/misc/async.rst.txt new file mode 100644 index 0000000..54e6061 --- /dev/null +++ b/docs/vendor/audible/raw/misc/async.rst.txt @@ -0,0 +1,14 @@ +==================== +Asynchronous requests +==================== + +This app supports asynchronous requests using the httpx module. +You can instantiate an async client with:: + + async with audible.AsyncClient(auth=...) as client: + ... + +Example +======= + +.. literalinclude:: ../../../examples/async.py diff --git a/docs/vendor/audible/raw/misc/changelog.md.txt b/docs/vendor/audible/raw/misc/changelog.md.txt new file mode 100644 index 0000000..efc5286 --- /dev/null +++ b/docs/vendor/audible/raw/misc/changelog.md.txt @@ -0,0 +1,3 @@ +```{include} ../../../CHANGELOG.md +:relative-docs: doc/src/misc +``` diff --git a/docs/vendor/audible/raw/misc/examples.rst.txt b/docs/vendor/audible/raw/misc/examples.rst.txt new file mode 100644 index 0000000..5160c8f --- /dev/null +++ b/docs/vendor/audible/raw/misc/examples.rst.txt @@ -0,0 +1,35 @@ +======== +Examples +======== + +Here are some examples and ideas how to use this app. Everyone who will +provide some examples are welcome. + +Print number of books for every marketplace:: + + import audible + + filename = "path/to/credentials.json" + auth = audible.Authenticator.from_file(filename) + client = audible.Client(auth) + country_codes = ["de", "us", "ca", "uk", "au", "fr", "jp", "it", "in"] + + for country in country_codes: + client.switch_marketplace(country) + library = client.get("library", num_results=1000) + asins = [book["asin"] for book in library["items"]] + print(f"Country: {client.marketplace.upper()} | Number of books: {len(asins)}") + print(34* "-") + +Get listening statistics aggregated month-over-month from 2021-03 to 2021-06:: + + import audible + + filename = "path/to/credentials.json" + auth = audible.Authenticator.from_file(filename) + with audible.Client(auth=auth) as client: + stats = client.get( + "1.0/stats/aggregates", + monthly_listening_interval_duration="3", #number of months to aggregate for + monthly_listening_interval_start_date="2021-03", #start month for aggregation + store="Audible") diff --git a/docs/vendor/audible/raw/misc/external_api.rst.txt b/docs/vendor/audible/raw/misc/external_api.rst.txt new file mode 100644 index 0000000..8b36a5d --- /dev/null +++ b/docs/vendor/audible/raw/misc/external_api.rst.txt @@ -0,0 +1,622 @@ +==================== +External Audible API +==================== + +Documentation +============= + +There is currently no publicly available documentation about the +Audible API. + +There is a node client `audible-api `_ +that has some endpoints documented, but does not provide information +on authentication. + +Luckily the Audible API is partially self-documenting, however the +parameter names need to be known. Error responses will look like: + +.. code-block:: json + + { + "message": "1 validation error detected: Value 'some_random_string123' at 'numResults' failed to satisfy constraint: Member must satisfy regular expression pattern: ^\\d+$" + } + +Few endpoints have been fully documented, as a large amount of functionality +is not testable from the app or functionality is unknown. Most calls need +to be authenticated. + +For `%s` substitutions the value is unknown or can be inferred from the +endpoint. `/1.0/catalog/products/%s` for example requires an `asin`, +as in `/1.0/catalog/products/B002V02KPU`. + +Each bullet below refers to a parameter for the request with the specified +method and URL. + +Responses will often provide very little info without `response_groups` +specified. Multiple response groups can be specified, for example: +`/1.0/catalog/products/B002V02KPU?response_groups=product_plan_details,media,review_attrs`. +When providing an invalid response group, the server will return an error +response but will not provide information on available response groups. + + +.. _api_endpoints: + +API Endpoints +============= + +.. http:get:: /0.0/library/books + :deprecated: + + This API endpoint is deprecated. Please use :http:get:`/1.0/library` instead. + + :query string purchaseAfterDate: mm/dd/yyyy + :query string sortByColumn: [SHORT_TITLE, strTitle, DOWNLOAD_STATUS, + RUNNING_TIME, sortPublishDate, SHORT_AUTHOR, + sortPurchDate, DATE_AVAILABLE] + :query bool sortInAscendingOrder: [true, false] + +Library +------- + +.. http:get:: /1.0/library + + The audible library of current user + + :query integer num_results: (max: 1000) + :query integer page: page + :query string purchased_after: [RFC3339](https://tools.ietf.org/html/rfc3339) + (e.g. `2000-01-01T00:00:00Z`) + :query string title: a title + :query string author: a author + :query string response_groups: [contributors, customer_rights, media, price, + product_attrs, product_desc, product_details, + product_extended_attrs, product_plan_details, + product_plans, rating, sample, sku, series, + reviews, ws4v, origin, relationships, + review_attrs, categories, badge_types, + category_ladders, claim_code_url, in_wishlist, is_archived, is_downloaded, + is_finished, is_playable, is_removable, + is_returnable, is_visible, listening_status, order_details, + origin_asin, pdf_url, percent_complete, periodicals, + provided_review] + :query string image_sizes: [1215,408,360,882,315,570,252,558,900,500] + :query string sort_by: [-Author, -Length, -Narrator, -PurchaseDate, -Title, + Author, Length, Narrator, PurchaseDate, Title] + :query string status: [Active, Revoked] ('Active' is the default, 'Revoked' + returns audiobooks the user has returned for a refund.) + :query string parent_asin: asin + :query string include_pending: [true, false] + :query string marketplace: [e.g. AN7V1F1VY261K] + :query string state_token: + +.. http:get:: /1.0/library/(string:asin) + + :param asin: The asin of the book + :type asin: string + :query string response_groups: [contributors, media, price, product_attrs, + product_desc, product_details, product_extended_attrs, + product_plan_details, product_plans, rating, + sample, sku, series, reviews, ws4v, origin, + relationships, review_attrs, categories, + badge_types, category_ladders, claim_code_url, + is_downloaded, is_finished, is_returnable, + origin_asin, pdf_url, percent_complete, + periodicals, provided_review] + +.. http:post:: /1.0/library/item + + :json collection_id: + :>json creation_date: + :>json customer_id: + :>json marketplace: + +.. http:get:: /1.0/collections/(collection_id) + + :param collection_id: + +.. http:put:: /1.0/collections/(collection_id) + + Modify a collection + + :param collection_id: + + :json state_token: + :>json collection_id: + :>json name: + :>json description: + +.. http:get:: /1.0/collections/(collection_id)/items + + :param collection_id: e.g __FAVORITES + :query response_groups: [always-returned] + +.. http:post:: /1.0/collections/(collection_id)/items + + Add item(s) to a collection + + :param collection_id: + :json description: + :>json name: + :>json int num_items_added: + :>json state_token: + +Orders +------ + +.. http:get:: /1.0/orders + + Returns order history from at least the past 6 months. Supports pagination. + + :query unknown: + +.. http:post:: /1.0/orders + + :json string license: The encrypted license + +.. http:get:: /1.0/content/FairPlay/certificate + + :>json string certificate: The base64 encoded FairPlay certificate + +Account +------- + +.. http:get:: /1.0/account/information + + :query response_groups: [delinquency_status, customer_benefits, customer_segments, subscription_details_payment_instrument, plan_summary, subscription_details, directed_ids] + :query source: [Credit, Enterprise, RodizioFreeBasic, AyceRomance, AllYouCanEat, AmazonEnglish, ComplimentaryOriginalMemberBenefit, Radio, SpecialBenefit, Rodizio] + + +Customer +-------- + +.. http:get:: /1.0/customer/information + + :query response_groups: [migration_details, subscription_details_rodizio, subscription_details_premium, customer_segment, subscription_details_channels] + +.. http:get:: /1.0/customer/status + + :query response_groups: [benefits_status, member_giving_status, prime_benefits_status, prospect_benefits_status] + +.. http:get:: /1.0/customer/freetrial/eligibility + +Stats +----- + +.. http:get:: /1.0/stats/aggregates + + :query daily_listening_interval_duration: ([012]?[0-9])|(30) (0 to 30, inclusive) + :query daily_listening_interval_start_date: YYYY-MM-DD (e.g. `2019-06-16`) + :query locale: en_US + :query monthly_listening_interval_duration: 0?[0-9]|1[012] (0 to 12, inclusive) + :query monthly_listening_interval_start_date: YYYY-MM (e.g. `2019-02`) + :query response_groups: [total_listening_stats] + :query store: [AudibleForInstitutions, Audible, AmazonEnglish, Rodizio] + +.. http:get:: /1.0/stats/status/finished + + :query asin: asin + :query start_date: [RFC3339](https://tools.ietf.org/html/rfc3339) (e.g. `2000-01-01T00:00:00Z`) + + +.. http:post:: /1.0/stats/status/finished + + :=8.0.0", "pytest-cov>=4.1.0", "pytest-asyncio>=1.3.0", diff --git a/requirements.txt b/requirements.txt index be9d103..724be3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,8 +17,15 @@ python-multipart itsdangerous idna cryptography +Pillow>=12.2.0 PyYAML # Structured logging structlog>=24.4.0 orjson>=3.10.0 + +# Core dependencies of the mkb79/Audible package (installed with --no-deps so these must be explicit). +# Supported Python range for the Audible install path is 3.11-3.13. +pbkdf2==1.3 +pyaes==1.6.1 +rsa==4.9.1 diff --git a/src/audible_client.py b/src/audible_client.py new file mode 100644 index 0000000..64e728d --- /dev/null +++ b/src/audible_client.py @@ -0,0 +1,128 @@ +"""Authenticated client management for the mkb79/Audible package.""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + import audible + +try: + import audible as _audible_mod + + _AUDIBLE_AVAILABLE = True +except ModuleNotFoundError: + _audible_mod = None # type: ignore[assignment] + _AUDIBLE_AVAILABLE = False + +from src.config import load_config +from src.logging_setup import get_logger + + +log = get_logger(__name__) + +__all__ = ["AudibleClientProvider"] + + +class AudibleClientProvider: + """Load encrypted Audible auth and reuse async clients per marketplace.""" + + def __init__( + self, + *, + auth_file: str | None = None, + auth_file_password: str | None = None, + ) -> None: + env_auth_file = os.getenv("AUDIBLE_AUTH_FILE") + env_auth_file_password = os.getenv("AUDIBLE_AUTH_FILE_PASSWORD") + audible_config: dict[str, str] = {} + if auth_file is None and env_auth_file is None: + config = load_config() + audible_config = config.get("metadata", {}).get("audible", {}) + + self.auth_file = auth_file or env_auth_file or audible_config.get("auth_file") + self.auth_file_password = auth_file_password or env_auth_file_password + + self._auth: audible.Authenticator | None = None + self._clients: dict[str, audible.AsyncClient] = {} + self._init_lock = asyncio.Lock() + self._shutting_down = False + + @property + def configured(self) -> bool: + """Return whether the encrypted auth file and decrypt password are set.""" + return bool(self.auth_file and self.auth_file_password) + + async def get_client(self, region: str) -> audible.AsyncClient | None: + """Return an authenticated Audible async client for a region.""" + if not _AUDIBLE_AVAILABLE: + log.warning("audible.library.package_not_installed") + return None + + if not self.auth_file: + log.warning("audible.library.no_auth_file") + return None + + if not self.auth_file_password: + log.warning("audible.library.no_auth_file_password") + return None + + auth_path = Path(self.auth_file).expanduser() + if not auth_path.exists(): + log.warning("audible.library.auth_file_missing", auth_file=auth_path.name) + return None + + async with self._init_lock: + if self._shutting_down: + log.warning("audible.library.shutting_down", region=region) + return None + + # Re-check inside the lock in case another coroutine already initialised this region. + if region in self._clients: + return self._clients[region] + + try: + if self._auth is None: + self._auth = _audible_mod.Authenticator.from_file( + auth_path, + password=self.auth_file_password, + ) + + client = _audible_mod.AsyncClient(auth=self._auth, country_code=region) + except Exception as exc: + log.warning("audible.library.auth_failed", error=str(exc)) + return None + + self._clients[region] = client + return client + + async def _close_all_clients(self) -> None: + """Close cached Audible clients without aborting on the first failure.""" + async with self._init_lock: + self._shutting_down = True + for client in list(self._clients.values()): + try: + await client.close() + except Exception as close_exc: + log.warning("audible.client.close_error", error=str(close_exc)) + self._clients.clear() + + async def aclose(self) -> None: + """Close any cached Audible async clients.""" + await self._close_all_clients() + + async def __aenter__(self) -> AudibleClientProvider: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: object, + ) -> None: + """Close all cached clients on context-manager exit.""" + await self._close_all_clients() diff --git a/src/audible_scraper.py b/src/audible_scraper.py index 61a101c..e2b6640 100644 --- a/src/audible_scraper.py +++ b/src/audible_scraper.py @@ -1,24 +1,18 @@ -""" -Audible.com metadata fallback scraper -Searches for audiobook metadata using Audible's search API - -This module provides async methods for searching Audible's catalog, -with support for title/author searches and ASIN lookups. -""" +"""Audible metadata search backend powered by mkb79/Audible.""" import argparse import asyncio +import os import re from typing import Any -from urllib.parse import urlencode +from src.audible_client import AudibleClientProvider from src.audnex_metadata import AudnexMetadata from src.config import load_config from src.http_client import ( REGION_MAP, AsyncHttpClient, get_default_client, - get_region_tld, ) from src.logging_setup import get_logger @@ -28,7 +22,7 @@ class AudibleScraper: """ - Async Audible metadata scraper with shared HTTP client. + Async Audible metadata backend with shared Audnex/HTTP helpers. Example usage: async with AudibleScraper() as scraper: @@ -36,9 +30,13 @@ class AudibleScraper: metadata = await scraper.search_by_asin("B08G9PRS1K") """ - def __init__(self, client: AsyncHttpClient | None = None) -> None: + def __init__( + self, + client: AsyncHttpClient | None = None, + audible_client_provider: AudibleClientProvider | None = None, + ) -> None: """ - Initialize the Audible scraper. + Initialize the Audible metadata backend. Args: client: Optional AsyncHttpClient instance. If not provided, uses the default shared client. @@ -46,8 +44,16 @@ def __init__(self, client: AsyncHttpClient | None = None) -> None: self._client = client self.config = load_config() self.audible_config = self.config.get("metadata", {}).get("audible", {}) - self.base_url = self.audible_config.get("base_url", "https://api.audible.com") self.search_endpoint = self.audible_config.get("search_endpoint", "/1.0/catalog/products") + if audible_client_provider is None: + self._audible_client_provider = AudibleClientProvider( + auth_file=os.getenv("AUDIBLE_AUTH_FILE") or self.audible_config.get("auth_file"), + auth_file_password=os.getenv("AUDIBLE_AUTH_FILE_PASSWORD"), + ) + self._owns_audible_provider = True + else: + self._audible_client_provider = audible_client_provider + self._owns_audible_provider = False # Use shared region map from http_client self.region_map = REGION_MAP @@ -83,6 +89,20 @@ async def __aexit__( Note: Does not close the HTTP client as it's managed by the application lifespan. The shared client is closed during app shutdown. """ + if self._owns_audible_provider: + await self._audible_client_provider.aclose() + + async def _get_audible_library_client(self, region: str) -> Any | None: + """Create or reuse an authenticated mkb79/Audible client for a region.""" + return await self._audible_client_provider.get_client(region) + + @staticmethod + def _is_english_language(language: str | None) -> bool: + """Accept common English language codes returned by Audible/Audnex.""" + if not isinstance(language, str): + return False + normalized = language.strip().lower() + return normalized == "english" or normalized.startswith("en") def _is_valid_asin(self, asin: str) -> bool: """Validate ASIN format (10 characters, alphanumeric).""" @@ -139,7 +159,7 @@ def _product_to_book(self, product: dict) -> dict: cover_url = None if product.get("product_images"): # Get the highest resolution image available - for size in ["500", "300", "200", "100"]: + for size in ["500", "408", "360", "252"]: if product["product_images"].get(size): cover_url = product["product_images"][size] break @@ -219,7 +239,7 @@ def _product_to_book(self, product: dict) -> dict: async def search_by_title_author(self, title: str, author: str = "", region: str = "us") -> list[dict[str, Any]]: """ - Search for audiobooks by title and author using Audible's catalog API. + Search for audiobooks by title and author using Audible's authenticated catalog API. Only returns English results. @@ -235,31 +255,30 @@ async def search_by_title_author(self, title: str, author: str = "", region: str log.error("audible.search.invalid_region", region=region) region = "us" - client = await self._get_client() - - # Add response_groups parameter to get full metadata directly from Audible + # Request enough groups to build user-facing metadata directly from Audible. params = { - "num_results": "10", + "num_results": 10, "products_sort_by": "Relevance", - "title": title, - "response_groups": "product_desc,media,contributors,series", + "keywords": title, + "response_groups": "product_desc,product_attrs,product_extended_attrs,media,contributors,series,rating", } if author: params["author"] = author - tld = get_region_tld(region) - url = f"https://api.audible{tld}{self.search_endpoint}?{urlencode(params)}" log.info("audible.search.start", title=title, author=author, region=region) - log.debug("audible.search.url", url=url) - - data = await client.get_json(url) + audible_client = await self._get_audible_library_client(region) + if audible_client is None: + log.warning("audible.library.not_configured") + return [] - if not data: - log.warning("audible.search.no_response") + try: + data = await audible_client.get(self.search_endpoint, params=params) + except Exception as exc: + log.warning("audible.library.search_failed", error=str(exc)) return [] - products = data.get("products", []) - log.info("audible.search.results", count=len(products)) + products = data.get("products", []) if isinstance(data, dict) else [] + log.info("audible.search.results", count=len(products), backend="library") if not products: log.warning("audible.search.no_products") @@ -270,8 +289,8 @@ async def search_by_title_author(self, title: str, author: str = "", region: str for product in products: # Only include English books - language = product.get("language", "").lower() - if language and language != "english": + language = product.get("language") + if language and not self._is_english_language(language): log.debug("audible.search.skip_non_english", language=language) continue @@ -293,7 +312,7 @@ async def search_by_title_author(self, title: str, author: str = "", region: str # Fallback: try Audnex for detailed metadata try: metadata = await audnex.get_book_by_asin(asin, region=region) - if metadata and metadata.get("language", "").lower() == "english": + if metadata and self._is_english_language(metadata.get("language")): audnex_book = self._product_to_book(metadata) if audnex_book: detailed_results.append(audnex_book) @@ -313,7 +332,7 @@ async def search_by_title_author(self, title: str, author: str = "", region: str async def search_by_asin(self, asin: str, region: str = "us") -> dict[str, Any] | None: """ - Search for audiobook by ASIN (delegates to Audnex). + Search for audiobook by ASIN via Audnex. Args: asin: Amazon Standard Identification Number @@ -328,7 +347,29 @@ async def search_by_asin(self, asin: str, region: str = "us") -> dict[str, Any] log.info("audible.asin_search.start", asin=asin, region=region) - # Use Audnex for ASIN lookups as it's more reliable + # Try the authenticated Audible catalog endpoint first β€” richer data than Audnex. + audible_client = await self._get_audible_library_client(region) + if audible_client is not None: + try: + data = await audible_client.get( + self.search_endpoint, + params={ + "asin": asin, + "response_groups": "contributors,media,product_attrs,product_desc,product_details,product_extended_attrs,series,rating,category_ladders", + }, + ) + products = data.get("products", []) if isinstance(data, dict) else [] + product = products[0] if products else data.get("product", {}) if isinstance(data, dict) else {} + if product: + book = self._product_to_book(product) + if book.get("title"): + log.info("audible.asin_search.found", asin=asin, source="audible_catalog") + return book + except Exception as exc: + log.warning("audible.asin_search.api_failed", asin=asin, error=str(exc)) + + # Fall back to Audnex when the Audible client is unavailable or the call failed. + log.info("audible.asin_search.audnex_fallback", asin=asin) audnex = await self._get_audnex() return await audnex.get_book_by_asin(asin, region=region) @@ -367,7 +408,7 @@ async def search( results.append(result) return results - # Strategy 3: Title/Author search via Audible catalog + # Strategy 3: Title/Author search via the authenticated Audible backend if title: log.info("audible.search.strategy3", title=title, author=author, strategy="title_author") results = await self.search_by_title_author(title, author, region=region) diff --git a/src/main.py b/src/main.py index 1f05fd7..2bae5c5 100644 --- a/src/main.py +++ b/src/main.py @@ -18,7 +18,6 @@ from src.db import save_request # switch to persistent DB store from src.http_client import AsyncHttpClient, close_default_client from src.logging_setup import clear_contextvars, configure_logging, get_logger -from src.metadata import fetch_metadata from src.metadata_coordinator import MetadataCoordinator from src.notify.discord import send_discord from src.notify.gotify import send_gotify @@ -309,38 +308,22 @@ async def webhook(request: Request): metadata = None last_error: Exception | None = None - # Try 1: Compatibility wrapper (tests often patch fetch_metadata) + # Primary metadata workflow: coordinator-managed async lookup try: - metadata = await fetch_metadata(payload) + metadata = await coordinator.get_metadata_from_webhook(payload) if metadata: - log.info("metadata.fetch.success", source="fetch_metadata_wrapper") metadata = await coordinator.get_enhanced_metadata(metadata) + log.info("metadata.fetch.success", source="coordinator") except ValueError as e: - # Expected when fetch_metadata returns None or finds no metadata - log.debug("metadata.fetch.no_result", source="fetch_metadata", reason=str(e)) + # Expected when coordinator finds no metadata + log.debug("metadata.fetch.no_result", source="coordinator", reason=str(e)) last_error = e except Exception as e: - # Unexpected exceptions from fetch_metadata - log and continue to next method - log.exception("metadata.fetch.error", source="fetch_metadata") + # Unexpected exceptions from coordinator - log and continue to fallback + log.exception("metadata.fetch.error", source="coordinator") last_error = e - # Try 2: Coordinator async workflow - if not metadata: - try: - metadata = await coordinator.get_metadata_from_webhook(payload) - if metadata: - metadata = await coordinator.get_enhanced_metadata(metadata) - log.info("metadata.fetch.success", source="coordinator") - except ValueError as e: - # Expected when coordinator finds no metadata - log.debug("metadata.fetch.no_result", source="coordinator", reason=str(e)) - last_error = e - except Exception as e: - # Network errors or other issues from coordinator - log.exception("metadata.fetch.error", source="coordinator") - last_error = e - - # Try 3: Fallback metadata + # Final fallback metadata if not metadata: metadata = _create_fallback_metadata( payload, token, last_error or Exception("No metadata sources available") diff --git a/src/metadata.py b/src/metadata.py index 3162380..c4b5e30 100644 --- a/src/metadata.py +++ b/src/metadata.py @@ -1,27 +1,16 @@ -""" -Metadata fetching and processing for audiobooks. +"""Metadata compatibility helpers and legacy Audnexus client.""" -This module provides async classes and functions for fetching audiobook metadata -from Audible and Audnex APIs, with support for multiple regions and fallback logic. -""" - -import os import re from typing import Any from urllib.parse import urlencode -import httpx - -from src.config import load_config +from src.audible_scraper import AudibleScraper from src.http_client import ( - REGION_MAP, AsyncHttpClient, get_default_client, - get_region_tld, - get_regions_priority, ) from src.logging_setup import get_logger -from src.utils import clean_author_list, validate_payload +from src.utils import clean_author_list log = get_logger(__name__) @@ -53,205 +42,145 @@ def levenshtein_distance(s1: str, s2: str) -> int: return dp[len2] -class Audible: - """ - Async Audible metadata client using shared HTTP client. - - Example usage: - async with Audible() as audible: - results = await audible.search(title="The Hobbit", author="Tolkien") - metadata = await audible.asin_search("B08G9PRS1K") - """ - - def __init__(self, client: AsyncHttpClient | None = None, response_timeout: int = 30000) -> None: - self._client = client - self.response_timeout = response_timeout - # Use shared region map from http_client - self.region_map = REGION_MAP - - async def _get_client(self) -> AsyncHttpClient: - """Get or create the HTTP client.""" - if self._client is None: - self._client = await get_default_client() - return self._client +def clean_series_sequence(series_name: str, sequence: Any) -> str: + """Normalize series numbering like "Book 1" to a plain numeric value.""" + if not sequence: + return "" - async def __aenter__(self) -> "Audible": - """Async context manager entry.""" - await self._get_client() - return self + sequence = str(sequence) - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc: BaseException | None, - tb: object, - ) -> None: - """Async context manager exit. + match = re.search(r"\.\d+|\d+(?:\.\d+)?", sequence) + updated_sequence = match.group(0) if match else sequence + if sequence != updated_sequence: + log.debug( + "metadata.series_sequence.cleaned", + series=series_name, + original=sequence, + cleaned=updated_sequence, + ) + return updated_sequence - Note: Does not close the HTTP client as it's managed by the application lifespan. - The shared client is closed during app shutdown. - """ - def clean_series_sequence(self, series_name: str, sequence: str) -> str: - """ - Audible will sometimes send sequences with "Book 1" or "2, Dramatized Adaptation" - Clean to extract just the number portion - """ - if not sequence: - return "" - # match any number with optional decimal (e.g, 1 or 1.5 or .5) - match = re.search(r"\.\d+|\d+(?:\.\d+)?", sequence) - updated_sequence = match.group(0) if match else sequence - if sequence != updated_sequence: - log.debug( - "metadata.series_sequence.cleaned", - series=series_name, - original=sequence, - cleaned=updated_sequence, - ) - return updated_sequence - - def clean_result(self, item: dict[str, Any]) -> dict[str, Any]: - """Clean and format the result from Audnex API""" - title = item.get("title") - subtitle = item.get("subtitle") - asin = item.get("asin") - authors = item.get("authors", []) - narrators = item.get("narrators", []) - publisher_name = item.get("publisherName") - summary = item.get("summary") - release_date = item.get("releaseDate") - image = item.get("image") - genres = item.get("genres", []) - series_primary = item.get("seriesPrimary") - series_secondary = item.get("seriesSecondary") - language = item.get("language") - runtime_length_min = item.get("runtimeLengthMin") - format_type = item.get("formatType") - isbn = item.get("isbn") - - series = [] - if series_primary: - series.append( - { - "series": series_primary.get("name"), - "sequence": self.clean_series_sequence( - series_primary.get("name", ""), series_primary.get("position", "") - ), - } - ) - if series_secondary: +def _extract_names(values: list[Any]) -> list[str]: + names: list[str] = [] + for value in values: + if isinstance(value, dict): + name = value.get("name", "") + elif isinstance(value, str): + name = value + else: + name = "" + + if name: + names.append(name) + + return names + + +def normalize_book_result(item: dict[str, Any]) -> dict[str, Any]: + """Normalize raw Audnex or Audible payloads into the app's common shape.""" + title = item.get("title") + subtitle = item.get("subtitle") + asin = item.get("asin") + authors = _extract_names(item.get("authors", [])) + narrators = _extract_names(item.get("narrators", [])) + publisher_name = item.get("publisherName") or item.get("publisher_name") or item.get("publisher") + summary = item.get("summary") or item.get("description") + release_date = item.get("releaseDate") or item.get("release_date") + image = item.get("image") or item.get("cover") or item.get("cover_url") + genres = item.get("genres", []) + language = item.get("language") + runtime_length_min = ( + item.get("runtimeLengthMin") + or item.get("runtime_length_min") + or item.get("runtime_minutes") + or item.get("duration") + or item.get("length") + ) + format_type = item.get("formatType") or item.get("format_type") + isbn = item.get("isbn") + + series: list[dict[str, str]] = [] + for existing_series in item.get("series", []): + if not isinstance(existing_series, dict): + continue + series_name = existing_series.get("series") or existing_series.get("title") or existing_series.get("name") or "" + sequence = existing_series.get("sequence") or existing_series.get("position") or "" + if series_name: series.append( { - "series": series_secondary.get("name"), - "sequence": self.clean_series_sequence( - series_secondary.get("name", ""), series_secondary.get("position", "") - ), + "series": series_name, + "sequence": clean_series_sequence(series_name, str(sequence)), } ) - genres_filtered = [g.get("name") for g in genres if g.get("type") == "genre"] - tags_filtered = [g.get("name") for g in genres if g.get("type") == "tag"] - - return { - "title": title, - "subtitle": subtitle or None, - "author": ", ".join([a.get("name", "") for a in authors]) if authors else None, - "narrator": ", ".join([n.get("name", "") for n in narrators]) if narrators else None, - "publisher": publisher_name, - "publishedYear": release_date.split("-")[0] if release_date else None, - "description": summary or None, - "cover": image, - "asin": asin, - "isbn": isbn, - "genres": genres_filtered if genres_filtered else None, - "tags": ", ".join(tags_filtered) if tags_filtered else None, - "series": series if series else None, - "language": language.capitalize() if language else None, - "duration": int(runtime_length_min) if runtime_length_min and str(runtime_length_min).isdigit() else 0, - "region": item.get("region") or None, - "rating": item.get("rating") or None, - "abridged": format_type == "abridged", - } - - async def asin_search(self, asin: str, region: str = "us", timeout: int | None = None) -> dict[str, Any] | None: - """Search for a book by ASIN - - Args: - asin: The ASIN to search for - region: The region code (default: us) - timeout: Request timeout in seconds (uses client default if None) - """ - if not asin: - return None - - client = await self._get_client() - asin = asin.upper() - region_query = f"?region={region}" if region else "" - url = f"https://api.audnex.us/books/{asin}{region_query}" - log.debug("metadata.audible.asin_url", url=url) - - data = await client.get_json(url, timeout=timeout) - if data and data.get("asin"): - return data - return None - - async def search( - self, title: str, author: str = "", asin: str = "", region: str = "us", timeout: int | None = None - ) -> list[dict[str, Any]]: - """Search for books using title, author, and/or ASIN - - Args: - title: Book title to search for - author: Author name (optional) - asin: ASIN code (optional) - region: Region code (default: us) - timeout: Request timeout in seconds (uses client default if None) - """ - if region and region not in self.region_map: - log.error("metadata.audible.invalid_region", region=region) - region = "us" + for series_key in ("seriesPrimary", "seriesSecondary"): + series_entry = item.get(series_key) + if not isinstance(series_entry, dict): + continue + series_name = series_entry.get("name", "") + if not series_name: + continue + series.append( + { + "series": series_name, + "sequence": clean_series_sequence(series_name, series_entry.get("position", "")), + } + ) - client = await self._get_client() - items = [] - - # Try ASIN search first if valid - if asin and is_valid_asin(asin.upper()): - item = await self.asin_search(asin, region, timeout) - if item: - items.append(item) - - # Try title as ASIN if no results and title looks like ASIN - if not items and is_valid_asin(title.upper()): - item = await self.asin_search(title, region, timeout) - if item: - items.append(item) - - # Fallback to catalog search - if not items: - query_obj = {"num_results": "10", "products_sort_by": "Relevance", "title": title} - if author: - query_obj["author"] = author - - query_string = urlencode(query_obj) - tld = get_region_tld(region) - url = f"https://api.audible{tld}/1.0/catalog/products?{query_string}" - log.debug("metadata.audible.search_url", url=url) - - data = await client.get_json(url) - - if data and data.get("products"): - # Get detailed info for each product - detailed_items = [] - for result in data["products"]: - if result.get("asin"): - detailed_item = await self.asin_search(result["asin"], region, timeout) - if detailed_item: - detailed_items.append(detailed_item) - items = detailed_items - - # Clean and return results - return [self.clean_result(item) for item in items if item] + genres_filtered: list[str] = [] + tags_filtered: list[str] = [] + for genre in genres: + if isinstance(genre, dict): + name = genre.get("name") + genre_type = genre.get("type") + if not name: + continue + if genre_type == "tag": + tags_filtered.append(name) + else: + genres_filtered.append(name) + elif isinstance(genre, str): + genres_filtered.append(genre) + + existing_tags = item.get("tags") + tags_value: str | None + if isinstance(existing_tags, str): + tags_value = existing_tags + else: + tags_value = ", ".join(tags_filtered) if tags_filtered else None + + duration = 0 + if runtime_length_min is not None: + try: + duration = int(float(runtime_length_min)) + except (TypeError, ValueError): + duration = 0 + + published_year = item.get("publishedYear") or None + if isinstance(release_date, str) and release_date: + published_year = release_date.split("-")[0] + + return { + "title": title, + "subtitle": subtitle or None, + "author": ", ".join(authors) if authors else item.get("author") or None, + "narrator": ", ".join(narrators) if narrators else item.get("narrator") or None, + "publisher": publisher_name, + "publishedYear": published_year, + "description": summary or None, + "cover": image, + "asin": asin, + "isbn": isbn, + "genres": genres_filtered if genres_filtered else None, + "tags": tags_value, + "series": series if series else None, + "language": language.capitalize() if isinstance(language, str) else None, + "duration": duration, + "region": item.get("region") or None, + "rating": item.get("rating") or None, + "abridged": format_type == "abridged", + } class Audnexus: @@ -405,180 +334,20 @@ async def get_chapters_by_asin(self, asin: str, region: str = "") -> dict[str, A return result -# Main fetch metadata function compatible with existing code -def get_cached_metadata(asin: str, region: str = "us", api_url: str | None = None) -> dict | None: # noqa: ARG001 - """Intentional stub kept for signature compatibility - tests are expected to patch/override this. - - This function is a placeholder whose parameters are currently unused in the default implementation. - The function signature is maintained for backwards compatibility with existing code and tests. - - Args: - asin: The Amazon Standard Identification Number (10 alphanumeric characters) to look up - region: The Audible region/marketplace (e.g., 'us', 'uk', 'ca') for regional content - api_url: Optional custom API endpoint URL for metadata lookup (typically None) - - Returns: - None by default. Tests should patch this function to return mock metadata dict when needed. - A real implementation might query a local cache or external API. - - Note: - This is an intentional no-op stub. The default behavior returns None to indicate no cached - metadata is available. Tests that require cached metadata should mock/patch this function - to return appropriate test data. - """ - # Default behavior: no cache. Tests may patch this to return values. - return None - - async def get_audible_asin(title: str, author: str = "") -> str | None: - """Try to extract an ASIN by scraping Audible search results. - - This function attempts to import BeautifulSoup and parse the page returned by - a simple Audible search. If bs4 is not available or parsing fails, return None. - """ - try: - import bs4 - - BeautifulSoup = bs4.BeautifulSoup - except ImportError: - return None + """Try to extract an ASIN using the package-backed Audible search backend.""" try: - client = await get_default_client() - - query = f"{title} {author}".strip() - # Simple Audible search URL - search_url = f"https://www.audible.com/search?keywords={query.replace(' ', '+')}" - - response = await client.get(search_url) - html = response.text - soup = BeautifulSoup(html, "html.parser") - - # Audible sometimes puts ASINs in adbl-impression-container data-asin - el = soup.find("div", class_="adbl-impression-container") - if el and hasattr(el, "get") and el.get("data-asin"): - asin = el.get("data-asin") - return str(asin) if asin and not isinstance(asin, list) else None - - # Fallback: look for data-asin attributes elsewhere - el2 = soup.find(attrs={"data-asin": True}) - if el2 and hasattr(el2, "get"): - asin2 = el2.get("data-asin") - return str(asin2) if asin2 and not isinstance(asin2, list) else None - + async with AudibleScraper() as scraper: + results = await scraper.search(title=title, author=author) + for result in results: + asin = result.get("asin") + if isinstance(asin, str) and is_valid_asin(asin.upper()): + return asin.upper() return None - except (AttributeError, ValueError, TypeError) as e: - # Expected parsing-related errors - log and return None + except Exception as e: log.debug("metadata.get_audible_asin.failed", error=str(e)) return None - except Exception as e: - # bs4.FeatureNotFound and other BeautifulSoup parsing exceptions - if "bs4" in type(e).__module__: - log.debug("metadata.get_audible_asin.bs4_failed", error=str(e)) - return None - # Unexpected exceptions should propagate - raise - - -async def fetch_metadata(payload: dict, regions: list[str] | None = None) -> dict: - """ - Enhanced metadata fetch using the new modular coordinator. - - Args: - payload: Dict containing 'name', 'url', 'download_url' - regions: Optional list of regions to try - - Returns: - Metadata dict - - Raises: - ValueError: If metadata cannot be fetched - """ - from src.metadata_coordinator import MetadataCoordinator - - # Validate payload early to fail fast for invalid input - config = load_config() - req_keys = config.get("payload", {}).get("required_keys") or ["name", "url", "download_url"] - if not validate_payload(payload, req_keys): - raise ValueError(f"Payload missing required keys: {req_keys}") - - # Optional test-mode guard to prevent real external API calls during CI/test runs - if os.getenv("DISABLE_EXTERNAL_API") == "1": - log.info("metadata.fetch.disabled_api") - raise ValueError("External API calls are disabled in this environment") - - coordinator = MetadataCoordinator() - - metadata = await coordinator.get_metadata_from_webhook(payload) - - if metadata: - # Get enhanced metadata with chapters - return await coordinator.get_enhanced_metadata(metadata) - else: - # Fallback to original logic for compatibility - name = payload.get("name", "") - title = payload.get("title") or name - author = payload.get("author", "") - - # Extract ASIN from name if present - asin_regex = config.get("payload", {}).get("asin_regex") - match = re.search(asin_regex, name) if asin_regex else None - asin = match.group(0) if match else None - - # Use provided regions or default sequence - if not regions: - regions = ["us", "ca", "uk", "au", "fr", "de", "jp", "it", "in", "es"] - - # If we have an ASIN, try to get cached metadata first - if asin: - cached = get_cached_metadata(asin, region="us", api_url=None) - if cached: - return cached - - # Attempt scraping to find an ASIN if none was extracted - if not asin: - scraped = await get_audible_asin(title, author) - if scraped: - asin = scraped - cached = get_cached_metadata(asin, region="us", api_url=None) - if cached: - return cached - - # If we still don't have an ASIN, try regions searching - async with Audible() as audible: - # Try searching with parallel regions (only if we have an ASIN) - if asin: - client = await get_default_client() - regions_to_try = get_regions_priority(regions[0] if regions else "us", max_regions=len(regions)) - - result, _found_region = await client.fetch_first_success( - regions=regions_to_try, - url_factory=lambda r: f"https://api.audnex.us/books/{asin}?region={r}", - validator=lambda d: bool(d.get("asin")), - ) - - if result: - return audible.clean_result(result) - - # Fallback to catalog search - for region in regions: - try: - if asin: - results = await audible.search(title=title, author=author, asin=asin, region=region) - else: - results = await audible.search(title=title, author=author, asin="", region=region) - - if results: - # Return the first (best) result - return results[0] - except (httpx.HTTPStatusError, httpx.RequestError, ValueError) as e: - log.warning("metadata.search.region_error", region=region, error=str(e)) - continue - - # Final error if we couldn't determine any metadata - if not asin: - raise ValueError("ASIN could not be determined") - raise ValueError(f"Could not fetch metadata for '{name}' [{asin}]") # Additional compatibility functions for existing tests and code @@ -589,8 +358,7 @@ def clean_metadata(item: dict[str, Any]) -> dict[str, Any]: - Ensures `narrators` is a list and `series` is an empty string when missing - Exposes `runtime_minutes` and both `cover` and `cover_url` """ - audible = Audible() - base = audible.clean_result(item) + base = normalize_book_result(item) result: dict[str, Any] = {} result["title"] = base.get("title") @@ -598,7 +366,7 @@ def clean_metadata(item: dict[str, Any]) -> dict[str, Any]: authors_raw = item.get("authors") or [] filtered_authors = clean_author_list(authors_raw) result["authors_raw"] = authors_raw - result["author"] = filtered_authors[0] if filtered_authors else None + result["author"] = filtered_authors[0] if filtered_authors else base.get("author") or item.get("author") # Narrators as list and narrator string narrator_raw = base.get("narrator") diff --git a/tests/conftest.py b/tests/conftest.py index a567db6..5f2db7a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ from src.db import delete_request, save_request from src.main import app +from src.metadata_coordinator import MetadataCoordinator from src.qbittorrent import QBittorrentManager from src.security import reset_rate_limit_buckets from src.token_gen import generate_token @@ -146,7 +147,9 @@ def mock_metadata(): Returns the mock object so callers can customize the return value: mock_metadata.return_value = {"title": "Custom Title"} """ - with patch("src.metadata.fetch_metadata", new_callable=AsyncMock) as mock: + with patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", new_callable=AsyncMock + ) as mock: mock.return_value = { "title": "Test Book", "author": "Test Author", @@ -298,6 +301,25 @@ def sample_authors(): return [{"name": "John Doe"}, {"name": "Jane Translator"}, {"name": "Alice Illustrator"}] +@pytest.fixture +def coordinator(): + """Pre-wired MetadataCoordinator with all external adapters replaced by mocks.""" + with ( + patch("src.metadata_coordinator.load_config", return_value={}), + patch("src.metadata_coordinator.MAMApiAdapter") as mock_mam, + patch("src.metadata_coordinator.AudnexMetadata") as mock_audnex, + patch("src.metadata_coordinator.AudibleScraper") as mock_audible, + ): + coord = MetadataCoordinator() + coord.mam_adapter = mock_mam.return_value + coord.audnex = mock_audnex.return_value + coord.audible = mock_audible.return_value + coord.mam_adapter.get_asin_from_url = AsyncMock(return_value=None) # type: ignore[method-assign] + coord.audnex.get_book_by_asin = AsyncMock(return_value=None) # type: ignore[method-assign] + coord.audible.search_from_webhook_name = AsyncMock(return_value=[]) # type: ignore[method-assign] + yield coord + + @pytest.fixture def sample_item(): return { diff --git a/tests/test_audible_client.py b/tests/test_audible_client.py new file mode 100644 index 0000000..5686208 --- /dev/null +++ b/tests/test_audible_client.py @@ -0,0 +1,152 @@ +"""Tests for the authenticated Audible client provider.""" + +import asyncio +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from src.audible_client import AudibleClientProvider + + +def test_explicit_auth_settings_skip_config_load(tmp_path: Path) -> None: + """Explicit auth settings should not require config/config.yaml to exist.""" + auth_file = tmp_path / "audible-auth.json" + auth_file.write_text("{}") + + with patch("src.audible_client.load_config", side_effect=AssertionError("load_config should not be called")): + provider = AudibleClientProvider( + auth_file=str(auth_file), + auth_file_password="test-password", + ) + + assert provider.auth_file == str(auth_file) + assert provider.auth_file_password == "test-password" + + +@pytest.mark.asyncio +async def test_get_client_loads_auth_file_and_caches_by_region(tmp_path: Path) -> None: + """Load the encrypted auth file once and reuse the same region client.""" + auth_file = tmp_path / "audible-auth.json" + auth_file.write_text("{}") + + mock_auth = MagicMock() + mock_client = MagicMock() + mock_client.close = AsyncMock() + + provider = AudibleClientProvider( + auth_file=str(auth_file), + auth_file_password="test-password", + ) + + with patch("src.audible_client._audible_mod.Authenticator.from_file", return_value=mock_auth) as mock_from_file: + with patch("src.audible_client._audible_mod.AsyncClient", return_value=mock_client) as mock_async_client: + first = await provider.get_client("us") + second = await provider.get_client("us") + + assert first is mock_client + assert second is mock_client + called_path = Path(mock_from_file.call_args.args[0]) + assert called_path == auth_file + assert mock_from_file.call_args.kwargs["password"] == "test-password" + mock_async_client.assert_called_once_with(auth=mock_auth, country_code="us") + + +@pytest.mark.asyncio +async def test_get_client_returns_none_without_decrypt_password(tmp_path: Path) -> None: + """Do not attempt auth loading when the decrypt password is missing.""" + auth_file = tmp_path / "audible-auth.json" + auth_file.write_text("{}") + + with patch.dict("os.environ", {"AUDIBLE_AUTH_FILE_PASSWORD": "", "AUDIBLE_AUTH_FILE": ""}, clear=False): + provider = AudibleClientProvider(auth_file=str(auth_file)) + + with patch("src.audible_client._audible_mod.Authenticator.from_file") as mock_from_file: + with patch("src.audible_client._audible_mod.AsyncClient") as mock_async_client: + client = await provider.get_client("us") + + assert client is None + mock_from_file.assert_not_called() + mock_async_client.assert_not_called() + + +@pytest.mark.asyncio +async def test_get_client_redacts_missing_auth_file_path(tmp_path: Path) -> None: + """Missing-file warnings should avoid logging the full auth path.""" + missing_auth_file = tmp_path / "nested" / "audible-auth.json" + provider = AudibleClientProvider( + auth_file=str(missing_auth_file), + auth_file_password="test-password", + ) + + with patch("src.audible_client.log.warning") as mock_warning: + client = await provider.get_client("us") + + assert client is None + mock_warning.assert_called_once_with( + "audible.library.auth_file_missing", + auth_file=missing_auth_file.name, + ) + + +@pytest.mark.asyncio +async def test_aclose_closes_cached_clients(tmp_path: Path) -> None: + """Close every cached Audible async client when the provider shuts down.""" + auth_file = tmp_path / "audible-auth.json" + auth_file.write_text("{}") + + first_client = MagicMock() + first_client.close = AsyncMock() + second_client = MagicMock() + second_client.close = AsyncMock() + + provider = AudibleClientProvider( + auth_file=str(auth_file), + auth_file_password="test-password", + ) + + with patch("src.audible_client._audible_mod.Authenticator.from_file", return_value=MagicMock()): + with patch("src.audible_client._audible_mod.AsyncClient", side_effect=[first_client, second_client]): + await provider.get_client("us") + await provider.get_client("ca") + + await provider.aclose() + + first_client.close.assert_awaited_once() + second_client.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_aclose_blocks_get_client_during_shutdown(tmp_path: Path) -> None: + """Do not hand out cached clients while shutdown is in progress.""" + auth_file = tmp_path / "audible-auth.json" + auth_file.write_text("{}") + + provider = AudibleClientProvider( + auth_file=str(auth_file), + auth_file_password="test-password", + ) + + close_started = asyncio.Event() + allow_close = asyncio.Event() + + async def _close_side_effect() -> None: + close_started.set() + await allow_close.wait() + + mock_client = MagicMock() + mock_client.close = AsyncMock(side_effect=_close_side_effect) + provider._clients["us"] = mock_client + + close_task = asyncio.create_task(provider.aclose()) + await close_started.wait() + + get_task = asyncio.create_task(provider.get_client("us")) + await asyncio.sleep(0) + allow_close.set() + + get_result = await get_task + await close_task + + assert get_result is None + assert provider._clients == {} diff --git a/tests/test_audible_scraper.py b/tests/test_audible_scraper.py new file mode 100644 index 0000000..1de747d --- /dev/null +++ b/tests/test_audible_scraper.py @@ -0,0 +1,159 @@ +"""Tests for the Audible scraper transport selection.""" + +import os +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from src.audible_client import AudibleClientProvider +from src.audible_scraper import AudibleScraper + + +@pytest.mark.asyncio +async def test_search_by_title_author_uses_audible_library_backend(tmp_path: Path) -> None: + """Use mkb79/Audible when an auth file and password are configured.""" + auth_file = tmp_path / "audible-auth.json" + auth_file.write_text("{}") + + mock_config = { + "metadata": { + "audible": { + "auth_file": str(auth_file), + "search_endpoint": "/1.0/catalog/products", + } + } + } + product = { + "asin": "B0TEST1234", + "title": "The Hobbit", + "authors": [{"name": "J.R.R. Tolkien"}], + "narrators": [{"name": "Andy Serkis"}], + "language": "en", + } + + mock_auth = MagicMock() + mock_client = MagicMock() + mock_client.get = AsyncMock(return_value={"products": [product]}) + mock_client.close = AsyncMock() + + with patch.dict(os.environ, {"AUDIBLE_AUTH_FILE_PASSWORD": "test-password"}, clear=False): + with patch("src.audible_scraper.load_config", return_value=mock_config): + with patch( + "src.audible_client._audible_mod.Authenticator.from_file", return_value=mock_auth + ) as mock_from_file: + with patch( + "src.audible_client._audible_mod.AsyncClient", return_value=mock_client + ) as mock_async_client: + scraper = AudibleScraper() + results = await scraper.search_by_title_author("The Hobbit", "J.R.R. Tolkien") + + assert len(results) == 1 + assert results[0]["asin"] == "B0TEST1234" + assert results[0]["source"] == "audible_api" + mock_from_file.assert_called_once_with( + auth_file, + password="test-password", + ) + mock_async_client.assert_called_once_with(auth=mock_auth, country_code="us") + mock_client.get.assert_awaited_once_with( + "/1.0/catalog/products", + params={ + "num_results": 10, + "products_sort_by": "Relevance", + "keywords": "The Hobbit", + "response_groups": "product_desc,product_attrs,product_extended_attrs,media,contributors,series,rating", + "author": "J.R.R. Tolkien", + }, + ) + + +@pytest.mark.asyncio +async def test_search_by_title_author_returns_empty_without_auth_config() -> None: + """The package-backed Audible backend requires an auth file and decrypt password.""" + mock_config = { + "metadata": { + "audible": { + "search_endpoint": "/1.0/catalog/products", + } + } + } + + with patch.dict(os.environ, {"AUDIBLE_AUTH_FILE_PASSWORD": "", "AUDIBLE_AUTH_FILE": ""}): + with patch("src.audible_scraper.load_config", return_value=mock_config): + scraper = AudibleScraper() + results = await scraper.search_by_title_author("The Hobbit", "J.R.R. Tolkien") + + assert results == [] + + +@pytest.mark.asyncio +async def test_search_by_asin_uses_catalog_params(tmp_path: Path) -> None: + """ASIN lookup should use the catalog search endpoint with params.""" + auth_file = tmp_path / "audible-auth.json" + auth_file.write_text("{}") + + mock_config = { + "metadata": { + "audible": { + "auth_file": str(auth_file), + "search_endpoint": "/1.0/catalog/products", + } + } + } + product = {"asin": "B0TEST1234", "title": "The Hobbit", "language": "en"} + + mock_client = MagicMock() + mock_client.get = AsyncMock(return_value={"products": [product]}) + mock_client.close = AsyncMock() + + with patch.dict(os.environ, {"AUDIBLE_AUTH_FILE_PASSWORD": "test-password"}, clear=False): + with patch("src.audible_scraper.load_config", return_value=mock_config): + with patch("src.audible_client._audible_mod.Authenticator.from_file", return_value=MagicMock()): + with patch("src.audible_client._audible_mod.AsyncClient", return_value=mock_client): + scraper = AudibleScraper() + result = await scraper.search_by_asin("B0TEST1234") + + assert result is not None + assert result["asin"] == "B0TEST1234" + mock_client.get.assert_awaited_once_with( + "/1.0/catalog/products", + params={ + "asin": "B0TEST1234", + "response_groups": "contributors,media,product_attrs,product_desc,product_details,product_extended_attrs,series,rating,category_ladders", + }, + ) + + +@pytest.mark.asyncio +async def test_shared_injected_provider_is_not_closed_on_scraper_exit() -> None: + """Injected Audible providers should remain usable after one scraper exits.""" + mock_config = {"metadata": {"audible": {"search_endpoint": "/1.0/catalog/products"}}} + product = {"asin": "B0TEST1234", "title": "Shared Provider Book", "language": "english"} + + shared_provider = MagicMock(spec=AudibleClientProvider) + shared_provider.get_client = AsyncMock() + shared_provider.aclose = AsyncMock() + + mock_client = MagicMock() + mock_client.get = AsyncMock(return_value={"products": [product]}) + shared_provider.get_client.return_value = mock_client + + with patch("src.audible_scraper.load_config", return_value=mock_config): + async with AudibleScraper(audible_client_provider=shared_provider) as first_scraper: + first_results = await first_scraper.search_by_title_author("Shared Provider Book") + + shared_provider.aclose.assert_not_awaited() + + async with AudibleScraper(audible_client_provider=shared_provider) as second_scraper: + second_results = await second_scraper.search_by_title_author("Shared Provider Book") + + assert first_results[0]["asin"] == "B0TEST1234" + assert second_results[0]["asin"] == "B0TEST1234" + + +def test_is_english_language_accepts_common_locale_variants() -> None: + assert AudibleScraper._is_english_language("english") + assert AudibleScraper._is_english_language("en-au") + assert AudibleScraper._is_english_language("en-ca") + assert not AudibleScraper._is_english_language("fr") diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 0de65d3..6177967 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -1,6 +1,7 @@ import concurrent.futures import time -from unittest.mock import patch +from typing import Any +from unittest.mock import AsyncMock, patch import pytest @@ -28,7 +29,6 @@ def test_complete_approval_workflow(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook") as mock_coord, - patch("src.metadata.fetch_metadata") as mock_fetch, patch("src.notify.pushover.send_pushover") as mock_pushover, patch("src.notify.discord.send_discord") as mock_discord, patch("src.config.load_config") as mock_config, @@ -44,13 +44,6 @@ def test_complete_approval_workflow(self): "cover_url": "http://example.com/cover.jpg", "asin": "B123456789", } - mock_fetch.return_value = { - "title": "E2E Test Book", - "author": "Test Author", - "series": "Test Series", - "cover_url": "http://example.com/cover.jpg", - "asin": "B123456789", - } # Mock notification responses mock_pushover.return_value = (200, {"status": 1}) @@ -101,7 +94,10 @@ def test_complete_rejection_workflow(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": "Rejection Book"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": "Rejection Book"}, + ), ): resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} @@ -129,7 +125,12 @@ def test_webhook_to_notification_pipeline(self): } # Track all notification calls - notification_calls = {"pushover": [], "discord": [], "gotify": [], "ntfy": []} + notification_calls: dict[str, list[tuple[tuple[Any, ...], dict[str, Any]]]] = { + "pushover": [], + "discord": [], + "gotify": [], + "ntfy": [], + } def track_pushover(*args, **kwargs): notification_calls["pushover"].append((args, kwargs)) @@ -158,7 +159,7 @@ def track_ntfy(*args, **kwargs): "DISABLE_WEBHOOK_NOTIFICATIONS": "0", # Enable notifications for this test }, ), - patch("src.metadata.fetch_metadata") as mock_fetch, + patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook") as mock_coord, patch("src.notify.pushover.send_pushover", side_effect=track_pushover), patch("src.notify.discord.send_discord", side_effect=track_discord), patch("src.notify.gotify.send_gotify", side_effect=track_gotify), @@ -176,7 +177,7 @@ def track_ntfy(*args, **kwargs): "payload": {"required_keys": ["name"]}, } - mock_fetch.return_value = {"title": "Pipeline Book", "author": "Pipeline Author"} + mock_coord.return_value = {"title": "Pipeline Book", "author": "Pipeline Author"} resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} @@ -210,7 +211,6 @@ def test_metadata_fetch_to_storage_pipeline(self): patch( "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", return_value=expected_metadata ), - patch("src.metadata.fetch_metadata", return_value=expected_metadata), ): resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} @@ -239,7 +239,10 @@ def test_token_lifecycle_complete(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": "Lifecycle Book"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": "Lifecycle Book"}, + ), ): # Step 1: Create token via webhook resp = self.client.post( @@ -288,16 +291,13 @@ def process_webhook(payload_data): # Move patching outside concurrent execution to avoid thread-safety issues with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata") as mock_fetch, + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", new_callable=AsyncMock + ) as mock_fetch, ): # Mock needs to return different values for different payloads - # Use async side_effect to match the real async function signature - async def _mock_fetch_metadata(*args, **kwargs): - # Extract payload from args or kwargs - payload = args[0] if args else kwargs.get("payload", {}) - if isinstance(payload, dict): - return {"title": payload.get("name", "Unknown")} - return {"title": "Unknown"} + async def _mock_fetch_metadata(webhook_payload: dict[str, Any]) -> dict[str, str]: + return {"title": webhook_payload.get("name", "Unknown")} mock_fetch.side_effect = _mock_fetch_metadata @@ -325,7 +325,10 @@ def test_error_recovery_in_pipeline(self): # Test with metadata fetch failure with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", side_effect=Exception("Metadata failed")), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + side_effect=Exception("Metadata failed"), + ), ): resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} @@ -354,7 +357,10 @@ def test_notification_failure_recovery(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": "Test Book"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": "Test Book"}, + ), patch("src.notify.pushover.send_pushover", side_effect=Exception("Pushover failed")), patch("src.notify.discord.send_discord", side_effect=Exception("Discord failed")), ): @@ -388,7 +394,6 @@ def test_qbittorrent_integration_workflow(self): "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", return_value={"title": "qBittorrent Book"}, ), - patch("src.metadata.fetch_metadata", return_value={"title": "qBittorrent Book"}), patch("src.config.load_config") as mock_config, ): mock_config.return_value = {"qbittorrent": {"enabled": True}} @@ -435,7 +440,10 @@ def test_token_expiration_workflow(self, monkeypatch): with ( patch("src.db._get_ttl", return_value=1), patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": "Expiration Book"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": "Expiration Book"}, + ), ): # Save with current time monkeypatch.setattr(time, "time", lambda: current_time) @@ -469,7 +477,10 @@ def test_malformed_data_recovery(self): for i, payload in enumerate(malformed_payloads): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": f"Malformed {i}"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": f"Malformed {i}"}, + ), ): resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} @@ -488,7 +499,7 @@ def test_unicode_handling_pipeline(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata") as mock_fetch, + patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook") as mock_fetch, ): mock_fetch.return_value = {"title": "測試書籍 πŸ“š", "author": "тСст Π°Π²Ρ‚ΠΎΡ€"} diff --git a/tests/test_error_recovery.py b/tests/test_error_recovery.py index fa8bff0..0e269a4 100644 --- a/tests/test_error_recovery.py +++ b/tests/test_error_recovery.py @@ -1,26 +1,28 @@ +import asyncio +import concurrent.futures import sqlite3 +import threading from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -from fastapi.testclient import TestClient import src.main from src.db import save_request -from src.main import app -from src.metadata import fetch_metadata from src.notify import pushover -client = TestClient(app) - - class TestErrorRecovery: """Test error recovery and resilience scenarios""" + @pytest.fixture(autouse=True) + def setup_client(self, test_client): + """Use the managed test client so FastAPI lifespan state is initialized.""" + self.client = test_client + @pytest.mark.asyncio - @patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", new_callable=AsyncMock) - async def test_network_timeout_recovery(self, mock_coord): + @pytest.mark.no_mock_external_apis + async def test_network_timeout_recovery(self, coordinator): """Test recovery from network timeouts during metadata fetch""" payload = { "name": "Test Book [B123456789]", @@ -28,16 +30,13 @@ async def test_network_timeout_recovery(self, mock_coord): "download_url": "http://example.com/download.torrent", } - # Configure coordinator to raise ValueError to simulate metadata fetch failure - mock_coord.side_effect = ValueError("Could not fetch metadata") + coordinator.audible.search_from_webhook_name = AsyncMock(side_effect=httpx.ConnectTimeout("Network timeout")) - with patch.dict("os.environ", {"DISABLE_EXTERNAL_API": "0"}): - # Should handle timeout gracefully - metadata service wraps all errors - with pytest.raises(ValueError) as exc_info: - await fetch_metadata(payload) + with pytest.raises(ValueError) as exc_info: + await coordinator.get_metadata_from_webhook(payload) - # Should be a controlled ValueError, not a crash - assert "could not fetch metadata" in str(exc_info.value).lower() + # Should be a controlled ValueError, not a crash + assert "could not fetch metadata" in str(exc_info.value).lower() def test_partial_notification_failure_recovery(self): """Test handling when some notifications fail but others succeed""" @@ -49,11 +48,15 @@ def test_partial_notification_failure_recovery(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", new_callable=AsyncMock) as mock_fetch, + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", new_callable=AsyncMock + ) as mock_coord, ): - mock_fetch.return_value = {"title": "Test Book"} + mock_coord.return_value = {"title": "Test Book"} - resp = client.post("/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"}) + resp = self.client.post( + "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} + ) # Should return 200 - notifications are mocked/disabled in tests assert resp.status_code == 200 @@ -88,8 +91,8 @@ def test_disk_space_exhaustion_handling(self): assert "space" in str(e).lower() @pytest.mark.asyncio - @patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", new_callable=AsyncMock) - async def test_api_rate_limit_handling(self, mock_coord): + @pytest.mark.no_mock_external_apis + async def test_api_rate_limit_handling(self, coordinator): """Test handling of API rate limits""" payload = { "name": "Test Book [B123456789]", @@ -97,41 +100,30 @@ async def test_api_rate_limit_handling(self, mock_coord): "download_url": "http://example.com/download.torrent", } - # Override autouse mock to raise ValueError (simulating rate limit error) - mock_coord.side_effect = ValueError("Could not fetch metadata") + coordinator.audible.search_from_webhook_name = AsyncMock(side_effect=ValueError("429 Too Many Requests")) - with patch.dict("os.environ", {"DISABLE_EXTERNAL_API": "0"}): - # Should handle rate limits gracefully - metadata service wraps all errors - with pytest.raises(ValueError) as exc_info: - await fetch_metadata(payload) + with pytest.raises(ValueError) as exc_info: + await coordinator.get_metadata_from_webhook(payload) - # Should be a controlled ValueError - assert "could not fetch metadata" in str(exc_info.value).lower() + # Should be a controlled ValueError + assert "could not fetch metadata" in str(exc_info.value).lower() @pytest.mark.asyncio - async def test_service_unavailable_fallback(self): - """Test fallback when external services are unavailable""" + @pytest.mark.no_mock_external_apis + async def test_service_unavailable_recovery(self, coordinator): + """Test controlled failure when external services are unavailable.""" payload = { "name": "Test Book [B123456789]", "url": "http://example.com/view", "download_url": "http://example.com/download.torrent", } - with ( - patch.dict("os.environ", {"DISABLE_EXTERNAL_API": "0"}), - patch("src.metadata.get_cached_metadata") as mock_cached, - ): - # All API calls fail - mock_cached.side_effect = httpx.ConnectError("Service unavailable") + coordinator.audible.search_from_webhook_name = AsyncMock(side_effect=httpx.ConnectError("Service unavailable")) - # Should handle service unavailability - try: - result = await fetch_metadata(payload) - # Should return minimal data or handle gracefully - assert isinstance(result, dict) - except Exception as e: - # Should be a controlled exception, not a crash - assert "connection" in str(e).lower() or "unavailable" in str(e).lower() + with pytest.raises(ValueError) as exc_info: + await coordinator.get_metadata_from_webhook(payload) + + assert "could not fetch metadata" in str(exc_info.value).lower() def test_concurrent_error_handling(self): """Test error handling under concurrent load""" @@ -141,27 +133,46 @@ def test_concurrent_error_handling(self): "download_url": "http://example.com/download.torrent", } - with patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}): - # Send multiple concurrent requests with some failing - responses = [] - for _i in range(5): - try: - resp = client.post( - "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} - ) - responses.append(resp.status_code) - except Exception as e: - responses.append(str(e)) - - # At least some requests should succeed or fail gracefully - assert len(responses) == 5 - # Should not have any unhandled exceptions (would be strings) - successful_responses = [r for r in responses if isinstance(r, int)] - assert len(successful_responses) > 0 + in_flight = 0 + max_in_flight = 0 + counter_lock = threading.Lock() + + async def slow_metadata_handler(_payload): + nonlocal in_flight, max_in_flight + with counter_lock: + in_flight += 1 + max_in_flight = max(max_in_flight, in_flight) + try: + await asyncio.sleep(0.05) + return {"title": "Test Book"} + finally: + with counter_lock: + in_flight -= 1 + + def send_request() -> int: + resp = self.client.post( + "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} + ) + return int(resp.status_code) + + with ( + patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + new_callable=AsyncMock, + side_effect=slow_metadata_handler, + ), + ): + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + responses = list(executor.map(lambda _unused: send_request(), range(5))) + + assert len(responses) == 5 + assert all(status_code == 200 for status_code in responses) + assert max_in_flight >= 2 @pytest.mark.asyncio - @patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", new_callable=AsyncMock) - async def test_malformed_response_handling(self, mock_coord): + @pytest.mark.no_mock_external_apis + async def test_malformed_response_handling(self, coordinator): """Test handling of malformed API responses""" payload = { "name": "Test Book [B123456789]", @@ -169,16 +180,13 @@ async def test_malformed_response_handling(self, mock_coord): "download_url": "http://example.com/download.torrent", } - # Override autouse mock to raise ValueError (simulating malformed response) - mock_coord.side_effect = ValueError("Could not fetch metadata") + coordinator.audible.search_from_webhook_name = AsyncMock(side_effect=ValueError("Malformed response")) - with patch.dict("os.environ", {"DISABLE_EXTERNAL_API": "0"}): - # Should handle malformed responses gracefully - metadata service wraps all errors - with pytest.raises(ValueError) as exc_info: - await fetch_metadata(payload) + with pytest.raises(ValueError) as exc_info: + await coordinator.get_metadata_from_webhook(payload) - # Should be a controlled ValueError - assert "could not fetch metadata" in str(exc_info.value).lower() + # Should be a controlled ValueError + assert "could not fetch metadata" in str(exc_info.value).lower() def test_memory_pressure_handling(self): """Test behavior under memory pressure""" @@ -198,9 +206,13 @@ def test_memory_pressure_handling(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", new_callable=AsyncMock, return_value={"title": "Test"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + new_callable=AsyncMock, + return_value={"title": "Test"}, + ), ): - resp = client.post( + resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} ) diff --git a/tests/test_integration.py b/tests/test_integration.py index 6ae4341..b44cb71 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -29,7 +29,6 @@ def test_complete_approval_workflow(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook") as mock_coord, - patch("src.metadata.fetch_metadata") as mock_fetch, patch("src.notify.pushover.send_pushover") as mock_pushover, patch("src.notify.discord.send_discord") as mock_discord, patch("src.config.load_config") as mock_config, @@ -45,13 +44,6 @@ def test_complete_approval_workflow(self): "cover_url": "http://example.com/cover.jpg", "asin": "B123456789", } - mock_fetch.return_value = { - "title": "E2E Test Book", - "author": "Test Author", - "series": "Test Series", - "cover_url": "http://example.com/cover.jpg", - "asin": "B123456789", - } # Mock notification responses mock_pushover.return_value = (200, {"status": 1}) @@ -102,7 +94,10 @@ def test_complete_rejection_workflow(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": "Rejection Book"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": "Rejection Book"}, + ), ): resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} @@ -130,7 +125,12 @@ def test_webhook_to_notification_pipeline(self): } # Track all notification calls - notification_calls = {"pushover": [], "discord": [], "gotify": [], "ntfy": []} + notification_calls: dict[str, list[tuple[tuple[Any, ...], dict[str, Any]]]] = { + "pushover": [], + "discord": [], + "gotify": [], + "ntfy": [], + } def track_pushover(*args: Any, **kwargs: Any) -> tuple[int, dict]: notification_calls["pushover"].append((args, kwargs)) @@ -159,7 +159,7 @@ def track_ntfy(*args: Any, **kwargs: Any) -> tuple[int, dict]: "DISABLE_WEBHOOK_NOTIFICATIONS": "0", # Enable notifications for this test }, ), - patch("src.metadata.fetch_metadata") as mock_fetch, + patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook") as mock_coord, patch("src.main.send_pushover", side_effect=track_pushover), patch("src.main.send_discord", side_effect=track_discord), patch("src.main.send_gotify", side_effect=track_gotify), @@ -177,30 +177,25 @@ def track_ntfy(*args: Any, **kwargs: Any) -> tuple[int, dict]: "payload": {"required_keys": ["name"]}, } - mock_fetch.return_value = {"title": "Pipeline Book", "author": "Pipeline Author"} + mock_coord.return_value = {"title": "Pipeline Book", "author": "Pipeline Author"} - # Also mock the metadata coordinator - with patch( - "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", - return_value={"title": "Pipeline Book", "author": "Pipeline Author"}, - ): - resp = self.client.post( - "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} - ) + resp = self.client.post( + "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} + ) - assert resp.status_code == 200 + assert resp.status_code == 200 - # Verify notifications were actually sent by the workflow - # (Based on config, notifications should be triggered) - total_notifications = ( - len(notification_calls["pushover"]) - + len(notification_calls["discord"]) - + len(notification_calls["gotify"]) - + len(notification_calls["ntfy"]) - ) + # Verify notifications were actually sent by the workflow + # (Based on config, notifications should be triggered) + total_notifications = ( + len(notification_calls["pushover"]) + + len(notification_calls["discord"]) + + len(notification_calls["gotify"]) + + len(notification_calls["ntfy"]) + ) - # With all notifications enabled in mock config, at least one should be sent - assert total_notifications >= 1, f"Expected at least 1 notification, got {total_notifications}" + # With all notifications enabled in mock config, at least one should be sent + assert total_notifications >= 1, f"Expected at least 1 notification, got {total_notifications}" def test_metadata_fetch_to_storage_pipeline(self): """Test metadata fetching and storage pipeline""" @@ -222,7 +217,6 @@ def test_metadata_fetch_to_storage_pipeline(self): patch( "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", return_value=expected_metadata ), - patch("src.metadata.fetch_metadata", return_value=expected_metadata), ): resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} @@ -249,7 +243,10 @@ def test_token_lifecycle_complete(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": "Lifecycle Book"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": "Lifecycle Book"}, + ), ): # Step 1: Create token via webhook resp = self.client.post( @@ -290,20 +287,31 @@ def test_concurrent_webhook_processing(self): ) def process_webhook(payload_data): - with ( - patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": payload_data["name"]}), - ): - resp = self.client.post( - "/webhook/audiobook-requests", json=payload_data, headers={"X-Autobrr-Token": "test_token"} - ) + resp = self.client.post( + "/webhook/audiobook-requests", json=payload_data, headers={"X-Autobrr-Token": "test_token"} + ) + + return {"status_code": resp.status_code, "payload": payload_data, "success": resp.status_code == 200} + + with ( + patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), + patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook") as mock_coord, + ): + + async def _mock_get_metadata(*args: Any, **kwargs: Any) -> dict[str, str]: + payload = kwargs.get("webhook_payload") or kwargs.get("payload", {}) + if not payload and args and isinstance(args[-1], dict): + payload = args[-1] + if isinstance(payload, dict): + return {"title": payload.get("name", "Unknown")} + return {"title": "Unknown"} - return {"status_code": resp.status_code, "payload": payload_data, "success": resp.status_code == 200} + mock_coord.side_effect = _mock_get_metadata - # Process webhooks concurrently - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - futures = [executor.submit(process_webhook, payload) for payload in payloads] - results = [future.result() for future in concurrent.futures.as_completed(futures)] + # Process webhooks concurrently + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + futures = [executor.submit(process_webhook, payload) for payload in payloads] + results = [future.result() for future in concurrent.futures.as_completed(futures)] # Verify all succeeded successful_results = [r for r in results if r["success"]] @@ -324,7 +332,10 @@ def test_error_recovery_in_pipeline(self): # Test with metadata fetch failure with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", side_effect=Exception("Metadata failed")), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + side_effect=Exception("Metadata failed"), + ), ): resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} @@ -352,7 +363,10 @@ def test_notification_failure_recovery(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": "Test Book"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": "Test Book"}, + ), patch("src.notify.pushover.send_pushover", side_effect=Exception("Pushover failed")), patch("src.notify.discord.send_discord", side_effect=Exception("Discord failed")), ): @@ -385,7 +399,6 @@ def test_qbittorrent_integration_workflow(self): "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", return_value={"title": "qBittorrent Book"}, ), - patch("src.metadata.fetch_metadata", return_value={"title": "qBittorrent Book"}), patch("src.config.load_config") as mock_config, ): mock_config.return_value = {"qbittorrent": {"enabled": True}} @@ -433,7 +446,10 @@ def test_token_expiration_workflow(self, monkeypatch): with ( patch("src.db._get_ttl", return_value=1), patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": "Expiration Book"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": "Expiration Book"}, + ), ): # Save with current time monkeypatch.setattr(time, "time", lambda: current_time) @@ -467,7 +483,10 @@ def test_malformed_data_recovery(self): for i, payload in enumerate(malformed_payloads): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": f"Malformed {i}"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": f"Malformed {i}"}, + ), ): resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} @@ -486,7 +505,7 @@ def test_unicode_handling_pipeline(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata") as mock_fetch, + patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook") as mock_fetch, ): mock_fetch.return_value = {"title": "測試書籍 πŸ“š", "author": "тСст Π°Π²Ρ‚ΠΎΡ€"} diff --git a/tests/test_main_integration.py b/tests/test_main_integration.py index ebf5706..14778d0 100644 --- a/tests/test_main_integration.py +++ b/tests/test_main_integration.py @@ -1,14 +1,14 @@ from unittest.mock import patch -from fastapi.testclient import TestClient - -from src.main import app - - -client = TestClient(app) +import pytest class TestMainAppIntegration: + @pytest.fixture(autouse=True) + def setup_client(self, test_client): + """Use the managed FastAPI client so lifespan state is initialized.""" + self.client = test_client + @patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}) def test_webhook_endpoint_valid_token(self): # Test the main webhook endpoint with valid token @@ -21,7 +21,7 @@ def test_webhook_endpoint_valid_token(self): } with ( - patch("src.metadata.fetch_metadata") as mock_fetch, + patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook") as mock_fetch, patch("src.notify.pushover.send_pushover") as mock_pushover, patch("src.notify.discord.send_discord") as mock_discord, ): @@ -37,7 +37,9 @@ def test_webhook_endpoint_valid_token(self): mock_pushover.return_value = (200, {"status": 1}) mock_discord.return_value = (204, {}) - resp = client.post("/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"}) + resp = self.client.post( + "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} + ) assert resp.status_code == 200 response_data = resp.json() @@ -56,7 +58,9 @@ def test_webhook_endpoint_invalid_token(self): "download_url": "http://example.com/download.torrent", } - resp = client.post("/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "invalid_token"}) + resp = self.client.post( + "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "invalid_token"} + ) assert resp.status_code == 401 @@ -68,7 +72,7 @@ def test_webhook_endpoint_missing_token(self): "download_url": "http://example.com/download.torrent", } - resp = client.post("/webhook/audiobook-requests", json=payload) + resp = self.client.post("/webhook/audiobook-requests", json=payload) assert resp.status_code == 401 @@ -80,7 +84,7 @@ def test_webhook_endpoint_missing_required_fields(self): # Missing url and download_url } - resp = client.post("/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"}) + resp = self.client.post("/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"}) assert resp.status_code == 400 @@ -94,7 +98,10 @@ def test_webhook_endpoint_metadata_failure(self): } with ( - patch("src.metadata.fetch_metadata", side_effect=Exception("Metadata service down")), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + side_effect=Exception("Metadata service down"), + ), patch("src.notify.pushover.send_pushover") as mock_pushover, patch("src.notify.discord.send_discord") as mock_discord, ): @@ -102,7 +109,9 @@ def test_webhook_endpoint_metadata_failure(self): mock_pushover.return_value = (200, {"status": 1}) mock_discord.return_value = (204, {}) - resp = client.post("/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"}) + resp = self.client.post( + "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} + ) # Should still succeed but with empty metadata assert resp.status_code == 200 @@ -119,20 +128,22 @@ def test_webhook_endpoint_notification_failure(self): } with ( - patch("src.metadata.fetch_metadata") as mock_fetch, + patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook") as mock_fetch, patch("src.notify.pushover.send_pushover", side_effect=Exception("Pushover down")), patch("src.notify.discord.send_discord", side_effect=Exception("Discord down")), ): mock_fetch.return_value = {"title": "Test Book"} - resp = client.post("/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"}) + resp = self.client.post( + "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} + ) # Should still succeed despite notification failures assert resp.status_code == 200 def test_request_id_logging(self): # Test that request IDs are generated and logged - resp = client.get("/") + resp = self.client.get("/") assert resp.status_code == 200 # Should have generated a request ID and returned it in headers assert "X-Request-ID" in resp.headers @@ -140,13 +151,13 @@ def test_request_id_logging(self): def test_client_ip_logging(self): # Test that client IPs are captured via X-Forwarded-For header # IP logging happens in request_id_middleware which is tested separately - resp = client.get("/", headers={"X-Forwarded-For": "1.2.3.4, 5.6.7.8"}) + resp = self.client.get("/", headers={"X-Forwarded-For": "1.2.3.4, 5.6.7.8"}) assert resp.status_code == 200 # Verify the request was processed successfully with the forwarded IP header assert "X-Request-ID" in resp.headers def test_cors_headers(self): # Test CORS headers if enabled - resp = client.options("/") + resp = self.client.options("/") # Should handle OPTIONS request assert resp.status_code in (200, 405) # 405 if CORS not enabled diff --git a/tests/test_metadata_extended.py b/tests/test_metadata_extended.py index 621d544..2e52479 100644 --- a/tests/test_metadata_extended.py +++ b/tests/test_metadata_extended.py @@ -2,7 +2,13 @@ import pytest -from src.metadata import clean_metadata, fetch_metadata, get_audible_asin, levenshtein_distance +from src.metadata import ( + clean_metadata, + clean_series_sequence, + get_audible_asin, + levenshtein_distance, + normalize_book_result, +) class TestMetadataModule: @@ -41,6 +47,27 @@ def test_clean_metadata_missing_fields(self): assert result["series"] == "" assert result["narrators"] == [] + def test_clean_metadata_scalar_author_fallback(self): + item = {"title": "Minimal Book", "author": "Brandon Sanderson"} + + result = clean_metadata(item) + + assert result["author"] == "Brandon Sanderson" + + def test_clean_series_sequence_numeric_inputs(self): + assert clean_series_sequence("Test Series", 1) == "1" + assert clean_series_sequence("Test Series", 1.5) == "1.5" + + def test_normalize_book_result_accepts_decimal_runtime_values(self): + assert normalize_book_result({"title": "Test Book", "runtimeLengthMin": 360})["duration"] == 360 + assert normalize_book_result({"title": "Test Book", "runtimeLengthMin": "360.5"})["duration"] == 360 + assert normalize_book_result({"title": "Test Book", "runtimeLengthMin": 360.5})["duration"] == 360 + + def test_normalize_book_result_handles_non_string_release_date(self): + result = normalize_book_result({"title": "Test Book", "releaseDate": 1700000000, "publishedYear": "2024"}) + + assert result["publishedYear"] == "2024" + def test_clean_metadata_genres_and_tags(self): item = { "title": "Test Book", @@ -57,146 +84,99 @@ def test_clean_metadata_genres_and_tags(self): assert result["tags"] == "Epic, Magic" @pytest.mark.asyncio - @patch("src.metadata.get_default_client", new_callable=AsyncMock) - @patch("builtins.__import__") - async def test_get_audible_asin_success(self, mock_import, mock_get_client): - # Mock HTTP client response - mock_response = MagicMock() - mock_response.text = '
' - - mock_client = MagicMock() - mock_client.get = AsyncMock(return_value=mock_response) - mock_get_client.return_value = mock_client - - # Mock the bs4 import - mock_bs4 = MagicMock() - mock_soup = MagicMock() - mock_bs4.BeautifulSoup.return_value = mock_soup - mock_soup.find.return_value = {"data-asin": "B123456789"} - - # Store original import - original_import = __builtins__["__import__"] - - def import_side_effect(name, *args, **kwargs): - if name == "bs4": - return mock_bs4 - elif name == "bs4.element": - mock_element = MagicMock() - return mock_element - return original_import(name, *args, **kwargs) - - mock_import.side_effect = import_side_effect + @patch("src.metadata.AudibleScraper") + async def test_get_audible_asin_success(self, mock_scraper_cls): + mock_scraper = MagicMock() + mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper) + mock_scraper.__aexit__ = AsyncMock(return_value=False) + mock_scraper.search = AsyncMock(return_value=[{"asin": "B123456789"}]) + mock_scraper_cls.return_value = mock_scraper asin = await get_audible_asin("Test Title", "Test Author") assert asin == "B123456789" @pytest.mark.asyncio - @patch("src.metadata.get_default_client", new_callable=AsyncMock) - @patch("builtins.__import__") - async def test_get_audible_asin_not_found(self, mock_import, mock_get_client): - mock_response = MagicMock() - mock_response.text = "
No ASIN here
" - - mock_client = MagicMock() - mock_client.get = AsyncMock(return_value=mock_response) - mock_get_client.return_value = mock_client - - # Mock the bs4 import - mock_bs4 = MagicMock() - mock_soup = MagicMock() - mock_bs4.BeautifulSoup.return_value = mock_soup - mock_soup.find.return_value = None - - # Store original import - original_import = __builtins__["__import__"] - - def import_side_effect(name, *args, **kwargs): - if name == "bs4": - return mock_bs4 - elif name == "bs4.element": - mock_element = MagicMock() - return mock_element - return original_import(name, *args, **kwargs) - - mock_import.side_effect = import_side_effect + @patch("src.metadata.AudibleScraper") + async def test_get_audible_asin_not_found(self, mock_scraper_cls): + mock_scraper = MagicMock() + mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper) + mock_scraper.__aexit__ = AsyncMock(return_value=False) + mock_scraper.search = AsyncMock(return_value=[{"title": "Unknown Title"}]) + mock_scraper_cls.return_value = mock_scraper asin = await get_audible_asin("Unknown Title", "Unknown Author") assert asin is None @pytest.mark.asyncio - async def test_get_audible_asin_no_beautifulsoup(self): - # Test when BeautifulSoup is not available - with patch("builtins.__import__", side_effect=ImportError("No module named 'bs4'")): - asin = await get_audible_asin("Test Title", "Test Author") - assert asin is None + @patch("src.metadata.AudibleScraper") + async def test_get_audible_asin_search_error(self, mock_scraper_cls): + mock_scraper = MagicMock() + mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper) + mock_scraper.__aexit__ = AsyncMock(return_value=False) + mock_scraper.search = AsyncMock(side_effect=RuntimeError("search failed")) + mock_scraper_cls.return_value = mock_scraper - @pytest.mark.asyncio - @patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", new_callable=AsyncMock) - @patch("src.metadata.get_cached_metadata") - async def test_fetch_metadata_success(self, mock_cached, mock_coord): - # Mock successful metadata fetch - expected_metadata = {"title": "Test Book", "authors": [{"name": "Test Author"}], "asin": "B123456789"} - mock_cached.return_value = expected_metadata - mock_coord.return_value = expected_metadata + asin = await get_audible_asin("Test Title", "Test Author") + assert asin is None + @pytest.mark.asyncio + @pytest.mark.no_mock_external_apis + async def test_coordinator_webhook_success(self, coordinator): payload = { "name": "Test Book by Test Author [B123456789]", - "url": "http://example.com/view", + "url": "https://www.myanonamouse.net/t/12345", "download_url": "http://example.com/download.torrent", } - result = await fetch_metadata(payload) + + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value="B123456789") + coordinator.audnex.get_book_by_asin = AsyncMock( + return_value={"title": "Test Book", "authors": [{"name": "Test Author"}], "asin": "B123456789"} + ) + + result = await coordinator.get_metadata_from_webhook(payload) assert result["title"] == "Test Book" - assert "asin" in result + assert result["asin"] == "B123456789" + assert result["source"] == "audnex" @pytest.mark.asyncio - @patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", new_callable=AsyncMock) - @patch("src.metadata.get_cached_metadata") - @patch("src.metadata.get_audible_asin") - async def test_fetch_metadata_fallback_to_scraping(self, mock_scrape, mock_cached, mock_coord): - # Test fallback when API fails but scraping succeeds - mock_cached.return_value = None - mock_scrape.return_value = "B987654321" - - # Mock successful API call with scraped ASIN - def cached_side_effect(asin, region, api_url): - if asin == "B987654321": - return {"title": "Scraped Book", "asin": asin} - return None - - mock_cached.side_effect = cached_side_effect - mock_coord.return_value = {"title": "Scraped Book", "asin": "B987654321"} - + @pytest.mark.no_mock_external_apis + async def test_coordinator_fallback_to_audible_search(self, coordinator): payload = { "name": "Unknown Book by Unknown Author", "url": "http://example.com/view", "download_url": "http://example.com/download.torrent", } - result = await fetch_metadata(payload) - assert "title" in result - assert result.get("asin") == "B987654321" + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value=None) + coordinator.audnex.get_book_by_asin = AsyncMock(return_value=None) + coordinator.audible.search_from_webhook_name = AsyncMock( + return_value=[{"title": "Resolved Book", "asin": "B987654321"}] + ) + + result = await coordinator.get_metadata_from_webhook(payload) + + assert result is not None + assert result["title"] == "Resolved Book" + assert result["asin"] == "B987654321" + assert result["source"] == "audible" @pytest.mark.asyncio - @patch("src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", new_callable=AsyncMock) - async def test_fetch_metadata_no_asin_found(self, mock_coord): - # Test when no ASIN can be extracted or found + @pytest.mark.no_mock_external_apis + async def test_coordinator_returns_none_when_no_metadata_found(self, coordinator): payload = { "name": "Very Obscure Book", "url": "http://example.com/view", "download_url": "http://example.com/download.torrent", } - # Override autouse mock to raise ValueError - mock_coord.return_value = None + coordinator.mam_adapter.get_asin_from_url = AsyncMock(return_value=None) + coordinator.audnex.get_book_by_asin = AsyncMock(return_value=None) + coordinator.audible.search_from_webhook_name = AsyncMock(return_value=[]) + + result = await coordinator.get_metadata_from_webhook(payload) - with ( - patch("src.metadata.get_cached_metadata", return_value=None), - patch("src.metadata.get_audible_asin", return_value=None), - pytest.raises(ValueError, match="ASIN could not be determined"), - ): - # Should raise ValueError when no ASIN can be found - await fetch_metadata(payload) + assert result is None def test_clean_metadata_runtime_conversion(self): # Test runtime conversion from minutes to readable format diff --git a/tests/test_security.py b/tests/test_security.py index b866b64..13d526b 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -1,19 +1,20 @@ import time from unittest.mock import patch -from fastapi.testclient import TestClient +import pytest -from src.main import app from src.token_gen import generate_token from src.utils import strip_html_tags -client = TestClient(app) - - class TestSecurity: """Test security and input validation""" + @pytest.fixture(autouse=True) + def setup_client(self, test_client): + """Use the managed FastAPI client so lifespan state is initialized.""" + self.client = test_client + def test_sql_injection_attempts(self): """Test protection against SQL injection""" malicious_payloads = [ @@ -33,9 +34,12 @@ def test_sql_injection_attempts(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": "Safe Title"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": "Safe Title"}, + ), ): - resp = client.post( + resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} ) @@ -75,9 +79,12 @@ def test_xss_payload_sanitization(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": f"Title {xss_payload}"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": f"Title {xss_payload}"}, + ), ): - resp = client.post( + resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} ) @@ -91,7 +98,7 @@ def test_token_brute_force_protection(self): failure_count = 0 for token in invalid_tokens: - resp = client.get(f"/approve/{token}") + resp = self.client.get(f"/approve/{token}") # Application returns 410 for invalid/expired tokens if resp.status_code in [404, 410]: failure_count += 1 @@ -102,7 +109,7 @@ def test_token_brute_force_protection(self): # Test rate limiting (if implemented) rapid_requests = [] for _i in range(20): - resp = client.post( + resp = self.client.post( "/webhook/audiobook-requests", json={"name": "test"}, headers={"X-Autobrr-Token": "invalid_token"} ) rapid_requests.append(resp.status_code) @@ -121,7 +128,7 @@ def test_request_size_limits(self): } with patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}): - resp = client.post( + resp = self.client.post( "/webhook/audiobook-requests", json=large_payload, headers={"X-Autobrr-Token": "test_token"} ) @@ -140,12 +147,12 @@ def test_path_traversal_prevention(self): for malicious_path in path_traversal_attempts: # Test in various fields - resp = client.get(f"/approve/{malicious_path}") + resp = self.client.get(f"/approve/{malicious_path}") # Should not expose file system - app returns 410 for invalid tokens assert resp.status_code in [404, 400, 422, 410] # Test as URL parameter - resp = client.get(f"/?file={malicious_path}") + resp = self.client.get(f"/?file={malicious_path}") assert resp.status_code in [200, 404, 400] # Should not crash def test_header_injection_prevention(self): @@ -165,7 +172,9 @@ def test_header_injection_prevention(self): with patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}): for header_name, header_value in malicious_headers.items(): - resp = client.post("/webhook/audiobook-requests", json=payload, headers={header_name: header_value}) + resp = self.client.post( + "/webhook/audiobook-requests", json=payload, headers={header_name: header_value} + ) # Should handle malicious headers safely assert resp.status_code in [200, 400, 401, 422] @@ -190,16 +199,12 @@ def test_json_injection_attempts(self): with patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}): for malicious_json in malicious_jsons: - try: - resp = client.post( - "/webhook/audiobook-requests", json=malicious_json, headers={"X-Autobrr-Token": "test_token"} - ) + resp = self.client.post( + "/webhook/audiobook-requests", json=malicious_json, headers={"X-Autobrr-Token": "test_token"} + ) - # Should handle malicious JSON safely - assert resp.status_code in [200, 400, 422, 500] - except Exception as e: - # Should not cause unhandled exceptions - assert "json" in str(e).lower() or "decode" in str(e).lower() + # Should handle malicious JSON safely without server errors + assert resp.status_code in [200, 400, 422] def test_unicode_security(self): """Test handling of dangerous Unicode characters""" @@ -220,10 +225,13 @@ def test_unicode_security(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": "Safe Title"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": "Safe Title"}, + ), ): try: - resp = client.post( + resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} ) @@ -255,9 +263,12 @@ def test_command_injection_prevention(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": "Safe Title"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": "Safe Title"}, + ), ): - resp = client.post( + resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} ) @@ -277,9 +288,12 @@ def test_ldap_injection_prevention(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": "Safe Title"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": "Safe Title"}, + ), ): - resp = client.post( + resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} ) @@ -306,9 +320,12 @@ def test_regex_dos_prevention(self): with ( patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}), - patch("src.metadata.fetch_metadata", return_value={"title": "Safe Title"}), + patch( + "src.metadata_coordinator.MetadataCoordinator.get_metadata_from_webhook", + return_value={"title": "Safe Title"}, + ), ): - resp = client.post( + resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} ) @@ -327,11 +344,16 @@ def test_csrf_protection(self): "download_url": "http://example.com/download.torrent", } - # Request without Origin header should be treated carefully - resp = client.post("/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"}) + with patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}): + matching_resp = self.client.post( + "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} + ) + mismatched_resp = self.client.post( + "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "wrong-token"} + ) - # Should still work for API endpoints, but web endpoints should be protected - assert resp.status_code in [200, 401, 403] + assert matching_resp.status_code == 200 + assert mismatched_resp.status_code == 401 def test_input_length_validation(self): """Test validation of input field lengths""" @@ -350,7 +372,7 @@ def test_input_length_validation(self): payload[field_name] = long_value with patch.dict("os.environ", {"AUTOBRR_TOKEN": "test_token"}): - resp = client.post( + resp = self.client.post( "/webhook/audiobook-requests", json=payload, headers={"X-Autobrr-Token": "test_token"} ) diff --git a/tests/test_utils_metadata.py b/tests/test_utils_metadata.py index 1a765ac..5ed1e9f 100644 --- a/tests/test_utils_metadata.py +++ b/tests/test_utils_metadata.py @@ -1,6 +1,4 @@ -import pytest - -from src.metadata import clean_metadata, fetch_metadata +from src.metadata import clean_metadata from src.utils import build_notification_message, clean_author_list, strip_html_tags, validate_payload @@ -57,9 +55,3 @@ def test_build_notification_message(sample_item, sample_payload): assert "Summary paragraph." in msg # Approve link present assert "/approve/token123" in msg - - -@pytest.mark.asyncio -async def test_fetch_metadata_invalid(): - with pytest.raises(ValueError, match="Payload missing required keys"): - await fetch_metadata({})