Stage 1: Add dotenv-based config system with backward-compatible credentials adapter

.env.docker

@@ -0,0 +1,18 @@
+# Docker-internal overrides
+# Loaded AFTER .env by Docker Compose (env_file directive).
+# only override values that differ inside the Docker network.
+
+# Inside Docker, services use container hostnames instead of localhost
+WP10DB_HOST=dev-database
+WP10DB_PORT=3306
+REDIS_HOST=redis
+REDIS_PORT=6379
+
+# Storage uses the minio container hostname inside Docker
+STORAGE_URL=http://minio:9000
+
+# Backend and Zimfarm URLs use Docker service names
+CLIENT_BACKEND_URL=http://wp1bot-web-dev:5000
+CLIENT_BACKEND_S3_URL=http://minio:9000/org-kiwix-dev-wp1
+ZIMFARM_URL=http://zimfarm-api/v2
+ZIMFARM_S3_URL=https://minio:9000/org-kiwix-dev-zims

.env.example

@@ -0,0 +1,172 @@
+# WP1 Configuration
+# Copy this file to .env and edit as needed.
+#
+# Your application database (WP10DB) should work immediately after you run:
+#   docker compose -f docker-compose-dev.yml up -d
+# in the root directory. The Wiki replica database (WIKIDB) requires actual
+# Toolforge credentials (see the WIKIDB section below).
+# The test database will be available when you run:
+#   docker compose -f docker-compose-test.yml up -d
+
+
+# --- Environment ---
+# The environment the app is running in. Controls which behavior and checks are active.
+# Values: development, production, test
+# Note: Never manually set this to 'test'. That is handled automatically by config.py
+# when pytest is detected.
+WP1_ENV=development
+
+# The directory under conf/ to look for the conf.json file in. So if this is
+# 'foo', the file will be loaded as conf/foo/conf.json. It is okay to check in
+# conf.json files, they don't contain any sensitive information.
+WP1_CONF_LANG=en
+
+
+# --- WIKIDB (Wikipedia replica database) ---
+# Database credentials for the Wikipedia replica database hosted on Wikimedia's
+# Toolforge infrastructure. This is a read-only replica of English Wikipedia.
+#
+# There are two ways to access the Wikipedia replicca database in development :
+#
+# Option A — SOCKS5 proxy :
+#   ssh -D 1080 login.toolforge.org
+# (This assumes you have set up your SSH credentials for Toolforge.)
+# Database traffic is tunneled through the proxy so *.eqiad.wmflabs can resolve.
+# Use the values below as-is.
+#
+# Option B — SSH port-forwarding (useful if running inside Docker):
+#   ssh -L 4711:enwiki.analytics.db.svc.eqiad.wmflabs:3306 login.toolforge.org
+# Then override in your .env:
+#   WIKIDB_HOST=localhost
+#   WIKIDB_PORT=4711
+WIKIDB_USER=someuser          # EDIT: your Toolforge username
+WIKIDB_PASSWORD=somepass      # EDIT: your Toolforge password
+WIKIDB_HOST=enwiki.analytics.db.svc.eqiad.wmflabs
+WIKIDB_DB=enwiki_p
+# WIKIDB_PORT=                # Not needed for Option A
+
+
+# --- WP10DB (application database) ---
+# Database credentials for the enwp10 project/application database.
+# For development, use docker-compose-dev.yml to spin up a local database that
+# has some (potentially out of date) data in it.
+WP10DB_USER=root
+WP10DB_PASSWORD=wikipedia
+WP10DB_HOST=localhost
+WP10DB_PORT=6300
+WP10DB_DB=enwp10_dev
+
+
+# --- Redis ---
+# Credentials for connecting to Redis. In development, this is also run as a
+# docker-compose service .
+REDIS_HOST=localhost
+REDIS_PORT=9736
+
+
+# --- API (Wikipedia bot credentials) ---
+# Credentials used by the bot backend to log in to English Wikipedia and edit
+# articles. Not necessary in development (there's no wiki to edit locally).
+# API_USER=
+# API_PASSWORD=
+
+
+# --- Overlay (development mock queue) ---
+# Options for the "Development overlay" which mocks queue functionality in the
+# development environment, so you can test update flows without a real job queue.
+OVERLAY_UPDATE_WAIT_TIME=40
+OVERLAY_JOB_ELAPSED_TIME=10
+OVERLAY_BASIC_INCOME_TOTAL_TIME=60
+
+
+# --- MWOAUTH (Wikimedia OAuth) ---
+# Credentials for authentication to Wikimedia through mwoauth. Used by the
+# frontend/backend to enable the "Login" functionality on the web app.
+#
+# In DEVELOPMENT mode, OAuth is automatically skipped and a fake user account
+# is created, so these credentials are NOT needed for local development.
+#
+# For production, register your own app with callback URL:
+#   http://localhost:5000/v1/oauth/complete
+# Register at: https://meta.wikimedia.org/wiki/Special:OAuthConsumerRegistration/propose
+MWOAUTH_CONSUMER_KEY=
+MWOAUTH_CONSUMER_SECRET=
+
+
+# --- Session ---
+# Secret key for Flask session encryption. If this ever changes, all currently
+# logged-in users will be logged out (their session cookies become invalid).
+# To generate a secure random key for production:
+#   python3 -c "import os; print(os.urandom(24).hex())"
+SESSION_SECRET_KEY=WP1_secret_key    # EDIT: set a random secret in production
+
+
+# --- Client URLs ---
+# URLs used for redirections, CORS, and internal service communication.
+# Comma-separated list of allowed origins for CORS.
+CLIENT_DOMAINS=http://localhost:5173
+CLIENT_HOMEPAGE=http://localhost:5173/#/
+CLIENT_S3_URL=http://localhost:9000/org-kiwix-dev-wp1
+CLIENT_API_URL=http://localhost:5000
+CLIENT_BACKEND_URL=http://localhost:5000
+# In development with Docker, override this via .env.docker so the zimfarm
+# worker can reach MinIO using its Docker hostname.
+CLIENT_BACKEND_S3_URL=http://localhost:9000/org-kiwix-dev-wp1
+
+
+# --- Storage (S3/MinIO) ---
+# Configuration for the storage backend for storing selection list files.
+# For development, you can use the local MinIO instance that's included in the
+# docker-compose setup (settings below are pre-configured for this).
+#
+# If you need to use an external service instead, you have two options:
+#   1. Use the Kiwix S3 backend on Wasabi. Request dev/prod credentials from the team.
+#      See https://github.com/openzim/zimfarm/wiki/S3-Cache-Policy for how.
+#   2. Use another S3-compatible storage service (AWS S3, etc.).
+# In either external case, edit the settings below accordingly.
+STORAGE_URL=http://localhost:9000/
+STORAGE_KEY=minio_key        # username
+STORAGE_SECRET=minio_secret  # password
+STORAGE_BUCKET=org-kiwix-dev-wp1
+
+
+# --- Zimfarm ---
+# Server URL and credentials for the Zim Farm that creates ZIM files from
+# materialized selections.
+ZIMFARM_URL=http://localhost:8003/v2
+ZIMFARM_S3_URL=https://localhost:9000/org-kiwix-dev-zims  # if using minio
+ZIMFARM_USER=admin
+ZIMFARM_PASSWORD=admin
+# A simple token secret exchanged between the WP1 server and the zimfarm
+# server, to ensure requests to the webhook endpoint are valid.
+
+# ZIMFARM_HOOK_TOKEN=          # EDIT: set a strong secret in production
+
+# Update this to the latest version at the time of your deployment.
+ZIMFARM_IMAGE=ghcr.io/openzim/mwoffliner:1.17.2
+ZIMFARM_DEFINITION_VERSION=1.17.2
+
+# ZIMFARM_CACHE_URL=           # Production only (Wasabi cache URL)
+
+
+# --- Mailgun ---
+# Credentials for the Mailgun service, used to send email notifications to users
+# when their requested/scheduled ZIM files are ready. Not required for development.
+
+# EDIT the api_key for production deployments.
+MAILGUN_URL=https://api.eu.mailgun.net/v3/mg.wp1.openzim.org/messages
+MAILGUN_API_KEY=INSERT_YOUR_MAILGUN_API_KEY_HERE  # EDIT this line for production.
+
+
+# --- File paths ---
+# Path where the pageviews.bz2 file (~3GB) will be downloaded and cached.
+# This file is used to calculate article view statistics.
+FILE_PATH_PAGEVIEWS=/tmp/pageviews
+
+
+# --- Logging ---
+# Logging directives are currently hardcoded as an empty dict in config.py.
+ # Logging directives. Keys are the names of the loggers, values are dictionaries
+# with the logging configuration. The special key '*' is used to set the
+# default logging configuration. Logging is always done to stdout, and is redirected
+# and rotated by the supervisor process.

.env.test

@@ -0,0 +1,73 @@
+# WP1 Test Environment Configuration
+# Used by pytest (auto-loaded by config.py when pytest is detected).
+#
+# WARNING: Do NOT remove lines from this file. Every key from .env must
+# be explicitly overridden here, even if set to empty, because Pipenv
+# auto-loads .env before our code runs. Any key not set here will leak
+# from the development .env values.
+
+WP1_ENV=test
+WP1_CONF_LANG=en
+
+# Test databases (docker-compose-test.yml exposes port 6600)
+WIKIDB_USER=root
+WIKIDB_PASSWORD=
+WIKIDB_HOST=localhost
+WIKIDB_PORT=6600
+WIKIDB_DB=enwikip_test
+
+WP10DB_USER=root
+WP10DB_PASSWORD=
+WP10DB_HOST=localhost
+WP10DB_PORT=6600
+WP10DB_DB=enwp10_test
+
+# Test Redis (docker-compose-test.yml)
+REDIS_HOST=localhost
+REDIS_PORT=9777
+
+# Not used in tests
+API_USER=
+API_PASSWORD=
+
+# Not used in tests
+OVERLAY_UPDATE_WAIT_TIME=40
+OVERLAY_JOB_ELAPSED_TIME=10
+OVERLAY_BASIC_INCOME_TOTAL_TIME=60
+
+# Not used in tests
+MWOAUTH_CONSUMER_KEY=
+MWOAUTH_CONSUMER_SECRET=
+
+# Test session
+SESSION_SECRET_KEY=WP1
+
+# Client URLs
+CLIENT_DOMAINS=
+CLIENT_HOMEPAGE=
+CLIENT_S3_URL=
+CLIENT_API_URL=http://test.server.fake
+CLIENT_BACKEND_URL=http://test.server.fake
+CLIENT_BACKEND_S3_URL=
+
+# Test storage
+STORAGE_URL=
+STORAGE_KEY=test_key
+STORAGE_SECRET=test_secret
+STORAGE_BUCKET=org-kiwix-dev-wp1
+
+# Fake Zimfarm values for tests
+ZIMFARM_URL=https://fake.farm/v2
+ZIMFARM_S3_URL=https://fake.wasabisys.com/org-kiwix-zimit
+ZIMFARM_USER=farmuser
+ZIMFARM_PASSWORD=farmpass
+ZIMFARM_HOOK_TOKEN=hook-token-abc
+ZIMFARM_IMAGE=
+ZIMFARM_DEFINITION_VERSION=
+ZIMFARM_CACHE_URL=
+
+# Mailgun
+MAILGUN_URL=https://api.eu.mailgun.net/v3/mg.wp1.openzim.org/messages
+MAILGUN_API_KEY=INSERT_YOUR_MAILGUN_API_KEY_HERE
+
+FILE_PATH_PAGEVIEWS=/tmp/pageviews

.gitignore

@@ -3,10 +3,10 @@ venv
 docs-venv/
 .tool-versions
 
-# Don't commit credentials, credential backup files, or login cookies!
-wp1/credentials.py
-wp1/credentials.py.bak
+#credential.py is now a secret-free adapter and is committed.
+#credential.py.dev contains hardcoded secrets (used by Docker volume mounts)
 wp1/credentials.py.dev
+wp1/credentials.py.bak
 wp1/credentials.py.dev.bak
 cookies.txt
 
@@ -18,6 +18,10 @@ cookies.txt
 # Don't commit logs
 log/**
 
+# Don't commit the active .env file (contains secrets).
+# .env.example, .env.test, and .env.docker are committed and safe to share.
+.env
+
 
 # Don't commit SSH keys
 id_ed25519

Pipfile

@@ -21,7 +21,8 @@ pyparsing = "==3.0.9"
 pytz = "==2022.7.1"
 pyyaml = "==6.0.1"
 redis = "==4.4.4"
-requests = "~=2.33.0"
+python-dotenv = "~=1.0"
+requests = "~=2.32.4"
 rdflib = "~=6.0"
 rq = "==2.2.0"
 rq-scheduler = "==0.14.0"