Reduce cold-start overhead and simplify single-service deployment

.github/app.template.yaml

@@ -1,8 +1,13 @@
 service: ${SERVICE_NAME}
 runtime: python313
-entrypoint: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app
-instance_class: F4
+entrypoint: ${ENTRYPOINT}
 service_account: "${CLOUD_SQL_USER}.gserviceaccount.com"
+instance_class: F4
+inbound_services:
+  - warmup
+automatic_scaling:
+  min_instances: ${MIN_INSTANCES}
+  max_instances: ${MAX_INSTANCES}
 handlers:
   - url: /.*
     secure: always

.github/workflows/CD_production.yml

@@ -8,7 +8,7 @@ permissions:
   contents: write
 
 jobs:
-    staging-deploy:
+    production-deploy:
 
         runs-on: ubuntu-latest
         environment: production
@@ -64,9 +64,8 @@ jobs:
               sudo apt-get install -y gettext-base
             fi
 
-        - name: Render app.yaml
+        - name: Render App Engine configs
           env:
-            SERVICE_NAME: "ocotillo-api"
             ENVIRONMENT: "production"
             CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}"
             CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}"
@@ -87,25 +86,59 @@ jobs:
             SESSION_SECRET_KEY: "${{ secrets.SESSION_SECRET_KEY }}"
             APITALLY_CLIENT_ID: "${{ vars.APITALLY_CLIENT_ID }}"
           run: |
+            export MAX_INSTANCES="10"
+            export SERVICE_NAME="ocotillo-api"
+            export ENTRYPOINT="gunicorn -w 1 -k uvicorn.workers.UvicornWorker main:app"
+            export MIN_INSTANCES="0"
             envsubst < .github/app.template.yaml > app.yaml
 
         - name: Deploy to Google Cloud
           run: |
-            gcloud app deploy app.yaml --quiet --project ${{ vars.GCP_PROJECT_ID }}
+            gcloud app deploy \
+              app.yaml \
+              --quiet \
+              --project ${{ vars.GCP_PROJECT_ID }}
 
-        # Clean up old versions - delete only the oldest version, one created and one destroyed
-        - name: Clean up oldest version
+        - name: Clean up oldest versions
           run: |
-            OLDEST_VERSION=$(gcloud app versions list --service=ocotillo-api --project=${{ vars.GCP_PROJECT_ID}} --format="value(id)" --sort-by="version.createTime" | head -n 1)
-            if [ ! -z "$OLDEST_VERSION" ]; then
-              echo "Deleting oldest version: $OLDEST_VERSION"
-              gcloud app versions delete $OLDEST_VERSION --service=ocotillo-api --project=${{ vars.GCP_PROJECT_ID }} --quiet
-              echo "Deleted oldest version: $OLDEST_VERSION"
+            SERVICE="ocotillo-api"
+            VERSIONS_JSON="$(gcloud app versions list --service="$SERVICE" --project=${{ vars.GCP_PROJECT_ID }} --format=json --sort-by="version.createTime" 2>/dev/null || printf '[]')"
+            export VERSIONS_JSON
+            DELETE_VERSION="$(python - <<'PY'
+            import json
+            import os
+
+            versions = json.loads(os.environ.get("VERSIONS_JSON", "[]") or "[]")
+            if len(versions) <= 1:
+                print("")
+                raise SystemExit(0)
+
+            def traffic_split(version):
+                for key in ("traffic_split", "trafficSplit"):
+                    value = version.get(key)
+                    if value is not None:
+                        try:
+                            return float(value)
+                        except (TypeError, ValueError):
+                            return 0.0
+                return 0.0
+
+            for version in versions:
+                if traffic_split(version) == 0.0:
+                    print(version.get("id", ""))
+                    break
+            else:
+                print("")
+            PY
+            )"
+            if [ -n "$DELETE_VERSION" ]; then
+              echo "Deleting old non-serving version for $SERVICE: $DELETE_VERSION"
+              gcloud app versions delete "$DELETE_VERSION" --service="$SERVICE" --project=${{ vars.GCP_PROJECT_ID }} --quiet
             else
-              echo "No versions to delete"
+              echo "No old non-serving versions to delete for $SERVICE"
             fi
 
-        - name: Remove app.yaml
+        - name: Remove rendered configs
           run: |
             rm app.yaml
 
@@ -118,5 +151,5 @@ jobs:
         # ":" are not alloed in git tags, so replace with "-"
         - name: Tag commit
           run: |
-            git tag -a "production-deploy-$(date -u +%Y-%m-%d)T$(date -u +%H-%M-%S%z)" -m "staging gcloud deployment: $(date -u +%Y-%m-%d)T$(date -u +%H:%M:%S%z)"
+            git tag -a "production-deploy-$(date -u +%Y-%m-%d)T$(date -u +%H-%M-%S%z)" -m "production gcloud deployment: $(date -u +%Y-%m-%d)T$(date -u +%H:%M:%S%z)"
             git push origin --tags

.github/workflows/CD_staging.yml

@@ -64,9 +64,8 @@ jobs:
               sudo apt-get install -y gettext-base
             fi
 
-        - name: Render app.yaml
+        - name: Render App Engine configs
           env:
-            SERVICE_NAME: "ocotillo-api-staging"
             ENVIRONMENT: "staging"
             CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}"
             CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}"
@@ -87,25 +86,59 @@ jobs:
             SESSION_SECRET_KEY: "${{ secrets.SESSION_SECRET_KEY }}"
             APITALLY_CLIENT_ID: "${{ vars.APITALLY_CLIENT_ID }}"
           run: |
+            export MAX_INSTANCES="10"
+            export SERVICE_NAME="ocotillo-api-staging"
+            export ENTRYPOINT="gunicorn -w 1 -k uvicorn.workers.UvicornWorker main:app"
+            export MIN_INSTANCES="0"
             envsubst < .github/app.template.yaml > app.yaml
 
         - name: Deploy to Google Cloud
           run: |
-            gcloud app deploy app.yaml --quiet --project ${{ vars.GCP_PROJECT_ID }}
+            gcloud app deploy \
+              app.yaml \
+              --quiet \
+              --project ${{ vars.GCP_PROJECT_ID }}
 
-        # Clean up old versions - delete only the oldest version, one created and one destroyed
-        - name: Clean up oldest version
+        - name: Clean up oldest versions
           run: |
-            OLDEST_VERSION=$(gcloud app versions list --service=ocotillo-api-staging --project=${{ vars.GCP_PROJECT_ID}} --format="value(id)" --sort-by="version.createTime" | head -n 1)
-            if [ ! -z "$OLDEST_VERSION" ]; then
-              echo "Deleting oldest version: $OLDEST_VERSION"
-              gcloud app versions delete $OLDEST_VERSION --service=ocotillo-api-staging --project=${{ vars.GCP_PROJECT_ID }} --quiet
-              echo "Deleted oldest version: $OLDEST_VERSION"
+            SERVICE="ocotillo-api-staging"
+            VERSIONS_JSON="$(gcloud app versions list --service="$SERVICE" --project=${{ vars.GCP_PROJECT_ID }} --format=json --sort-by="version.createTime" 2>/dev/null || printf '[]')"
+            export VERSIONS_JSON
+            DELETE_VERSION="$(python - <<'PY'
+            import json
+            import os
+
+            versions = json.loads(os.environ.get("VERSIONS_JSON", "[]") or "[]")
+            if len(versions) <= 1:
+                print("")
+                raise SystemExit(0)
+
+            def traffic_split(version):
+                for key in ("traffic_split", "trafficSplit"):
+                    value = version.get(key)
+                    if value is not None:
+                        try:
+                            return float(value)
+                        except (TypeError, ValueError):
+                            return 0.0
+                return 0.0
+
+            for version in versions:
+                if traffic_split(version) == 0.0:
+                    print(version.get("id", ""))
+                    break
+            else:
+                print("")
+            PY
+            )"
+            if [ -n "$DELETE_VERSION" ]; then
+              echo "Deleting old non-serving version for $SERVICE: $DELETE_VERSION"
+              gcloud app versions delete "$DELETE_VERSION" --service="$SERVICE" --project=${{ vars.GCP_PROJECT_ID }} --quiet
             else
-              echo "No versions to delete"
+              echo "No old non-serving versions to delete for $SERVICE"
             fi
 
-        - name: Remove app.yaml
+        - name: Remove rendered configs
           run: |
             rm app.yaml
 

AGENTS.MD

@@ -25,6 +25,13 @@ these transfers, keep the following rules in mind to avoid hour-long runs:
 - Data migrations should be safe to re-run without creating duplicate rows or corrupting data.
 - Use upserts or duplicate checks and update source fields only after successful inserts.
 
+## 4. Do a cleanup and code analysis pass after code changes
+- After completing any code modification, do a cleanup and code analysis pass adjusted to the size and risk of the change.
+- Check for obvious regressions, dead code, inconsistent config/docs/tests, and adjacent issues introduced by the change.
+- Fix any concrete issues you find in that pass instead of stopping at implementation.
+- After code cleanup, run `black` on the touched Python files and run `flake8` on the touched Python files before wrapping up.
+- Run targeted validation for the modified area after cleanup; use broader validation when the change affects shared boot, deploy, or database paths.
+
 Following this playbook keeps ETL runs measured in seconds/minutes instead of hours. EOF
 
 ## Activate python venv

README.md

@@ -28,6 +28,10 @@ supports research, field operations, and public data delivery for the Bureau of
 ## 🗺️ OGC API - Features
 
 The API exposes OGC API - Features endpoints under `/ogcapi` using `pygeoapi`.
+In App Engine deployments, `/admin` and `/ogcapi` are served from the same
+application as the primary API. The service is intended to scale to zero
+outside business hours and be kept warm during the workday with Cloud Scheduler
+hits to `/_ah/warmup`.
 
 ### Landing & metadata
 
@@ -147,7 +151,7 @@ Minimum vars to set in `.env` for local development:
 * `POSTGRES_HOST` (`localhost` for local psql/pytest against mapped Docker port)
 * `POSTGRES_PORT` (`5432`)
 * `MODE` (`development` recommended locally)
-* `SESSION_SECRET_KEY`
+* `SESSION_SECRET_KEY` (required if you want to use `/admin`)
 
 Auth-related vars (required when auth is enabled, optional when `AUTHENTIK_DISABLE_AUTHENTICATION=1`):
 * `AUTHENTIK_DISABLE_AUTHENTICATION`
@@ -199,15 +203,18 @@ docker compose up --build
 
 Notes:
 * Requires Docker Desktop.
-* By default, spins up two containers: `db` (PostGIS/PostgreSQL) and `app` (FastAPI API service).
+* By default, spins up two containers:
+  * `db` for PostGIS/PostgreSQL
+  * `app` for the primary API, admin UI, and OGC API on `http://localhost:8000`
 * `db` initializes both application databases in the same Postgres service:
   * `ocotilloapi_dev`
   * `ocotilloapi_test`
-* `alembic upgrade head` runs on app startup after `docker compose up`.
+* `alembic upgrade head` runs in the `app` container on startup.
 * Compose uses hardcoded DB names:
   * dev: `ocotilloapi_dev`
   * test: `ocotilloapi_test` (created by init SQL in `docker/db/init/01-create-test-db.sql`)
 * The database listens on port `5432` both inside the container and on your host. Ensure `POSTGRES_PORT=5432` and `POSTGRES_DB=ocotilloapi_dev` in your `.env` to run local commands against the Docker dev DB (e.g., `uv run pytest`, `uv run python -m transfers.transfer`).
+* `SESSION_SECRET_KEY` only needs to be set in `.env` if you plan to use `/admin`; without it, the API and `/ogcapi` still boot, but `/admin` will be unavailable.
 
 #### Staging Data
 

alembic/env.py

@@ -7,7 +7,7 @@
 from dotenv import load_dotenv
 from sqlalchemy import create_engine, engine_from_config, pool, text
 
-from services.util import get_bool_env
+from services.env import get_bool_env
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.