Merge pull request #578 from DataIntegrationGroup/staging

Maintenance Push

Author: jirhiker

.env.example

@@ -4,7 +4,7 @@ POSTGRES_USER=admin
 POSTGRES_PASSWORD=password
 POSTGRES_DB=ocotillo
 POSTGRES_HOST=localhost
-POSTGRES_PORT=54321
+POSTGRES_PORT=5432
 
 # Connection pool configuration for parallel transfers
 # pool_size: number of persistent connections to maintain
@@ -20,6 +20,7 @@ TRANSFER_PARALLEL=1
 TRANSFER_WELL_SCREENS=True
 TRANSFER_SENSORS=True
 TRANSFER_CONTACTS=True
+TRANSFER_PERMISSIONS=True
 TRANSFER_WATERLEVELS=True
 TRANSFER_WATERLEVELS_PRESSURE=True
 TRANSFER_WATERLEVELS_ACOUSTIC=True
@@ -46,6 +47,9 @@ MODE=development
 # disable authentication (for development only)
 AUTHENTIK_DISABLE_AUTHENTICATION=1
 
+# erase and rebuild the database for step tests
+REBUILD_DB=1
+
 # authentik
 AUTHENTIK_URL=
 AUTHENTIK_CLIENT_ID=

.github/app.template.yaml

@@ -0,0 +1,34 @@
+service: ${SERVICE_NAME}
+runtime: python313
+entrypoint: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app
+instance_class: F4
+service_account: "${CLOUD_SQL_USER}.gserviceaccount.com"
+handlers:
+  - url: /.*
+    secure: always
+    script: auto
+env_variables:
+  MODE: "production"
+  ENVIRONMENT: "${ENVIRONMENT}"
+  DB_DRIVER: "cloudsql"
+  CLOUD_SQL_INSTANCE_NAME: "${CLOUD_SQL_INSTANCE_NAME}"
+  CLOUD_SQL_DATABASE: "${CLOUD_SQL_DATABASE}"
+  CLOUD_SQL_USER: "${CLOUD_SQL_USER}"
+  PYGEOAPI_POSTGRES_DB: "${PYGEOAPI_POSTGRES_DB}"
+  PYGEOAPI_POSTGRES_USER: "${PYGEOAPI_POSTGRES_USER}"
+  PYGEOAPI_POSTGRES_HOST: "${PYGEOAPI_POSTGRES_HOST}"
+  PYGEOAPI_POSTGRES_PORT: "${PYGEOAPI_POSTGRES_PORT}"
+  PYGEOAPI_POSTGRES_PASSWORD: |-
+    ${PYGEOAPI_POSTGRES_PASSWORD}
+  PYGEOAPI_SERVER_URL: "${PYGEOAPI_SERVER_URL}"
+  CLOUD_SQL_IAM_AUTH: "${CLOUD_SQL_IAM_AUTH}"
+  GCS_SERVICE_ACCOUNT_KEY: |-
+    ${GCS_SERVICE_ACCOUNT_KEY}
+  GCS_BUCKET_NAME: "${GCS_BUCKET_NAME}"
+  AUTHENTIK_URL: "${AUTHENTIK_URL}"
+  AUTHENTIK_CLIENT_ID: "${AUTHENTIK_CLIENT_ID}"
+  AUTHENTIK_AUTHORIZE_URL: "${AUTHENTIK_AUTHORIZE_URL}"
+  AUTHENTIK_TOKEN_URL: "${AUTHENTIK_TOKEN_URL}"
+  SESSION_SECRET_KEY: |-
+    ${SESSION_SECRET_KEY}
+  APITALLY_CLIENT_ID: "${APITALLY_CLIENT_ID}"

.github/dependabot.yml

@@ -5,8 +5,44 @@
 
 version: 2
 updates:
-  - package-ecosystem: "uv" # See documentation for possible values
-    directory: "/" # Location of package manifests
+  - package-ecosystem: "uv"
+    directory: "/"
     schedule:
       interval: "weekly"
+      day: "monday"
+      time: "09:00"
+      timezone: "America/Denver"
     target-branch: "staging"
+    open-pull-requests-limit: 5
+    rebase-strategy: "auto"
+    labels:
+      - "dependencies"
+      - "python"
+    groups:
+      uv-non-major:
+        patterns:
+          - "*"
+        update-types:
+          - "minor"
+          - "patch"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+      time: "09:00"
+      timezone: "America/Denver"
+    target-branch: "staging"
+    open-pull-requests-limit: 5
+    rebase-strategy: "auto"
+    labels:
+      - "dependencies"
+      - "github-actions"
+    groups:
+      gha-minor-and-patch:
+        patterns:
+          - "*"
+        update-types:
+          - "minor"
+          - "patch"

.github/workflows/CD_production.yml

@@ -15,12 +15,12 @@ jobs:
 
         steps:
         - name: Check out source repository
-          uses: actions/checkout@v4
+          uses: actions/checkout@v6.0.2
           with:
             fetch-depth: 0
 
         - name: Install uv in container
-          uses: astral-sh/setup-uv@v6
+          uses: astral-sh/setup-uv@v7.3.1
           with:
             version: "latest"
 
@@ -33,7 +33,7 @@ jobs:
               --output-file requirements.txt
 
         - name: Authenticate to Google Cloud
-          uses: 'google-github-actions/auth@v2'
+          uses: 'google-github-actions/auth@v3'
           with:
             credentials_json: ${{ secrets.CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY }}
 
@@ -47,33 +47,37 @@ jobs:
           run: |
             uv run alembic upgrade head
 
-        - name: Create app.yaml
+        - name: Ensure envsubst is available
           run: |
-            cat <<EOF > app.yaml
-            service: ocotillo-api
-            runtime: python313
-            entrypoint: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app
-            instance_class: F4
-            service_account: "${{ secrets.CLOUD_SQL_USER }}.gserviceaccount.com"
-            handlers:
-              - url: /.*
-                secure: always
-                script: auto
-            env_variables:
-              MODE: "production"
-              DB_DRIVER: "cloudsql"
-              CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}"
-              CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}"
-              CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}"
-              CLOUD_SQL_IAM_AUTH: true
-              GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}"
-              GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}"
-              AUTHENTIK_URL: "${{ vars.AUTHENTIK_URL }}"
-              AUTHENTIK_CLIENT_ID: "${{ vars.AUTHENTIK_CLIENT_ID }}"
-              AUTHENTIK_AUTHORIZE_URL: "${{ vars.AUTHENTIK_AUTHORIZE_URL }}"
-              AUTHENTIK_TOKEN_URL: "${{ vars.AUTHENTIK_TOKEN_URL }}"
-              SESSION_SECRET_KEY: "${{ secrets.SESSION_SECRET_KEY }}"
-            EOF
+            if ! command -v envsubst >/dev/null 2>&1; then
+              sudo apt-get update
+              sudo apt-get install -y gettext-base
+            fi
+
+        - name: Render app.yaml
+          env:
+            SERVICE_NAME: "ocotillo-api"
+            ENVIRONMENT: "production"
+            CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}"
+            CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}"
+            CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}"
+            PYGEOAPI_POSTGRES_DB: "${{ vars.CLOUD_SQL_DATABASE }}"
+            PYGEOAPI_POSTGRES_USER: "${{ secrets.PYGEOAPI_POSTGRES_USER }}"
+            PYGEOAPI_POSTGRES_HOST: "${{ vars.PYGEOAPI_POSTGRES_HOST || '127.0.0.1' }}"
+            PYGEOAPI_POSTGRES_PORT: "${{ vars.PYGEOAPI_POSTGRES_PORT || '5432' }}"
+            PYGEOAPI_POSTGRES_PASSWORD: "${{ secrets.PYGEOAPI_POSTGRES_PASSWORD }}"
+            PYGEOAPI_SERVER_URL: "${{ vars.PYGEOAPI_SERVER_URL }}"
+            CLOUD_SQL_IAM_AUTH: "true"
+            GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}"
+            GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}"
+            AUTHENTIK_URL: "${{ vars.AUTHENTIK_URL }}"
+            AUTHENTIK_CLIENT_ID: "${{ vars.AUTHENTIK_CLIENT_ID }}"
+            AUTHENTIK_AUTHORIZE_URL: "${{ vars.AUTHENTIK_AUTHORIZE_URL }}"
+            AUTHENTIK_TOKEN_URL: "${{ vars.AUTHENTIK_TOKEN_URL }}"
+            SESSION_SECRET_KEY: "${{ secrets.SESSION_SECRET_KEY }}"
+            APITALLY_CLIENT_ID: "${{ vars.APITALLY_CLIENT_ID }}"
+          run: |
+            envsubst < .github/app.template.yaml > app.yaml
 
         - name: Deploy to Google Cloud
           run: |

.github/workflows/CD_staging.yml

@@ -15,12 +15,12 @@ jobs:
 
         steps:
         - name: Check out source repository
-          uses: actions/checkout@v4
+          uses: actions/checkout@v6.0.2
           with:
             fetch-depth: 0
 
         - name: Install uv in container
-          uses: astral-sh/setup-uv@v6
+          uses: astral-sh/setup-uv@v7.3.1
           with:
             version: "latest"
 
@@ -33,7 +33,7 @@ jobs:
               --output-file requirements.txt
 
         - name: Authenticate to Google Cloud
-          uses: 'google-github-actions/auth@v2'
+          uses: 'google-github-actions/auth@v3'
           with:
             credentials_json: ${{ secrets.CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY }}
 
@@ -47,34 +47,37 @@ jobs:
           run: |
             uv run alembic upgrade head
 
-        # Uses Google Cloud Secret Manager to store secret credentials
-        - name: Create app.yaml
+        - name: Ensure envsubst is available
           run: |
-            cat <<EOF > app.yaml
-            service: ocotillo-api-staging
-            runtime: python313
-            entrypoint: gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app
-            service_account: "${{ secrets.CLOUD_SQL_USER }}.gserviceaccount.com"
-            instance_class: F4
-            handlers:
-              - url: /.*
-                secure: always
-                script: auto
-            env_variables:
-              MODE: "production"
-              DB_DRIVER: "cloudsql"
-              CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}"
-              CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}"
-              CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}"
-              CLOUD_SQL_IAM_AUTH: true
-              GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}"
-              GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}"
-              AUTHENTIK_URL: "${{ vars.AUTHENTIK_URL }}"
-              AUTHENTIK_CLIENT_ID: "${{ vars.AUTHENTIK_CLIENT_ID }}"
-              AUTHENTIK_AUTHORIZE_URL: "${{ vars.AUTHENTIK_AUTHORIZE_URL }}"
-              AUTHENTIK_TOKEN_URL: "${{ vars.AUTHENTIK_TOKEN_URL }}"
-              SESSION_SECRET_KEY: "${{ secrets.SESSION_SECRET_KEY }}"
-            EOF
+            if ! command -v envsubst >/dev/null 2>&1; then
+              sudo apt-get update
+              sudo apt-get install -y gettext-base
+            fi
+
+        - name: Render app.yaml
+          env:
+            SERVICE_NAME: "ocotillo-api-staging"
+            ENVIRONMENT: "staging"
+            CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}"
+            CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}"
+            CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}"
+            PYGEOAPI_POSTGRES_DB: "${{ vars.CLOUD_SQL_DATABASE }}"
+            PYGEOAPI_POSTGRES_USER: "${{ secrets.PYGEOAPI_POSTGRES_USER }}"
+            PYGEOAPI_POSTGRES_HOST: "${{ vars.PYGEOAPI_POSTGRES_HOST || '127.0.0.1' }}"
+            PYGEOAPI_POSTGRES_PORT: "${{ vars.PYGEOAPI_POSTGRES_PORT || '5432' }}"
+            PYGEOAPI_POSTGRES_PASSWORD: "${{ secrets.PYGEOAPI_POSTGRES_PASSWORD }}"
+            PYGEOAPI_SERVER_URL: "${{ vars.PYGEOAPI_SERVER_URL }}"
+            CLOUD_SQL_IAM_AUTH: "true"
+            GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}"
+            GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}"
+            AUTHENTIK_URL: "${{ vars.AUTHENTIK_URL }}"
+            AUTHENTIK_CLIENT_ID: "${{ vars.AUTHENTIK_CLIENT_ID }}"
+            AUTHENTIK_AUTHORIZE_URL: "${{ vars.AUTHENTIK_AUTHORIZE_URL }}"
+            AUTHENTIK_TOKEN_URL: "${{ vars.AUTHENTIK_TOKEN_URL }}"
+            SESSION_SECRET_KEY: "${{ secrets.SESSION_SECRET_KEY }}"
+            APITALLY_CLIENT_ID: "${{ vars.APITALLY_CLIENT_ID }}"
+          run: |
+            envsubst < .github/app.template.yaml > app.yaml
 
         - name: Deploy to Google Cloud
           run: |

.github/workflows/dependabot_automerge.yml

@@ -0,0 +1,58 @@
+name: Dependabot auto-merge
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize, ready_for_review]
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  automerge:
+    if: github.actor == 'dependabot[bot]' && github.event.pull_request.user.login == 'dependabot[bot]'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Fetch Dependabot metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@v2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      # Auto-approve (only matters if your branch protection requires reviews)
+      - name: Approve PR
+        if: steps.metadata.outputs.update-type != 'version-update:semver-major'
+        uses: actions/github-script@v8
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const { data: reviews } = await github.rest.pulls.listReviews({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.payload.pull_request.number,
+            });
+
+            const alreadyApprovedByBot = reviews.some(
+              (review) =>
+                review.state === "APPROVED" &&
+                review.user?.login === "github-actions[bot]"
+            );
+
+            if (!alreadyApprovedByBot) {
+              await github.rest.pulls.createReview({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                pull_number: context.payload.pull_request.number,
+                event: "APPROVE",
+              });
+            }
+
+      # Enable GitHub auto-merge; it will merge once required checks (your Test Suite) are green
+      - name: Enable auto-merge (squash)
+        if: steps.metadata.outputs.update-type != 'version-update:semver-major'
+        uses: peter-evans/enable-pull-request-automerge@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          pull-request-number: ${{ github.event.pull_request.number }}
+          merge-method: squash

.github/workflows/format_code.yml

@@ -17,14 +17,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out source repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6.0.2
       - name: Set up Python environment - 3.12
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6.2.0
         with:
           python-version: "3.12"
           cache: "pip"
       - name: Run flake8
-        uses: py-actions/flake8@v1
+        uses: py-actions/flake8@v2
         with:
           ignore: "F401,E501"
           args: "--exit-zero --select=E"
@@ -34,15 +34,15 @@ jobs:
       contents: write
       pull-requests: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6.0.2
         with:
           ref: ${{ github.head_ref }}
       - uses: psf/black@stable
         with:
           options: "--verbose"
 
       - name: Commit changes
-        uses: stefanzweifel/git-auto-commit-action@v4.1.2
+        uses: stefanzweifel/git-auto-commit-action@v7.1.0
         with:
           commit_message: Formatting changes
           branch: ${{ github.head_ref }}