feat: parameterize tooling for multi-env support

.github/workflows/pr-tests.yaml

@@ -10,25 +10,31 @@ on:
 
 jobs:
   sync-versions-check:
-    name: Chart versions in sync
+    name: Chart versions in sync (${{ matrix.env }})
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        env: [dev]  # TODO: add prod once clusters/prod/ has apps
     steps:
       - uses: actions/checkout@v4
 
       - uses: mikefarah/yq@v4
 
-      - run: make sync-versions
+      - run: make ENV=${{ matrix.env }} sync-versions
 
       - name: Fail if Chart.yaml files drifted
         run: |
-          if ! git diff --exit-code -- 'clusters/dev/apps/*/Chart.yaml'; then
-            echo "::error::Chart.yaml versions out of sync with versions.yaml. Run 'make sync-versions' and commit."
+          if ! git diff --exit-code -- 'clusters/${{ matrix.env }}/apps/*/Chart.yaml'; then
+            echo "::error::Chart.yaml versions out of sync with versions.yaml. Run 'make ENV=${{ matrix.env }} sync-versions' and commit."
             exit 1
           fi
 
   helm-tests:
-    name: Helm template tests
+    name: Helm template tests (${{ matrix.env }})
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        env: [dev]  # TODO: add prod once clusters/prod/ has apps
     steps:
       - uses: actions/checkout@v4
 
@@ -41,13 +47,13 @@ jobs:
         with:
           path: |
             ~/.cache/helm
-            clusters/dev/apps/*/charts
-          key: helm-deps-${{ hashFiles('clusters/dev/apps/*/Chart.yaml', 'clusters/dev/apps/*/Chart.lock') }}
-          restore-keys: helm-deps-
+            clusters/${{ matrix.env }}/apps/*/charts
+          key: helm-deps-${{ matrix.env }}-${{ hashFiles(format('clusters/{0}/apps/*/Chart.yaml', matrix.env), format('clusters/{0}/apps/*/Chart.lock', matrix.env)) }}
+          restore-keys: helm-deps-${{ matrix.env }}-
 
       - name: Add Helm repos
         run: |
           helm repo add pilot https://pilotdataplatform.github.io/helm-charts/
           helm repo add nfs-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner
 
-      - run: make test
+      - run: make ENV=${{ matrix.env }} test

Makefile

@@ -1,9 +1,25 @@
-APPS_DIR := clusters/dev/apps
-APPS := registry-secrets greenroom-storage core-storage nfs-provisioner postgresql keycloak-postgresql kong-postgresql redis kafka elasticsearch message-bus-greenroom keycloak auth metadata project dataops dataset approval kong bff minio mailhog notification portal queue-consumer queue-producer queue-socketio pipelinewatch upload-greenroom upload-core download-greenroom download-core metadata-event-handler search kg-integration bff-cli workspace xwiki
-REGISTRY_DIR := clusters/dev
-VERSIONS_FILE := clusters/dev/versions.yaml
-WORKBENCH_DIR := clusters/dev/workbench
-WORKBENCH_CHARTS := guacamole-stack superset jupyterhub
+ENV ?= dev
+$(if $(filter $(ENV),dev prod),,$(error ENV=$(ENV) is invalid. Valid: dev prod))
+CLUSTER_DIR := clusters/$(ENV)
+APPS_DIR := $(CLUSTER_DIR)/apps
+REGISTRY_DIR := $(CLUSTER_DIR)
+VERSIONS_FILE := $(CLUSTER_DIR)/versions.yaml
+WORKBENCH_DIR := $(CLUSTER_DIR)/workbench
+
+# Full curated app list — filtered to existing dirs so partial envs work
+_APPS := registry-secrets greenroom-storage core-storage nfs-provisioner postgresql keycloak-postgresql kong-postgresql redis kafka elasticsearch message-bus-greenroom keycloak auth metadata project dataops dataset approval kong bff minio mailhog notification portal queue-consumer queue-producer queue-socketio pipelinewatch upload-greenroom upload-core download-greenroom download-core metadata-event-handler search kg-integration bff-cli workspace xwiki
+APPS := $(strip $(foreach app,$(_APPS),$(if $(wildcard $(APPS_DIR)/$(app)),$(app))))
+
+_WORKBENCH_CHARTS := guacamole-stack superset jupyterhub
+WORKBENCH_CHARTS := $(strip $(foreach c,$(_WORKBENCH_CHARTS),$(if $(wildcard $(WORKBENCH_DIR)/$(c)),$(c))))
+
+ifeq ($(ENV),prod)
+  DOMAIN := hdc.ebrains.eu
+else
+  DOMAIN := dev.hdc.ebrains.eu
+endif
+
+export ENV
 
 .PHONY: helm-deps helm-deps-workbench helm-test-eso helm-test-image helm-test-versions helm-test-envdup helm-test-pullsecrets helm-test-envvars-rendered helm-test-regsecret-coverage helm-test-workbench sync-versions sync-rsa-key test clean switch-registry which-registry
 
@@ -75,9 +91,11 @@ sync-rsa-key:
 # Verify image tags rendered by helm template match versions.yaml
 helm-test-versions: helm-deps
 	@echo "Testing image tags from versions.yaml..."
-	@failed=0; \
+	@if [ ! -f $(VERSIONS_FILE) ]; then echo "⊘ No versions.yaml (skipped)"; exit 0; fi; \
+	failed=0; \
 	check_tag() { \
 		app=$$1; values_key=$$2; dir=$$3; \
+		if [ ! -d $(APPS_DIR)/$$dir ]; then echo "⊘ $$app: not present (skipped)"; return 0; fi; \
 		expected=$$(yq ".\"$$values_key\".image.tag" $(VERSIONS_FILE)); \
 		rendered=$$(helm template test $(APPS_DIR)/$$dir \
 			-f $(REGISTRY_DIR)/registry.yaml \
@@ -116,23 +134,23 @@ helm-test-versions: helm-deps
 # Detect duplicate env var names that ServerSideApply would reject
 helm-test-envdup: helm-deps
 	@echo "Testing for duplicate env vars..."
-	@bash scripts/check-duplicate-env.sh $(APPS)
+	@if [ -n "$(APPS)" ]; then bash scripts/check-duplicate-env.sh $(APPS); else echo "⊘ No apps to test"; fi
 
 # Ensure every pod spec has imagePullSecrets for private registry access
 helm-test-pullsecrets: helm-deps
 	@echo "Testing imagePullSecrets on all pod specs..."
-	@bash scripts/check-pull-secrets.sh $(APPS)
+	@if [ -n "$(APPS)" ]; then bash scripts/check-pull-secrets.sh $(APPS); else echo "⊘ No apps to test"; fi
 
 # Verify env vars defined in values.yaml are actually rendered in helm template
 # Catches chart bugs where extraEnvVars aren't picked up (e.g., Kong migration job)
 helm-test-envvars-rendered: helm-deps
 	@echo "Testing env vars defined in values.yaml are rendered..."
-	@bash scripts/check-envvars-rendered.sh $(APPS)
+	@if [ -n "$(APPS)" ]; then bash scripts/check-envvars-rendered.sh $(APPS); else echo "⊘ No apps to test"; fi
 
 # Ensure every namespace that uses docker-registry-secret is covered by registry-secrets
 helm-test-regsecret-coverage: helm-deps
 	@echo "Testing registry-secret namespace coverage..."
-	@bash scripts/check-registry-secret-coverage.sh $(APPS)
+	@if [ -n "$(APPS)" ]; then bash scripts/check-registry-secret-coverage.sh $(APPS); else echo "⊘ No apps to test"; fi
 
 # Test workbench charts render correctly
 helm-test-workbench: helm-deps-workbench
@@ -144,7 +162,7 @@ helm-test-workbench: helm-deps-workbench
 			-f $(REGISTRY_DIR)/registry.yaml \
 			-f $(WORKBENCH_DIR)/$$chart/values.yaml \
 			--set projectName=testproject \
-			--set domain=dev.hdc.ebrains.eu \
+			--set domain=$(DOMAIN) \
 			--skip-tests 2>&1); \
 		if [ $$? -ne 0 ]; then \
 			echo "✗ $$chart: helm template failed"; \

scripts/check-duplicate-env.sh

@@ -4,9 +4,10 @@
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-APPS_DIR="$REPO_ROOT/clusters/dev/apps"
-REGISTRY_DIR="$REPO_ROOT/clusters/dev"
-VERSIONS_FILE="$REPO_ROOT/clusters/dev/versions.yaml"
+ENV="${ENV:-dev}"
+APPS_DIR="$REPO_ROOT/clusters/$ENV/apps"
+REGISTRY_DIR="$REPO_ROOT/clusters/$ENV"
+VERSIONS_FILE="$REPO_ROOT/clusters/$ENV/versions.yaml"
 
 if [[ $# -eq 0 ]]; then
   echo "Usage: $0 app1 [app2 ...]" >&2

scripts/check-envvars-rendered.sh

@@ -4,9 +4,10 @@
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-APPS_DIR="$REPO_ROOT/clusters/dev/apps"
-REGISTRY_DIR="$REPO_ROOT/clusters/dev"
-VERSIONS_FILE="$REPO_ROOT/clusters/dev/versions.yaml"
+ENV="${ENV:-dev}"
+APPS_DIR="$REPO_ROOT/clusters/$ENV/apps"
+REGISTRY_DIR="$REPO_ROOT/clusters/$ENV"
+VERSIONS_FILE="$REPO_ROOT/clusters/$ENV/versions.yaml"
 
 if [[ $# -eq 0 ]]; then
   echo "Usage: $0 app1 [app2 ...]" >&2

scripts/check-pull-secrets.sh

@@ -4,9 +4,10 @@
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-APPS_DIR="$REPO_ROOT/clusters/dev/apps"
-REGISTRY_DIR="$REPO_ROOT/clusters/dev"
-VERSIONS_FILE="$REPO_ROOT/clusters/dev/versions.yaml"
+ENV="${ENV:-dev}"
+APPS_DIR="$REPO_ROOT/clusters/$ENV/apps"
+REGISTRY_DIR="$REPO_ROOT/clusters/$ENV"
+VERSIONS_FILE="$REPO_ROOT/clusters/$ENV/versions.yaml"
 
 if [[ $# -eq 0 ]]; then
   echo "Usage: $0 app1 [app2 ...]" >&2

scripts/check-registry-secret-coverage.sh

@@ -5,9 +5,10 @@
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-APPS_DIR="$REPO_ROOT/clusters/dev/apps"
-REGISTRY_DIR="$REPO_ROOT/clusters/dev"
-VERSIONS_FILE="$REPO_ROOT/clusters/dev/versions.yaml"
+ENV="${ENV:-dev}"
+APPS_DIR="$REPO_ROOT/clusters/$ENV/apps"
+REGISTRY_DIR="$REPO_ROOT/clusters/$ENV"
+VERSIONS_FILE="$REPO_ROOT/clusters/$ENV/versions.yaml"
 REG_SECRET_TMPL="$APPS_DIR/registry-secrets/templates/docker-registry-secret.yaml"
 
 if [[ $# -eq 0 ]]; then

scripts/sync-chart-versions.sh

@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
-# Syncs chart dependency versions from clusters/dev/versions.yaml into each app's Chart.yaml
+# Syncs chart dependency versions from versions.yaml into each app's Chart.yaml
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-VERSIONS_FILE="$REPO_ROOT/clusters/dev/versions.yaml"
-APPS_DIR="$REPO_ROOT/clusters/dev/apps"
+ENV="${ENV:-dev}"
+VERSIONS_FILE="$REPO_ROOT/clusters/$ENV/versions.yaml"
+APPS_DIR="$REPO_ROOT/clusters/$ENV/apps"
 
 if ! command -v yq &>/dev/null; then
   echo "ERROR: yq is required. Install: https://github.com/mikefarah/yq" >&2

scripts/sync-rsa-public-key.sh

@@ -9,7 +9,8 @@
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-APPS_DIR="$REPO_ROOT/clusters/dev/apps"
+ENV="${ENV:-dev}"
+APPS_DIR="$REPO_ROOT/clusters/$ENV/apps"
 OVH_INFRA="${OVH_INFRA:-$REPO_ROOT/../pilot-hdc-ovh-infra}"
 TF_DIR="$OVH_INFRA/terraform/keycloak"
 

scripts/update-pilot-cli.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # Deploy pilotcli binary to shared-tools NFS PVC in all project namespaces.
-# Discovers projects dynamically from clusters/dev/workbench/projects/*.yaml.
+# Discovers projects dynamically from the active environment's workbench/projects/*.yaml.
 # Run from repo root.
 
 set -euo pipefail
@@ -10,7 +10,8 @@ PILOTCLI_PATH="/tmp/pilotcli"
 COPY_DESTINATION="/opt/shared"
 OWNER="PilotDataPlatform"
 REPO="cli"
-PROJECTS_DIR="clusters/dev/workbench/projects"
+ENV="${ENV:-dev}"
+PROJECTS_DIR="clusters/$ENV/workbench/projects"
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 cd "$REPO_ROOT"