feat: add prod apps for waves 4-5

README.md

@@ -155,32 +155,19 @@ vault write auth/kubernetes/role/external-secrets \
 ### Verify Integration
 
 ```bash
-# Create test secret
 vault kv put secret/test foo=bar
+vault kv get secret/test
 
 # Check ESO synced it (ClusterSecretStore "vault" is pre-configured)
 kubectl get externalsecret -A
-```
-
-## Required Vault Secrets
-
-These secrets must exist in Vault before the corresponding apps can sync.
-
-### MinIO (`secret/minio`)
-
-```bash
-# Generate 256-bit encryption key
-ENCRYPTION_KEY=$(openssl rand -base64 32)
 
-vault kv put secret/minio \
-  access_key="minio-admin" \
-  secret_key="$(openssl rand -hex 16)" \
-  kms_secret_key="minio-encryption-key:${ENCRYPTION_KEY}"
+# Clean up
+vault kv delete secret/test
 ```
 
-**Important**: Back up `kms_secret_key` - losing it means losing access to encrypted data.
+## Required Vault Secrets
 
-### Other Secrets
+These secrets must exist in Vault before the corresponding apps can sync. See [`docs/vault-secrets.md`](docs/vault-secrets.md) for ready-to-run provisioning commands.
 
 | Path | Keys | Used By |
 |------|------|---------|
@@ -199,16 +186,6 @@ vault kv put secret/minio \
 | `secret/docker-registry/ovh` | username, password | registry-secrets |
 | `secret/github-runner` | github_app_id, github_app_installation_id, github_app_private_key | arc-runners-public |
 
-### Docker Registry (`secret/docker-registry/ovh`)
-
-```bash
-vault kv put secret/docker-registry/ovh \
-  username='<registry-robot-account>' \
-  password='<registry-robot-token>'
-```
-
-To add or update a service password: `vault kv patch secret/postgresql <service>-user-password=<value>`
-
 ## Platform Architecture (WIP)
 
 HDC splits workloads across namespaces by trust boundary and function:

clusters/prod/apps/elasticsearch/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: elasticsearch
+version: 0.1.0
+dependencies:
+  - name: elasticsearch
+    version: "17.9.29"
+    repository: https://pilotdataplatform.github.io/helm-charts/

clusters/prod/apps/elasticsearch/application.yaml

@@ -0,0 +1,27 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: elasticsearch
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "5"
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/PilotDataPlatform/pilot-hdc-platform-gitops.git
+    targetRevision: main
+    path: clusters/prod/apps/elasticsearch
+    helm:
+      valueFiles:
+        - ../../registry.yaml
+        - values.yaml
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: utility
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

clusters/prod/apps/elasticsearch/values.yaml

@@ -0,0 +1,51 @@
+elasticsearch:
+  global:
+    imageRegistry: n47w5524.c1.de1.container-registry.ovh.net
+    imagePullSecrets:
+      - docker-registry-secret
+
+  image:
+    repository: hdc-services-external/bitnami/elasticsearch
+    tag: 7.17.3-debian-10-r29
+
+  # Single-node master (no split-brain risk with 1 node)
+  master:
+    replicas: 1
+    heapSize: 1024m
+    resources:
+      requests:
+        cpu: 25m
+        memory: 2048Mi
+    persistence:
+      accessModes:
+        - ReadWriteOnce
+
+  # Single data node
+  data:
+    replicas: 1
+    heapSize: 1024m
+    resources:
+      requests:
+        cpu: 25m
+        memory: 2048Mi
+    persistence:
+      accessModes:
+        - ReadWriteOnce
+      size: 5Gi
+
+  # Disabled node types (same as CSCS)
+  coordinating:
+    replicas: 0
+  ingest:
+    replicas: 0
+  curator:
+    enabled: false
+  metrics:
+    enabled: false
+
+  # OVH nodes have vm.max_map_count=65530 (too low for ES, needs >=262144)
+  # Init container runs sysctl -w vm.max_map_count=262144 as privileged
+  sysctlImage:
+    enabled: true
+    repository: hdc-services-external/bitnami/bitnami-shell-archived
+    tag: 10-debian-10-r403

clusters/prod/apps/kafka/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: kafka
+version: 0.1.0
+dependencies:
+  - name: kafka
+    version: "20.0.3"
+    repository: https://pilotdataplatform.github.io/helm-charts/

clusters/prod/apps/kafka/application.yaml

@@ -0,0 +1,27 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: kafka
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "5"
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/PilotDataPlatform/pilot-hdc-platform-gitops.git
+    targetRevision: main
+    path: clusters/prod/apps/kafka
+    helm:
+      valueFiles:
+        - ../../registry.yaml
+        - values.yaml
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: utility
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

clusters/prod/apps/kafka/values.yaml

@@ -0,0 +1,112 @@
+kafka:
+  global:
+    imageRegistry: n47w5524.c1.de1.container-registry.ovh.net
+    imagePullSecrets:
+      - docker-registry-secret
+
+  image:
+    repository: hdc-services-external/bitnami/kafka
+
+  heapOpts: -Xms256M -Xmx256M
+  deleteTopicEnable: true
+
+  replicaCount: 1
+  defaultReplicationFactor: 1
+  offsetsTopicReplicationFactor: 1
+  transactionStateLogReplicationFactor: 1
+  transactionStateLogMinIsr: 1
+
+  persistence:
+    enabled: true
+    size: 2Gi
+
+  zookeeper:
+    image:
+      repository: hdc-services-external/bitnami/zookeeper
+    replicaCount: 1
+    heapSize: 256
+    persistence:
+      enabled: true
+      size: 1Gi
+    resources:
+      limits:
+        cpu: 500m
+        memory: 512Mi
+
+  service:
+    type: ClusterIP
+
+  extraDeploy:
+    - |
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: {{ include "kafka.name" . }}-connect
+        labels: {{- include "common.labels.standard" . | nindent 4 }}
+          app.kubernetes.io/component: connector
+      spec:
+        replicas: 1
+        selector:
+          matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }}
+            app.kubernetes.io/component: connector
+        template:
+          metadata:
+            labels: {{- include "common.labels.standard" . | nindent 8 }}
+              app.kubernetes.io/component: connector
+          spec:
+            imagePullSecrets:
+              - name: docker-registry-secret
+            # initContainers: plugins-downloader removed — add back when CDC/ES connectors needed
+            containers:
+              - name: connect
+                image: n47w5524.c1.de1.container-registry.ovh.net/hdc-services-external/debezium/connect:1.1
+                imagePullPolicy: IfNotPresent
+                ports:
+                  - name: connector
+                    containerPort: 8083
+                volumeMounts:
+                  - name: configuration
+                    mountPath: /bitnami/kafka/config
+                  - name: kafka-connect-plugins-dir
+                    mountPath: /tmp
+                env:
+                  - name: BOOTSTRAP_SERVERS
+                    value: "kafka:9092"
+                  - name: GROUP_ID
+                    value: "sde_group"
+                  - name: CONFIG_STORAGE_TOPIC
+                    value: "sde_storage_topic"
+                  - name: OFFSET_STORAGE_TOPIC
+                    value: "sde_offset_topic"
+                  - name: KAFKA_CONNECT_PLUGINS_DIR
+                    value: "/kafka/connect,/tmp"
+            volumes:
+              - name: configuration
+                configMap:
+                  name: {{ include "kafka.name" . }}-connect
+              - name: kafka-connect-plugins-dir
+                emptyDir: {}
+    - |
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: {{ include "kafka.name" . }}-connect
+        labels: {{- include "common.labels.standard" . | nindent 4 }}
+          app.kubernetes.io/component: connector
+      data:
+        connect-standalone.properties: |-
+          bootstrap.servers = {{ include "kafka.name" . }}-0.{{ include "kafka.name" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}:{{ .Values.service.port }}
+    - |
+      apiVersion: v1
+      kind: Service
+      metadata:
+        name: {{ include "kafka.name" . }}-connect
+        labels: {{- include "common.labels.standard" . | nindent 4 }}
+          app.kubernetes.io/component: connector
+      spec:
+        ports:
+          - protocol: TCP
+            port: 8083
+            targetPort: connector
+        selector: {{- include "common.labels.matchLabels" . | nindent 4 }}
+          app.kubernetes.io/component: connector

clusters/prod/apps/keycloak-postgresql/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: keycloak-postgresql
+version: 0.1.0
+dependencies:
+  - name: postgresql
+    version: "15.5.17"
+    repository: https://pilotdataplatform.github.io/helm-charts/

clusters/prod/apps/keycloak-postgresql/application.yaml

@@ -0,0 +1,27 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: keycloak-postgresql
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "4"
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/PilotDataPlatform/pilot-hdc-platform-gitops.git
+    targetRevision: main
+    path: clusters/prod/apps/keycloak-postgresql
+    helm:
+      valueFiles:
+        - ../../registry.yaml
+        - values.yaml
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: keycloak
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

clusters/prod/apps/keycloak-postgresql/templates/external-secret.yaml

@@ -0,0 +1,21 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: keycloak-postgresql-credentials
+  namespace: keycloak
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  target:
+    name: keycloak-postgresql-credentials
+  data:
+    - secretKey: postgres-password
+      remoteRef:
+        key: secret/data/keycloak
+        property: postgres-password
+    - secretKey: password
+      remoteRef:
+        key: secret/data/keycloak
+        property: keycloak-user-password

clusters/prod/apps/keycloak-postgresql/values.yaml

@@ -0,0 +1,30 @@
+postgresql:
+  fullnameOverride: keycloak-postgresql
+
+  global:
+    imagePullSecrets:
+      - docker-registry-secret
+    postgresql:
+      auth:
+        database: bitnami_keycloak
+        username: bn_keycloak
+        existingSecret: keycloak-postgresql-credentials
+        secretKeys:
+          adminPasswordKey: postgres-password
+          userPasswordKey: password
+
+  image:
+    repository: hdc-services-external/bitnami/postgresql
+
+  primary:
+    persistence:
+      size: 5Gi
+      storageClass: "csi-cinder-high-speed"
+
+    resources:
+      limits:
+        cpu: 500m
+        memory: 256Mi
+      requests:
+        cpu: 10m
+        memory: 64Mi

clusters/prod/apps/message-bus-greenroom/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: message-bus-greenroom
+version: 0.1.0
+dependencies:
+  - name: rabbitmq
+    version: "10.1.12"
+    repository: https://pilotdataplatform.github.io/helm-charts/