From 44191daf727be0356d1fef6ca1240c22a0bb04c4 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 18 Feb 2026 18:55:50 +0100 Subject: [PATCH 01/66] feat: get changes from the PoC (platform secrets as sealed secrest) branches --- chart/apl/templates/deployment.yaml | 4 - charts/external-secrets/Chart.yaml | 16 + .../crds/clusterexternalsecrets.yaml | 51 ++ .../crds/clustersecretstores.yaml | 48 ++ .../crds/externalsecrets.yaml | 51 ++ .../crds/generatorstates.yaml | 39 ++ charts/external-secrets/crds/pushsecrets.yaml | 43 ++ .../external-secrets/crds/secretstores.yaml | 48 ++ .../external-secrets/templates/_helpers.tpl | 41 ++ .../templates/clusterrole.yaml | 37 ++ .../templates/deployment.yaml | 33 + .../templates/serviceaccount.yaml | 9 + charts/external-secrets/values.yaml | 31 + .../ingress-nginx/templates/clusterrole.yaml | 1 + charts/otomi-api/templates/deployment.yaml | 8 + charts/otomi-api/values.yaml | 1 - helmfile.d/helmfile-01.init.yaml.gotmpl | 8 + helmfile.d/helmfile-04.init.yaml.gotmpl | 6 + helmfile.d/helmfile-60.teams.yaml.gotmpl | 83 ++- helmfile.d/helmfile-70.shared.yaml.gotmpl | 7 + helmfile.d/snippets/alertmanager-teams.gotmpl | 8 +- helmfile.d/snippets/alertmanager.gotmpl | 8 +- helmfile.d/snippets/defaults.gotmpl | 10 +- helmfile.d/snippets/defaults.yaml | 10 + helmfile.d/snippets/derived.gotmpl | 7 +- helmfile.d/snippets/grafana.gotmpl | 4 +- helmfile.d/snippets/sops-env.gotmpl | 24 - package-lock.json | 37 +- src/cmd/bootstrap.test.ts | 85 ++- src/cmd/bootstrap.ts | 54 +- src/cmd/commit.ts | 50 +- src/cmd/install.test.ts | 35 +- src/cmd/install.ts | 154 ++++- src/cmd/migrate.ts | 9 +- src/cmd/validate-values.ts | 53 +- src/common/bootstrap.ts | 13 +- src/common/git-config.test.ts | 76 ++- src/common/git-config.ts | 26 +- src/common/repo.ts | 1 - src/common/sealed-secrets.test.ts | 575 ++++++++++++++++ src/common/sealed-secrets.ts | 628 ++++++++++++++++++ src/common/values.ts | 29 +- src/operator/installer.test.ts | 72 +- src/operator/installer.ts | 41 +- values-schema.yaml | 13 +- .../apl-gitea-operator-raw.gotmpl | 33 +- .../apl-harbor-operator-raw.gotmpl | 35 +- .../apl-keycloak-operator-raw.gotmpl | 65 +- values/apl-operator/apl-operator-raw.gotmpl | 25 + values/apl-operator/apl-operator.gotmpl | 9 - values/argocd/argocd-raw.gotmpl | 102 ++- values/argocd/argocd.gotmpl | 3 +- values/cert-manager/cert-manager-raw.gotmpl | 139 +++- values/external-dns/external-dns-raw.gotmpl | 241 +++++-- values/external-dns/external-dns.gotmpl | 10 +- .../external-secrets-raw.gotmpl | 47 ++ .../external-secrets/external-secrets.gotmpl | 9 + .../gitea-db-secret-raw.gotmpl | 27 +- values/gitea/gitea-raw.gotmpl | 77 ++- values/gitea/gitea.gotmpl | 9 +- values/harbor/harbor-raw.gotmpl | 217 ++++-- values/harbor/harbor.gotmpl | 3 +- values/ingress-nginx/ingress-nginx-raw.gotmpl | 34 +- values/k8s/k8s-raw.gotmpl | 9 - values/keycloak/keycloak-raw.gotmpl | 52 +- values/loki/loki-raw.gotmpl | 65 +- values/oauth2-proxy/oauth2-proxy-raw.gotmpl | 50 +- values/oauth2-proxy/oauth2-proxy.gotmpl | 5 +- values/otomi-api/otomi-api-raw.gotmpl | 24 + values/otomi-api/otomi-api.gotmpl | 5 +- .../prometheus-operator-raw.gotmpl | 138 +++- .../prometheus-operator.gotmpl | 27 +- versions.yaml | 2 +- 73 files changed, 3594 insertions(+), 455 deletions(-) create mode 100644 charts/external-secrets/Chart.yaml create mode 100644 charts/external-secrets/crds/clusterexternalsecrets.yaml create mode 100644 charts/external-secrets/crds/clustersecretstores.yaml create mode 100644 charts/external-secrets/crds/externalsecrets.yaml create mode 100644 charts/external-secrets/crds/generatorstates.yaml create mode 100644 charts/external-secrets/crds/pushsecrets.yaml create mode 100644 charts/external-secrets/crds/secretstores.yaml create mode 100644 charts/external-secrets/templates/_helpers.tpl create mode 100644 charts/external-secrets/templates/clusterrole.yaml create mode 100644 charts/external-secrets/templates/deployment.yaml create mode 100644 charts/external-secrets/templates/serviceaccount.yaml create mode 100644 charts/external-secrets/values.yaml create mode 100644 src/common/sealed-secrets.test.ts create mode 100644 src/common/sealed-secrets.ts create mode 100644 values/apl-operator/apl-operator-raw.gotmpl create mode 100644 values/external-secrets/external-secrets-raw.gotmpl create mode 100644 values/external-secrets/external-secrets.gotmpl create mode 100644 values/otomi-api/otomi-api-raw.gotmpl diff --git a/chart/apl/templates/deployment.yaml b/chart/apl/templates/deployment.yaml index 7f6bff1de9..0661532ea4 100644 --- a/chart/apl/templates/deployment.yaml +++ b/chart/apl/templates/deployment.yaml @@ -75,13 +75,9 @@ spec: value: {{ .Values.operator.pollIntervalMs | default "30000" | quote }} - name: RECONCILE_INTERVAL_MS value: {{ .Values.operator.reconcileIntervalMs | default "300000" | quote }} - {{- if hasKey $kms "sops" }} envFrom: - - secretRef: - name: apl-sops-secrets - secretRef: name: apl-git-credentials - {{- end }} volumeMounts: - name: otomi-values mountPath: /home/app/stack/env diff --git a/charts/external-secrets/Chart.yaml b/charts/external-secrets/Chart.yaml new file mode 100644 index 0000000000..f166cf473d --- /dev/null +++ b/charts/external-secrets/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +appVersion: 0.14.3 +description: External Secrets Operator for Kubernetes +home: https://external-secrets.io +keywords: +- secrets +- external-secrets +kubeVersion: '>=1.19.0-0' +maintainers: +- name: External Secrets Community + url: https://github.com/external-secrets/external-secrets +name: external-secrets +sources: +- https://github.com/external-secrets/external-secrets +type: application +version: 0.14.3 diff --git a/charts/external-secrets/crds/clusterexternalsecrets.yaml b/charts/external-secrets/crds/clusterexternalsecrets.yaml new file mode 100644 index 0000000000..bda5779fe9 --- /dev/null +++ b/charts/external-secrets/crds/clusterexternalsecrets.yaml @@ -0,0 +1,51 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterexternalsecrets.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: ClusterExternalSecret + listKind: ClusterExternalSecretList + plural: clusterexternalsecrets + shortNames: + - ces + singular: clusterexternalsecret + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.externalSecretSpec.secretStoreRef.name + name: Store + type: string + - jsonPath: .spec.refreshTime + name: Refresh Interval + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: ClusterExternalSecret creates ExternalSecrets across namespaces + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/clustersecretstores.yaml b/charts/external-secrets/crds/clustersecretstores.yaml new file mode 100644 index 0000000000..568ca452f1 --- /dev/null +++ b/charts/external-secrets/crds/clustersecretstores.yaml @@ -0,0 +1,48 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clustersecretstores.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: ClusterSecretStore + listKind: ClusterSecretStoreList + plural: clustersecretstores + shortNames: + - css + singular: clustersecretstore + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: ClusterSecretStore represents a cluster-wide secret store + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/externalsecrets.yaml b/charts/external-secrets/crds/externalsecrets.yaml new file mode 100644 index 0000000000..07ba80b4e5 --- /dev/null +++ b/charts/external-secrets/crds/externalsecrets.yaml @@ -0,0 +1,51 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: externalsecrets.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: ExternalSecret + listKind: ExternalSecretList + plural: externalsecrets + shortNames: + - es + singular: externalsecret + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.secretStoreRef.name + name: Store + type: string + - jsonPath: .spec.refreshInterval + name: Refresh Interval + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: ExternalSecret reads secret data from external secret stores + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/generatorstates.yaml b/charts/external-secrets/crds/generatorstates.yaml new file mode 100644 index 0000000000..0f50ec67aa --- /dev/null +++ b/charts/external-secrets/crds/generatorstates.yaml @@ -0,0 +1,39 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: generatorstates.generators.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: generators.external-secrets.io + names: + categories: + - externalsecrets + kind: GeneratorState + listKind: GeneratorStateList + plural: generatorstates + singular: generatorstate + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: GeneratorState tracks the state of generators + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/pushsecrets.yaml b/charts/external-secrets/crds/pushsecrets.yaml new file mode 100644 index 0000000000..7595969ca1 --- /dev/null +++ b/charts/external-secrets/crds/pushsecrets.yaml @@ -0,0 +1,43 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: pushsecrets.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: PushSecret + listKind: PushSecretList + plural: pushsecrets + singular: pushsecret + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: PushSecret pushes secrets to external secret stores + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/crds/secretstores.yaml b/charts/external-secrets/crds/secretstores.yaml new file mode 100644 index 0000000000..a61ca3b805 --- /dev/null +++ b/charts/external-secrets/crds/secretstores.yaml @@ -0,0 +1,48 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: secretstores.external-secrets.io + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 +spec: + group: external-secrets.io + names: + categories: + - externalsecrets + kind: SecretStore + listKind: SecretStoreList + plural: secretstores + shortNames: + - ss + singular: secretstore + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: SecretStore represents a source of secrets + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/external-secrets/templates/_helpers.tpl b/charts/external-secrets/templates/_helpers.tpl new file mode 100644 index 0000000000..d090bbc336 --- /dev/null +++ b/charts/external-secrets/templates/_helpers.tpl @@ -0,0 +1,41 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "external-secrets.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "external-secrets.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "external-secrets.labels" -}} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} +app.kubernetes.io/name: {{ include "external-secrets.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "external-secrets.selectorLabels" -}} +app.kubernetes.io/name: {{ include "external-secrets.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/charts/external-secrets/templates/clusterrole.yaml b/charts/external-secrets/templates/clusterrole.yaml new file mode 100644 index 0000000000..daee3d4cd9 --- /dev/null +++ b/charts/external-secrets/templates/clusterrole.yaml @@ -0,0 +1,37 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "external-secrets.fullname" . }} + labels: + {{- include "external-secrets.labels" . | nindent 4 }} +rules: + - apiGroups: ["external-secrets.io"] + resources: ["externalsecrets", "externalsecrets/status", "externalsecrets/finalizers", "secretstores", "secretstores/status", "secretstores/finalizers", "clustersecretstores", "clustersecretstores/status", "clustersecretstores/finalizers", "clusterexternalsecrets", "clusterexternalsecrets/status", "clusterexternalsecrets/finalizers", "pushsecrets", "pushsecrets/status", "pushsecrets/finalizers"] + verbs: ["get", "list", "watch", "update", "patch", "create", "delete"] + - apiGroups: ["generators.external-secrets.io"] + resources: ["generatorstates", "generatorstates/status", "generatorstates/finalizers"] + verbs: ["get", "list", "watch", "update", "patch", "create", "delete"] + - apiGroups: [""] + resources: ["secrets", "configmaps", "serviceaccounts", "serviceaccounts/token", "events"] + verbs: ["get", "list", "watch", "update", "patch", "create", "delete"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "watch"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "external-secrets.fullname" . }} + labels: + {{- include "external-secrets.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "external-secrets.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.name | default (include "external-secrets.fullname" .) }} + namespace: {{ .Release.Namespace }} diff --git a/charts/external-secrets/templates/deployment.yaml b/charts/external-secrets/templates/deployment.yaml new file mode 100644 index 0000000000..2b2b4ea088 --- /dev/null +++ b/charts/external-secrets/templates/deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "external-secrets.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "external-secrets.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "external-secrets.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "external-secrets.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ .Values.serviceAccount.name | default (include "external-secrets.fullname" .) }} + containers: + - name: external-secrets + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --concurrent=1 + - --metrics-addr=:8080 + ports: + - containerPort: 8080 + name: metrics + protocol: TCP + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/charts/external-secrets/templates/serviceaccount.yaml b/charts/external-secrets/templates/serviceaccount.yaml new file mode 100644 index 0000000000..c12da9edf8 --- /dev/null +++ b/charts/external-secrets/templates/serviceaccount.yaml @@ -0,0 +1,9 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.name | default (include "external-secrets.fullname" .) }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "external-secrets.labels" . | nindent 4 }} +{{- end }} diff --git a/charts/external-secrets/values.yaml b/charts/external-secrets/values.yaml new file mode 100644 index 0000000000..5f2abea1d5 --- /dev/null +++ b/charts/external-secrets/values.yaml @@ -0,0 +1,31 @@ +## External Secrets Operator values + +## @param replicaCount Number of ESO controller replicas +replicaCount: 1 + +## @param image ESO controller image configuration +image: + repository: ghcr.io/external-secrets/external-secrets + tag: v0.14.3 + pullPolicy: IfNotPresent + +## @param serviceAccount Service account configuration +serviceAccount: + create: true + name: external-secrets + +## @param installCRDs Install ESO CRDs +installCRDs: true + +## @param resources Resource limits and requests +resources: {} + +## @param webhook Webhook configuration +webhook: + replicaCount: 1 + resources: {} + +## @param certController Cert controller configuration +certController: + replicaCount: 1 + resources: {} diff --git a/charts/ingress-nginx/templates/clusterrole.yaml b/charts/ingress-nginx/templates/clusterrole.yaml index 51bc5002cc..15b2ac4b55 100644 --- a/charts/ingress-nginx/templates/clusterrole.yaml +++ b/charts/ingress-nginx/templates/clusterrole.yaml @@ -27,6 +27,7 @@ rules: - namespaces {{- end}} verbs: + - get - list - watch - apiGroups: diff --git a/charts/otomi-api/templates/deployment.yaml b/charts/otomi-api/templates/deployment.yaml index 527ca0a1bc..b4ba47d821 100644 --- a/charts/otomi-api/templates/deployment.yaml +++ b/charts/otomi-api/templates/deployment.yaml @@ -38,6 +38,10 @@ spec: envFrom: - secretRef: name: {{ include "otomi-api.fullname" . }} + {{- with .Values.existingSecret }} + - secretRef: + name: {{ . }} + {{- end }} - configMapRef: name: {{ include "otomi-api.fullname" . }} livenessProbe: @@ -75,6 +79,10 @@ spec: envFrom: - secretRef: name: {{ include "otomi-api.fullname" . }} + {{- with .Values.existingSecret }} + - secretRef: + name: {{ . }} + {{- end }} - configMapRef: name: {{ include "otomi-api.fullname" . }} ports: diff --git a/charts/otomi-api/values.yaml b/charts/otomi-api/values.yaml index 8c7793b4fe..8cdfe5787a 100644 --- a/charts/otomi-api/values.yaml +++ b/charts/otomi-api/values.yaml @@ -87,7 +87,6 @@ affinity: {} secrets: GIT_USER: GIT_EMAIL: - GIT_PASSWORD: env: GIT_REPO_URL: diff --git a/helmfile.d/helmfile-01.init.yaml.gotmpl b/helmfile.d/helmfile-01.init.yaml.gotmpl index 6287a13d27..c31194a595 100644 --- a/helmfile.d/helmfile-01.init.yaml.gotmpl +++ b/helmfile.d/helmfile-01.init.yaml.gotmpl @@ -34,6 +34,14 @@ releases: installed: {{ $a | get "sealed-secrets.enabled" }} namespace: sealed-secrets <<: *default + - name: external-secrets + installed: {{ $a | get "sealed-secrets.enabled" }} + namespace: external-secrets + <<: *default + - name: external-secrets-artifacts + installed: {{ $a | get "sealed-secrets.enabled" }} + namespace: external-secrets + <<: *raw - name: cert-manager installed: true namespace: cert-manager diff --git a/helmfile.d/helmfile-04.init.yaml.gotmpl b/helmfile.d/helmfile-04.init.yaml.gotmpl index 2e6800ce8e..df2adc1e35 100644 --- a/helmfile.d/helmfile-04.init.yaml.gotmpl +++ b/helmfile.d/helmfile-04.init.yaml.gotmpl @@ -25,6 +25,12 @@ releases: labels: pkg: apl-operator <<: *default + - name: apl-operator-artifacts + installed: true + namespace: apl-operator + labels: + pkg: apl-operator + <<: *raw - name: otomi-operator installed: true namespace: otomi-operator diff --git a/helmfile.d/helmfile-60.teams.yaml.gotmpl b/helmfile.d/helmfile-60.teams.yaml.gotmpl index dfcac659cc..ce769d6040 100644 --- a/helmfile.d/helmfile-60.teams.yaml.gotmpl +++ b/helmfile.d/helmfile-60.teams.yaml.gotmpl @@ -24,6 +24,8 @@ releases: {{- $prometheusDomain := printf "prometheus-%s.%s" $teamId $domain }} {{- $grafanaDomain := printf "grafana-%s.%s" $teamId $domain }} {{- $teamApps := index $tc $teamId "apps" | default dict }} + {{- $teamReceivers := $teamSettings | get "alerts.receivers" ($v | get "alerts.receivers" (list "slack")) }} + {{- $teamAlertmanagerConfig := tpl (readFile "../helmfile.d/snippets/alertmanager-teams.gotmpl") (dict "instance" $teamSettings "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | toString }} - name: tekton-dashboard-{{ $teamId }} installed: true namespace: team-{{ $teamId }} @@ -61,8 +63,8 @@ releases: prometheus: system resources: {{- $teamApps.alertmanager.resources | toYaml | nindent 14 }} - # to do: load slackTpl and opsgenieTpl only if alerts.receicers = true - config: {{- tpl (readFile "../helmfile.d/snippets/alertmanager-teams.gotmpl") (dict "instance" $teamSettings "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | nindent 12 }} + useExistingSecret: true + configSecret: alertmanager-team-{{ $teamId }}-config defaultRules: rules: general: false @@ -73,7 +75,10 @@ releases: prometheusSpec: {} grafana: enabled: {{ $teamSettings | get "managedMonitoring.grafana" false }} - adminPassword: {{ $teamSettings.password | quote }} + admin: + existingSecret: team-{{ $teamId }}-grafana-admin + userKey: admin-user + passwordKey: admin-password resources: {{- $teamApps.grafana.resources.grafana | toYaml | nindent 12 }} namespaceOverride: null # team-{{ $teamId }} @@ -116,8 +121,6 @@ releases: url: http://loki-query-frontend-headless.monitoring:3101 basicAuth: true basicAuthUser: {{ $teamId }} - secureJsonData: - basicAuthPassword: {{ $teamSettings.password | quote }} {{- if has "msteams" ($teamSettings | get "alerts.receivers" list) }} - name: prometheus-msteams-{{ $teamId }} installed: {{ $teamSettings | get "managedMonitoring.alertmanager" false }} @@ -166,4 +169,74 @@ releases: pipeline: otomi-task-teams values: - ../values/team-ns/team-ns.gotmpl + - name: team-secrets-{{ $teamId }} + installed: true + namespace: team-{{ $teamId }} + chart: ../charts/raw + labels: + tag: teams + team: {{ $teamId }} + pipeline: otomi-task-teams + values: + - resources: + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: team-{{ $teamId }}-grafana-admin + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: team-{{ $teamId }}-grafana-admin + creationPolicy: Owner + template: + type: Opaque + data: + admin-user: {{ $teamId }} + admin-password: '{{ "{{ .password | toString }}" }}' + data: + - secretKey: password + remoteRef: + key: team-{{ $teamId }}-settings-secrets + property: settings_password + {{- if $teamSettings | get "managedMonitoring.alertmanager" false }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: alertmanager-team-{{ $teamId }}-config + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: alertmanager-team-{{ $teamId }}-config + creationPolicy: Owner + template: + type: Opaque + data: + alertmanager.yaml: | + {{- $teamAlertmanagerConfig | nindent 20 }} + data: + {{- if has "slack" $teamReceivers }} + - secretKey: slackUrl + remoteRef: + key: alerts-secrets + property: slack_url + {{- end }} + {{- if has "email" $teamReceivers }} + - secretKey: smtpAuthPassword + remoteRef: + key: smtp-secrets + property: auth_password + - secretKey: smtpAuthSecret + remoteRef: + key: smtp-secrets + property: auth_secret + {{- end }} + {{- end }} {{- end }} diff --git a/helmfile.d/helmfile-70.shared.yaml.gotmpl b/helmfile.d/helmfile-70.shared.yaml.gotmpl index 2b0f64b250..6b25e08103 100644 --- a/helmfile.d/helmfile-70.shared.yaml.gotmpl +++ b/helmfile.d/helmfile-70.shared.yaml.gotmpl @@ -35,6 +35,13 @@ releases: pkg: oauth2-proxy app: core <<: *raw + - name: otomi-api-artifacts + installed: true + namespace: otomi + labels: + pkg: otomi + app: core + <<: *raw - name: otomi-api installed: true namespace: otomi diff --git a/helmfile.d/snippets/alertmanager-teams.gotmpl b/helmfile.d/snippets/alertmanager-teams.gotmpl index aefb5e30af..5f2a51ba8d 100644 --- a/helmfile.d/snippets/alertmanager-teams.gotmpl +++ b/helmfile.d/snippets/alertmanager-teams.gotmpl @@ -2,10 +2,10 @@ {{- $suffix := (true | ternary "" ".monitoring.svc.cluster.local") }} global: {{- if (has "slack" $receivers ) }} - slack_api_url: {{ .instance | get "alerts.slack.url" (.root | get "alerts.slack.url" (.root | get "home.slack.url" nil)) }} + slack_api_url: {{ "{{ .slackUrl | toString }}" }} {{- end }} {{- if (has "opsgenie" $receivers ) }} - opsgenie_api_key: {{ .instance | get "alerts.opsgenie.apiKey" (.root | get "alerts.opsgenie.apiKey" (.root | get "home.opsgenie.apiKey" nil)) }} + opsgenie_api_key: {{ "{{ .opsgenieApiKey | toString }}" }} opsgenie_api_url: {{ .instance | get "alerts.opsgenie.url" (.root | get "alerts.opsgenie.url" (.root | get "home.opsgenie.url" nil)) }} {{- end }} {{- if or (has "email" $receivers) }} @@ -13,8 +13,8 @@ global: smtp_hello: {{ .root | get "smtp.hello" .root.cluster.domainSuffix }} smtp_from: {{ .root | get "smtp.from" (print "alerts@" .root.cluster.domainSuffix) }} smtp_auth_username: {{ .root | get "smtp.auth_username" nil }} - smtp_auth_password: {{ .root | get "smtp.auth_password" nil | quote }} - smtp_auth_secret: {{ .root | get "smtp.auth_secret" nil | quote }} + smtp_auth_password: '{{ "{{ .smtpAuthPassword | toString }}" }}' + smtp_auth_secret: '{{ "{{ .smtpAuthSecret | toString }}" }}' smtp_auth_identity: {{ .root | get "smtp.auth_identity" nil }} {{- end }} route: diff --git a/helmfile.d/snippets/alertmanager.gotmpl b/helmfile.d/snippets/alertmanager.gotmpl index a4eb2346bc..24448df09d 100644 --- a/helmfile.d/snippets/alertmanager.gotmpl +++ b/helmfile.d/snippets/alertmanager.gotmpl @@ -2,10 +2,10 @@ {{- $suffix := (true | ternary "" ".monitoring.svc.cluster.local") }} global: {{- if (has "slack" $receivers ) }} - slack_api_url: {{ .instance | get "alerts.slack.url" (.root | get "alerts.slack.url" (.root | get "home.slack.url" nil)) }} + slack_api_url: {{ "{{ .slackUrl | toString }}" }} {{- end }} {{- if (has "opsgenie" $receivers ) }} - opsgenie_api_key: {{ .instance | get "alerts.opsgenie.apiKey" (.root | get "alerts.opsgenie.apiKey" (.root | get "home.opsgenie.apiKey" nil)) }} + opsgenie_api_key: {{ "{{ .opsgenieApiKey | toString }}" }} opsgenie_api_url: {{ .instance | get "alerts.opsgenie.url" (.root | get "alerts.opsgenie.url" (.root | get "home.opsgenie.url" nil)) }} {{- end }} {{- if or (has "email" $receivers) }} @@ -13,8 +13,8 @@ global: smtp_hello: {{ .root | get "smtp.hello" .root.cluster.domainSuffix }} smtp_from: {{ .root | get "smtp.from" (print "alerts@" .root.cluster.domainSuffix) }} smtp_auth_username: {{ .root | get "smtp.auth_username" nil }} - smtp_auth_password: {{ .root | get "smtp.auth_password" nil | quote }} - smtp_auth_secret: {{ .root | get "smtp.auth_secret" nil | quote }} + smtp_auth_password: '{{ "{{ .smtpAuthPassword | toString }}" }}' + smtp_auth_secret: '{{ "{{ .smtpAuthSecret | toString }}" }}' smtp_auth_identity: {{ .root | get "smtp.auth_identity" nil }} {{- end }} route: diff --git a/helmfile.d/snippets/defaults.gotmpl b/helmfile.d/snippets/defaults.gotmpl index df5cca08c3..e271a1f656 100644 --- a/helmfile.d/snippets/defaults.gotmpl +++ b/helmfile.d/snippets/defaults.gotmpl @@ -22,8 +22,8 @@ environments: default: values: - apps: - kubeflow-pipelines: - rootPassword: {{ randAlphaNum 32 }} + kubeflow-pipelines: {} + gitea: {} {{- range $index,$ingressClassName := $ingressClassNames }} ingress-nginx-{{ $ingressClassName}}: autoscaling: @@ -267,13 +267,9 @@ environments: useCloudShell: true downloadKubeconfig: true downloadDockerLogin: true - password: {{ randAlphaNum 32 }} {{- end }} {{- end }} - otomi: - adminPassword: {{ randAlphaNum 32 }} - git: - password: {{ randAlphaNum 20 }} + otomi: {} cluster: owner: customer name: apl diff --git a/helmfile.d/snippets/defaults.yaml b/helmfile.d/snippets/defaults.yaml index ae92ed7377..1f60127045 100644 --- a/helmfile.d/snippets/defaults.yaml +++ b/helmfile.d/snippets/defaults.yaml @@ -977,6 +977,16 @@ environments: cpu: "2" memory: 1Gi _rawValues: {} + external-secrets: + resources: + operator: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 512Mi + _rawValues: {} rabbitmq: enabled: false resources: diff --git a/helmfile.d/snippets/derived.gotmpl b/helmfile.d/snippets/derived.gotmpl index 569f30c8b6..0a7c209f99 100644 --- a/helmfile.d/snippets/derived.gotmpl +++ b/helmfile.d/snippets/derived.gotmpl @@ -220,15 +220,10 @@ environments: {{- end }} external-dns: enabled: {{ $v.otomi.hasExternalDNS }} - harbor: - adminPassword: {{ $a | get "harbor.adminPassword" $v.otomi.adminPassword | quote }} - registry: - credentials: - password: {{ $a | get "harbor.registry.credentials.password" $v.otomi.adminPassword | quote }} + harbor: {} keycloak: enabled: true address: {{ $keycloakBaseUrl }} - adminPassword: {{ $a | get "keycloak.adminPassword" $v.otomi.adminPassword | quote }} ingress-nginx: enabled: true istio: diff --git a/helmfile.d/snippets/grafana.gotmpl b/helmfile.d/snippets/grafana.gotmpl index 9128cca891..4490a0f2fd 100644 --- a/helmfile.d/snippets/grafana.gotmpl +++ b/helmfile.d/snippets/grafana.gotmpl @@ -9,8 +9,8 @@ analytics: org_role: Admin allow_sign_up: true oauth_auto_login: true - client_id: {{ .keycloak.clientID }} - client_secret: {{ .keycloak.clientSecret | quote }} + client_id: $__env{GF_AUTH_GENERIC_OAUTH_CLIENT_ID} + client_secret: $__env{GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET} scopes: email profile openid email_attribute_path: email login_attribute_path: username diff --git a/helmfile.d/snippets/sops-env.gotmpl b/helmfile.d/snippets/sops-env.gotmpl index aa9e2ff866..e69de29bb2 100644 --- a/helmfile.d/snippets/sops-env.gotmpl +++ b/helmfile.d/snippets/sops-env.gotmpl @@ -1,24 +0,0 @@ -{{- with . | get "azure" nil }} -AZURE_CLIENT_ID: {{ .clientId }} -AZURE_CLIENT_SECRET: {{ .clientSecret }} -{{- with . | get "tenantId" nil }} -AZURE_TENANT_ID: {{ . }}{{ end }} -{{- with . | get "environment" nil }} -AZURE_ENVIRONMENT: {{ . }}{{ end }} -{{- end }} -{{- with . | get "aws" nil }} -AWS_ACCESS_KEY_ID: {{ .accessKey }} -AWS_SECRET_ACCESS_KEY: {{ .secretKey }} -{{- with . | get "region" nil }} -AWS_REGION: {{ . }}{{ end }} -{{- end }} -{{- with . | get "age" nil }} -SOPS_AGE_KEY: {{ .privateKey }} -{{- end }} -{{- with . | get "google" nil }} -GCLOUD_SERVICE_KEY: '{{ .accountJson | replace "\n" "" }}' -{{- with . | get "project" nil }} -GOOGLE_PROJECT: {{ . }}{{ end }} -{{- with . | get "region" nil }} -GOOGLE_REGION: {{ . }}{{ end }} -{{- end }} diff --git a/package-lock.json b/package-lock.json index e1ace274da..8bdaab5dfb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -200,6 +200,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -2465,6 +2466,7 @@ "integrity": "sha512-lf6d+BdMkJIFCxx2FpajLpqVGGyaGUNFU6jhEM6QUPeGuoA5et2kJXrL0NSY2uWLOVyYYc/FPjzlbe8trA9tBQ==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=20" } @@ -2546,7 +2548,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-css/-/dict-css-4.0.19.tgz", "integrity": "sha512-VYHtPnZt/Zd/ATbW3rtexWpBnHUohUrQOHff/2JBhsVgxOrksAxJnLAO43Q1ayLJBJUUwNVo+RU0sx0aaysZfg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-dart": { "version": "2.3.2", @@ -2686,14 +2689,16 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-html/-/dict-html-4.0.14.tgz", "integrity": "sha512-2bf7n+kS92g+cMKV0wr9o/Oq9n8JzU7CcrB96gIh2GHgnF+0xDOqO2W/1KeFAqOfqosoOVE48t+4dnEMkkoJ2Q==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-html-symbol-entities": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@cspell/dict-html-symbol-entities/-/dict-html-symbol-entities-4.0.5.tgz", "integrity": "sha512-429alTD4cE0FIwpMucvSN35Ld87HCyuM8mF731KU5Rm4Je2SG6hmVx7nkBsLyrmH3sQukTcr1GaiZsiEg8svPA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-java": { "version": "5.0.12", @@ -2891,7 +2896,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-typescript/-/dict-typescript-3.2.3.tgz", "integrity": "sha512-zXh1wYsNljQZfWWdSPYwQhpwiuW0KPW1dSd8idjMRvSD0aSvWWHoWlrMsmZeRl4qM4QCEAjua8+cjflm41cQBg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-vue": { "version": "3.0.5", @@ -4983,6 +4989,7 @@ "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -6572,7 +6579,8 @@ "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/json5": { "version": "0.0.29", @@ -6638,6 +6646,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -6837,6 +6846,7 @@ "integrity": "sha512-4z2nCSBfVIMnbuu8uinj+f0o4qOeggYJLbjpPHka3KH1om7e+H9yLKTYgksTaHcGco+NClhhY2vyO3HsMH1RGw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.55.0", "@typescript-eslint/types": "8.55.0", @@ -7377,6 +7387,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -8258,6 +8269,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -10655,6 +10667,7 @@ "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", @@ -12049,6 +12062,7 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -12109,6 +12123,7 @@ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", "dev": true, "license": "MIT", + "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -12243,6 +12258,7 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -15752,6 +15768,7 @@ "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "30.2.0", "@jest/types": "30.2.0", @@ -17519,6 +17536,7 @@ "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", "license": "MIT", + "peer": true, "engines": { "node": ">= 10.16.0" } @@ -18562,6 +18580,7 @@ "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", "dev": true, "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -21666,6 +21685,7 @@ "dev": true, "inBundle": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -22721,6 +22741,7 @@ "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -23722,6 +23743,7 @@ "integrity": "sha512-WRgl5GcypwramYX4HV+eQGzUbD7UUbljVmS+5G1uMwX/wLgYuJAxGeerXJDMO2xshng4+FXqCgyB5QfClV6WjA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@semantic-release/commit-analyzer": "^13.0.1", "@semantic-release/error": "^4.0.0", @@ -25582,6 +25604,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -25785,6 +25808,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -26082,6 +26106,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -26261,6 +26286,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "napi-postinstall": "^0.2.4" }, @@ -26763,6 +26789,7 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz", "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=10.0.0" }, diff --git a/src/cmd/bootstrap.test.ts b/src/cmd/bootstrap.test.ts index 1417769ca7..6cfb0f22a1 100644 --- a/src/cmd/bootstrap.test.ts +++ b/src/cmd/bootstrap.test.ts @@ -12,6 +12,8 @@ import { processValues, } from './bootstrap' +jest.mock('@linode/kubeseal-encrypt') + const { terminal } = stubs jest.mock('src/common/envalid', () => ({ @@ -37,6 +39,7 @@ describe('Bootstrapping values', () => { }), }), bootstrapSops: jest.fn(), + bootstrapSealedSecrets: jest.fn(), copyBasicFiles: jest.fn(), copyFile: jest.fn(), createCustomCA: jest.fn(), @@ -59,14 +62,14 @@ describe('Bootstrapping values', () => { } }) it('should call relevant sub routines', async () => { - deps.processValues.mockReturnValue(values) + deps.processValues.mockReturnValue({ originalInput: values, allSecrets: {} }) deps.hfValues.mockReturnValue(values) await bootstrap(deps) expect(deps.copyBasicFiles).toHaveBeenCalled() - expect(deps.bootstrapSops).toHaveBeenCalled() + expect(deps.bootstrapSealedSecrets).toHaveBeenCalled() }) it('should copy only skeleton files to env dir if it is empty or nonexisting', async () => { - deps.processValues.mockReturnValue(undefined) + deps.processValues.mockReturnValue({ originalInput: undefined, allSecrets: {} }) await bootstrap(deps) expect(deps.hfValues).toHaveBeenCalledTimes(0) }) @@ -280,6 +283,18 @@ describe('Bootstrapping values', () => { { id: 'user1', initialPassword: 'existing-password' }, { id: 'user2', initialPassword: generatedPassword }, ] + // Pre-processed users (as stored in allSecrets for sealed secret generation) + const processedUsers = usersWithPasswords.map((u: any) => ({ + email: u.email, + firstName: u.firstName, + lastName: u.lastName, + initialPassword: u.initialPassword, + groups: [ + ...(u.isPlatformAdmin ? ['platform-admin'] : []), + ...(u.isTeamAdmin ? ['team-admin'] : []), + ...(u.teams || []).map((t: string) => `team-${t}`), + ], + })) const ca = { a: 'cert' } const mergedSecretsWithCa = merge(cloneDeep(secrets), cloneDeep(ca)) const mergedSecretsWithGen = merge(cloneDeep(secrets), cloneDeep(generatedSecrets)) @@ -304,6 +319,8 @@ describe('Bootstrapping values', () => { addInitialPasswords: jest.fn().mockReturnValue(usersWithPasswords), addPlatformAdmin: jest.fn().mockReturnValue(usersWithPasswords), pathExists: jest.fn().mockReturnValue(true), + getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), + stripAllSecrets: jest.fn().mockImplementation((v) => v), } }) describe('Creating CA', () => { @@ -331,7 +348,8 @@ describe('Bootstrapping values', () => { deps.getStoredClusterSecrets.mockReturnValue(secrets) deps.generateSecrets.mockReturnValue(allSecrets) await processValues(deps) - expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', allSecrets) + const expected = { ...allSecrets, users: processedUsers } + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', expected) expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) }) it('should create a custom ca if issuer is custom-ca or undefined and no CA yet exists', async () => { @@ -351,11 +369,10 @@ describe('Bootstrapping values', () => { deps.generateSecrets.mockReturnValue(generatedSecrets) deps.createCustomCA.mockReturnValue(ca) await processValues(deps) - expect(deps.createK8sSecret).toHaveBeenCalledWith( - 'otomi-generated-passwords', - 'otomi', - mergedSecretsWithGenAndCa, - ) + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { + ...mergedSecretsWithGenAndCa, + users: processedUsers, + }) }) it('should not overwrite stored secrets', async () => { deps.loadYaml.mockReturnValue({}) @@ -364,37 +381,47 @@ describe('Bootstrapping values', () => { deps.generateSecrets.mockReturnValue(generatedSecrets) await processValues(deps) expect(deps.generateSecrets).toHaveBeenCalledWith(generatedSecrets) - expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', generatedSecrets) + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { + ...generatedSecrets, + users: processedUsers, + }) }) - it('should only write and return original values', async () => { + it('should merge allSecrets into disk values so non-secret fields like customRootCA are preserved', async () => { deps.loadYaml.mockReturnValue({ cluster: { name: 'bla', provider: 'dida' }, }) - deps.createCustomCA.mockReturnValue({ a: 'cert' }) deps.getStoredClusterSecrets.mockReturnValue({ users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], }) deps.generateSecrets.mockReturnValue({ gen: 'x' }) deps.createCustomCA.mockReturnValue(ca) const res = await processValues(deps) + // mergedForDisk includes allSecrets (stripAllSecrets mock is identity, real impl strips x-secret paths) + // processedUsers adds groups:[] to each user via element-wise lodash merge expect(deps.writeValues).toHaveBeenNthCalledWith(1, { - cluster: { name: 'bla', provider: 'dida' }, a: 'cert', gen: 'x', + cluster: { name: 'bla', provider: 'dida' }, users: [ - { id: 'user1', initialPassword: 'existing-password' }, - { id: 'user2', initialPassword: 'generated-password' }, + { id: 'user1', initialPassword: 'existing-password', groups: [] }, + { id: 'user2', initialPassword: 'generated-password', groups: [] }, ], }) - expect(res).toEqual({ + expect(res.originalInput).toEqual({ cluster: { name: 'bla', provider: 'dida' }, users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], }) }) - it('should merge original with generated values and write them to env dir', async () => { - const writtenValues = merge( + it('should merge originalInput + allSecrets + users for disk (stripAllSecrets removes x-secret paths)', async () => { + // mergedForDisk = merge(originalInput, allSecrets, { users }) + // allSecrets = merge(ca, storedSecrets, generatedSecrets, kmsValues) + users: processedUsers + const allSecretsExpected = merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { + users: processedUsers, + }) + const expectedDiskValues = merge( + cloneDeep(secrets), cloneDeep(values), - cloneDeep(mergedSecretsWithGenAndCa), + cloneDeep(allSecretsExpected), cloneDeep({ users: usersWithPasswords }), ) deps.loadYaml.mockReturnValue({ ...values, users }) @@ -402,7 +429,25 @@ describe('Bootstrapping values', () => { deps.generateSecrets.mockReturnValue(generatedSecrets) deps.getUsers.mockReturnValue(usersWithPasswords) await processValues(deps) - expect(deps.writeValues).toHaveBeenNthCalledWith(1, writtenValues) + expect(deps.writeValues).toHaveBeenNthCalledWith(1, expectedDiskValues) + }) + it('should call stripAllSecrets before writing values to disk', async () => { + deps.loadYaml.mockReturnValue(values) + deps.getSchemaSecretsPaths.mockResolvedValue(['apps.gitea.adminPassword', 'apps.harbor.adminPassword']) + await processValues(deps) + expect(deps.stripAllSecrets).toHaveBeenCalledTimes(1) + expect(deps.getSchemaSecretsPaths).toHaveBeenCalledTimes(1) + }) + it('should still return full allSecrets for bootstrapSealedSecrets', async () => { + deps.loadYaml.mockReturnValue(values) + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(generatedSecrets) + deps.createCustomCA.mockReturnValue(ca) + const result = await processValues(deps) + // allSecrets should contain full unstripped secrets including pre-processed users + expect(result.allSecrets).toEqual( + merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { users: processedUsers }), + ) }) }) }) diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index 96e30c60c7..f5d4c13f60 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -21,7 +21,16 @@ import { secretId, } from 'src/common/k8s' import { getKmsSettings } from 'src/common/repo' -import { ensureTeamGitOpsDirectories, getFilename, gucci, isCore, loadYaml, rootDir } from 'src/common/utils' +import { bootstrapSealedSecrets, stripAllSecrets } from 'src/common/sealed-secrets' +import { + ensureTeamGitOpsDirectories, + getFilename, + getSchemaSecretsPaths, + gucci, + isCore, + loadYaml, + rootDir, +} from 'src/common/utils' import { generateSecrets, writeValues } from 'src/common/values' import { BasicArguments, setParsedArgs } from 'src/common/yargs' import { Argv } from 'yargs' @@ -99,11 +108,6 @@ export const bootstrapSops = async ( await deps.writeFile(targetPath, output) d.log(`Ready generating sops files. The configuration is written to: ${targetPath}`) - d.info('Copying sops related files') - // add sops related files - const file = '.gitattributes' - await deps.copyFile(`${rootDir}/.values/${file}`, `${envDir}/${file}`) - // prepare some credential files the first time and crypt some if (!exists) { if (isCli || env.OTOMI_DEV) { @@ -305,8 +309,10 @@ export const processValues = async ( generatePassword, addInitialPasswords, addPlatformAdmin, + getSchemaSecretsPaths, + stripAllSecrets, }, -): Promise> => { +): Promise<{ originalInput: Record; allSecrets: Record }> => { const d = deps.terminal(`cmd:${cmdName}:processValues`) const { VALUES_INPUT } = env d.log(`Loading app values from ${VALUES_INPUT}`) @@ -334,12 +340,32 @@ export const processValues = async ( ) // add default platform admin & generate initial passwords for users if they don't have one const users = deps.getUsers(originalInput) - // we have generated all we need, now store everything by merging the original values over all the secrets - await deps.writeValues(merge(cloneDeep(allSecrets), cloneDeep(originalInput), cloneDeep({ users }))) + // Pre-process users into keycloak-operator format (with groups resolved) for sealed secret storage + const processedUsers = users.map((user: any) => { + const groups: string[] = [] + if (user.isPlatformAdmin) groups.push('platform-admin') + if (user.isTeamAdmin) groups.push('team-admin') + for (const team of user.teams || []) groups.push(`team-${team}`) + return { + email: user.email, + firstName: user.firstName, + lastName: user.lastName, + initialPassword: user.initialPassword, + groups, + } + }) + // Store processed users in allSecrets so they flow into sealed secret generation + allSecrets.users = processedUsers + // Write only non-secret values to disk — secrets are stored exclusively in SealedSecrets + // Include allSecrets so non-secret fields like customRootCA are preserved (stripAllSecrets removes only x-secret paths) + const mergedForDisk = merge(cloneDeep(originalInput), cloneDeep(allSecrets), cloneDeep({ users })) + const secretPaths = await deps.getSchemaSecretsPaths(Object.keys(get(mergedForDisk, 'teamConfig', {}))) + const valuesForDisk = deps.stripAllSecrets(mergedForDisk, secretPaths) + await deps.writeValues(valuesForDisk) // and do some context dependent post processing: // to support potential failing chart install we store secrets on cluster if (!(env.isDev && env.DISABLE_SYNC)) await deps.createK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, 'otomi', allSecrets) - return originalInput + return { originalInput, allSecrets } } // create file structure based on file entry @@ -434,6 +460,7 @@ export const bootstrap = async ( hfValues, writeValues, bootstrapSops, + bootstrapSealedSecrets, migrate, encrypt, decrypt, @@ -449,10 +476,10 @@ export const bootstrap = async ( } await deps.copyBasicFiles() await deps.migrate() - const originalValues = await deps.processValues() + const { originalInput, allSecrets } = await deps.processValues() await deps.handleFileEntry() - await deps.bootstrapSops() - await ensureTeamGitOpsDirectories(ENV_DIR, originalValues) + await deps.bootstrapSealedSecrets(allSecrets, ENV_DIR, originalInput) + await ensureTeamGitOpsDirectories(ENV_DIR, originalInput) d.log(`Done bootstrapping values`) } @@ -471,7 +498,6 @@ export const module = { handler: async (argv: BasicArguments): Promise => { setParsedArgs(argv) await prepareEnvironment({ skipAllPreChecks: true }) - await decrypt() await bootstrap() await bootstrapGit() }, diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 1605e9a3a8..96f427920e 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -4,11 +4,11 @@ import { prepareEnvironment } from 'src/common/cli' import { encrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { env } from 'src/common/envalid' -import { hfValues } from 'src/common/hf' +import { getRepo, GitRepoConfig } from 'src/common/git-config' import { waitTillGitRepoAvailable } from 'src/common/gitea' -import { createUpdateConfigMap, createUpdateGenericSecret, k8s } from 'src/common/k8s' +import { hfValues } from 'src/common/hf' +import { createUpdateConfigMap, createUpdateGenericSecret, getK8sSecret, k8s } from 'src/common/k8s' import { getFilename } from 'src/common/utils' -import { getRepo, GitRepoConfig } from 'src/common/git-config' import { HelmArguments, setParsedArgs } from 'src/common/yargs' import { Argv } from 'yargs' import { $, cd } from 'zx' @@ -63,7 +63,7 @@ const commitAndPush = async ( const d = terminal(`cmd:${cmdName}:commitAndPush`) d.info('Committing values') const message = initialInstall ? 'otomi commit' : 'updated values [ci skip]' - const { password } = gitConfig ?? getRepo(values) + const { password } = gitConfig ?? (await getRepo(values)) cd(env.ENV_DIR) try { try { @@ -84,8 +84,9 @@ const commitAndPush = async ( } await $`git commit -m ${message} --no-verify`.quiet() } catch (e) { - d.log('commitAndPush error ', e?.message?.replace(password, '****')) - return + const errorMsg = `commitAndPush error: ${e?.message?.replace(password, '****')}` + d.error(errorMsg) + throw new Error(errorMsg) } if (values._derived?.untrustedCA) process.env.GIT_SSL_NO_VERIFY = '1' await retry( @@ -142,11 +143,16 @@ export const commit = async ( await validateValues(overrideArgs) d.info('Preparing values') const values = (await hfValues()) as Record - // Use provided gitConfig if available (operator mode), otherwise read from values (bootstrap/install mode) - const { branch, authenticatedUrl: remote, username, email } = gitConfig ?? getRepo(values) + const { branch, authenticatedUrl: remote, username, email } = gitConfig ?? (await getRepo(values)) if (initialInstall) { // we call this here again, as we might not have completed (happens upon first install): await bootstrapGit(values) + // Always update the remote URL after bootstrap - the initial bootstrapGit() (called during + // the bootstrap phase before install) may have set the URL with unresolved placeholder + // passwords because K8s secrets didn't exist yet. Now that secrets are decrypted, + // we need to update the URL with the real credentials. + cd(env.ENV_DIR) + await $`git remote set-url origin ${remote}`.nothrow().quiet() } else { cd(env.ENV_DIR) await setIdentity(username, email) @@ -166,23 +172,35 @@ export async function initialSetupData(): Promise { const values = (await hfValues()) as Record const { domainSuffix } = values.cluster const { hasExternalIDP } = values.otomi - - const defaultPlatformAdminEmail = `platform-admin@${domainSuffix}` - const platformAdmin = values.users.find((user: any) => user.email === defaultPlatformAdminEmail) const secretName = hasExternalIDP ? 'root-credentials' : 'platform-admin-initial-credentials' - if (platformAdmin && !hasExternalIDP) { + if (!hasExternalIDP) { + // Read the platform admin's initialPassword from users-secrets (set by keycloak-operator) + const usersSecret = await getK8sSecret('users-secrets', 'sealed-secrets') + let platformAdminPassword = '' + if (usersSecret?.usersJson) { + // getK8sSecret already parses JSON/YAML values, so usersJson may be an array or a string + const users = Array.isArray(usersSecret.usersJson) + ? usersSecret.usersJson + : JSON.parse(String(usersSecret.usersJson)) + const defaultEmail = `platform-admin@${domainSuffix}` + const platformAdmin = users.find((u: any) => u.email === defaultEmail) + platformAdminPassword = platformAdmin?.initialPassword ?? '' + } return { domainSuffix, - username: platformAdmin.email, - password: platformAdmin.initialPassword, + username: `platform-admin@${domainSuffix}`, + password: platformAdminPassword, secretName, } } else { + // External IDP: show Keycloak admin credentials (keycloak-initial-admin uses otomi-platform-secrets.adminPassword) + const otomiSecret = await getK8sSecret('otomi-platform-secrets', 'sealed-secrets') + const adminPassword = otomiSecret?.adminPassword ? String(otomiSecret.adminPassword) : '' return { domainSuffix, - username: values.apps.keycloak.adminUsername, - password: values.apps.keycloak.adminPassword, + username: 'otomi-admin', + password: adminPassword, secretName, } } diff --git a/src/cmd/install.test.ts b/src/cmd/install.test.ts index 86d69a4a5a..89967147a5 100644 --- a/src/cmd/install.test.ts +++ b/src/cmd/install.test.ts @@ -17,6 +17,7 @@ jest.mock('src/common/k8s', () => ({ applyServerSide: jest.fn(), restartOtomiApiDeployment: jest.fn(), waitForCRD: jest.fn(), + getK8sSecret: jest.fn().mockResolvedValue({ password: 'test', username: 'test' }), k8s: { app: jest.fn(), }, @@ -39,9 +40,31 @@ jest.mock('src/common/git-config', () => ({ setGitConfig: jest.fn(), })) -jest.mock('zx', () => ({ - $: jest.fn(), - cd: jest.fn(), +jest.mock('zx', () => { + const mockResult = { exitCode: 0, stdout: '', stderr: '' } + const createMockProcessPromise = () => { + const promise = Promise.resolve(mockResult) + const chainable: any = promise + chainable.nothrow = jest.fn().mockReturnValue(chainable) + chainable.quiet = jest.fn().mockReturnValue(chainable) + return chainable + } + return { + $: jest.fn().mockImplementation(() => createMockProcessPromise()), + cd: jest.fn(), + } +}) + +jest.mock('src/common/sealed-secrets', () => ({ + applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue(undefined), + restartSealedSecretsController: jest.fn().mockResolvedValue(undefined), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), +})) + +jest.mock('src/common/utils', () => ({ + ...jest.requireActual('src/common/utils'), + rootDir: '/test/root', + getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), })) jest.mock('./commit', () => ({ @@ -64,11 +87,6 @@ jest.mock('src/common/cli', () => ({ prepareEnvironment: jest.fn(), })) -jest.mock('src/common/utils', () => ({ - ...jest.requireActual('src/common/utils'), - rootDir: '/test/root', -})) - jest.mock('src/common/yargs', () => ({ getParsedArgs: jest.fn().mockReturnValue({}), setParsedArgs: jest.fn(), @@ -111,7 +129,6 @@ describe('Install command', () => { stderr: '', }) mockDeps.deployEssential.mockResolvedValue(true) - mockDeps.$.mockResolvedValue(undefined) }) describe('module configuration', () => { diff --git a/src/cmd/install.ts b/src/cmd/install.ts index b91de56f4e..8a221696cf 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -5,8 +5,20 @@ import { logLevelString, terminal } from 'src/common/debug' import { env } from 'src/common/envalid' import { setGitConfig } from 'src/common/git-config' import { deployEssential, hf, HF_DEFAULT_SYNC_ARGS, hfValues } from 'src/common/hf' -import { applyServerSide, getDeploymentState, getHelmReleases, setDeploymentState, waitForCRD } from 'src/common/k8s' -import { getFilename, rootDir } from 'src/common/utils' +import { + applyServerSide, + getDeploymentState, + getHelmReleases, + getK8sSecret, + setDeploymentState, + waitForCRD, +} from 'src/common/k8s' +import { + applySealedSecretManifestsFromDir, + buildSecretToNamespaceMap, + restartSealedSecretsController, +} from 'src/common/sealed-secrets' +import { getFilename, getSchemaSecretsPaths, rootDir } from 'src/common/utils' import { getImageTagFromValues, getPackageVersion, writeValuesToFile } from 'src/common/values' import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs' import { Argv, CommandModule } from 'yargs' @@ -45,6 +57,65 @@ const retryInstallStep = async ( ) } +/** + * Wait for SealedSecrets controller to decrypt SealedSecret resources into K8s Secrets. + * Derives the list of secrets to wait for from schema x-secret fields via buildSecretToNamespaceMap(). + */ +const waitForSealedSecrets = async ( + timeoutMs = 120000, + intervalMs = 3000, + deps = { getK8sSecret, terminal, buildSecretToNamespaceMap, getSchemaSecretsPaths }, +): Promise => { + const d = deps.terminal(`cmd:${cmdName}:waitForSealedSecrets`) + + // Build list of secrets to wait for from schema-driven mappings + // We pass empty secrets/teams since we just need the secret names and namespaces + const mappings = await deps.buildSecretToNamespaceMap({}, [], undefined, { + getSchemaSecretsPaths: deps.getSchemaSecretsPaths, + }) + + // Deduplicate by namespace/secretName + const secretsToWait = new Map() + for (const mapping of mappings) { + const key = `${mapping.namespace}/${mapping.secretName}` + if (!secretsToWait.has(key)) { + secretsToWait.set(key, { namespace: mapping.namespace, secretName: mapping.secretName }) + } + } + + if (secretsToWait.size === 0) { + d.info('No sealed secrets to wait for') + return + } + + d.info(`Waiting for ${secretsToWait.size} sealed secrets to be decrypted`) + const start = Date.now() + + while (Date.now() - start < timeoutMs) { + const pending: string[] = [] + for (const { namespace, secretName } of secretsToWait.values()) { + try { + const secret = await deps.getK8sSecret(secretName, namespace) + if (!secret) { + pending.push(`${namespace}/${secretName}`) + } + } catch { + pending.push(`${namespace}/${secretName}`) + } + } + + if (pending.length === 0) { + d.info('All sealed secrets have been decrypted') + return + } + + d.info(`Still waiting for sealed secrets: ${pending.join(', ')}`) + await new Promise((resolve) => setTimeout(resolve, intervalMs)) + } + + throw new Error(`Timed out waiting for sealed secrets to be decrypted after ${timeoutMs}ms`) +} + export const installAll = async () => { const d = terminal(`cmd:${cmdName}:installAll`) const prevState = await getDeploymentState() @@ -65,16 +136,65 @@ export const installAll = async () => { throw new Error('Failed to deploy essential manifests') } + // Deploy sealed-secrets controller right after essentials + d.info('Deploying sealed-secrets controller') + await hf( + { + fileOpts: 'helmfile.d/helmfile-01.init.yaml.gotmpl', + labelOpts: ['name=sealed-secrets'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + + d.info('Waiting for SealedSecret CRD to be ready') + await retryInstallStep(waitForCRD, 'sealedsecrets.bitnami.com') + + d.info('Applying SealedSecret manifests') + await applySealedSecretManifestsFromDir(env.ENV_DIR) + + d.info('Restarting sealed-secrets controller') + await restartSealedSecretsController() + + d.info('Waiting for sealed secrets to be decrypted into K8s Secrets') + await waitForSealedSecrets() + + // Deploy ESO (External Secrets Operator) + d.info('Deploying external-secrets operator') + await hf( + { + fileOpts: 'helmfile.d/helmfile-01.init.yaml.gotmpl', + labelOpts: ['name=external-secrets'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + + d.info('Waiting for ExternalSecret CRD to be ready') + await retryInstallStep(waitForCRD, 'externalsecrets.external-secrets.io') + + d.info('Deploying ESO ClusterSecretStore') + await hf( + { + fileOpts: 'helmfile.d/helmfile-01.init.yaml.gotmpl', + labelOpts: ['name=external-secrets-artifacts'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + + // Deploy CRDs d.info('Deploying CRDs') await retryInstallStep(applyServerSide, 'charts/kube-prometheus-stack/charts/crds/crds') - // Wait for ServiceMonitor CRD to be established before deploying nginx await retryInstallStep(waitForCRD, 'servicemonitors.monitoring.coreos.com') await retryInstallStep(async () => $`kubectl apply -f charts/tekton-triggers/crds --server-side`) d.info('Deploying charts containing label stage=prep') await hf( { - // 'fileOpts' limits the hf scope and avoids parse errors (we only have basic values at this stage): fileOpts: 'helmfile.d/helmfile-02.init.yaml.gotmpl', labelOpts: ['stage=prep'], logLevel: logLevelString(), @@ -96,18 +216,42 @@ export const installAll = async () => { { streams: { stdout: d.stream.log, stderr: d.stream.error } }, ) + // Deploy cert-manager artifacts (ExternalSecrets, ClusterIssuers, Certificates) + // Must be after app=core (cert-manager CRDs) and after ESO + ClusterSecretStore + d.info('Deploying cert-manager artifacts') + await hf( + { + fileOpts: 'helmfile.d/helmfile-07.init.yaml.gotmpl', + labelOpts: ['name=cert-manager-artifacts'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + if (!(env.isDev && env.DISABLE_SYNC)) { // Get the git configuration from values const values = (await hfValues()) as Record // Commit to Git repository await commit(true) + const gitBranch = values?.otomi?.git?.branch ?? 'main' await setGitConfig({ repoUrl: values?.otomi?.git?.repoUrl, - branch: values?.otomi?.git?.branch ?? 'main', + branch: gitBranch, email: values?.otomi?.git?.email, }) + // Verify the git push actually succeeded by checking the remote branch exists + d.info('Verifying git push succeeded') + const verifyResult = await $`git -C ${env.ENV_DIR} ls-remote --exit-code --heads origin ${gitBranch}` + .nothrow() + .quiet() + if (verifyResult.exitCode !== 0) { + throw new Error(`Git push verification failed: remote branch ${gitBranch} does not exist after commit`) + } + d.info('Git push verified successfully') + const initialData = await initialSetupData() await retryInstallStep(createCredentialsSecret, initialData.secretName, initialData.username, initialData.password) await retryInstallStep(createWelcomeConfigMap, initialData.secretName, initialData.domainSuffix) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index bc4e12d6d4..0ad054f1f0 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -21,7 +21,7 @@ import { parse } from 'yaml' import { Argv } from 'yargs' import { $, cd } from 'zx' import { ARGOCD_APP_PARAMS } from '../common/constants' -import { getSealedSecretsPEM, k8s } from '../common/k8s' +import { getK8sSecret, getSealedSecretsPEM, k8s } from '../common/k8s' const cmdName = getFilename(__filename) @@ -712,7 +712,12 @@ const setDefaultAplCatalog = async (values: Record): Promise let secretCreated = false if (useGiteaCatalog) { try { - await createCatalogSealedSecret(d, gitea as { adminUsername: string; adminPassword: string }) + const giteaSecrets = await getK8sSecret('gitea-secrets', 'sealed-secrets') + const resolvedGitea = { + adminUsername: giteaSecrets?.adminUsername ? String(giteaSecrets.adminUsername) : String(gitea!.adminUsername), + adminPassword: giteaSecrets?.adminPassword ? String(giteaSecrets.adminPassword) : String(gitea!.adminPassword), + } + await createCatalogSealedSecret(d, resolvedGitea) secretCreated = true } catch (error) { d.error('Failed to create catalog sealed secret, continuing without it:', error) diff --git a/src/cmd/validate-values.ts b/src/cmd/validate-values.ts index 63d604954b..47eecbe98b 100644 --- a/src/cmd/validate-values.ts +++ b/src/cmd/validate-values.ts @@ -1,5 +1,5 @@ import Ajv, { ValidateFunction } from 'ajv' -import { unset } from 'lodash' +import { cloneDeep, unset } from 'lodash' import { prepareEnvironment } from 'src/common/cli' import { terminal } from 'src/common/debug' import { env } from 'src/common/envalid' @@ -13,6 +13,53 @@ const cmdName = getFilename(__filename) const internalPaths: string[] = ['k8s', 'adminApps', 'teamApps'] +/** + * Remove x-secret properties from `required` arrays throughout the schema. + * Disk values have secrets stripped, so requiring them would always fail validation. + */ +export function removeSecretRequirements(schema: Record): Record { + const cleaned = cloneDeep(schema) + removeSecretRequirementsInPlace(cleaned) + return cleaned +} + +function removeSecretRequirementsInPlace(schema: Record): void { + if (!schema || typeof schema !== 'object') return + + // Collect property names that have x-secret in this node + const secretProps = new Set() + if (schema.properties) { + for (const [propName, propSchema] of Object.entries(schema.properties as Record)) { + if (propSchema && typeof propSchema === 'object' && 'x-secret' in propSchema) { + secretProps.add(propName) + } + } + } + + // Remove secret properties from required array + if (Array.isArray(schema.required) && secretProps.size > 0) { + const filtered = schema.required.filter((r: string) => !secretProps.has(r)) + if (filtered.length === 0) { + // eslint-disable-next-line no-param-reassign + delete schema.required + } else { + // eslint-disable-next-line no-param-reassign + schema.required = filtered + } + } + + // Recurse into all sub-schemas + for (const value of Object.values(schema)) { + if (Array.isArray(value)) { + for (const item of value) { + removeSecretRequirementsInPlace(item) + } + } else if (value && typeof value === 'object') { + removeSecretRequirementsInPlace(value) + } + } +} + // TODO: Accept json path to validate - on empty, validate all export const validateValues = async (argv: HelmArguments = getParsedArgs(), envDir = env.ENV_DIR): Promise => { const d = terminal(`cmd:${cmdName}:validateValues`) @@ -33,12 +80,14 @@ export const validateValues = async (argv: HelmArguments = getParsedArgs(), envD d.info('Loading values-schema.yaml') const valuesSchema = (await loadYaml(`${rootDir}/values-schema.yaml`)) as Record + // Disk values have secrets stripped — remove x-secret fields from required arrays + const adjustedSchema = removeSecretRequirements(valuesSchema) d.debug('Initializing Ajv') const ajv = new Ajv({ allErrors: true, strict: false, strictTypes: false, verbose: true }) d.debug('Compiling Ajv validation') let validate: ValidateFunction try { - validate = ajv.compile(valuesSchema) + validate = ajv.compile(adjustedSchema) } catch (error) { throw new Error(`Schema is invalid: ${chalk.italic(error.message)}`) } diff --git a/src/common/bootstrap.ts b/src/common/bootstrap.ts index 7fc12aa385..7f1ce9d1d7 100644 --- a/src/common/bootstrap.ts +++ b/src/common/bootstrap.ts @@ -1,10 +1,12 @@ import { existsSync } from 'fs' +import { get } from 'lodash' import { decrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { env, isCli } from 'src/common/envalid' -import { hfValues } from 'src/common/hf' -import { getFilename } from 'src/common/utils' import { getRepo } from 'src/common/git-config' +import { hfValues } from 'src/common/hf' +import { stripAllSecrets } from 'src/common/sealed-secrets' +import { getFilename, getSchemaSecretsPaths } from 'src/common/utils' import { writeValues } from 'src/common/values' import { $, cd } from 'zx' @@ -23,7 +25,7 @@ export const bootstrapGit = async (inValues?: Record): Promise) - const { authenticatedUrl: remote, branch, email, username, password } = getRepo(values) + const { authenticatedUrl: remote, branch, email, username, password } = await getRepo(values) cd(env.ENV_DIR) if (existsSync(`${env.ENV_DIR}/.git`)) { d.info(`Git repo was already bootstrapped, setting identity just in case`) @@ -62,9 +64,12 @@ export const bootstrapGit = async (inValues?: Record): Promise + // Strip ALL secrets before writing to disk — secrets are in SealedSecrets only + const secretPaths = await getSchemaSecretsPaths(Object.keys(get(defaultValues, 'teamConfig', {}))) + const strippedValues = stripAllSecrets(defaultValues, secretPaths) // finally write back the new values without overwriting existing values d.info('Write default values to env repo') - await writeValues(defaultValues) + await writeValues(strippedValues) } if (!existsSync(`${env.ENV_DIR}/.git`)) { diff --git a/src/common/git-config.test.ts b/src/common/git-config.test.ts index 6d2e23888f..4883b81d94 100644 --- a/src/common/git-config.test.ts +++ b/src/common/git-config.test.ts @@ -289,7 +289,7 @@ describe('git-config', () => { }) describe('getRepo', () => { - it('should return full config from values', () => { + it('should return full config from values', async () => { const values = { otomi: { git: { @@ -302,7 +302,7 @@ describe('git-config', () => { }, } - const result = getRepo(values) + const result = await getRepo(values) expect(result).toEqual({ repoUrl: 'https://github.com/org/repo.git', authenticatedUrl: 'https://admin:s3cret@github.com/org/repo.git', @@ -313,19 +313,19 @@ describe('git-config', () => { }) }) - it('should throw when repoUrl is missing', () => { - expect(() => getRepo({ otomi: { git: {} } })).toThrow('No otomi.git.repoUrl config was given.') + it('should throw when repoUrl is missing', async () => { + await expect(getRepo({ otomi: { git: {} } })).rejects.toThrow('No otomi.git.repoUrl config was given.') }) - it('should throw when otomi.git is missing', () => { - expect(() => getRepo({ otomi: {} })).toThrow('No otomi.git.repoUrl config was given.') + it('should throw when otomi.git is missing', async () => { + await expect(getRepo({ otomi: {} })).rejects.toThrow('No otomi.git.repoUrl config was given.') }) - it('should throw when values is empty', () => { - expect(() => getRepo({})).toThrow('No otomi.git.repoUrl config was given.') + it('should throw when values is empty', async () => { + await expect(getRepo({})).rejects.toThrow('No otomi.git.repoUrl config was given.') }) - it('should use GIT_REPO_URL env var in development mode', () => { + it('should use GIT_REPO_URL env var in development mode', async () => { process.env.NODE_ENV = 'development' process.env.GIT_REPO_URL = 'http://localhost:3000/dev/repo.git' @@ -341,9 +341,65 @@ describe('git-config', () => { }, } - const result = getRepo(values) + const result = await getRepo(values) expect(result.repoUrl).toBe('http://localhost:3000/dev/repo.git') expect(result.authenticatedUrl).toBe('http://admin:s3cret@localhost:3000/dev/repo.git') }) + + it('should fallback to K8s secret when password is a sealed placeholder', async () => { + const secretMock = jest.fn().mockResolvedValue({ git_password: 'real-password' }) + const values = { + otomi: { + git: { + repoUrl: 'https://github.com/org/repo.git', + username: 'admin', + password: 'sealed:sealed-secrets/otomi-platform-secrets/git_password', + branch: 'main', + email: 'pipeline@cluster.local', + }, + }, + } + + const result = await getRepo(values, { getK8sSecret: secretMock }) + expect(secretMock).toHaveBeenCalledWith('otomi-platform-secrets', 'sealed-secrets') + expect(result.password).toBe('real-password') + expect(result.authenticatedUrl).toContain('real-password') + }) + + it('should fallback to K8s secret when password is empty', async () => { + const secretMock = jest.fn().mockResolvedValue({ git_password: 'from-k8s' }) + const values = { + otomi: { + git: { + repoUrl: 'https://github.com/org/repo.git', + username: 'admin', + password: '', + branch: 'main', + email: 'pipeline@cluster.local', + }, + }, + } + + const result = await getRepo(values, { getK8sSecret: secretMock }) + expect(result.password).toBe('from-k8s') + }) + + it('should keep empty password when K8s secret also has no password', async () => { + const secretMock = jest.fn().mockResolvedValue(null) + const values = { + otomi: { + git: { + repoUrl: 'https://github.com/org/repo.git', + username: 'admin', + password: '', + branch: 'main', + email: 'pipeline@cluster.local', + }, + }, + } + + const result = await getRepo(values, { getK8sSecret: secretMock }) + expect(result.password).toBe('') + }) }) }) diff --git a/src/common/git-config.ts b/src/common/git-config.ts index b92d318b85..3c18e86333 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -1,6 +1,6 @@ +import type { CoreV1Api } from '@kubernetes/client-node' import { terminal } from './debug' import { createUpdateConfigMap, getK8sConfigMap, getK8sSecret, k8s } from './k8s' -import type { CoreV1Api } from '@kubernetes/client-node' const d = terminal('common:git-config') @@ -111,7 +111,7 @@ export async function getStoredGitRepoConfig(): Promise, coreV1Api?: C /** * Gets repository configuration from values, constructing the authenticated URL with embedded credentials. + * If password is missing or is an unresolved sealed-secret placeholder, falls back to reading + * the real password from the K8s secret (populated by ESO from SealedSecrets). */ -export const getRepo = (values: Record): GitRepoConfig => { +export const getRepo = async (values: Record, deps = { getK8sSecret }): Promise => { const otomiGit = values?.otomi?.git if (!otomiGit?.repoUrl) { @@ -142,15 +144,29 @@ export const getRepo = (values: Record): GitRepoConfig => { otomiGit.repoUrl = process.env.GIT_REPO_URL } const username = otomiGit?.username - const password = otomiGit?.password + let password = otomiGit?.password ?? '' const email = otomiGit?.email const branch = otomiGit?.branch + // If password is missing or is an unresolved sealed-secret placeholder, + // try reading the real password from the K8s secret (populated by ESO from SealedSecrets) + if (!password || (typeof password === 'string' && password.startsWith('sealed:'))) { + try { + const secret = await deps.getK8sSecret('otomi-platform-secrets', 'sealed-secrets') + if (secret?.git_password) { + password = String(secret.git_password) + d.debug('Read git password from K8s secret (ESO)') + } + } catch { + d.warn('Could not read git password from K8s secret, using value from config') + } + } + const repoUrl = otomiGit?.repoUrl as string const url = new URL(repoUrl) url.username = username url.password = password const authenticatedUrl = url.toString() - return { repoUrl, authenticatedUrl, branch, email, username, password } + return { repoUrl, authenticatedUrl, branch, email, username, password } as GitRepoConfig } diff --git a/src/common/repo.ts b/src/common/repo.ts index f37993ca7b..6257c89f60 100644 --- a/src/common/repo.ts +++ b/src/common/repo.ts @@ -571,7 +571,6 @@ export async function setValuesFile( deps = { pathExists: existsSync, loadValues, writeFile }, ): Promise { const valuesPath = path.join(envDir, 'values-repo.yaml') - // if (await deps.pathExists(valuesPath)) return valuesPath const allValues = await deps.loadValues(envDir) await deps.writeFile(valuesPath, objectToYaml(allValues)) return valuesPath diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts new file mode 100644 index 0000000000..0fa226e9b8 --- /dev/null +++ b/src/common/sealed-secrets.test.ts @@ -0,0 +1,575 @@ +import { pki } from 'node-forge' +import stubs from 'src/test-stubs' +import { + APP_NAMESPACE_MAP, + bootstrapSealedSecrets, + buildSecretToNamespaceMap, + createSealedSecretManifest, + createSealedSecretsKeySecret, + generateSealedSecretsKeyPair, + getPemFromCertificate, + SECRET_NAME_MAP, + stripAllSecrets, + writeSealedSecretManifests, +} from './sealed-secrets' + +const { terminal } = stubs + +jest.mock('@linode/kubeseal-encrypt', () => ({ + encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), +})) + +jest.mock('zx', () => ({ + $: jest.fn().mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ stderr: '', exitCode: 0 }), + }), + }), +})) + +jest.mock('src/common/envalid', () => ({ + env: {}, +})) + +describe('sealed-secrets', () => { + describe('generateSealedSecretsKeyPair', () => { + it('should generate a valid key pair with certificate and private key', () => { + const mockCert = { + publicKey: {}, + serialNumber: '', + validity: { notBefore: new Date(), notAfter: new Date() }, + sign: jest.fn(), + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + } + const mockKeys = { + publicKey: { n: {}, e: {} }, + privateKey: { d: {}, p: {}, q: {} }, + } + const deps = { + terminal, + pki: { + rsa: { generateKeyPair: jest.fn().mockReturnValue(mockKeys) }, + createCertificate: jest.fn().mockReturnValue(mockCert), + certificateToPem: jest.fn().mockReturnValue('-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n'), + privateKeyToPem: jest + .fn() + .mockReturnValue('-----BEGIN RSA PRIVATE KEY-----\nfake\n-----END RSA PRIVATE KEY-----\n'), + } as unknown as typeof pki, + } + + const result = generateSealedSecretsKeyPair(deps) + + expect(deps.pki.rsa.generateKeyPair).toHaveBeenCalledWith(4096) + expect(deps.pki.createCertificate).toHaveBeenCalled() + expect(mockCert.sign).toHaveBeenCalled() + expect(result.certificate).toContain('BEGIN CERTIFICATE') + expect(result.privateKey).toContain('BEGIN RSA PRIVATE KEY') + }) + + it('should set 10-year validity', () => { + const mockCert = { + publicKey: {}, + serialNumber: '', + validity: { notBefore: new Date(), notAfter: new Date() }, + sign: jest.fn(), + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + } + const mockKeys = { + publicKey: {}, + privateKey: {}, + } + const deps = { + terminal, + pki: { + rsa: { generateKeyPair: jest.fn().mockReturnValue(mockKeys) }, + createCertificate: jest.fn().mockReturnValue(mockCert), + certificateToPem: jest.fn().mockReturnValue('cert'), + privateKeyToPem: jest.fn().mockReturnValue('key'), + } as unknown as typeof pki, + } + + generateSealedSecretsKeyPair(deps) + + const notBefore = mockCert.validity.notBefore.getFullYear() + const notAfter = mockCert.validity.notAfter.getFullYear() + expect(notAfter - notBefore).toBe(10) + }) + }) + + describe('getPemFromCertificate', () => { + it('should extract SPKI public key from a certificate', () => { + // Generate a real key pair and certificate for this test + const keys = pki.rsa.generateKeyPair(2048) + const cert = pki.createCertificate() + cert.publicKey = keys.publicKey + cert.serialNumber = '01' + cert.validity.notBefore = new Date() + cert.validity.notAfter = new Date() + cert.validity.notAfter.setFullYear(cert.validity.notBefore.getFullYear() + 1) + const attrs = [{ name: 'commonName', value: 'test' }] + cert.setSubject(attrs) + cert.setIssuer(attrs) + cert.sign(keys.privateKey) + const certPem = pki.certificateToPem(cert) + + const result = getPemFromCertificate(certPem) + + expect(result).toContain('BEGIN PUBLIC KEY') + expect(result).toContain('END PUBLIC KEY') + }) + }) + + describe('createSealedSecretsKeySecret', () => { + it('should create secret if it does not exist', async () => { + const mockQuiet = jest.fn().mockResolvedValue({ stderr: '', exitCode: 0 }) + const mockNothrow = jest.fn().mockReturnValue({ quiet: mockQuiet }) + // First call (namespace): success, Second call (check exists): not found (exitCode 1) + // Third call (create): success, Fourth call (label): success + const mock$ = jest + .fn() + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // namespace + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 1 }) }), + }) // check exists - NOT FOUND + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // create + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // label + const deps = { + $: mock$ as any, + terminal, + writeFile: jest.fn(), + mkdir: jest.fn(), + } + + await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) + + expect(mock$).toHaveBeenCalledTimes(4) + expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.crt', 'cert-pem') + expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.key', 'key-pem') + }) + + it('should skip creation if secret already exists', async () => { + const mock$ = jest + .fn() + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // namespace + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // check exists - FOUND + const deps = { + $: mock$ as any, + terminal, + writeFile: jest.fn(), + mkdir: jest.fn(), + } + + await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) + + // Should only call namespace and check exists, not create or label + expect(mock$).toHaveBeenCalledTimes(2) + expect(deps.writeFile).not.toHaveBeenCalled() + }) + }) + + describe('buildSecretToNamespaceMap', () => { + it('should group secrets by namespace and secret name with leaf key naming', async () => { + const secrets = { + apps: { + harbor: { adminPassword: 'harbor-pass', secretKey: 'harbor-secret' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.adminPassword', 'apps.harbor.secretKey']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + // All secrets now go to sealed-secrets namespace + const harborMapping = result.find((m) => m.secretName === 'harbor-secrets') + expect(harborMapping).toBeDefined() + expect(harborMapping!.namespace).toBe('sealed-secrets') + expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') + expect(harborMapping!.data).toHaveProperty('secretKey', 'harbor-secret') + }) + + it('should skip kms.sops paths', async () => { + const secrets = { + kms: { sops: { provider: 'age', age: { publicKey: 'pk', privateKey: 'sk' } } }, + apps: { harbor: { adminPassword: 'pass' } }, + } + const deps = { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue(['kms.sops.provider', 'kms.sops.age.publicKey', 'apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].namespace).toBe('sealed-secrets') + }) + + it('should serialize users array as single JSON value in users-secrets', async () => { + const secrets = { + users: [ + { + email: 'admin@example.com', + firstName: 'Admin', + lastName: 'User', + initialPassword: 'pass', + groups: ['platform-admin'], + }, + ], + apps: { harbor: { adminPassword: 'pass' } }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['users', 'apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(2) + const usersMapping = result.find((m) => m.secretName === 'users-secrets') + expect(usersMapping).toBeDefined() + expect(usersMapping!.namespace).toBe('sealed-secrets') + expect(usersMapping!.data.usersJson).toBe(JSON.stringify(secrets.users)) + }) + + it('should handle teamConfig dynamic paths in sealed-secrets namespace', async () => { + const secrets = { + teamConfig: { + 'team-alpha': { someSecret: 'value' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['teamConfig.team-alpha.someSecret']), + } + + const result = await buildSecretToNamespaceMap(secrets, ['team-alpha'], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].namespace).toBe('sealed-secrets') + expect(result[0].secretName).toBe('team-team-alpha-settings-secrets') + }) + + it('should filter out mappings with no data', async () => { + const secrets = {} + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(0) + }) + + it('should use leaf key naming for nested secret paths', async () => { + const secrets = { + apps: { + harbor: { core: { secret: 'core-secret-val' } }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.core.secret']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].data).toHaveProperty('core_secret', 'core-secret-val') + }) + + it('should put gitea secrets in sealed-secrets namespace using convention naming', async () => { + const secrets = { + apps: { + gitea: { adminPassword: 'gitea-pass', postgresqlPassword: 'pg-pass' }, + harbor: { adminPassword: 'harbor-pass' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue([ + 'apps.gitea.adminPassword', + 'apps.gitea.postgresqlPassword', + 'apps.harbor.adminPassword', + ]), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + // Harbor should use convention naming in sealed-secrets ns + const harborMapping = result.find((m) => m.secretName === 'harbor-secrets') + expect(harborMapping).toBeDefined() + expect(harborMapping!.namespace).toBe('sealed-secrets') + expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') + + // Gitea should have a gitea-secrets mapping in sealed-secrets ns + const giteaMapping = result.find((m) => m.secretName === 'gitea-secrets') + expect(giteaMapping).toBeDefined() + expect(giteaMapping!.namespace).toBe('sealed-secrets') + expect(giteaMapping!.data).toHaveProperty('adminPassword', 'gitea-pass') + expect(giteaMapping!.data).toHaveProperty('postgresqlPassword', 'pg-pass') + }) + }) + + describe('createSealedSecretManifest', () => { + it('should produce correct SealedSecret structure', async () => { + const mapping = { + namespace: 'sealed-secrets', + secretName: 'harbor-secrets', + data: { adminPassword: 'my-password', secretKey: 'my-secret' }, + } + const deps = { + encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), + } + + const result = await createSealedSecretManifest('mock-pem', mapping, deps) + + expect(result.apiVersion).toBe('bitnami.com/v1alpha1') + expect(result.kind).toBe('SealedSecret') + expect(result.metadata.name).toBe('harbor-secrets') + expect(result.metadata.namespace).toBe('sealed-secrets') + expect(result.metadata.annotations['sealedsecrets.bitnami.com/namespace-wide']).toBe('true') + expect(result.spec.encryptedData.adminPassword).toBe('encrypted-value') + expect(result.spec.encryptedData.secretKey).toBe('encrypted-value') + expect(result.spec.template.type).toBe('Opaque') + expect(result.spec.template.metadata.name).toBe('harbor-secrets') + expect(result.spec.template.metadata.namespace).toBe('sealed-secrets') + }) + + it('should call encryptSecretItem for each data key', async () => { + const mapping = { + namespace: 'sealed-secrets', + secretName: 'gitea-secrets', + data: { key1: 'val1', key2: 'val2', key3: 'val3' }, + } + const deps = { + encryptSecretItem: jest.fn().mockResolvedValue('enc'), + } + + await createSealedSecretManifest('pem', mapping, deps) + + expect(deps.encryptSecretItem).toHaveBeenCalledTimes(3) + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'sealed-secrets', 'val1') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'sealed-secrets', 'val2') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'sealed-secrets', 'val3') + }) + }) + + describe('writeSealedSecretManifests', () => { + it('should write each manifest to the correct directory', async () => { + const manifests = [ + { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: 'harbor-secrets', + namespace: 'sealed-secrets', + }, + spec: { + encryptedData: { key: 'enc' }, + template: { + immutable: false, + metadata: { name: 'harbor-secrets', namespace: 'sealed-secrets' }, + type: 'Opaque', + }, + }, + }, + ] + const deps = { + mkdir: jest.fn(), + writeFile: jest.fn(), + objectToYaml: jest.fn().mockReturnValue('yaml-content'), + terminal, + } + + await writeSealedSecretManifests(manifests, '/test', deps) + + expect(deps.mkdir).toHaveBeenCalledWith('/test/env/manifests/ns/sealed-secrets', { recursive: true }) + expect(deps.writeFile).toHaveBeenCalledWith( + '/test/env/manifests/ns/sealed-secrets/harbor-secrets.yaml', + 'yaml-content', + ) + }) + }) + + describe('bootstrapSealedSecrets', () => { + it('should generate new key pair when no existing cert found', async () => { + const secrets = { + apps: { harbor: { adminPassword: 'pass' } }, + } + const mockMapping = { + namespace: 'sealed-secrets', + secretName: 'harbor-secrets', + data: { adminPassword: 'pass' }, + } + const mockManifest = { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: 'harbor-secrets', + namespace: 'sealed-secrets', + }, + spec: { + encryptedData: { adminPassword: 'encrypted' }, + template: { + immutable: false, + metadata: { name: 'harbor-secrets', namespace: 'sealed-secrets' }, + type: 'Opaque', + }, + }, + } + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), // No existing cert + generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ + certificate: 'cert-pem', + privateKey: 'key-pem', + }), + getPemFromCertificate: jest.fn().mockReturnValue('spki-pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), + createSealedSecretManifest: jest.fn().mockResolvedValue(mockManifest), + writeSealedSecretManifests: jest.fn(), + encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), + } + + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + + expect(deps.getExistingSealedSecretsCert).toHaveBeenCalled() + expect(deps.generateSealedSecretsKeyPair).toHaveBeenCalled() + expect(deps.createSealedSecretsKeySecret).toHaveBeenCalledWith('cert-pem', 'key-pem') + expect(deps.getPemFromCertificate).toHaveBeenCalledWith('cert-pem') + expect(deps.writeSealedSecretManifests).toHaveBeenCalledWith([mockManifest], '/test') + }) + + it('should use existing cert when found', async () => { + const secrets = { + apps: { harbor: { adminPassword: 'pass' } }, + } + const mockMapping = { + namespace: 'sealed-secrets', + secretName: 'harbor-secrets', + data: { adminPassword: 'pass' }, + } + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue('existing-cert-pem'), // Existing cert found + generateSealedSecretsKeyPair: jest.fn(), + getPemFromCertificate: jest.fn().mockReturnValue('existing-spki-pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), + createSealedSecretManifest: jest.fn().mockResolvedValue({}), + writeSealedSecretManifests: jest.fn(), + encryptSecretItem: jest.fn(), + } + + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + + expect(deps.getExistingSealedSecretsCert).toHaveBeenCalled() + expect(deps.generateSealedSecretsKeyPair).not.toHaveBeenCalled() // Should NOT generate new key + expect(deps.createSealedSecretsKeySecret).not.toHaveBeenCalled() // Should NOT create secret + expect(deps.getPemFromCertificate).toHaveBeenCalledWith('existing-cert-pem') + }) + + it('should extract team names from secrets', async () => { + const secrets = { + teamConfig: { + alpha: { secret: 'val' }, + beta: { secret: 'val' }, + }, + } + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), + generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ + certificate: 'cert', + privateKey: 'key', + }), + getPemFromCertificate: jest.fn().mockReturnValue('pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), + createSealedSecretManifest: jest.fn(), + writeSealedSecretManifests: jest.fn(), + encryptSecretItem: jest.fn(), + } + + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + + expect(deps.buildSecretToNamespaceMap).toHaveBeenCalledWith(secrets, ['alpha', 'beta'], undefined) + }) + }) + + describe('APP_NAMESPACE_MAP', () => { + it('should have expected mappings', () => { + expect(APP_NAMESPACE_MAP['apps.harbor']).toBe('harbor') + expect(APP_NAMESPACE_MAP['apps.gitea']).toBe('gitea') + expect(APP_NAMESPACE_MAP['apps.oauth2-proxy']).toBe('istio-system') + expect(APP_NAMESPACE_MAP['apps.loki']).toBe('monitoring') + expect(APP_NAMESPACE_MAP['otomi']).toBe('otomi') + expect(APP_NAMESPACE_MAP['dns']).toBe('external-dns') + expect(APP_NAMESPACE_MAP['cluster']).toBe('cert-manager') + }) + }) + + describe('SECRET_NAME_MAP', () => { + it('should have expected secret name mappings', () => { + expect(SECRET_NAME_MAP['apps.harbor']).toBe('harbor-secrets') + expect(SECRET_NAME_MAP['apps.gitea']).toBe('gitea-secrets') + expect(SECRET_NAME_MAP['apps.keycloak']).toBe('keycloak-secrets') + expect(SECRET_NAME_MAP['otomi']).toBe('otomi-platform-secrets') + expect(SECRET_NAME_MAP['oidc']).toBe('oidc-secrets') + expect(SECRET_NAME_MAP['dns']).toBe('dns-secrets') + }) + }) + + describe('stripAllSecrets', () => { + it('should remove all secret paths from values', () => { + const values = { + apps: { + gitea: { adminPassword: 'secret', postgresqlPassword: 'pg-secret', resources: { cpu: '100m' } }, + }, + oidc: { clientID: 'otomi', clientSecret: 'my-secret', issuer: 'https://example.com' }, + } + const secretPaths = ['apps.gitea.adminPassword', 'apps.gitea.postgresqlPassword', 'oidc.clientSecret'] + + const result = stripAllSecrets(values, secretPaths) + + // Secret values should be removed + expect(result.apps.gitea.adminPassword).toBeUndefined() + expect(result.apps.gitea.postgresqlPassword).toBeUndefined() + expect(result.oidc.clientSecret).toBeUndefined() + // Non-secret values should be preserved + expect(result.apps.gitea.resources).toEqual({ cpu: '100m' }) + expect(result.oidc.clientID).toBe('otomi') + expect(result.oidc.issuer).toBe('https://example.com') + }) + + it('should not modify the original values object', () => { + const values = { + apps: { gitea: { adminPassword: 'secret' } }, + } + const secretPaths = ['apps.gitea.adminPassword'] + + stripAllSecrets(values, secretPaths) + + expect(values.apps.gitea.adminPassword).toBe('secret') + }) + }) +}) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts new file mode 100644 index 0000000000..4c364a4946 --- /dev/null +++ b/src/common/sealed-secrets.ts @@ -0,0 +1,628 @@ +import { encryptSecretItem } from '@linode/kubeseal-encrypt' +import { X509Certificate } from 'crypto' +import { existsSync } from 'fs' +import { mkdir, readdir, readFile, writeFile } from 'fs/promises' +import { cloneDeep, get, unset } from 'lodash' +import { pki } from 'node-forge' +import { join } from 'path' +import { terminal } from 'src/common/debug' +import { flattenObject, getSchemaSecretsPaths } from 'src/common/utils' +import { objectToYaml } from 'src/common/values' +import { $ } from 'zx' + +const cmdName = 'sealed-secrets' + +/** + * Strip ALL x-secret fields from values before writing to disk. + * Secrets are stored exclusively in SealedSecrets and delivered to apps via ExternalSecrets. + * The values repo contains zero secret values. + */ +export function stripAllSecrets(values: Record, secretPaths: string[]): Record { + const stripped = cloneDeep(values) + for (const secretPath of secretPaths) { + unset(stripped, secretPath) + } + return stripped +} + +/** + * Ensure a namespace exists. If it doesn't exist, create it with proper labels. + * This avoids overwriting labels on existing namespaces that were created by k8s-raw.gotmpl. + */ +export const ensureNamespaceExists = async (namespace: string, deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:ensureNamespaceExists`) + + // Check if namespace already exists + const existingNs = await deps.$`kubectl get namespace ${namespace}`.nothrow().quiet() + if (existingNs.exitCode === 0) { + d.debug(`Namespace ${namespace} already exists`) + return + } + + // Create namespace with proper label + d.info(`Creating namespace ${namespace}`) + const nsYaml = `apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} + labels: + name: ${namespace}` + + await deps.$`echo ${nsYaml} | kubectl apply -f -`.nothrow().quiet() +} + +export interface SecretMapping { + namespace: string + secretName: string + data: Record +} + +export interface SealedSecretManifest { + apiVersion: string + kind: string + metadata: { + annotations: Record + name: string + namespace: string + } + spec: { + encryptedData: Record + template: { + immutable: boolean + metadata: { name: string; namespace: string } + type: string + } + } +} + +/** + * Mapping from secret path prefix to target Kubernetes namespace. + * Dynamic entries like `teamConfig.{teamId}` are handled separately. + */ +export const APP_NAMESPACE_MAP: Record = { + 'apps.harbor': 'harbor', + 'apps.gitea': 'gitea', + 'apps.keycloak': 'keycloak', + 'apps.grafana': 'grafana', + 'apps.loki': 'monitoring', + 'apps.oauth2-proxy': 'istio-system', + 'apps.oauth2-proxy-redis': 'istio-system', + 'apps.prometheus': 'monitoring', + 'apps.otomi-api': 'otomi', + 'apps.cert-manager': 'cert-manager', + 'apps.kubeflow-pipelines': 'kfp', + otomi: 'otomi', + oidc: 'otomi', + smtp: 'otomi', + dns: 'external-dns', + obj: 'otomi', + license: 'otomi', + users: 'keycloak', + alerts: 'monitoring', + cluster: 'cert-manager', +} + +/** + * Generate an RSA 4096-bit key pair and self-signed X.509 certificate for Sealed Secrets. + * Follows the pattern from createCustomCA() in bootstrap.ts. + */ +export const generateSealedSecretsKeyPair = (deps = { terminal, pki }): { certificate: string; privateKey: string } => { + const d = deps.terminal(`common:${cmdName}:generateSealedSecretsKeyPair`) + d.info('Generating sealed-secrets RSA key pair') + + const keys = deps.pki.rsa.generateKeyPair(4096) + const cert = deps.pki.createCertificate() + cert.publicKey = keys.publicKey + cert.serialNumber = '01' + cert.validity.notBefore = new Date() + cert.validity.notAfter = new Date() + cert.validity.notAfter.setFullYear(cert.validity.notBefore.getFullYear() + 10) + + const attrs = [ + { name: 'countryName', value: 'NL' }, + { shortName: 'ST', value: 'Utrecht' }, + { name: 'localityName', value: 'Utrecht' }, + { name: 'organizationName', value: 'APL' }, + { shortName: 'OU', value: 'SealedSecrets' }, + ] + cert.setSubject(attrs) + cert.setIssuer(attrs) + cert.setExtensions([ + { + name: 'basicConstraints', + cA: true, + }, + { + name: 'keyUsage', + keyCertSign: true, + digitalSignature: true, + keyEncipherment: true, + }, + ]) + cert.sign(keys.privateKey) + + const certificate = deps.pki.certificateToPem(cert).replaceAll('\r\n', '\n') + const privateKey = deps.pki.privateKeyToPem(keys.privateKey).replaceAll('\r\n', '\n') + + d.info('Generated sealed-secrets key pair') + return { certificate, privateKey } +} + +/** + * Extract SPKI PEM public key from a PEM-encoded X.509 certificate. + * Uses Node.js crypto.X509Certificate (same approach as getSealedSecretsPEM() in k8s.ts). + */ +export const getPemFromCertificate = (certificate: string): string => { + const x509 = new X509Certificate(certificate) + const exported = x509.publicKey.export({ format: 'pem', type: 'spki' }) + return typeof exported === 'string' ? exported : exported.toString('utf-8') +} + +/** + * Get the existing sealed-secrets certificate from the cluster if it exists. + * Returns the certificate PEM string or undefined if not found. + */ +export const getExistingSealedSecretsCert = async (deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:getExistingSealedSecretsCert`) + + const result = + await deps.$`kubectl get secret sealed-secrets-key -n sealed-secrets -o jsonpath='{.data.tls\\.crt}' 2>/dev/null` + .nothrow() + .quiet() + + if (result.exitCode !== 0 || !result.stdout || result.stdout === '') { + d.info('No existing sealed-secrets-key found') + return undefined + } + + try { + const certBase64 = result.stdout.replace(/'/g, '') + const cert = Buffer.from(certBase64, 'base64').toString('utf-8') + d.info('Found existing sealed-secrets-key certificate') + return cert + } catch { + d.warn('Failed to decode existing certificate') + return undefined + } +} + +/** + * Create the sealed-secrets namespace and TLS secret in Kubernetes. + * The controller will pick up this pre-created key on startup. + * IMPORTANT: This only creates the secret if it doesn't already exist. + */ +export const createSealedSecretsKeySecret = async ( + certificate: string, + privateKey: string, + deps = { $, terminal, writeFile, mkdir }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:createSealedSecretsKeySecret`) + + // Create namespace if it doesn't exist + await ensureNamespaceExists('sealed-secrets', { $: deps.$, terminal: deps.terminal }) + + // Check if secret already exists + const existingSecret = await deps.$`kubectl get secret sealed-secrets-key -n sealed-secrets`.nothrow().quiet() + if (existingSecret.exitCode === 0) { + d.info('sealed-secrets-key already exists, skipping creation') + return + } + + d.info('Creating sealed-secrets TLS secret') + + // Write temp files for kubectl create secret tls + const tmpDir = '/tmp/sealed-secrets-bootstrap' + await deps.mkdir(tmpDir, { recursive: true }) + const certPath = `${tmpDir}/tls.crt` + const keyPath = `${tmpDir}/tls.key` + await deps.writeFile(certPath, certificate) + await deps.writeFile(keyPath, privateKey) + + // Create the TLS secret (only if it doesn't exist) + const result = + await deps.$`kubectl create secret tls sealed-secrets-key -n sealed-secrets --cert=${certPath} --key=${keyPath}` + .nothrow() + .quiet() + if (result.exitCode !== 0) { + d.error(`Failed to create sealed-secrets-key: ${result.stderr}`) + return + } + + // Label the secret so the controller picks it up + const labelResult = + await deps.$`kubectl label secret sealed-secrets-key -n sealed-secrets sealedsecrets.bitnami.com/sealed-secrets-key=active --overwrite` + .nothrow() + .quiet() + if (labelResult.stderr) d.error(labelResult.stderr) + + d.info('Created sealed-secrets TLS secret with key label') +} + +/** + * Resolve the namespace for a given secret path. + * All core secrets go to 'sealed-secrets' namespace for ESO access. + * APP_NAMESPACE_MAP is kept for reference but not used for SealedSecret placement. + */ +const resolveNamespace = (secretPath: string): string | undefined => { + // Check for teamConfig dynamic paths + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) { + return 'sealed-secrets' + } + + // Check if this path matches any known prefix + const sortedKeys = Object.keys(APP_NAMESPACE_MAP).sort((a, b) => b.length - a.length) + for (const prefix of sortedKeys) { + if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { + return 'sealed-secrets' + } + } + + return undefined +} + +// Map specific path prefixes to secret names +export const SECRET_NAME_MAP: Record = { + 'apps.harbor': 'harbor-secrets', + 'apps.gitea': 'gitea-secrets', + 'apps.keycloak': 'keycloak-secrets', + 'apps.grafana': 'grafana-secrets', + 'apps.loki': 'loki-secrets', + 'apps.oauth2-proxy': 'oauth2-proxy-secrets', + 'apps.oauth2-proxy-redis': 'oauth2-proxy-redis-secrets', + 'apps.prometheus': 'prometheus-secrets', + 'apps.otomi-api': 'otomi-api-secrets', + 'apps.cert-manager': 'cert-manager-secrets', + 'apps.kubeflow-pipelines': 'kubeflow-pipelines-secrets', + otomi: 'otomi-platform-secrets', + oidc: 'oidc-secrets', + smtp: 'smtp-secrets', + dns: 'dns-secrets', + obj: 'obj-storage-secrets', + license: 'license-secrets', + users: 'users-secrets', + alerts: 'alerts-secrets', + cluster: 'cluster-secrets', +} + +/** + * Find the group prefix for a secret path. + * Returns the prefix that maps to the secret name (e.g., 'apps.harbor' for 'apps.harbor.adminPassword'). + */ +const findGroupPrefix = (secretPath: string): string | undefined => { + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) { + return `teamConfig.${teamMatch[1]}` + } + + const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) + for (const prefix of sortedKeys) { + if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { + return prefix + } + } + + // Fallback: use first two path segments + const parts = secretPath.split('.') + if (parts.length >= 2) { + return parts.slice(0, 2).join('.') + } + return undefined +} + +/** + * Derive a K8s secret name from the secret path prefix. + */ +const deriveSecretName = (secretPath: string): string => { + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) { + return `team-${teamMatch[1]}-settings-secrets` + } + + const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) + for (const prefix of sortedKeys) { + if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { + return SECRET_NAME_MAP[prefix] + } + } + + // Fallback: derive from first two path segments + const parts = secretPath.split('.') + return `${parts.slice(0, 2).join('-')}-secrets` +} + +/** + * Build a mapping from secrets to their target namespaces and K8s secret names. + * Groups secret paths by namespace and secret name. + */ +export const buildSecretToNamespaceMap = async ( + secrets: Record, + teams: string[], + allValues?: Record, + deps = { getSchemaSecretsPaths }, +): Promise => { + const secretPaths = await deps.getSchemaSecretsPaths(teams) + const flat = flattenObject(secrets) + const allFlat = allValues ? flattenObject(allValues) : flat + + // Group by namespace + secretName + const groupMap = new Map() + + for (const secretPath of secretPaths) { + // Skip SOPS-related paths + if (secretPath.startsWith('kms.sops')) continue + // Handle 'users' path specially — serialize pre-processed users array as single JSON value + if (secretPath === 'users') { + const usersData = secrets.users + if (Array.isArray(usersData) && usersData.length > 0) { + const namespace = 'sealed-secrets' + const secretName = 'users-secrets' + const groupKey = `${namespace}/${secretName}` + if (!groupMap.has(groupKey)) { + groupMap.set(groupKey, { namespace, secretName, data: {} }) + } + const mapping = groupMap.get(groupKey)! + mapping.data.usersJson = JSON.stringify(usersData) + } + continue + } + + const namespace = resolveNamespace(secretPath) + if (!namespace) continue + + const secretName = deriveSecretName(secretPath) + const groupKey = `${namespace}/${secretName}` + + if (!groupMap.has(groupKey)) { + groupMap.set(groupKey, { namespace, secretName, data: {} }) + } + + const mapping = groupMap.get(groupKey)! + + // Find the group prefix (e.g., 'apps.harbor' for 'apps.harbor.adminPassword') + const groupPrefix = findGroupPrefix(secretPath) + + // Find all flat keys that match this secret path + for (const [flatKey, value] of Object.entries(flat)) { + if (flatKey === secretPath || flatKey.startsWith(`${secretPath}.`)) { + // Use leaf key: strip the group prefix to get relative path + const relativePath = + groupPrefix && (flatKey === groupPrefix || flatKey.startsWith(`${groupPrefix}.`)) + ? flatKey.slice(groupPrefix.length + 1) + : flatKey + const dataKey = relativePath.replace(/\./g, '_') + if (value !== undefined && value !== null && value !== '') { + mapping.data[dataKey] = String(value) + } + } + } + } + + // Filter out empty mappings + return Array.from(groupMap.values()).filter((m) => Object.keys(m.data).length > 0) +} + +/** + * Create a SealedSecret manifest by encrypting each data value. + * Follows the pattern from createCatalogSealedSecret() in migrate.ts. + */ +export const createSealedSecretManifest = async ( + pem: string, + mapping: SecretMapping, + deps = { encryptSecretItem }, +): Promise => { + const encryptedData: Record = {} + for (const [key, value] of Object.entries(mapping.data)) { + encryptedData[key] = await deps.encryptSecretItem(pem, mapping.namespace, value) + } + + return { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: mapping.secretName, + namespace: mapping.namespace, + }, + spec: { + encryptedData, + template: { + immutable: false, + metadata: { name: mapping.secretName, namespace: mapping.namespace }, + type: 'Opaque', + }, + }, + } +} + +/** + * Write SealedSecret manifests to the env/manifests/ns directory. + */ +export const writeSealedSecretManifests = async ( + manifests: SealedSecretManifest[], + envDir: string, + deps = { mkdir, writeFile, objectToYaml, terminal }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:writeSealedSecretManifests`) + + for (const manifest of manifests) { + // /env/manifests/ns/argocd/ + const dir = `${envDir}/env/manifests/ns/${manifest.metadata.namespace}` + await deps.mkdir(dir, { recursive: true }) + const filePath = `${dir}/${manifest.metadata.name}.yaml` + d.info(`Writing sealed secret to ${filePath}`) + await deps.writeFile(filePath, deps.objectToYaml(manifest)) + } +} + +/** + * Apply SealedSecret manifests to the Kubernetes cluster. + * Creates namespaces if needed and applies the SealedSecret resources. + */ +export const applySealedSecretManifests = async ( + manifests: SealedSecretManifest[], + deps = { $, terminal, objectToYaml }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:applySealedSecretManifests`) + + // Group manifests by namespace + const byNamespace = new Map() + for (const manifest of manifests) { + const ns = manifest.metadata.namespace + if (!byNamespace.has(ns)) { + byNamespace.set(ns, []) + } + byNamespace.get(ns)!.push(manifest) + } + + // Ensure namespaces exist and apply manifests + for (const [namespace, nsManifests] of byNamespace) { + await ensureNamespaceExists(namespace, { $: deps.$, terminal: deps.terminal }) + + for (const manifest of nsManifests) { + d.info(`Applying SealedSecret ${manifest.metadata.name} to namespace ${namespace}`) + const yaml = deps.objectToYaml(manifest) + const result = await deps.$`echo ${yaml} | kubectl apply -f -`.nothrow().quiet() + if (result.exitCode !== 0) { + d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${result.stderr}`) + } + } + } + + d.info(`Applied ${manifests.length} SealedSecret manifests to cluster`) +} + +/** + * Read and apply all SealedSecret manifests from the env/manifests/ns directory. + * This should be called during install, after the sealed-secrets controller is deployed. + */ +export const applySealedSecretManifestsFromDir = async ( + envDir: string, + deps = { $, terminal, readdir, readFile, existsSync }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) + const manifestsDir = join(envDir, 'env/manifests/ns') + + if (!deps.existsSync(manifestsDir)) { + d.info(`No SealedSecret manifests directory found at ${manifestsDir}`) + return + } + + d.info(`Applying SealedSecret manifests from ${manifestsDir}`) + + // Read all namespace directories + const namespaces = await deps.readdir(manifestsDir, { withFileTypes: true }) + let appliedCount = 0 + + for (const nsEntry of namespaces) { + if (!nsEntry.isDirectory()) continue + const namespace = nsEntry.name + const nsDir = join(manifestsDir, namespace) + + // Ensure namespace exists with proper labels + await ensureNamespaceExists(namespace, { $: deps.$, terminal: deps.terminal }) + + // Read all YAML files in the namespace directory + const files = await deps.readdir(nsDir) + for (const file of files) { + if (!file.endsWith('.yaml') && !file.endsWith('.yml')) continue + const filePath = join(nsDir, file) + d.info(`Applying SealedSecret from ${filePath}`) + + const result = await deps.$`kubectl apply -f ${filePath}`.nothrow().quiet() + if (result.exitCode !== 0) { + d.error(`Failed to apply SealedSecret from ${filePath}: ${result.stderr}`) + } else { + appliedCount += 1 + } + } + } + + d.info(`Applied ${appliedCount} SealedSecret manifests from directory`) +} + +/** + * Restart the sealed-secrets controller to ensure it uses the correct key. + * This is needed because if the controller starts before the sealed-secrets-key secret exists, + * it will generate its own key. Restarting forces it to pick up the existing key. + */ +export const restartSealedSecretsController = async (deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:restartSealedSecretsController`) + d.info('Restarting sealed-secrets controller to ensure correct key is used') + + const result = await deps.$`kubectl rollout restart deployment/sealed-secrets -n sealed-secrets`.nothrow().quiet() + if (result.exitCode !== 0) { + d.warn(`Failed to restart sealed-secrets controller: ${result.stderr}`) + return + } + + d.info('Waiting for sealed-secrets controller rollout') + const waitResult = await deps.$`kubectl rollout status deployment/sealed-secrets -n sealed-secrets --timeout=120s` + .nothrow() + .quiet() + if (waitResult.exitCode !== 0) { + d.warn(`Rollout status check failed: ${waitResult.stderr}`) + } else { + d.info('Sealed-secrets controller restarted successfully') + } +} + +/** + * Orchestrator: bootstrap sealed secrets for the platform. + * Replaces bootstrapSops(). + */ +export const bootstrapSealedSecrets = async ( + secrets: Record, + envDir: string, + allValues?: Record, + deps = { + terminal, + generateSealedSecretsKeyPair, + getPemFromCertificate, + createSealedSecretsKeySecret, + getExistingSealedSecretsCert, + buildSecretToNamespaceMap, + createSealedSecretManifest, + writeSealedSecretManifests, + encryptSecretItem, + }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:bootstrapSealedSecrets`) + d.info('Bootstrapping sealed secrets') + + // 1. Check if there's an existing sealed-secrets key in the cluster + const existingCert = await deps.getExistingSealedSecretsCert() + + let pem: string + if (existingCert) { + // Use existing certificate for encryption + d.info('Using existing sealed-secrets certificate') + pem = deps.getPemFromCertificate(existingCert) + } else { + // Generate new key pair and create the secret + d.info('Generating new sealed-secrets key pair') + const { certificate, privateKey } = deps.generateSealedSecretsKeyPair() + await deps.createSealedSecretsKeySecret(certificate, privateKey) + pem = deps.getPemFromCertificate(certificate) + } + + // 5. Build secret-to-namespace mapping + const teams = Object.keys(get(secrets, 'teamConfig', {}) as Record) + const mappings = await deps.buildSecretToNamespaceMap(secrets, teams, allValues) + + // 6. Create SealedSecret manifests + const manifests: SealedSecretManifest[] = [] + for (const mapping of mappings) { + const manifest = await deps.createSealedSecretManifest(pem, mapping, { + encryptSecretItem: deps.encryptSecretItem, + }) + manifests.push(manifest) + } + + // 7. Write SealedSecret manifests to disk + // Note: These manifests are applied later during install, after the sealed-secrets + // controller is deployed and the SealedSecret CRD is available. + await deps.writeSealedSecretManifests(manifests, envDir) + + d.info(`Bootstrapped ${manifests.length} sealed secret manifests`) +} diff --git a/src/common/values.ts b/src/common/values.ts index 157ee2fc2c..de44be2dbf 100644 --- a/src/common/values.ts +++ b/src/common/values.ts @@ -1,6 +1,6 @@ import { existsSync } from 'fs' import { mkdir, unlink, writeFile } from 'fs/promises' -import { cloneDeep, get, isEmpty, isEqual, merge, mergeWith, omit, pick, set } from 'lodash' +import { cloneDeep, isEmpty, isEqual, merge, mergeWith, pick, set } from 'lodash' import path from 'path' import { supportedK8sVersions } from 'src/supportedK8sVersions.json' import { stringify } from 'yaml' @@ -9,18 +9,8 @@ import { decrypt, encrypt } from './crypt' import { terminal } from './debug' import { env } from './envalid' import { hfValues } from './hf' -import { - extract, - flattenObject, - getSchemaSecretsPaths, - getValuesSchema, - gucci, - loadYaml, - pkg, - removeBlankAttributes, -} from './utils' - import { saveValues } from './repo' +import { extract, flattenObject, getValuesSchema, gucci, loadYaml, pkg, removeBlankAttributes } from './utils' import { HelmArguments } from './yargs' export const objectToYaml = (obj: Record, indent = 4, lineWidth = 200): string => { @@ -114,22 +104,13 @@ export const writeValuesToFile = async ( /** * Writes new values to the repo. Will keep the original values if `overwrite` is `false`. + * Secret values are written as-is — they are protected by SealedSecrets on the cluster side, + * and child secrets are derived via ESO ExternalSecret CRs. */ export const writeValues = async (inValues: Record, overwrite = false): Promise => { const d = terminal('common:values:writeValues') d.debug('Writing values: ', inValues) - hasSops = existsSync(`${env.ENV_DIR}/.sops.yaml`) - const values = inValues - const teams = Object.keys(get(inValues, 'teamConfig', {})) - const cleanSecretPaths = await getSchemaSecretsPaths(teams) - d.debug('cleanSecretPaths: ', cleanSecretPaths) - // separate out the secrets - const secrets = removeBlankAttributes(pick(values, cleanSecretPaths)) - d.debug('secrets: ', JSON.stringify(secrets, null, 2)) - // from the plain values - const plainValues = omit(values, cleanSecretPaths) as any - await saveValues(env.ENV_DIR, plainValues, secrets) - + await saveValues(env.ENV_DIR, inValues, {}) d.info('All values were written to ENV_DIR') } diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index 55809ea5d5..02cb411730 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -4,6 +4,31 @@ import * as k8s from '../common/k8s' import { AplOperations } from './apl-operations' import { Installer } from './installer' +const mockZx = jest.fn().mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }), + }), +}) + +jest.mock('zx', () => ({ + $: (...args: any[]) => mockZx(...args), +})) + +jest.mock('../common/envalid', () => ({ + env: { + GIT_PROTOCOL: 'http', + GIT_URL: 'gitea-http.gitea.svc.cluster.local', + GIT_PORT: '3000', + }, +})) + +jest.mock('./validators', () => ({ + operatorEnv: { + GIT_ORG: 'otomi', + GIT_REPO: 'values', + }, +})) + jest.mock('../common/debug', () => ({ terminal: jest.fn().mockImplementation(() => ({ info: jest.fn(), @@ -236,10 +261,18 @@ describe('Installer', () => { }) describe('isInstalled', () => { - test('should return completed status when ConfigMap exists', async () => { + test('should return completed status when ConfigMap exists and git repo has main branch', async () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ data: { status: 'completed' }, }) + // getK8sSecret returns credentials for git verification + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ GIT_USERNAME: 'admin', GIT_PASSWORD: 'pass' }) + // git ls-remote succeeds (main branch exists) + mockZx.mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 0 }), + }), + }) const isInstalled = await installer.isInstalled() @@ -248,6 +281,23 @@ describe('Installer', () => { expect(mockAplOps.install).not.toHaveBeenCalled() }) + test('should return false when status is completed but git repo has no main branch', async () => { + ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ + data: { status: 'completed' }, + }) + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ GIT_USERNAME: 'admin', GIT_PASSWORD: 'pass' }) + // git ls-remote fails (main branch does not exist) + mockZx.mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 2 }), + }), + }) + + const isInstalled = await installer.isInstalled() + + expect(isInstalled).toBe(false) + }) + test('should return true when ConfigMap does not exist', async () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue(null) @@ -280,6 +330,19 @@ describe('Installer', () => { expect(k8s.getK8sConfigMap).toHaveBeenCalledWith('apl-operator', 'apl-installation-status', mockCoreApi) expect(isInstalled).toBe(false) }) + + test('should return true when git verification fails (gitea not ready)', async () => { + ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ + data: { status: 'completed' }, + }) + // getK8sSecret throws (cluster issues) + ;(k8s.getK8sSecret as jest.Mock).mockRejectedValue(new Error('connection refused')) + + const isInstalled = await installer.isInstalled() + + // Should assume installed when verification can't be performed + expect(isInstalled).toBe(true) + }) }) describe('setEnvAndCreateSecrets', () => { @@ -292,10 +355,13 @@ describe('Installer', () => { expect(process.env.SOPS_AGE_KEY).toBe('existing-sops-key') }) - test('should handle failure when SOPS key not found in secret', async () => { + test('should skip gracefully when SOPS key not found in secret (SealedSecrets in use)', async () => { ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(null) - await expect(installer.setEnvAndCreateSecrets()).rejects.toThrow('SOPS_AGE_KEY not found in secret') + await installer.setEnvAndCreateSecrets() + + // Should not throw — SOPS is no longer required (replaced by SealedSecrets + ESO) + expect(process.env.SOPS_AGE_KEY).toBe('') }) }) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 72cb74b749..5bd01f228b 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -1,4 +1,5 @@ import * as process from 'node:process' +import { $ } from 'zx' import { terminal } from '../common/debug' import { getGitConfigData, @@ -11,6 +12,7 @@ import { hfValues } from '../common/hf' import { createUpdateConfigMap, createUpdateGenericSecret, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' +import { operatorEnv } from './validators' export class Installer { private d = terminal('operator:installer') @@ -28,7 +30,34 @@ export class Installer { await this.updateInstallationStatus('completed', -1) return true } - return installStatus === 'completed' + if (installStatus === 'completed') { + // Verify the git repo actually has content - the previous install may have + // marked status as completed but the pod was killed before the git push finished + const gitRepoHasContent = await this.verifyGitRepoHasMainBranch() + if (!gitRepoHasContent) { + this.d.warn('Installation marked as completed but git repo has no main branch - will re-install') + return false + } + return true + } + return false + } + + private async verifyGitRepoHasMainBranch(): Promise { + try { + // Get credentials from K8s secret (created by Helm at deploy time) + const creds = await getK8sSecret('gitea-credentials', 'apl-operator') + const username = creds?.GIT_USERNAME ?? 'otomi-admin' + const password = creds?.GIT_PASSWORD ?? '' + const repoUrl = `${process.env.GIT_PROTOCOL}://${username}:${password}@${process.env.GIT_URL}:${process.env.GIT_PORT}/${operatorEnv.GIT_ORG}/${operatorEnv.GIT_REPO}.git` + const result = await $`git ls-remote --exit-code --heads ${repoUrl} main`.nothrow().quiet() + return result.exitCode === 0 + } catch { + // If we can't check (e.g. gitea not ready yet), assume it's fine + // The operator will detect the issue later during git polling + this.d.warn('Could not verify git repo - gitea may not be ready yet') + return true + } } public async initialize() { @@ -102,10 +131,14 @@ export class Installer { private async setupSopsEnvironment() { const aplSopsSecret = await getK8sSecret('apl-sops-secrets', 'apl-operator') - if (!aplSopsSecret?.SOPS_AGE_KEY) { - throw new Error('SOPS_AGE_KEY not found in secret') + if (aplSopsSecret?.SOPS_AGE_KEY) { + process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY + this.d.debug('Using existing sops credentials from secret') + } else { + // SOPS is no longer used (replaced by SealedSecrets + ESO). + // Skip hfValues() call which requires the git repo that may not exist yet. + this.d.debug('SOPS Age key not found in secret, skipping (SealedSecrets in use)') } - process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY } // public for testing. This method should only be used if you are certain there are values locally. diff --git a/values-schema.yaml b/values-schema.yaml index 770310ae14..6efea064ae 100644 --- a/values-schema.yaml +++ b/values-schema.yaml @@ -903,7 +903,7 @@ definitions: secretTemplates: definitions: otomiAdminUsername: - x-secret: 'admin' + default: 'admin' securityContext: additionalProperties: uniqueItems: true @@ -1644,7 +1644,6 @@ properties: To be used with issuer externally-managed-tls-secret. $ref: '#/definitions/idName' customRootCA: - x-secret: '' type: string description: CA that is used to create and verify self-signed certificates. Leave it empty to generate one automatically. customRootCAKey: @@ -1704,6 +1703,12 @@ properties: $ref: '#/definitions/rawValues' enabled: type: boolean + adminUsername: + type: string + default: otomi-admin + adminPassword: + type: string + x-secret: '{{ randAlphaNum 20 }}' postgresqlPassword: type: string description: This password was generated and cannot be changed without manual intervention. @@ -1791,7 +1796,6 @@ properties: type: string x-secret: '{{ randAlphaNum 32 }}' required: - - secret - credentials databaseMaxConnections: type: number @@ -1899,6 +1903,7 @@ properties: x-secret: '' adminUsername: type: string + default: otomi-admin theme: type: string default: otomi @@ -2729,6 +2734,7 @@ properties: $ref: '#/definitions/url' clientID: $ref: '#/definitions/wordCharacterPattern' + x-secret: '' clientSecret: type: string x-secret: '' @@ -2748,7 +2754,6 @@ properties: default: sub required: - clientID - - clientSecret - issuer otomi: additionalProperties: false diff --git a/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl b/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl index 36fa0f144a..a89537c3ca 100644 --- a/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl +++ b/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl @@ -19,13 +19,32 @@ resources: hasArgocd: "{{ $v.apps.argocd.enabled | toString }}" domainSuffix: "{{ $v.cluster.domainSuffix }}" teamConfig: {{ $teamConfig | toJson }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: apl-gitea-operator-secret namespace: apl-gitea-operator - data: - giteaPassword: {{ $v.otomi.git.password | b64enc }} - oidcClientId: {{ $k.idp.clientID | b64enc }} - oidcClientSecret: {{ $k.idp.clientSecret | b64enc }} - oidcEndpoint: {{ $v._derived.oidcBaseUrl | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: apl-gitea-operator-secret + creationPolicy: Owner + template: + type: Opaque + data: + giteaPassword: '{{ "{{ .gitPassword | toString }}" }}' + oidcClientId: {{ $k.idp.clientID }} + oidcClientSecret: '{{ "{{ .keycloakClientSecret | toString }}" }}' + oidcEndpoint: {{ $v._derived.oidcBaseUrl }} + data: + - secretKey: gitPassword + remoteRef: + key: otomi-platform-secrets + property: git_password + - secretKey: keycloakClientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret diff --git a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl index 1a58d1fec9..72ad5eb791 100644 --- a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl +++ b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl @@ -11,17 +11,36 @@ {{- $teamNamespaces := $teamNamespaces | sortAlpha | toJson }} resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: apl-harbor-operator-secret namespace: apl-harbor-operator - data: - harborPassword: {{ $h.adminPassword | b64enc }} - harborUser: {{ "admin" | b64enc }} - oidcEndpoint: {{ $v._derived.oidcBaseUrl | b64enc }} - oidcClientId: {{ $k.idp.clientID | b64enc }} - oidcClientSecret: {{ $k.idp.clientSecret | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: apl-harbor-operator-secret + creationPolicy: Owner + template: + type: Opaque + data: + harborPassword: '{{ "{{ .harborAdminPassword | toString }}" }}' + harborUser: admin + oidcEndpoint: {{ $v._derived.oidcBaseUrl }} + oidcClientId: {{ $k.idp.clientID }} + oidcClientSecret: '{{ "{{ .keycloakClientSecret | toString }}" }}' + data: + - secretKey: harborAdminPassword + remoteRef: + key: harbor-secrets + property: adminPassword + - secretKey: keycloakClientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret - apiVersion: v1 kind: ConfigMap metadata: diff --git a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl index 5a3a8609c4..2516d98edf 100644 --- a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl +++ b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl @@ -21,31 +21,54 @@ {{- $k := $c | get "keycloak" }} {{- $doms := tpl (readFile "../../helmfile.d/snippets/domains.gotmpl") $v | fromYaml }} {{- $joinTpl := readFile "../../helmfile.d/utils/joinListWithSep.gotmpl" }} -{{ $users := list }} -{{- range $user := $v.users }} - {{ $groups := list }} - {{- if $user.isPlatformAdmin }}{{ $groups = append $groups "platform-admin" }}{{ end }} - {{- if $user.isTeamAdmin }}{{ $groups = append $groups "team-admin" }}{{ end }} - {{- range $team := $user | get "teams" list }}{{ $groups = append $groups (print "team-" $team) }}{{ end }} - {{- $users = append $users (dict "email" $user.email "firstName" $user.firstName "lastName" $user.lastName "initialPassword" $user.initialPassword "groups" $groups) }} -{{- end }} -{{- $users := $users | toJson }} - resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: apl-keycloak-operator-secret namespace: apl-keycloak-operator - data: - KEYCLOAK_ADMIN: {{ .Values.apps.keycloak.adminUsername | b64enc }} - KEYCLOAK_ADMIN_PASSWORD: {{ $k.adminPassword | b64enc }} - KEYCLOAK_CLIENT_SECRET: {{ $k.idp.clientSecret | b64enc }} - USERS: {{ $users | b64enc }} - {{- if $v.otomi.hasExternalIDP }} - IDP_CLIENT_ID: {{ $oi.clientID | b64enc}} - IDP_CLIENT_SECRET: {{ $oi.clientSecret | b64enc }} - {{- end }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: apl-keycloak-operator-secret + creationPolicy: Owner + template: + type: Opaque + data: + KEYCLOAK_ADMIN: {{ $k.adminUsername }} + KEYCLOAK_ADMIN_PASSWORD: '{{ "{{ .adminPassword | toString }}" }}' + KEYCLOAK_CLIENT_SECRET: '{{ "{{ .idpClientSecret | toString }}" }}' + USERS: '{{ "{{ .usersJson | toString }}" }}' + {{- if $v.otomi.hasExternalIDP }} + IDP_CLIENT_ID: '{{ "{{ .oidcClientID | toString }}" }}' + IDP_CLIENT_SECRET: '{{ "{{ .oidcClientSecret | toString }}" }}' + {{- end }} + data: + - secretKey: adminPassword + remoteRef: + key: otomi-platform-secrets + property: adminPassword + - secretKey: idpClientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret + - secretKey: usersJson + remoteRef: + key: users-secrets + property: usersJson + {{- if $v.otomi.hasExternalIDP }} + - secretKey: oidcClientID + remoteRef: + key: oidc-secrets + property: clientID + - secretKey: oidcClientSecret + remoteRef: + key: oidc-secrets + property: clientSecret + {{- end }} - apiVersion: v1 kind: ConfigMap metadata: diff --git a/values/apl-operator/apl-operator-raw.gotmpl b/values/apl-operator/apl-operator-raw.gotmpl new file mode 100644 index 0000000000..1ad5c4a39b --- /dev/null +++ b/values/apl-operator/apl-operator-raw.gotmpl @@ -0,0 +1,25 @@ +{{- $v := .Values }} + +resources: +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: gitea-credentials + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: gitea-credentials + creationPolicy: Owner + template: + type: Opaque + data: + GIT_USERNAME: {{ $v.otomi.git.username | default "otomi-admin" }} + GIT_PASSWORD: '{{ "{{ .git_password | toString }}" }}' + data: + - secretKey: git_password + remoteRef: + key: otomi-platform-secrets + property: git_password diff --git a/values/apl-operator/apl-operator.gotmpl b/values/apl-operator/apl-operator.gotmpl index d3bc151b2a..03463b273a 100644 --- a/values/apl-operator/apl-operator.gotmpl +++ b/values/apl-operator/apl-operator.gotmpl @@ -2,7 +2,6 @@ {{- $o := $v.apps | get "apl-operator" }} {{- $version := $v.otomi.version }} {{- $isSemver := regexMatch "^[0-9.]+" $version }} -{{- $g := $v.apps.gitea }} {{- $kms := $v | get "kms" dict }} image: {{- if $v.otomi.linodeLkeImageRepository }} @@ -19,11 +18,3 @@ imagePullSecrets: resources: {{- toYaml $o.resources.operator | nindent 2 }} kms: {{- $kms | toYaml | nindent 2 }} - -git: - password: {{ $v.otomi.git.password | quote }} - username: {{ $v.otomi.git.username | quote }} - email: {{ $v.otomi.git.email | quote }} - repoUrl: {{ $v.otomi.git.repoUrl | quote }} - branch: {{ $v.otomi.git.branch | quote }} - diff --git a/values/argocd/argocd-raw.gotmpl b/values/argocd/argocd-raw.gotmpl index 8fdc599015..478058fdc1 100644 --- a/values/argocd/argocd-raw.gotmpl +++ b/values/argocd/argocd-raw.gotmpl @@ -10,41 +10,89 @@ resources: custom-ca-certificates.crt: {{ .Values._derived.caCert | b64enc }} {{- end }} {{- if contains "gitea-http.gitea.svc.cluster.local" $v.otomi.git.repoUrl }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: argocd-repo-creds-gitea namespace: argocd - labels: - argocd.argoproj.io/secret-type: repo-creds - data: - type: {{ print "git" | b64enc | quote }} - url: {{ printf "https://%s" $v._derived.giteaDomain | b64enc }} - username: {{ $v.otomi.git.username | b64enc }} - password: {{ $v.otomi.git.password| b64enc }} - - apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-repo-creds-gitea + creationPolicy: Owner + template: + metadata: + labels: + argocd.argoproj.io/secret-type: repo-creds + type: Opaque + data: + type: {{ print "git" | quote }} + url: {{ printf "https://%s" $v._derived.giteaDomain }} + username: {{ $v.otomi.git.username }} + password: '{{ "{{ .gitPassword | toString }}" }}' + data: + - secretKey: gitPassword + remoteRef: + key: otomi-platform-secrets + property: git_password + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: argocd-repo-creds-gitea-internal namespace: argocd - labels: - argocd.argoproj.io/secret-type: repo-creds - data: - type: {{ print "git" | b64enc | quote }} - url: {{ "http://gitea-http.gitea.svc.cluster.local:3000/otomi/values.git" | b64enc | quote }} - username: {{ $v.otomi.git.username | b64enc }} - password: {{ $v.otomi.git.password| b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-repo-creds-gitea-internal + creationPolicy: Owner + template: + metadata: + labels: + argocd.argoproj.io/secret-type: repo-creds + type: Opaque + data: + type: {{ print "git" | quote }} + url: {{ "http://gitea-http.gitea.svc.cluster.local:3000/otomi/values.git" | quote }} + username: {{ $v.otomi.git.username }} + password: '{{ "{{ .gitPassword | toString }}" }}' + data: + - secretKey: gitPassword + remoteRef: + key: otomi-platform-secrets + property: git_password {{- else }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: argocd-repo-creds-git namespace: argocd - labels: - argocd.argoproj.io/secret-type: repo-creds - data: - type: {{ print "git" | b64enc | quote }} - url: {{ $v.otomi.git.repoUrl | b64enc }} - username: {{ $v.otomi.git.username | b64enc }} - password: {{ $v.otomi.git.password | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-repo-creds-git + creationPolicy: Owner + template: + metadata: + labels: + argocd.argoproj.io/secret-type: repo-creds + type: Opaque + data: + type: {{ print "git" | quote }} + url: {{ $v.otomi.git.repoUrl }} + username: {{ $v.otomi.git.username }} + password: '{{ "{{ .gitPassword | toString }}" }}' + data: + - secretKey: gitPassword + remoteRef: + key: otomi-platform-secrets + property: git_password {{- end }} diff --git a/values/argocd/argocd.gotmpl b/values/argocd/argocd.gotmpl index 6017fee452..c356f32639 100644 --- a/values/argocd/argocd.gotmpl +++ b/values/argocd/argocd.gotmpl @@ -205,8 +205,7 @@ configs: {{- end }} secret: - extra: - oidc.clientSecret: {{ $k.idp.clientSecret | quote }} + extra: {} params: server.insecure: true # nginx terminates tls # -- Number of application status processors diff --git a/values/cert-manager/cert-manager-raw.gotmpl b/values/cert-manager/cert-manager-raw.gotmpl index 6f7b113888..e7f0df883f 100644 --- a/values/cert-manager/cert-manager-raw.gotmpl +++ b/values/cert-manager/cert-manager-raw.gotmpl @@ -6,31 +6,96 @@ resources: {{- if and $v.otomi.hasExternalDNS (or (not (hasKey $p "aws")) ($p | get "aws.credentials.secretKey" nil)) }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: - name: "external-dns" - data: - {{- if hasKey $p "google" }} - secret: "{{ $p.google.serviceAccountKey | b64enc }}" - {{- else if hasKey $p "akamai" }} - access_token: {{ $p.akamai.accessToken | b64enc | quote }} - client_token: {{ $p.akamai.clientToken | b64enc | quote }} - client_secret: {{ $p.akamai.clientSecret | b64enc | quote }} - {{- else if hasKey $p "azure-private-dns" }} - secret: "{{ $p | get "azure-private-dns.aadClientSecret" | b64enc }}" - {{- else if hasKey $p "azure" }} - secret: "{{ $p.azure.aadClientSecret | b64enc }}" - {{- else if and (hasKey $p "aws") ($p | get "aws.credentials.secretKey" nil) }} - secret: "{{ $p.aws.credentials.secretKey | b64enc }}" - {{- else if hasKey $p "digitalocean" }} - secret: "{{ $p.digitalocean.apiToken | b64enc }}" - {{- else if hasKey $p "cloudflare" }} - secret: "{{ $p.cloudflare.apiToken | b64enc }}" - {{- else if hasKey $p "linode" }} - secret: "{{ $p.linode.apiToken | b64enc }}" - {{- end }} + name: external-dns + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: external-dns + creationPolicy: Owner + template: + type: Opaque + data: + {{- if hasKey $p "google" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "akamai" }} + access_token: '{{ "{{ .access_token | toString }}" }}' + client_token: '{{ "{{ .client_token | toString }}" }}' + client_secret: '{{ "{{ .client_secret | toString }}" }}' + {{- else if hasKey $p "azure-private-dns" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "azure" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if and (hasKey $p "aws") ($p | get "aws.credentials.secretKey" nil) }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "digitalocean" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "cloudflare" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- else if hasKey $p "linode" }} + secret: '{{ "{{ .secret | toString }}" }}' + {{- end }} + data: + - secretKey: secret + remoteRef: + key: dns-secrets + {{- if hasKey $p "google" }} + property: provider_google_serviceAccountKey + {{- else if hasKey $p "akamai" }} + property: provider_akamai_clientSecret + - secretKey: access_token + remoteRef: + key: dns-secrets + property: provider_akamai_accessToken + - secretKey: client_token + remoteRef: + key: dns-secrets + property: provider_akamai_clientToken + - secretKey: client_secret + remoteRef: + key: dns-secrets + property: provider_akamai_clientSecret + {{- else if hasKey $p "azure-private-dns" }} + property: provider_azure-private-dns_aadClientSecret + {{- else if hasKey $p "azure" }} + property: provider_azure_aadClientSecret + {{- else if and (hasKey $p "aws") ($p | get "aws.credentials.secretKey" nil) }} + property: provider_aws_credentials_secretKey + {{- else if hasKey $p "digitalocean" }} + property: provider_digitalocean_apiToken + {{- else if hasKey $p "cloudflare" }} + property: provider_cloudflare_apiToken + {{- else if hasKey $p "linode" }} + property: provider_linode_apiToken + {{- end }} {{- end }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: custom-ca + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: custom-ca + creationPolicy: Owner + template: + type: Opaque + data: + tls.crt: {{ $cm.customRootCA | quote }} + tls.key: '{{ "{{ .customRootCAKey | toString }}" }}' + data: + - secretKey: customRootCAKey + remoteRef: + key: cert-manager-secrets + property: customRootCAKey - apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: @@ -147,15 +212,29 @@ resources: {{- end }} {{- end }} {{- if eq $cm.issuer "byo-wildcard-cert" }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: otomi-byo-wildcard-cert namespace: istio-system - type: kubernetes.io/tls - data: - tls.crt: {{ $cm.byoWildcardCert | b64enc }} - tls.key: {{ $cm.byoWildcardCertKey | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: otomi-byo-wildcard-cert + creationPolicy: Owner + template: + type: kubernetes.io/tls + data: + tls.crt: {{ $cm.byoWildcardCert | quote }} + tls.key: '{{ "{{ .byoWildcardCertKey | toString }}" }}' + data: + - secretKey: byoWildcardCertKey + remoteRef: + key: cert-manager-secrets + property: byoWildcardCertKey {{- end }} {{- if or (eq $cm.issuer "letsencrypt" ) (eq $cm.issuer "custom-ca" ) }} - apiVersion: cert-manager.io/v1 @@ -179,4 +258,4 @@ resources: - key encipherment - ocsp signing {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/values/external-dns/external-dns-raw.gotmpl b/values/external-dns/external-dns-raw.gotmpl index e70774372c..9f699a9c1a 100644 --- a/values/external-dns/external-dns-raw.gotmpl +++ b/values/external-dns/external-dns-raw.gotmpl @@ -3,103 +3,252 @@ {{- with $d.provider | get "akamai" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: akamai-dns - data: - EXTERNAL_DNS_AKAMAI_CLIENT_SECRET: "{{.clientSecret | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: akamai-dns + creationPolicy: Owner + template: + type: Opaque + data: + EXTERNAL_DNS_AKAMAI_CLIENT_SECRET: '{{ "{{ .clientSecret | toString }}" }}' + EXTERNAL_DNS_AKAMAI_CLIENT_TOKEN: '{{ "{{ .clientToken | toString }}" }}' + EXTERNAL_DNS_AKAMAI_ACCESS_TOKEN: '{{ "{{ .accessToken | toString }}" }}' + data: + - secretKey: clientSecret + remoteRef: + key: dns-secrets + property: provider_akamai_clientSecret + - secretKey: clientToken + remoteRef: + key: dns-secrets + property: provider_akamai_clientToken + - secretKey: accessToken + remoteRef: + key: dns-secrets + property: provider_akamai_accessToken {{- end }} {{- with $d.provider | get "linode" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: linode-dns-api-token - data: - LINODE_TOKEN: "{{ .apiToken | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: linode-dns-api-token + creationPolicy: Owner + template: + type: Opaque + data: + LINODE_TOKEN: '{{ "{{ .apiToken | toString }}" }}' + data: + - secretKey: apiToken + remoteRef: + key: dns-secrets + property: provider_linode_apiToken {{- end }} {{- with $d.provider | get "digitalocean" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: do-token - data: - DO_TOKEN: "{{ .apiToken | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: do-token + creationPolicy: Owner + template: + type: Opaque + data: + DO_TOKEN: '{{ "{{ .apiToken | toString }}" }}' + data: + - secretKey: apiToken + remoteRef: + key: dns-secrets + property: provider_digitalocean_apiToken {{- end }} {{- with $d.provider | get "cloudflare" nil }} {{- with .apiToken }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: cloudflare-api-key - data: - CF_API_TOKEN: "{{ . | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: cloudflare-api-key + creationPolicy: Owner + template: + type: Opaque + data: + CF_API_TOKEN: '{{ "{{ .apiToken | toString }}" }}' + data: + - secretKey: apiToken + remoteRef: + key: dns-secrets + property: provider_cloudflare_apiToken {{- end }} {{- with .apiSecret }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: cloudflare-api-key - data: - CF_API_KEY: "{{ . | b64enc }}" - CF_API_EMAIL: "{{ $d.provider.cloudflare.email | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: cloudflare-api-key + creationPolicy: Owner + template: + type: Opaque + data: + CF_API_KEY: '{{ "{{ .apiSecret | toString }}" }}' + CF_API_EMAIL: {{ $d.provider.cloudflare.email | quote }} + data: + - secretKey: apiSecret + remoteRef: + key: dns-secrets + property: provider_cloudflare_apiSecret {{- end }} {{- end }} {{- with $d.provider | get "aws" nil }} {{- with .credentials }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: aws-dns-credentials - data: - AWS_ACCESS_KEY_ID: "{{ .accessKey | b64enc }}" - AWS_SECRET_ACCESS_KEY: "{{ .secretKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: aws-dns-credentials + creationPolicy: Owner + template: + type: Opaque + data: + AWS_ACCESS_KEY_ID: '{{ "{{ .accessKey | toString }}" }}' + AWS_SECRET_ACCESS_KEY: '{{ "{{ .secretKey | toString }}" }}' + data: + - secretKey: accessKey + remoteRef: + key: dns-secrets + property: provider_aws_credentials_accessKey + - secretKey: secretKey + remoteRef: + key: dns-secrets + property: provider_aws_credentials_secretKey {{- end }} {{- end }} {{- with $d.provider | get "azure" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: azure-dns - data: - AZURE_TENANT_ID: "{{ .tenantId | b64enc }}" - AZURE_SUBSCRIPTION_ID: "{{ .subscriptionId | b64enc }}" - AZURE_RESOURCE_GROUP: "{{ .resourceGroup | b64enc }}" - AZURE_CLIENT_ID: "{{ .aadClientId | b64enc }}" - AZURE_CLIENT_SECRET: "{{ .aadClientSecret | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: azure-dns + creationPolicy: Owner + template: + type: Opaque + data: + AZURE_TENANT_ID: {{ .tenantId | quote }} + AZURE_SUBSCRIPTION_ID: {{ .subscriptionId | quote }} + AZURE_RESOURCE_GROUP: {{ .resourceGroup | quote }} + AZURE_CLIENT_ID: {{ .aadClientId | quote }} + AZURE_CLIENT_SECRET: '{{ "{{ .aadClientSecret | toString }}" }}' + data: + - secretKey: aadClientSecret + remoteRef: + key: dns-secrets + property: provider_azure_aadClientSecret {{- end }} {{- with $d.provider | get "azure-private-dns" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: azure-private-dns - data: - AZURE_TENANT_ID: "{{ .tenantId | b64enc }}" - AZURE_SUBSCRIPTION_ID: "{{ .subscriptionId | b64enc }}" - AZURE_RESOURCE_GROUP: "{{ .resourceGroup | b64enc }}" - AZURE_CLIENT_ID: "{{ .aadClientId | b64enc }}" - AZURE_CLIENT_SECRET: "{{ .aadClientSecret | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: azure-private-dns + creationPolicy: Owner + template: + type: Opaque + data: + AZURE_TENANT_ID: {{ .tenantId | quote }} + AZURE_SUBSCRIPTION_ID: {{ .subscriptionId | quote }} + AZURE_RESOURCE_GROUP: {{ .resourceGroup | quote }} + AZURE_CLIENT_ID: {{ .aadClientId | quote }} + AZURE_CLIENT_SECRET: '{{ "{{ .aadClientSecret | toString }}" }}' + data: + - secretKey: aadClientSecret + remoteRef: + key: dns-secrets + property: provider_azure-private-dns_aadClientSecret {{- end }} {{- with $d.provider | get "google" nil }} resources: -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: google-dns - data: - GOOGLE_APPLICATION_CREDENTIALS: "{{ .serviceAccountKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: google-dns + creationPolicy: Owner + template: + type: Opaque + data: + GOOGLE_APPLICATION_CREDENTIALS: '{{ "{{ .serviceAccountKey | toString }}" }}' + data: + - secretKey: serviceAccountKey + remoteRef: + key: dns-secrets + property: provider_google_serviceAccountKey {{- end }} diff --git a/values/external-dns/external-dns.gotmpl b/values/external-dns/external-dns.gotmpl index a3737c9c57..e5f1d5e83c 100644 --- a/values/external-dns/external-dns.gotmpl +++ b/values/external-dns/external-dns.gotmpl @@ -50,9 +50,15 @@ env: - name: EXTERNAL_DNS_AKAMAI_SERVICECONSUMERDOMAIN value: "{{ $dns.provider.akamai.host }}" - name: EXTERNAL_DNS_AKAMAI_CLIENT_TOKEN - value: "{{ $dns.provider.akamai.clientToken }}" + valueFrom: + secretKeyRef: + name: akamai-dns + key: EXTERNAL_DNS_AKAMAI_CLIENT_TOKEN - name: EXTERNAL_DNS_AKAMAI_ACCESS_TOKEN - value: "{{ $dns.provider.akamai.accessToken }}" + valueFrom: + secretKeyRef: + name: akamai-dns + key: EXTERNAL_DNS_AKAMAI_ACCESS_TOKEN - name: EXTERNAL_DNS_AKAMAI_CLIENT_SECRET valueFrom: secretKeyRef: diff --git a/values/external-secrets/external-secrets-raw.gotmpl b/values/external-secrets/external-secrets-raw.gotmpl new file mode 100644 index 0000000000..c543f54b1e --- /dev/null +++ b/values/external-secrets/external-secrets-raw.gotmpl @@ -0,0 +1,47 @@ +{{- $v := .Values }} + +resources: + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: eso-store-sa + namespace: external-secrets + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: eso-core-secret-reader + rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: eso-core-secret-reader-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: eso-core-secret-reader + subjects: + - kind: ServiceAccount + name: eso-store-sa + namespace: external-secrets + - apiVersion: external-secrets.io/v1beta1 + kind: ClusterSecretStore + metadata: + name: core-secrets-store + spec: + provider: + kubernetes: + remoteNamespace: sealed-secrets + server: + url: "https://kubernetes.default.svc" + caProvider: + type: ConfigMap + name: kube-root-ca.crt + namespace: external-secrets + key: ca.crt + auth: + serviceAccount: + name: eso-store-sa + namespace: external-secrets diff --git a/values/external-secrets/external-secrets.gotmpl b/values/external-secrets/external-secrets.gotmpl new file mode 100644 index 0000000000..30ba6ada8c --- /dev/null +++ b/values/external-secrets/external-secrets.gotmpl @@ -0,0 +1,9 @@ +{{- $v := .Values }} +{{- $app := $v.apps | get "sealed-secrets" }} + +replicaCount: 1 + +resources: {{- $app.resources.operator | toYaml | nindent 2 }} + +image: + repository: ghcr.io/external-secrets/external-secrets diff --git a/values/gitea-db-secret/gitea-db-secret-raw.gotmpl b/values/gitea-db-secret/gitea-db-secret-raw.gotmpl index 792c355e5f..067dfb8644 100644 --- a/values/gitea-db-secret/gitea-db-secret-raw.gotmpl +++ b/values/gitea-db-secret/gitea-db-secret-raw.gotmpl @@ -1,12 +1,25 @@ {{- $v := .Values }} -{{- $g := $v.apps.gitea }} resources: -- apiVersion: v1 - kind: Secret - type: kubernetes.io/basic-auth +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: gitea-db-secret - data: - username: "{{ "gitea" | b64enc }}" - password: "{{ $g.postgresqlPassword | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: gitea-db-secret + creationPolicy: Owner + template: + type: kubernetes.io/basic-auth + data: + username: gitea + password: '{{ "{{ .postgresqlPassword | toString }}" }}' + data: + - secretKey: postgresqlPassword + remoteRef: + key: gitea-secrets + property: postgresqlPassword diff --git a/values/gitea/gitea-raw.gotmpl b/values/gitea/gitea-raw.gotmpl index 02b2cc15e8..2098889f8d 100644 --- a/values/gitea/gitea-raw.gotmpl +++ b/values/gitea/gitea-raw.gotmpl @@ -13,6 +13,28 @@ {{- end }} resources: +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: gitea-admin-secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: gitea-admin-secret + creationPolicy: Owner + template: + type: Opaque + data: + username: {{ $v.otomi.git.username | default "otomi-admin" }} + password: '{{ "{{ .git_password | toString }}" }}' + data: + - secretKey: git_password + remoteRef: + key: otomi-platform-secrets + property: git_password {{- if $v._derived.untrustedCA }} - apiVersion: v1 kind: Secret @@ -21,22 +43,53 @@ resources: data: ca-certificates.crt: {{ .Values._derived.caCert | b64enc }} {{- end }} -- apiVersion: v1 - kind: Secret - metadata: - name: gitea-admin-secret - data: - username: "{{ $v.otomi.git.username | b64enc }}" - password: "{{ $v.otomi.git.password | b64enc }}" # DB / app backup resources {{- if eq $obj.type "linode" }} -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: linode-creds - data: - S3_STORAGE_ACCOUNT: "{{ $obj.linode.accessKeyId | b64enc }}" - S3_STORAGE_KEY: "{{ $obj.linode.secretAccessKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: linode-creds + creationPolicy: Owner + template: + type: Opaque + data: + S3_STORAGE_ACCOUNT: {{ $obj.linode.accessKeyId }} + S3_STORAGE_KEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-storage-secrets + property: provider_linode_secretAccessKey +{{- end }} +{{- with $v | get "smtp" nil }} +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: gitea-smtp-secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: gitea-smtp-secret + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .authPassword | toString }}" }}' + data: + - secretKey: authPassword + remoteRef: + key: smtp-secrets + property: auth_password {{- end }} {{- if ne $v.cluster.provider "custom" }} # Application backup resources diff --git a/values/gitea/gitea.gotmpl b/values/gitea/gitea.gotmpl index d7c9bdee21..2f697e0ce2 100644 --- a/values/gitea/gitea.gotmpl +++ b/values/gitea/gitea.gotmpl @@ -63,7 +63,7 @@ gitea: HELO_HOSTNAME: {{ .hello }} FROM: {{ .from }} USER: {{ .auth_username }} - PASSWD: {{ .auth_password | quote }} + PASSWD: placeholder-overridden-by-env MAILER_TYPE: smtp IS_TLS_ENABLED: true SUBJECT_PREFIX: 'Otomi[Gitea]: ' @@ -125,6 +125,13 @@ gitea: secretKeyRef: name: gitea-db-secret key: password + {{- with $v | get "smtp" nil }} + - name: GITEA__MAILER__PASSWD + valueFrom: + secretKeyRef: + name: gitea-smtp-secret + key: password + {{- end }} - name: GITEA__DATABASE__MAX_OPEN_CONNS value: {{ $g.databaseMaxConnections | quote }} - name: GITEA__DATABASE__MAX_IDLE_CONNS diff --git a/values/harbor/harbor-raw.gotmpl b/values/harbor/harbor-raw.gotmpl index 623148ef62..feb9862641 100644 --- a/values/harbor/harbor-raw.gotmpl +++ b/values/harbor/harbor-raw.gotmpl @@ -33,58 +33,201 @@ resources: issuerRef: name: custom-ca kind: ClusterIssuer -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-admin-password - data: - HARBOR_ADMIN_PASSWORD: "{{ $h.adminPassword | b64enc }}" -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-admin-password + creationPolicy: Owner + template: + type: Opaque + data: + HARBOR_ADMIN_PASSWORD: '{{ "{{ .adminPassword | toString }}" }}' + data: + - secretKey: adminPassword + remoteRef: + key: harbor-secrets + property: adminPassword +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-registry-credentials - data: - REGISTRY_PASSWD: "{{ $h.registry.credentials.password | b64enc }}" - REGISTRY_HTPASSWD: "{{ $h.registry.credentials.htpasswd | b64enc }}" -{{- if ne $h.secretKey nil }} -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-registry-credentials + creationPolicy: Owner + template: + type: Opaque + data: + REGISTRY_PASSWD: '{{ "{{ .password | toString }}" }}' + REGISTRY_HTPASSWD: '{{ "{{ .htpasswd | toString }}" }}' + data: + - secretKey: password + remoteRef: + key: harbor-secrets + property: registry_credentials_password + - secretKey: htpasswd + remoteRef: + key: harbor-secrets + property: registry_credentials_htpasswd +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-secret-key - data: - secretKey: "{{ $h.secretKey | b64enc }}" -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-secret-key + creationPolicy: Owner + template: + type: Opaque + data: + secretKey: '{{ "{{ .secretKey | toString }}" }}' + data: + - secretKey: secretKey + remoteRef: + key: harbor-secrets + property: secretKey +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-core-secret - data: - secret: "{{ $h.core.secret | b64enc }}" -{{- end }} -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-core-secret + creationPolicy: Owner + template: + type: Opaque + data: + secret: '{{ "{{ .coreSecret | toString }}" }}' + data: + - secretKey: coreSecret + remoteRef: + key: harbor-secrets + property: core_secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: harbor-core-xsrf-secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-core-xsrf-secret + creationPolicy: Owner + template: + type: Opaque + data: + CSRF_KEY: '{{ "{{ .xsrfKey | toString }}" }}' + data: + - secretKey: xsrfKey + remoteRef: + key: harbor-secrets + property: core_xsrfKey +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-jobservice-secret - data: - JOBSERVICE_SECRET: "{{ $h.jobservice.secret | default "" | b64enc }}" -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-jobservice-secret + creationPolicy: Owner + template: + type: Opaque + data: + JOBSERVICE_SECRET: '{{ "{{ .jobserviceSecret | toString }}" }}' + data: + - secretKey: jobserviceSecret + remoteRef: + key: harbor-secrets + property: jobservice_secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: harbor-registry-http - data: - REGISTRY_HTTP_SECRET: "{{ $h.registry.secret | default "" | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: harbor-registry-http + creationPolicy: Owner + template: + type: Opaque + data: + REGISTRY_HTTP_SECRET: '{{ "{{ .registrySecret | toString }}" }}' + data: + - secretKey: registrySecret + remoteRef: + key: harbor-secrets + property: registry_secret {{- if eq $obj.type "linode" }} -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: linode-creds - data: - S3_STORAGE_ACCOUNT: "{{ $obj.linode.accessKeyId | b64enc }}" - S3_STORAGE_KEY: "{{ $obj.linode.secretAccessKey | b64enc }}" -- apiVersion: v1 - kind: Secret + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: linode-creds + creationPolicy: Owner + template: + type: Opaque + data: + S3_STORAGE_ACCOUNT: {{ $obj.linode.accessKeyId }} + S3_STORAGE_KEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-storage-secrets + property: provider_linode_secretAccessKey +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: registry-storage-credentials - data: - REGISTRY_STORAGE_S3_ACCESSKEY: "{{ $obj.linode.accessKeyId | b64enc }}" - REGISTRY_STORAGE_S3_SECRETKEY: "{{ $obj.linode.secretAccessKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: registry-storage-credentials + creationPolicy: Owner + template: + type: Opaque + data: + REGISTRY_STORAGE_S3_ACCESSKEY: {{ $obj.linode.accessKeyId }} + REGISTRY_STORAGE_S3_SECRETKEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-storage-secrets + property: provider_linode_secretAccessKey {{- end }} diff --git a/values/harbor/harbor.gotmpl b/values/harbor/harbor.gotmpl index 24048fef1f..794364489e 100644 --- a/values/harbor/harbor.gotmpl +++ b/values/harbor/harbor.gotmpl @@ -31,7 +31,8 @@ core: resources: {{- $h.resources.core | toYaml | nindent 4 }} existingSecret: harbor-core-secret - xsrfKey: {{ $h | get "core.xsrfKey" nil }} + existingXsrfSecret: harbor-core-xsrf-secret + existingXsrfSecretKey: CSRF_KEY database: maxOpenConns: {{ $h.databaseMaxConnections }} diff --git a/values/ingress-nginx/ingress-nginx-raw.gotmpl b/values/ingress-nginx/ingress-nginx-raw.gotmpl index 28e89c9963..366b4d6389 100644 --- a/values/ingress-nginx/ingress-nginx-raw.gotmpl +++ b/values/ingress-nginx/ingress-nginx-raw.gotmpl @@ -7,10 +7,42 @@ resources: labels: app.kubernetes.io/component: controller name: {{ $ingress.className }} - {{- if eq $ingress.className $v.ingress.platformClass.className }} + {{- if eq $ingress.className $v.ingress.platformClass.className }} annotations: ingressclass.kubernetes.io/is-default-class: "true" {{- end }} spec: controller: "k8s.io/{{ $ingress.className }}" +{{- end }} +# ClusterRole to allow ingress controller to read TLS secrets from all namespaces +{{- range $ingress := $v.ingress.classes }} +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: ingress-nginx-{{ $ingress.className }}-secrets-reader + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx-{{ $ingress.className }} + rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: ingress-nginx-{{ $ingress.className }}-secrets-reader + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx-{{ $ingress.className }} + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-{{ $ingress.className }}-secrets-reader + subjects: + - kind: ServiceAccount + name: ingress-nginx-{{ $ingress.className }} + namespace: ingress {{- end }} \ No newline at end of file diff --git a/values/k8s/k8s-raw.gotmpl b/values/k8s/k8s-raw.gotmpl index eb25208035..65631e6f26 100644 --- a/values/k8s/k8s-raw.gotmpl +++ b/values/k8s/k8s-raw.gotmpl @@ -72,12 +72,3 @@ resources: value: 1000000 globalDefault: false description: "This priority class should be used for Otomi High priority service pods only." - - - apiVersion: v1 - kind: Secret - metadata: - name: custom-ca - namespace: cert-manager - data: - tls.crt: {{ $cm.customRootCA | b64enc }} - tls.key: {{ $cm.customRootCAKey | b64enc}} diff --git a/values/keycloak/keycloak-raw.gotmpl b/values/keycloak/keycloak-raw.gotmpl index 3e483fd345..dcf7214cc4 100644 --- a/values/keycloak/keycloak-raw.gotmpl +++ b/values/keycloak/keycloak-raw.gotmpl @@ -1,5 +1,5 @@ {{- $v := .Values }} -{{- $otomiAdmin := "otomi-admin" }} +{{- $k := $v.apps.keycloak }} {{- $obj := $v.obj.provider }} resources: @@ -9,20 +9,50 @@ resources: name: custom-ca data: custom-ca.pem: {{ .Values._derived.caCert | b64enc }} -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: keycloak-initial-admin namespace: keycloak - data: - password: {{ .Values.otomi.adminPassword | b64enc }} - username: {{ .Values.apps.keycloak.adminUsername | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: keycloak-initial-admin + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .adminPassword | toString }}" }}' + username: {{ $k.adminUsername }} + data: + - secretKey: adminPassword + remoteRef: + key: otomi-platform-secrets + property: adminPassword {{- if eq $obj.type "linode" }} -- apiVersion: v1 - kind: Secret +- apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: linode-creds - data: - S3_STORAGE_ACCOUNT: "{{ $obj.linode.accessKeyId | b64enc }}" - S3_STORAGE_KEY: "{{ $obj.linode.secretAccessKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: linode-creds + creationPolicy: Owner + template: + type: Opaque + data: + S3_STORAGE_ACCOUNT: {{ $obj.linode.accessKeyId }} + S3_STORAGE_KEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-storage-secrets + property: provider_linode_secretAccessKey {{- end }} diff --git a/values/loki/loki-raw.gotmpl b/values/loki/loki-raw.gotmpl index a252ef3dce..3ca9d69809 100644 --- a/values/loki/loki-raw.gotmpl +++ b/values/loki/loki-raw.gotmpl @@ -4,24 +4,69 @@ {{- $obj := $v.obj.provider }} {{- if $v.otomi.isMultitenant }} resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: labels: app: loki name: reverse-proxy-auth-config - data: - authn.yaml: {{ tpl (readFile "auth-config.gotmpl") (dict "adminPassword" $l.adminPassword "teams" $v.teamConfig) | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: reverse-proxy-auth-config + creationPolicy: Owner + template: + type: Opaque + data: + authn.yaml: | + {{ "{{ " }}$adminPassword := .adminPassword | toString{{ " }}" }} + users: + - username: otomi-admin + password: {{ "\"{{ $adminPassword }}\"" }} + orgid: admins + {{- range $id, $team := $v.teamConfig }} + - username: {{ $id }} + password: {{ printf "\"{{ .team_%s_password | toString }}\"" $id }} + orgid: {{ $id }} + {{- end }} + data: + - secretKey: adminPassword + remoteRef: + key: loki-secrets + property: adminPassword + {{- range $id, $team := $v.teamConfig }} + - secretKey: team_{{ $id }}_password + remoteRef: + key: team-{{ $id }}-settings-secrets + property: settings_password + {{- end }} {{- if eq $obj.type "linode" }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: labels: app: loki name: loki-s3-linode-credentials - type: Opaque - data: - AWS_ACCESS_KEY_ID: "{{ $obj.linode.accessKeyId | b64enc }}" - AWS_SECRET_ACCESS_KEY: "{{ $obj.linode.secretAccessKey | b64enc }}" + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: loki-s3-linode-credentials + creationPolicy: Owner + template: + type: Opaque + data: + AWS_ACCESS_KEY_ID: {{ $obj.linode.accessKeyId }} + AWS_SECRET_ACCESS_KEY: '{{ "{{ .secretAccessKey | toString }}" }}' + data: + - secretKey: secretAccessKey + remoteRef: + key: obj-storage-secrets + property: provider_linode_secretAccessKey {{- end }} {{- end }} diff --git a/values/oauth2-proxy/oauth2-proxy-raw.gotmpl b/values/oauth2-proxy/oauth2-proxy-raw.gotmpl index bf137b073e..17696000bf 100644 --- a/values/oauth2-proxy/oauth2-proxy-raw.gotmpl +++ b/values/oauth2-proxy/oauth2-proxy-raw.gotmpl @@ -7,15 +7,50 @@ {{- $ingress := $v.ingress.platformClass }} resources: - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: oauth2-proxy-redis-password + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: oauth2-proxy-redis-password + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .password | toString }}" }}' + data: + - secretKey: password + remoteRef: + key: oauth2-proxy-redis-secrets + property: password + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: name: oauth2-proxy-client-access - type: Opaque - data: - client-id: {{ $k.idp.clientID | b64enc }} - client-secret: {{ $k.idp.clientSecret | b64enc }} - cookie-secret: {{ $oauth2 | get "config.cookieSecret" (randAlpha 32) | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: oauth2-proxy-client-access + creationPolicy: Owner + template: + type: Opaque + data: + client-id: {{ $k.idp.clientID }} + client-secret: '{{ "{{ .clientSecret | toString }}" }}' + cookie-secret: {{ $oauth2 | get "config.cookieSecret" (randAlpha 32) }} + data: + - secretKey: clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret - apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -204,4 +239,3 @@ resources: - diff --git a/values/oauth2-proxy/oauth2-proxy.gotmpl b/values/oauth2-proxy/oauth2-proxy.gotmpl index e8ab163ce2..7a76a7321c 100644 --- a/values/oauth2-proxy/oauth2-proxy.gotmpl +++ b/values/oauth2-proxy/oauth2-proxy.gotmpl @@ -104,7 +104,8 @@ sessionStorage: standalone: connectionUrl: "redis://oauth2-proxy-redis-ha-haproxy.istio-system.svc.cluster.local:6379" {{- end }} - password: {{ $r | get "password" | quote }} + existingSecret: oauth2-proxy-redis-password + passwordKey: password redis-ha: global: @@ -114,7 +115,7 @@ redis-ha: repository: "{{- $v.otomi.linodeLkeImageRepository }}/docker/redis" {{- end }} enabled: true - redisPassword: {{ $r | get "password" | quote }} + existingSecret: oauth2-proxy-redis-password replicas: {{ $r.replicas }} redis: resources: {{- $r.resources.master | toYaml | nindent 6 }} diff --git a/values/otomi-api/otomi-api-raw.gotmpl b/values/otomi-api/otomi-api-raw.gotmpl new file mode 100644 index 0000000000..51e4f6023b --- /dev/null +++ b/values/otomi-api/otomi-api-raw.gotmpl @@ -0,0 +1,24 @@ +{{- $v := .Values }} +resources: + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: otomi-api-git-credentials + namespace: otomi + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: otomi-api-git-credentials + creationPolicy: Owner + template: + type: Opaque + data: + GIT_PASSWORD: '{{ "{{ .gitPassword | toString }}" }}' + data: + - secretKey: gitPassword + remoteRef: + key: otomi-platform-secrets + property: git_password diff --git a/values/otomi-api/otomi-api.gotmpl b/values/otomi-api/otomi-api.gotmpl index 6a21424911..52849c396b 100644 --- a/values/otomi-api/otomi-api.gotmpl +++ b/values/otomi-api/otomi-api.gotmpl @@ -2,11 +2,9 @@ {{- $c := $v.cluster }} {{- $o := $v.apps | get "otomi-api" }} {{- $cm := $v.apps | get "cert-manager" }} -{{- $sops := $v | get "kms.sops" dict }} {{- $giteaValuesPublicUrl := printf "https://gitea.%s/otomi/values" $v.cluster.domainSuffix }} {{- $git := $v.otomi.git }} {{- $defaultPlatformAdminEmail := printf "platform-admin@%s" $v.cluster.domainSuffix }} -{{- $sopsEnv := tpl (readFile "../../helmfile.d/snippets/sops-env.gotmpl") $sops }} {{- $version := $v.versions | get "api" }} {{- $isSemver := regexMatch "^[0-9.]+" $version }} {{- $coreVersion := $v.otomi.version }} @@ -39,8 +37,7 @@ tools: secrets: GIT_USER: {{ $git.username | quote }} GIT_EMAIL: {{ $git.email | quote }} - GIT_PASSWORD: {{ $git.password | quote }} - {{- $sopsEnv | nindent 2 }} +existingSecret: otomi-api-git-credentials env: DEFAULT_PLATFORM_ADMIN_EMAIL: {{ $defaultPlatformAdminEmail }} diff --git a/values/prometheus-operator/prometheus-operator-raw.gotmpl b/values/prometheus-operator/prometheus-operator-raw.gotmpl index fff4e04c3b..b74bf5873b 100644 --- a/values/prometheus-operator/prometheus-operator-raw.gotmpl +++ b/values/prometheus-operator/prometheus-operator-raw.gotmpl @@ -1,17 +1,141 @@ {{- $v := .Values }} {{- $p := $v.apps | get "prometheus" }} +{{- $a := $v.apps | get "alertmanager" }} +{{- $k := $v.apps.keycloak }} {{- $obj := $v.obj.provider }} -{{- if or ($p | get "remoteWrite.rwConfig.basicAuth.enabled" false) }} +{{- $receivers := $v | get "alerts.receivers" (list "slack") }} +{{- $slackTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/slack.gotmpl") $v | toString }} +{{- $opsgenieTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/opsgenie.gotmpl") $v | toString }} +{{- $alertmanagerConfig := tpl (readFile "../../helmfile.d/snippets/alertmanager.gotmpl") (dict "instance" $v "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | toString }} resources: + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-admin-secret + namespace: monitoring + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-admin-secret + creationPolicy: Owner + template: + type: Opaque + data: + admin-user: otomi-admin + admin-password: '{{ "{{ .adminPassword | toString }}" }}' + data: + - secretKey: adminPassword + remoteRef: + key: otomi-platform-secrets + property: adminPassword + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-oidc-secret + namespace: monitoring + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-oidc-secret + creationPolicy: Owner + template: + type: Opaque + data: + client_id: {{ $k.idp.clientID }} + client_secret: '{{ "{{ .clientSecret | toString }}" }}' + data: + - secretKey: clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-loki-datasource-secret + namespace: monitoring + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-loki-datasource-secret + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .adminPassword | toString }}" }}' + data: + - secretKey: adminPassword + remoteRef: + key: loki-secrets + property: adminPassword {{- if $p | get "remoteWrite.rwConfig.basicAuth.enabled" false }} - - apiVersion: v1 - kind: Secret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret metadata: labels: app: prometheus name: prometheus-remote-write-basic-auth - data: - username: {{ $p.remoteWrite.rwConfig.basicAuth.username | b64enc }} - password: {{ $p.remoteWrite.rwConfig.basicAuth.password | b64enc }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: prometheus-remote-write-basic-auth + creationPolicy: Owner + template: + type: Opaque + data: + username: {{ $p.remoteWrite.rwConfig.basicAuth.username }} + password: '{{ "{{ .password | toString }}" }}' + data: + - secretKey: password + remoteRef: + key: prometheus-secrets + property: remoteWrite_rwConfig_basicAuth_password + {{- end }} + {{- if $a.enabled }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: alertmanager-platform-config + namespace: monitoring + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: alertmanager-platform-config + creationPolicy: Owner + template: + type: Opaque + data: + alertmanager.yaml: | + {{- $alertmanagerConfig | nindent 14 }} + data: + {{- if has "slack" $receivers }} + - secretKey: slackUrl + remoteRef: + key: alerts-secrets + property: slack_url + {{- end }} + {{- if has "email" $receivers }} + - secretKey: smtpAuthPassword + remoteRef: + key: smtp-secrets + property: auth_password + - secretKey: smtpAuthSecret + remoteRef: + key: smtp-secrets + property: auth_secret + {{- end }} {{- end }} -{{- end }} diff --git a/values/prometheus-operator/prometheus-operator.gotmpl b/values/prometheus-operator/prometheus-operator.gotmpl index c90b770d14..bc7a7199e5 100644 --- a/values/prometheus-operator/prometheus-operator.gotmpl +++ b/values/prometheus-operator/prometheus-operator.gotmpl @@ -12,9 +12,7 @@ {{- $alertmanagerDomain := printf "alertmanager.%s" $domain }} {{- $prometheusDomain := printf "prometheus.%s" $domain }} {{- $grafanaDomain := printf "grafana.%s" $domain }} -{{- $slackTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/slack.gotmpl") $v | toString }} -{{- $opsgenieTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/opsgenie.gotmpl") $v | toString }} -{{- $grafanaIni := tpl (readFile "../../helmfile.d/snippets/grafana.gotmpl") (dict "keycloakBase" $v._derived.oidcBaseUrl "untrustedCA" $v._derived.untrustedCA "keycloak" ($k | get "idp")) | toString }} +{{- $grafanaIni := tpl (readFile "../../helmfile.d/snippets/grafana.gotmpl") (dict "keycloakBase" $v._derived.oidcBaseUrl "untrustedCA" $v._derived.untrustedCA "keycloak" (dict "clientID" ($k | get "idp.clientID" "otomi"))) | toString }} {{- $hasServices := false }} {{- range $teamId, $team := $v.teamConfig }} {{- if gt (len ($team | get "services" list)) 0 }}{{ $hasServices = true }}{{ end }} @@ -199,7 +197,8 @@ alertmanager: priorityClassName: otomi-critical resources: {{- $a.resources.alertmanager | toYaml | nindent 6 }} externalUrl: https://{{ $alertmanagerDomain }} - config: {{- tpl (readFile "../../helmfile.d/snippets/alertmanager.gotmpl") (dict "instance" $v "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | nindent 4 }} + useExistingSecret: true + configSecret: alertmanager-platform-config grafana: enabled: {{ $g.enabled }} defaultDashboardsEnabled: false @@ -248,11 +247,25 @@ grafana: {{- end }} basicAuth: true basicAuthUser: otomi-admin - secureJsonData: - basicAuthPassword: {{ $v.apps.loki.adminPassword | quote }} {{- end }} {{- end }} - adminPassword: {{ $g | get "adminPassword" $v.otomi.adminPassword | quote }} + admin: + existingSecret: grafana-admin-secret + userKey: admin-user + passwordKey: admin-password + envValueFrom: + GF_AUTH_GENERIC_OAUTH_CLIENT_ID: + secretKeyRef: + name: grafana-oidc-secret + key: client_id + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: + secretKeyRef: + name: grafana-oidc-secret + key: client_secret + GF_LOKI_BASIC_AUTH_PASSWORD: + secretKeyRef: + name: grafana-loki-datasource-secret + key: password grafana.ini: {{- $grafanaIni | nindent 4 }} server: root_url: https://{{ $grafanaDomain }} diff --git a/versions.yaml b/versions.yaml index d54e4c2e07..cba4e40597 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,5 +1,5 @@ api: main console: main consoleLogin: main -tasks: main +tasks: APL-1476-1 tools: main From 8775f5ac70975976c36bb3d03eb6aa47151205a8 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 18 Feb 2026 20:50:11 +0100 Subject: [PATCH 02/66] feat: create core secrets in apl-secrets namespace --- src/common/git-config.test.ts | 4 +- src/common/git-config.ts | 2 +- src/common/sealed-secrets.test.ts | 52 +++++++++---------- src/common/sealed-secrets.ts | 8 +-- .../external-secrets-raw.gotmpl | 2 +- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/common/git-config.test.ts b/src/common/git-config.test.ts index 4883b81d94..9ffce19d00 100644 --- a/src/common/git-config.test.ts +++ b/src/common/git-config.test.ts @@ -353,7 +353,7 @@ describe('git-config', () => { git: { repoUrl: 'https://github.com/org/repo.git', username: 'admin', - password: 'sealed:sealed-secrets/otomi-platform-secrets/git_password', + password: 'sealed:apl-secrets/otomi-platform-secrets/git_password', branch: 'main', email: 'pipeline@cluster.local', }, @@ -361,7 +361,7 @@ describe('git-config', () => { } const result = await getRepo(values, { getK8sSecret: secretMock }) - expect(secretMock).toHaveBeenCalledWith('otomi-platform-secrets', 'sealed-secrets') + expect(secretMock).toHaveBeenCalledWith('otomi-platform-secrets', 'apl-secrets') expect(result.password).toBe('real-password') expect(result.authenticatedUrl).toContain('real-password') }) diff --git a/src/common/git-config.ts b/src/common/git-config.ts index 3c18e86333..66cefab43b 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -152,7 +152,7 @@ export const getRepo = async (values: Record, deps = { getK8sSecret // try reading the real password from the K8s secret (populated by ESO from SealedSecrets) if (!password || (typeof password === 'string' && password.startsWith('sealed:'))) { try { - const secret = await deps.getK8sSecret('otomi-platform-secrets', 'sealed-secrets') + const secret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') if (secret?.git_password) { password = String(secret.git_password) d.debug('Read git password from K8s secret (ESO)') diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 0fa226e9b8..cadefb92cc 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -194,10 +194,10 @@ describe('sealed-secrets', () => { const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) - // All secrets now go to sealed-secrets namespace + // All secrets now go to apl-secrets namespace const harborMapping = result.find((m) => m.secretName === 'harbor-secrets') expect(harborMapping).toBeDefined() - expect(harborMapping!.namespace).toBe('sealed-secrets') + expect(harborMapping!.namespace).toBe('apl-secrets') expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') expect(harborMapping!.data).toHaveProperty('secretKey', 'harbor-secret') }) @@ -216,7 +216,7 @@ describe('sealed-secrets', () => { const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) expect(result).toHaveLength(1) - expect(result[0].namespace).toBe('sealed-secrets') + expect(result[0].namespace).toBe('apl-secrets') }) it('should serialize users array as single JSON value in users-secrets', async () => { @@ -241,11 +241,11 @@ describe('sealed-secrets', () => { expect(result).toHaveLength(2) const usersMapping = result.find((m) => m.secretName === 'users-secrets') expect(usersMapping).toBeDefined() - expect(usersMapping!.namespace).toBe('sealed-secrets') + expect(usersMapping!.namespace).toBe('apl-secrets') expect(usersMapping!.data.usersJson).toBe(JSON.stringify(secrets.users)) }) - it('should handle teamConfig dynamic paths in sealed-secrets namespace', async () => { + it('should handle teamConfig dynamic paths in apl-secrets namespace', async () => { const secrets = { teamConfig: { 'team-alpha': { someSecret: 'value' }, @@ -258,7 +258,7 @@ describe('sealed-secrets', () => { const result = await buildSecretToNamespaceMap(secrets, ['team-alpha'], undefined, deps) expect(result).toHaveLength(1) - expect(result[0].namespace).toBe('sealed-secrets') + expect(result[0].namespace).toBe('apl-secrets') expect(result[0].secretName).toBe('team-team-alpha-settings-secrets') }) @@ -289,7 +289,7 @@ describe('sealed-secrets', () => { expect(result[0].data).toHaveProperty('core_secret', 'core-secret-val') }) - it('should put gitea secrets in sealed-secrets namespace using convention naming', async () => { + it('should put gitea secrets in apl-secrets namespace using convention naming', async () => { const secrets = { apps: { gitea: { adminPassword: 'gitea-pass', postgresqlPassword: 'pg-pass' }, @@ -308,16 +308,16 @@ describe('sealed-secrets', () => { const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) - // Harbor should use convention naming in sealed-secrets ns + // Harbor should use convention naming in apl-secrets ns const harborMapping = result.find((m) => m.secretName === 'harbor-secrets') expect(harborMapping).toBeDefined() - expect(harborMapping!.namespace).toBe('sealed-secrets') + expect(harborMapping!.namespace).toBe('apl-secrets') expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') - // Gitea should have a gitea-secrets mapping in sealed-secrets ns + // Gitea should have a gitea-secrets mapping in apl-secrets ns const giteaMapping = result.find((m) => m.secretName === 'gitea-secrets') expect(giteaMapping).toBeDefined() - expect(giteaMapping!.namespace).toBe('sealed-secrets') + expect(giteaMapping!.namespace).toBe('apl-secrets') expect(giteaMapping!.data).toHaveProperty('adminPassword', 'gitea-pass') expect(giteaMapping!.data).toHaveProperty('postgresqlPassword', 'pg-pass') }) @@ -326,7 +326,7 @@ describe('sealed-secrets', () => { describe('createSealedSecretManifest', () => { it('should produce correct SealedSecret structure', async () => { const mapping = { - namespace: 'sealed-secrets', + namespace: 'apl-secrets', secretName: 'harbor-secrets', data: { adminPassword: 'my-password', secretKey: 'my-secret' }, } @@ -339,18 +339,18 @@ describe('sealed-secrets', () => { expect(result.apiVersion).toBe('bitnami.com/v1alpha1') expect(result.kind).toBe('SealedSecret') expect(result.metadata.name).toBe('harbor-secrets') - expect(result.metadata.namespace).toBe('sealed-secrets') + expect(result.metadata.namespace).toBe('apl-secrets') expect(result.metadata.annotations['sealedsecrets.bitnami.com/namespace-wide']).toBe('true') expect(result.spec.encryptedData.adminPassword).toBe('encrypted-value') expect(result.spec.encryptedData.secretKey).toBe('encrypted-value') expect(result.spec.template.type).toBe('Opaque') expect(result.spec.template.metadata.name).toBe('harbor-secrets') - expect(result.spec.template.metadata.namespace).toBe('sealed-secrets') + expect(result.spec.template.metadata.namespace).toBe('apl-secrets') }) it('should call encryptSecretItem for each data key', async () => { const mapping = { - namespace: 'sealed-secrets', + namespace: 'apl-secrets', secretName: 'gitea-secrets', data: { key1: 'val1', key2: 'val2', key3: 'val3' }, } @@ -361,9 +361,9 @@ describe('sealed-secrets', () => { await createSealedSecretManifest('pem', mapping, deps) expect(deps.encryptSecretItem).toHaveBeenCalledTimes(3) - expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'sealed-secrets', 'val1') - expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'sealed-secrets', 'val2') - expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'sealed-secrets', 'val3') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'apl-secrets', 'val1') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'apl-secrets', 'val2') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'apl-secrets', 'val3') }) }) @@ -376,13 +376,13 @@ describe('sealed-secrets', () => { metadata: { annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, name: 'harbor-secrets', - namespace: 'sealed-secrets', + namespace: 'apl-secrets', }, spec: { encryptedData: { key: 'enc' }, template: { immutable: false, - metadata: { name: 'harbor-secrets', namespace: 'sealed-secrets' }, + metadata: { name: 'harbor-secrets', namespace: 'apl-secrets' }, type: 'Opaque', }, }, @@ -397,9 +397,9 @@ describe('sealed-secrets', () => { await writeSealedSecretManifests(manifests, '/test', deps) - expect(deps.mkdir).toHaveBeenCalledWith('/test/env/manifests/ns/sealed-secrets', { recursive: true }) + expect(deps.mkdir).toHaveBeenCalledWith('/test/env/manifests/ns/apl-secrets', { recursive: true }) expect(deps.writeFile).toHaveBeenCalledWith( - '/test/env/manifests/ns/sealed-secrets/harbor-secrets.yaml', + '/test/env/manifests/ns/apl-secrets/harbor-secrets.yaml', 'yaml-content', ) }) @@ -411,7 +411,7 @@ describe('sealed-secrets', () => { apps: { harbor: { adminPassword: 'pass' } }, } const mockMapping = { - namespace: 'sealed-secrets', + namespace: 'apl-secrets', secretName: 'harbor-secrets', data: { adminPassword: 'pass' }, } @@ -421,13 +421,13 @@ describe('sealed-secrets', () => { metadata: { annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, name: 'harbor-secrets', - namespace: 'sealed-secrets', + namespace: 'apl-secrets', }, spec: { encryptedData: { adminPassword: 'encrypted' }, template: { immutable: false, - metadata: { name: 'harbor-secrets', namespace: 'sealed-secrets' }, + metadata: { name: 'harbor-secrets', namespace: 'apl-secrets' }, type: 'Opaque', }, }, @@ -462,7 +462,7 @@ describe('sealed-secrets', () => { apps: { harbor: { adminPassword: 'pass' } }, } const mockMapping = { - namespace: 'sealed-secrets', + namespace: 'apl-secrets', secretName: 'harbor-secrets', data: { adminPassword: 'pass' }, } diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 4c364a4946..33c37c95a5 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -240,21 +240,21 @@ export const createSealedSecretsKeySecret = async ( /** * Resolve the namespace for a given secret path. - * All core secrets go to 'sealed-secrets' namespace for ESO access. + * All core secrets go to 'apl-secrets' namespace for ESO access. * APP_NAMESPACE_MAP is kept for reference but not used for SealedSecret placement. */ const resolveNamespace = (secretPath: string): string | undefined => { // Check for teamConfig dynamic paths const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) if (teamMatch) { - return 'sealed-secrets' + return 'apl-secrets' } // Check if this path matches any known prefix const sortedKeys = Object.keys(APP_NAMESPACE_MAP).sort((a, b) => b.length - a.length) for (const prefix of sortedKeys) { if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { - return 'sealed-secrets' + return 'apl-secrets' } } @@ -355,7 +355,7 @@ export const buildSecretToNamespaceMap = async ( if (secretPath === 'users') { const usersData = secrets.users if (Array.isArray(usersData) && usersData.length > 0) { - const namespace = 'sealed-secrets' + const namespace = 'apl-secrets' const secretName = 'users-secrets' const groupKey = `${namespace}/${secretName}` if (!groupMap.has(groupKey)) { diff --git a/values/external-secrets/external-secrets-raw.gotmpl b/values/external-secrets/external-secrets-raw.gotmpl index c543f54b1e..c1b95b3111 100644 --- a/values/external-secrets/external-secrets-raw.gotmpl +++ b/values/external-secrets/external-secrets-raw.gotmpl @@ -33,7 +33,7 @@ resources: spec: provider: kubernetes: - remoteNamespace: sealed-secrets + remoteNamespace: apl-secrets server: url: "https://kubernetes.default.svc" caProvider: From 45a18611717f0904b9ae69c52c3d72a11f1a0853 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 18 Feb 2026 20:53:27 +0100 Subject: [PATCH 03/66] fix: add default value for the existingSecret --- charts/otomi-api/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/otomi-api/values.yaml b/charts/otomi-api/values.yaml index 8cdfe5787a..f4ddb4695e 100644 --- a/charts/otomi-api/values.yaml +++ b/charts/otomi-api/values.yaml @@ -24,6 +24,8 @@ serviceAccount: imagePullSecrets: {} +existingSecret: "" + rbac: # Specifies whether rbac should be set up create: true From cd503a9a1e6481ade53a8cf89232854c9d45cd9c Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 18 Feb 2026 22:31:39 +0100 Subject: [PATCH 04/66] fix: namespace changes --- charts/apl-operator/templates/deployment.yaml | 2 ++ src/cmd/commit.ts | 4 ++-- src/cmd/migrate.ts | 2 +- values/apl-operator/apl-operator.gotmpl | 6 ++++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/charts/apl-operator/templates/deployment.yaml b/charts/apl-operator/templates/deployment.yaml index c4f5011207..7683978046 100644 --- a/charts/apl-operator/templates/deployment.yaml +++ b/charts/apl-operator/templates/deployment.yaml @@ -44,8 +44,10 @@ spec: envFrom: - secretRef: name: apl-sops-secrets + optional: true - secretRef: name: apl-git-credentials + optional: true resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 96f427920e..a497aa8290 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -176,7 +176,7 @@ export async function initialSetupData(): Promise { if (!hasExternalIDP) { // Read the platform admin's initialPassword from users-secrets (set by keycloak-operator) - const usersSecret = await getK8sSecret('users-secrets', 'sealed-secrets') + const usersSecret = await getK8sSecret('users-secrets', 'apl-secrets') let platformAdminPassword = '' if (usersSecret?.usersJson) { // getK8sSecret already parses JSON/YAML values, so usersJson may be an array or a string @@ -195,7 +195,7 @@ export async function initialSetupData(): Promise { } } else { // External IDP: show Keycloak admin credentials (keycloak-initial-admin uses otomi-platform-secrets.adminPassword) - const otomiSecret = await getK8sSecret('otomi-platform-secrets', 'sealed-secrets') + const otomiSecret = await getK8sSecret('otomi-platform-secrets', 'apl-secrets') const adminPassword = otomiSecret?.adminPassword ? String(otomiSecret.adminPassword) : '' return { domainSuffix, diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 0ad054f1f0..f5c9cf5121 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -712,7 +712,7 @@ const setDefaultAplCatalog = async (values: Record): Promise let secretCreated = false if (useGiteaCatalog) { try { - const giteaSecrets = await getK8sSecret('gitea-secrets', 'sealed-secrets') + const giteaSecrets = await getK8sSecret('gitea-secrets', 'apl-secrets') const resolvedGitea = { adminUsername: giteaSecrets?.adminUsername ? String(giteaSecrets.adminUsername) : String(gitea!.adminUsername), adminPassword: giteaSecrets?.adminPassword ? String(giteaSecrets.adminPassword) : String(gitea!.adminPassword), diff --git a/values/apl-operator/apl-operator.gotmpl b/values/apl-operator/apl-operator.gotmpl index 03463b273a..1657090d50 100644 --- a/values/apl-operator/apl-operator.gotmpl +++ b/values/apl-operator/apl-operator.gotmpl @@ -18,3 +18,9 @@ imagePullSecrets: resources: {{- toYaml $o.resources.operator | nindent 2 }} kms: {{- $kms | toYaml | nindent 2 }} + +git: + repoUrl: {{ $v.otomi.git.repoUrl | quote }} + branch: {{ $v.otomi.git.branch | default "main" | quote }} + email: {{ $v.otomi.git.email | default "pipeline@cluster.local" | quote }} + username: {{ $v.otomi.git.username | default "otomi-admin" | quote }} From edc8ffdc540aeac0d4ce3bcde13237815e8e48b6 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 18 Feb 2026 23:35:34 +0100 Subject: [PATCH 05/66] test: sealed secrets with eso --- src/operator/installer.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 5bd01f228b..77aeca38a8 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -154,12 +154,14 @@ export class Installer { const agePrivateKey = values?.kms?.sops?.age?.privateKey // Ensure apl-git-credentials secret + // Only recreate if credentials are missing AND values has the password available + // (password may be undefined when secrets are stripped from disk and managed by SealedSecrets + ESO) const credentials = await getGitCredentials() - if (!credentials) { + if (!credentials && otomiGit?.username && otomiGit?.password) { this.d.info('Recreating apl-git-credentials secret') await createUpdateGenericSecret(k8s.core(), GIT_CONFIG_SECRET_NAME, GIT_CONFIG_NAMESPACE, { - username: otomiGit?.username, - password: otomiGit?.password, + username: otomiGit.username, + password: otomiGit.password, }) } From 0a4cc5a0fd53ec4bd8b9ef4b0db1cde876cafbc9 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 19 Feb 2026 09:12:04 +0100 Subject: [PATCH 06/66] fix: merge conflicts/changes --- src/common/git-config.test.ts | 9 + src/common/git-config.ts | 5 + src/operator/installer.test.ts | 253 +++--------------------- src/operator/installer.ts | 102 +++------- src/operator/main.ts | 22 ++- values/apl-operator/apl-operator.gotmpl | 1 + 6 files changed, 79 insertions(+), 313 deletions(-) diff --git a/src/common/git-config.test.ts b/src/common/git-config.test.ts index 9ffce19d00..ef0c5a2ab2 100644 --- a/src/common/git-config.test.ts +++ b/src/common/git-config.test.ts @@ -67,6 +67,15 @@ describe('git-config', () => { const result = await getGitCredentials() expect(result).toBeUndefined() }) + + it('should return undefined when password is a sealed-secret placeholder', async () => { + mockGetK8sSecret.mockResolvedValue({ + username: 'admin', + password: 'sealed:apl-secrets/otomi-platform-secrets/git_password', + }) + const result = await getGitCredentials() + expect(result).toBeUndefined() + }) }) describe('getOldGitCredentials', () => { diff --git a/src/common/git-config.ts b/src/common/git-config.ts index 66cefab43b..cfa824d535 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -40,6 +40,11 @@ export async function getGitCredentials(): Promise { return undefined } + // Reject unresolved sealed-secret placeholders (e.g. during first deploy before secrets are decrypted) + if (typeof secretData.password === 'string' && secretData.password.startsWith('sealed:')) { + return undefined + } + return { username: secretData.username, password: secretData.password, diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index 02cb411730..96a09eff0e 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -1,5 +1,4 @@ import * as gitConfig from '../common/git-config' -import * as hf from '../common/hf' import * as k8s from '../common/k8s' import { AplOperations } from './apl-operations' import { Installer } from './installer' @@ -14,21 +13,6 @@ jest.mock('zx', () => ({ $: (...args: any[]) => mockZx(...args), })) -jest.mock('../common/envalid', () => ({ - env: { - GIT_PROTOCOL: 'http', - GIT_URL: 'gitea-http.gitea.svc.cluster.local', - GIT_PORT: '3000', - }, -})) - -jest.mock('./validators', () => ({ - operatorEnv: { - GIT_ORG: 'otomi', - GIT_REPO: 'values', - }, -})) - jest.mock('../common/debug', () => ({ terminal: jest.fn().mockImplementation(() => ({ info: jest.fn(), @@ -39,7 +23,6 @@ jest.mock('../common/debug', () => ({ })) jest.mock('../common/k8s', () => ({ - deletePendingHelmReleases: jest.fn(), getK8sConfigMap: jest.fn(), getK8sSecret: jest.fn(), createUpdateConfigMap: jest.fn(), @@ -49,17 +32,8 @@ jest.mock('../common/k8s', () => ({ }, })) -jest.mock('../common/hf', () => ({ - hfValues: jest.fn(), -})) - jest.mock('../common/git-config', () => ({ - getGitCredentials: jest.fn().mockResolvedValue(undefined), - getGitConfigData: jest.fn().mockResolvedValue(undefined), - getStoredGitRepoConfig: jest.fn().mockResolvedValue(undefined), - setGitConfig: jest.fn().mockResolvedValue(undefined), - GIT_CONFIG_SECRET_NAME: 'apl-git-credentials', - GIT_CONFIG_NAMESPACE: 'apl-operator', + getStoredGitRepoConfig: jest.fn(), })) jest.mock('./utils', () => ({ @@ -173,7 +147,7 @@ describe('Installer', () => { }), ) - // Verify failed status was recorded + // Verify failed status was recorded with error message expect(k8s.createUpdateConfigMap).toHaveBeenCalledWith( mockCoreApi, 'apl-installation-status', @@ -181,6 +155,7 @@ describe('Installer', () => { expect.objectContaining({ status: 'failed', attempt: '1', + error: 'Install failed', }), ) @@ -265,8 +240,15 @@ describe('Installer', () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ data: { status: 'completed' }, }) - // getK8sSecret returns credentials for git verification - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ GIT_USERNAME: 'admin', GIT_PASSWORD: 'pass' }) + // getStoredGitRepoConfig returns valid config + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://admin:pass@gitea:3000/otomi/values.git', + repoUrl: 'https://gitea:3000/otomi/values.git', + branch: 'main', + email: 'test@test.com', + username: 'admin', + password: 'pass', + }) // git ls-remote succeeds (main branch exists) mockZx.mockReturnValue({ nothrow: jest.fn().mockReturnValue({ @@ -285,7 +267,14 @@ describe('Installer', () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ data: { status: 'completed' }, }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ GIT_USERNAME: 'admin', GIT_PASSWORD: 'pass' }) + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://admin:pass@gitea:3000/otomi/values.git', + repoUrl: 'https://gitea:3000/otomi/values.git', + branch: 'main', + email: 'test@test.com', + username: 'admin', + password: 'pass', + }) // git ls-remote fails (main branch does not exist) mockZx.mockReturnValue({ nothrow: jest.fn().mockReturnValue({ @@ -335,8 +324,8 @@ describe('Installer', () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ data: { status: 'completed' }, }) - // getK8sSecret throws (cluster issues) - ;(k8s.getK8sSecret as jest.Mock).mockRejectedValue(new Error('connection refused')) + // getStoredGitRepoConfig throws (cluster issues) + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockRejectedValue(new Error('connection refused')) const isInstalled = await installer.isInstalled() @@ -346,10 +335,10 @@ describe('Installer', () => { }) describe('setEnvAndCreateSecrets', () => { - test('should use existing credentials from apl-git-credentials secret when available', async () => { - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValueOnce({ SOPS_AGE_KEY: 'existing-sops-key' }) // apl-sops-secrets + test('should use existing SOPS key from secret', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValueOnce({ SOPS_AGE_KEY: 'existing-sops-key' }) - const result = await installer.setEnvAndCreateSecrets() + await installer.setEnvAndCreateSecrets() expect(k8s.getK8sSecret).toHaveBeenCalledWith('apl-sops-secrets', 'apl-operator') expect(process.env.SOPS_AGE_KEY).toBe('existing-sops-key') @@ -364,196 +353,4 @@ describe('Installer', () => { expect(process.env.SOPS_AGE_KEY).toBe('') }) }) - - describe('ensureSecretsAndConfig', () => { - const mockValues = { - otomi: { - git: { - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - username: 'admin', - password: 's3cret', - }, - }, - kms: { - sops: { - age: { - privateKey: 'AGE-SECRET-KEY-1234', - }, - }, - }, - } - - test('should skip when hfValues returns undefined', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(undefined) - - await installer.ensureSecretsAndConfig() - - expect(gitConfig.getGitCredentials).not.toHaveBeenCalled() - expect(k8s.getK8sSecret).not.toHaveBeenCalled() - expect(gitConfig.getGitConfigData).not.toHaveBeenCalled() - }) - - test('should not recreate resources when all exist', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - - await installer.ensureSecretsAndConfig() - - expect(k8s.createUpdateGenericSecret).not.toHaveBeenCalled() - expect(gitConfig.setGitConfig).not.toHaveBeenCalled() - }) - - test('should recreate git credentials when missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue(undefined) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - - await installer.ensureSecretsAndConfig() - - expect(k8s.createUpdateGenericSecret).toHaveBeenCalledWith(mockCoreApi, 'apl-git-credentials', 'apl-operator', { - username: 'admin', - password: 's3cret', - }) - }) - - test('should recreate sops secret when missing and age key exists in values', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - - await installer.ensureSecretsAndConfig() - - expect(k8s.createUpdateGenericSecret).toHaveBeenCalledWith(mockCoreApi, 'apl-sops-secrets', 'apl-operator', { - SOPS_AGE_KEY: 'AGE-SECRET-KEY-1234', - }) - }) - - test('should not recreate sops secret when missing but no age key in values', async () => { - const valuesWithoutSops = { - ...mockValues, - kms: { sops: { age: {} } }, - } - ;(hf.hfValues as jest.Mock).mockResolvedValue(valuesWithoutSops) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - - await installer.ensureSecretsAndConfig() - - expect(k8s.createUpdateGenericSecret).not.toHaveBeenCalled() - }) - - test('should recreate git config when repoUrl is missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - branch: 'main', - email: 'pipeline@cluster.local', - }) - - await installer.ensureSecretsAndConfig() - - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - }) - - test('should recreate git config when branch is missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - email: 'pipeline@cluster.local', - }) - - await installer.ensureSecretsAndConfig() - - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - }) - - test('should recreate git config when email is missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - }) - - await installer.ensureSecretsAndConfig() - - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - }) - - test('should recreate git config when configData is undefined', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue({ username: 'admin', password: 's3cret' }) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'existing-key' }) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue(undefined) - - await installer.ensureSecretsAndConfig() - - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - }) - - test('should recreate all resources when all are missing', async () => { - ;(hf.hfValues as jest.Mock).mockResolvedValue(mockValues) - ;(gitConfig.getGitCredentials as jest.Mock).mockResolvedValue(undefined) - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) - ;(gitConfig.getGitConfigData as jest.Mock).mockResolvedValue(undefined) - - await installer.ensureSecretsAndConfig() - - expect(k8s.createUpdateGenericSecret).toHaveBeenCalledWith(mockCoreApi, 'apl-git-credentials', 'apl-operator', { - username: 'admin', - password: 's3cret', - }) - expect(k8s.createUpdateGenericSecret).toHaveBeenCalledWith(mockCoreApi, 'apl-sops-secrets', 'apl-operator', { - SOPS_AGE_KEY: 'AGE-SECRET-KEY-1234', - }) - expect(gitConfig.setGitConfig).toHaveBeenCalledWith({ - repoUrl: 'https://github.com/org/repo.git', - branch: 'main', - email: 'pipeline@cluster.local', - }) - }) - }) }) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 77aeca38a8..682c2dabf1 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -1,18 +1,10 @@ import * as process from 'node:process' import { $ } from 'zx' import { terminal } from '../common/debug' -import { - getGitConfigData, - getGitCredentials, - GIT_CONFIG_NAMESPACE, - GIT_CONFIG_SECRET_NAME, - setGitConfig, -} from '../common/git-config' -import { hfValues } from '../common/hf' -import { createUpdateConfigMap, createUpdateGenericSecret, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' +import { getStoredGitRepoConfig } from '../common/git-config' +import { createUpdateConfigMap, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' -import { operatorEnv } from './validators' export class Installer { private d = terminal('operator:installer') @@ -45,17 +37,14 @@ export class Installer { private async verifyGitRepoHasMainBranch(): Promise { try { - // Get credentials from K8s secret (created by Helm at deploy time) - const creds = await getK8sSecret('gitea-credentials', 'apl-operator') - const username = creds?.GIT_USERNAME ?? 'otomi-admin' - const password = creds?.GIT_PASSWORD ?? '' - const repoUrl = `${process.env.GIT_PROTOCOL}://${username}:${password}@${process.env.GIT_URL}:${process.env.GIT_PORT}/${operatorEnv.GIT_ORG}/${operatorEnv.GIT_REPO}.git` - const result = await $`git ls-remote --exit-code --heads ${repoUrl} main`.nothrow().quiet() + const gitConfig = await getStoredGitRepoConfig() + if (!gitConfig) return true // Can't verify without config, assume fine + const result = await $`git ls-remote --exit-code --heads ${gitConfig.authenticatedUrl} main`.nothrow().quiet() return result.exitCode === 0 } catch { // If we can't check (e.g. gitea not ready yet), assume it's fine // The operator will detect the issue later during git polling - this.d.warn('Could not verify git repo - gitea may not be ready yet') + this.d.warn('Could not verify git repo - may not be ready yet') return true } } @@ -87,13 +76,14 @@ export class Installer { // Run the installation sequence await this.updateInstallationStatus('in-progress', attemptNumber) await this.aplOps.install() - await this.ensureSecretsAndConfig() await this.updateInstallationStatus('completed', attemptNumber) return } catch (error) { - await this.updateInstallationStatus('failed', attemptNumber) - this.d.warn(`Installation attempt ${attemptNumber} failed, retrying in 1 second...`, getErrorMessage(error)) + const errorMessage = getErrorMessage(error) + this.d.error(`Installation attempt ${attemptNumber} failed:`, errorMessage) + await this.updateInstallationStatus('failed', attemptNumber, errorMessage) + this.d.warn(`Installation attempt ${attemptNumber} failed, retrying in 1 second...`) // Wait 1 second before retrying await new Promise((resolve) => setTimeout(resolve, 1000)) @@ -109,12 +99,14 @@ export class Installer { return status } - private async updateInstallationStatus(status: string, attempt: number): Promise { + private async updateInstallationStatus(status: string, attempt: number, error?: string): Promise { try { const data = { status, attempt: attempt.toString(), timestamp: new Date().toISOString(), + // Always include error field to prevent stale values from StrategicMergePatch + error: error ?? '', } await createUpdateConfigMap(k8s.core(), 'apl-installation-status', 'apl-operator', data) @@ -129,60 +121,20 @@ export class Installer { } private async setupSopsEnvironment() { - const aplSopsSecret = await getK8sSecret('apl-sops-secrets', 'apl-operator') - - if (aplSopsSecret?.SOPS_AGE_KEY) { - process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY - this.d.debug('Using existing sops credentials from secret') - } else { - // SOPS is no longer used (replaced by SealedSecrets + ESO). - // Skip hfValues() call which requires the git repo that may not exist yet. - this.d.debug('SOPS Age key not found in secret, skipping (SealedSecrets in use)') - } - } - - // public for testing. This method should only be used if you are certain there are values locally. - async ensureSecretsAndConfig(): Promise { - this.d.info('Verifying secrets and config after installation') - const values = (await hfValues()) as Record - if (!values) { - this.d.warn('Could not retrieve hfValues, skipping secrets/config verification') - return - } - - const otomiGit = values?.otomi?.git - const agePrivateKey = values?.kms?.sops?.age?.privateKey - - // Ensure apl-git-credentials secret - // Only recreate if credentials are missing AND values has the password available - // (password may be undefined when secrets are stripped from disk and managed by SealedSecrets + ESO) - const credentials = await getGitCredentials() - if (!credentials && otomiGit?.username && otomiGit?.password) { - this.d.info('Recreating apl-git-credentials secret') - await createUpdateGenericSecret(k8s.core(), GIT_CONFIG_SECRET_NAME, GIT_CONFIG_NAMESPACE, { - username: otomiGit.username, - password: otomiGit.password, - }) - } - - // Ensure apl-sops-secrets secret - const sopsSecret = await getK8sSecret('apl-sops-secrets', GIT_CONFIG_NAMESPACE) - if (!sopsSecret?.SOPS_AGE_KEY && agePrivateKey) { - this.d.info('Recreating apl-sops-secrets secret') - await createUpdateGenericSecret(k8s.core(), 'apl-sops-secrets', GIT_CONFIG_NAMESPACE, { - SOPS_AGE_KEY: agePrivateKey, - }) - } - - // Ensure apl-git-config configmap - const configData = await getGitConfigData() - if (!configData?.repoUrl || !configData?.branch || !configData?.email) { - this.d.info('Recreating apl-git-config configmap') - await setGitConfig({ - repoUrl: otomiGit?.repoUrl, - branch: otomiGit?.branch, - email: otomiGit?.email, - }) + try { + const aplSopsSecret = await getK8sSecret('apl-sops-secrets', 'apl-operator') + + if (aplSopsSecret?.SOPS_AGE_KEY) { + process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY + this.d.debug('Using existing sops credentials from secret') + } else { + // SOPS is no longer used (replaced by SealedSecrets + ESO). + // Skip hfValues() call which requires the git repo that may not exist yet. + this.d.debug('SOPS Age key not found in secret, skipping (SealedSecrets in use)') + } + } catch (error) { + this.d.error('Failed to retrieve sops credentials:', getErrorMessage(error)) + throw error } } } diff --git a/src/operator/main.ts b/src/operator/main.ts index bc8e897e60..e601f85fbe 100644 --- a/src/operator/main.ts +++ b/src/operator/main.ts @@ -1,17 +1,17 @@ import * as dotenv from 'dotenv' -import { terminal } from '../common/debug' -import { AplOperator, AplOperatorConfig } from './apl-operator' -import { Installer } from './installer' -import { operatorEnv } from './validators' -import { env } from '../common/envalid' import fs from 'fs' +import process from 'node:process' import path from 'path' +import { runTraceCollectionLoop } from '../cmd/traces' +import { terminal } from '../common/debug' +import { env } from '../common/envalid' +import { getStoredGitRepoConfig } from '../common/git-config' import { AplOperations } from './apl-operations' -import { getErrorMessage } from './utils' +import { AplOperator, AplOperatorConfig } from './apl-operator' import { GitRepository } from './git-repository' -import { getStoredGitRepoConfig } from '../common/git-config' -import process from 'node:process' -import { runTraceCollectionLoop } from '../cmd/traces' +import { Installer } from './installer' +import { getErrorMessage } from './utils' +import { operatorEnv } from './validators' dotenv.config() @@ -78,13 +78,15 @@ async function main(): Promise { await installer.reconcileInstall() } + // Set up SOPS environment if applicable (no-op when SealedSecrets + ESO is in use) + await installer.setEnvAndCreateSecrets() + // Start trace collection in background (runs for 30 minutes from ConfigMap creation) runTraceCollectionLoop().catch((error) => { d.warn('Trace collection loop failed:', getErrorMessage(error)) }) // Phase 2: Set environment variables and start operator for GitOps operations - // await installer.setEnvAndCreateSecrets() const config = await loadConfig(aplOps) const operator = new AplOperator(config) handleTerminationSignals(operator) diff --git a/values/apl-operator/apl-operator.gotmpl b/values/apl-operator/apl-operator.gotmpl index 1657090d50..56acfc091b 100644 --- a/values/apl-operator/apl-operator.gotmpl +++ b/values/apl-operator/apl-operator.gotmpl @@ -24,3 +24,4 @@ git: branch: {{ $v.otomi.git.branch | default "main" | quote }} email: {{ $v.otomi.git.email | default "pipeline@cluster.local" | quote }} username: {{ $v.otomi.git.username | default "otomi-admin" | quote }} + password: {{ $v.otomi.git.password | default "" | quote }} From c5b029f8cb5b804ed82b819a9d6ddee13970e969 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 19 Feb 2026 10:47:44 +0100 Subject: [PATCH 07/66] fix: merge conflicts/changes --- helmfile.d/helmfile-04.init.yaml.gotmpl | 1 + src/common/k8s.test.ts | 6 +++--- src/common/k8s.ts | 16 +++++++++------- src/operator/installer.test.ts | 1 + src/operator/installer.ts | 13 ++++++++++--- values/apl-operator/apl-operator-raw.gotmpl | 2 +- values/apl-operator/apl-operator.gotmpl | 2 +- 7 files changed, 26 insertions(+), 15 deletions(-) diff --git a/helmfile.d/helmfile-04.init.yaml.gotmpl b/helmfile.d/helmfile-04.init.yaml.gotmpl index df2adc1e35..6a3d9121f7 100644 --- a/helmfile.d/helmfile-04.init.yaml.gotmpl +++ b/helmfile.d/helmfile-04.init.yaml.gotmpl @@ -30,6 +30,7 @@ releases: namespace: apl-operator labels: pkg: apl-operator + app: core <<: *raw - name: otomi-operator installed: true diff --git a/src/common/k8s.test.ts b/src/common/k8s.test.ts index bc35c28693..99d454b1d4 100644 --- a/src/common/k8s.test.ts +++ b/src/common/k8s.test.ts @@ -14,8 +14,8 @@ import { V1StatefulSet, V1Status, } from '@kubernetes/client-node' -import { X509Certificate } from 'crypto' import retry from 'async-retry' +import { X509Certificate } from 'crypto' import { ARGOCD_APP_PARAMS } from './constants' import { terminal } from './debug' import { env } from './envalid' @@ -733,8 +733,8 @@ describe('helm operations in progress check', () => { await k8s.deletePendingHelmReleases() expect(mockGetPendingHelmReleases).toHaveBeenCalled() - expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(1, 'release-1', 'ns-1') - expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(3, 'release-2', 'ns-2') + expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(2, 'release-1', 'ns-1', 2) + expect(mockDeleteSecretForHelmRelease).toHaveBeenNthCalledWith(3, 'release-2', 'ns-2', 1) }) }) diff --git a/src/common/k8s.ts b/src/common/k8s.ts index e701de4f7b..dcfc943300 100644 --- a/src/common/k8s.ts +++ b/src/common/k8s.ts @@ -16,9 +16,9 @@ import { V1Status, } from '@kubernetes/client-node' import retry, { Options } from 'async-retry' +import { X509Certificate } from 'crypto' import { AnyAaaaRecord, AnyARecord } from 'dns' import { resolveAny } from 'dns/promises' -import { X509Certificate } from 'crypto' import { access, mkdir, writeFile } from 'fs/promises' import { isEmpty, isEqual, map, mapValues } from 'lodash' import { dirname, join } from 'path' @@ -149,12 +149,12 @@ export const getK8sSecret = async (name: string, namespace: string): Promise { +export const deleteSecretForHelmRelease = async (releaseName: string, namespace: string, revision = 1) => { const d = terminal('common:k8s:deleteSecretForHelmRelease') - d.info(`Deleting secret for Helm release ${releaseName} in namespace ${namespace}`) + d.info(`Deleting secret for Helm release ${releaseName} revision ${revision} in namespace ${namespace}`) try { - await coreClient.deleteNamespacedSecret({ name: `sh.helm.release.v1.${releaseName}.v1`, namespace }) - d.debug(`Deleted secret for Helm release ${releaseName} in namespace ${namespace}`) + await coreClient.deleteNamespacedSecret({ name: `sh.helm.release.v1.${releaseName}.v${revision}`, namespace }) + d.debug(`Deleted secret for Helm release ${releaseName} revision ${revision} in namespace ${namespace}`) } catch (error) { if (error?.response?.statusCode !== 404) { throw error @@ -194,9 +194,11 @@ export const deletePendingHelmReleases = async (): Promise => { const d = terminal(`common:k8s:deletePendingHelmReleases`) const pendingHelmReleases = await getPendingHelmReleases() if (pendingHelmReleases.length > 0) { - d.info(`Pending Helm operations detected for releases: ${pendingHelmReleases.join(', ')}. removing secrets...`) + d.info( + `Pending Helm operations detected for releases: ${pendingHelmReleases.map((r) => `${r.namespace}/${r.name}:v${r.revision}`).join(', ')}. removing secrets...`, + ) for (const release of pendingHelmReleases) { - await deleteSecretForHelmRelease(release.name, release.namespace) + await deleteSecretForHelmRelease(release.name, release.namespace, release.revision) } } } diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index 96a09eff0e..22dece724d 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -27,6 +27,7 @@ jest.mock('../common/k8s', () => ({ getK8sSecret: jest.fn(), createUpdateConfigMap: jest.fn(), createUpdateGenericSecret: jest.fn(), + deletePendingHelmReleases: jest.fn().mockResolvedValue(undefined), k8s: { core: jest.fn(), }, diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 682c2dabf1..e55f28edfd 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -2,7 +2,7 @@ import * as process from 'node:process' import { $ } from 'zx' import { terminal } from '../common/debug' import { getStoredGitRepoConfig } from '../common/git-config' -import { createUpdateConfigMap, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' +import { createUpdateConfigMap, deletePendingHelmReleases, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' @@ -83,9 +83,16 @@ export class Installer { const errorMessage = getErrorMessage(error) this.d.error(`Installation attempt ${attemptNumber} failed:`, errorMessage) await this.updateInstallationStatus('failed', attemptNumber, errorMessage) - this.d.warn(`Installation attempt ${attemptNumber} failed, retrying in 1 second...`) - // Wait 1 second before retrying + // Clean up stuck Helm releases (e.g. pending-install, pending-upgrade) + // so the next retry can proceed without "another operation is in progress" errors + try { + await deletePendingHelmReleases() + } catch (cleanupError) { + this.d.warn('Failed to clean up pending Helm releases:', getErrorMessage(cleanupError)) + } + + this.d.warn(`Installation attempt ${attemptNumber} failed, retrying in 1 second...`) await new Promise((resolve) => setTimeout(resolve, 1000)) } } diff --git a/values/apl-operator/apl-operator-raw.gotmpl b/values/apl-operator/apl-operator-raw.gotmpl index 1ad5c4a39b..bd18d18d0e 100644 --- a/values/apl-operator/apl-operator-raw.gotmpl +++ b/values/apl-operator/apl-operator-raw.gotmpl @@ -16,7 +16,7 @@ resources: template: type: Opaque data: - GIT_USERNAME: {{ $v.otomi.git.username | default "otomi-admin" }} + GIT_USERNAME: {{ $v.otomi.git | get "username" "otomi-admin" }} GIT_PASSWORD: '{{ "{{ .git_password | toString }}" }}' data: - secretKey: git_password diff --git a/values/apl-operator/apl-operator.gotmpl b/values/apl-operator/apl-operator.gotmpl index 56acfc091b..bfe276488b 100644 --- a/values/apl-operator/apl-operator.gotmpl +++ b/values/apl-operator/apl-operator.gotmpl @@ -24,4 +24,4 @@ git: branch: {{ $v.otomi.git.branch | default "main" | quote }} email: {{ $v.otomi.git.email | default "pipeline@cluster.local" | quote }} username: {{ $v.otomi.git.username | default "otomi-admin" | quote }} - password: {{ $v.otomi.git.password | default "" | quote }} + password: {{ $v.otomi.git | get "password" "" | quote }} From 4fd4dfefccf19aadb0f762e316f587fd1b4be3ff Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 19 Feb 2026 14:29:35 +0100 Subject: [PATCH 08/66] fix: merge conflicts/changes --- src/cmd/bootstrap.test.ts | 31 +++++++++++++++++++++++++++++++ src/cmd/bootstrap.ts | 5 +++++ 2 files changed, 36 insertions(+) diff --git a/src/cmd/bootstrap.test.ts b/src/cmd/bootstrap.test.ts index 6cfb0f22a1..0a1b004fc7 100644 --- a/src/cmd/bootstrap.test.ts +++ b/src/cmd/bootstrap.test.ts @@ -449,6 +449,37 @@ describe('Bootstrapping values', () => { merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { users: processedUsers }), ) }) + it('should preserve existing groups when users are recovered from stored secrets (bootstrap retry)', async () => { + // Simulate bootstrap retry: stored secrets contain processed users (with groups, without isPlatformAdmin) + const storedProcessedUsers = [ + { + email: 'platform-admin@example.com', + firstName: 'platform', + lastName: 'admin', + initialPassword: 'existing-pass', + groups: ['platform-admin'], + }, + ] + deps.loadYaml.mockReturnValue({}) + deps.getStoredClusterSecrets.mockReturnValue({ users: storedProcessedUsers }) + deps.generateSecrets.mockReturnValue({}) + deps.createCustomCA.mockReturnValue({}) + // getUsers returns the stored processed users (no isPlatformAdmin flag) + deps.getUsers.mockReturnValue(storedProcessedUsers) + + const result = await processValues(deps) + + // Groups should be preserved from existing data, not reset to [] + expect(result.allSecrets.users).toEqual([ + { + email: 'platform-admin@example.com', + firstName: 'platform', + lastName: 'admin', + initialPassword: 'existing-pass', + groups: ['platform-admin'], + }, + ]) + }) }) }) }) diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index f5d4c13f60..99da59dac5 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -346,6 +346,11 @@ export const processValues = async ( if (user.isPlatformAdmin) groups.push('platform-admin') if (user.isTeamAdmin) groups.push('team-admin') for (const team of user.teams || []) groups.push(`team-${team}`) + // Preserve existing groups when boolean flags are absent (e.g., user recovered + // from stored secrets which uses the processed format without isPlatformAdmin/isTeamAdmin) + if (groups.length === 0 && Array.isArray(user.groups) && user.groups.length > 0) { + groups.push(...(user.groups as string[])) + } return { email: user.email, firstName: user.firstName, From 0d8ca8478eba61ba91e01d6b1f3bffb3f24365db Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 19 Feb 2026 16:00:55 +0100 Subject: [PATCH 09/66] feat: waiting for sealed secrets --- src/cmd/install.ts | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 8a221696cf..1416f0de83 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -89,31 +89,35 @@ const waitForSealedSecrets = async ( } d.info(`Waiting for ${secretsToWait.size} sealed secrets to be decrypted`) - const start = Date.now() - - while (Date.now() - start < timeoutMs) { - const pending: string[] = [] - for (const { namespace, secretName } of secretsToWait.values()) { - try { - const secret = await deps.getK8sSecret(secretName, namespace) - if (!secret) { + + await retry( + async () => { + const pending: string[] = [] + for (const { namespace, secretName } of secretsToWait.values()) { + try { + const secret = await deps.getK8sSecret(secretName, namespace) + if (!secret) { + pending.push(`${namespace}/${secretName}`) + } + } catch { pending.push(`${namespace}/${secretName}`) } - } catch { - pending.push(`${namespace}/${secretName}`) } - } - if (pending.length === 0) { - d.info('All sealed secrets have been decrypted') - return - } - - d.info(`Still waiting for sealed secrets: ${pending.join(', ')}`) - await new Promise((resolve) => setTimeout(resolve, intervalMs)) - } + if (pending.length > 0) { + d.info(`Still waiting for sealed secrets: ${pending.join(', ')}`) + throw new Error(`Sealed secrets not yet decrypted: ${pending.join(', ')}`) + } - throw new Error(`Timed out waiting for sealed secrets to be decrypted after ${timeoutMs}ms`) + d.info('All sealed secrets have been decrypted') + }, + { + retries: Math.ceil(timeoutMs / intervalMs), + minTimeout: intervalMs, + maxTimeout: intervalMs, + factor: 1, + }, + ) } export const installAll = async () => { From 431bc0fe84bdae144df4f6cd5876cb81a4a02d67 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 19 Feb 2026 16:11:06 +0100 Subject: [PATCH 10/66] feat: move function to k8s.ts --- src/common/k8s.ts | 26 ++++++++++++++++++++++++++ src/common/sealed-secrets.ts | 27 +-------------------------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/common/k8s.ts b/src/common/k8s.ts index dcfc943300..f6e0c6726c 100644 --- a/src/common/k8s.ts +++ b/src/common/k8s.ts @@ -962,6 +962,32 @@ export async function waitForCRD(crdName: string, timeoutSeconds: number = 60): } } +/** + * Ensure a namespace exists. If it doesn't exist, create it with proper labels. + * This avoids overwriting labels on existing namespaces that were created by k8s-raw.gotmpl. + */ +export const ensureNamespaceExists = async (namespace: string, deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:k8s:ensureNamespaceExists`) + + // Check if namespace already exists + const existingNs = await deps.$`kubectl get namespace ${namespace}`.nothrow().quiet() + if (existingNs.exitCode === 0) { + d.debug(`Namespace ${namespace} already exists`) + return + } + + // Create namespace with proper label + d.info(`Creating namespace ${namespace}`) + const nsYaml = `apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} + labels: + name: ${namespace}` + + await deps.$`echo ${nsYaml} | kubectl apply -f -`.nothrow().quiet() +} + export async function getSealedSecretsPEM(): Promise { const d = terminal('common:k8s:getSealedSecretsPEM') const namespace = 'sealed-secrets' diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 33c37c95a5..84d12fd242 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -6,6 +6,7 @@ import { cloneDeep, get, unset } from 'lodash' import { pki } from 'node-forge' import { join } from 'path' import { terminal } from 'src/common/debug' +import { ensureNamespaceExists } from 'src/common/k8s' import { flattenObject, getSchemaSecretsPaths } from 'src/common/utils' import { objectToYaml } from 'src/common/values' import { $ } from 'zx' @@ -25,32 +26,6 @@ export function stripAllSecrets(values: Record, secretPaths: string return stripped } -/** - * Ensure a namespace exists. If it doesn't exist, create it with proper labels. - * This avoids overwriting labels on existing namespaces that were created by k8s-raw.gotmpl. - */ -export const ensureNamespaceExists = async (namespace: string, deps = { $, terminal }): Promise => { - const d = deps.terminal(`common:${cmdName}:ensureNamespaceExists`) - - // Check if namespace already exists - const existingNs = await deps.$`kubectl get namespace ${namespace}`.nothrow().quiet() - if (existingNs.exitCode === 0) { - d.debug(`Namespace ${namespace} already exists`) - return - } - - // Create namespace with proper label - d.info(`Creating namespace ${namespace}`) - const nsYaml = `apiVersion: v1 -kind: Namespace -metadata: - name: ${namespace} - labels: - name: ${namespace}` - - await deps.$`echo ${nsYaml} | kubectl apply -f -`.nothrow().quiet() -} - export interface SecretMapping { namespace: string secretName: string From 8ac4c9696ac9b8813683104ea7bae42b4edabe84 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 20 Feb 2026 10:04:28 +0100 Subject: [PATCH 11/66] feat: use kubernetes package instead of kubectl --- src/common/k8s.ts | 35 ++--- src/common/sealed-secrets.test.ts | 67 ++++----- src/common/sealed-secrets.ts | 220 ++++++++++++++++++++---------- 3 files changed, 189 insertions(+), 133 deletions(-) diff --git a/src/common/k8s.ts b/src/common/k8s.ts index f6e0c6726c..6f37001e5c 100644 --- a/src/common/k8s.ts +++ b/src/common/k8s.ts @@ -966,26 +966,27 @@ export async function waitForCRD(crdName: string, timeoutSeconds: number = 60): * Ensure a namespace exists. If it doesn't exist, create it with proper labels. * This avoids overwriting labels on existing namespaces that were created by k8s-raw.gotmpl. */ -export const ensureNamespaceExists = async (namespace: string, deps = { $, terminal }): Promise => { - const d = deps.terminal(`common:k8s:ensureNamespaceExists`) +export const ensureNamespaceExists = async (namespace: string): Promise => { + const d = terminal(`common:k8s:ensureNamespaceExists`) - // Check if namespace already exists - const existingNs = await deps.$`kubectl get namespace ${namespace}`.nothrow().quiet() - if (existingNs.exitCode === 0) { + try { + await k8s.core().readNamespace({ name: namespace }) d.debug(`Namespace ${namespace} already exists`) - return + } catch (error) { + if (error instanceof ApiException && error.code === 404) { + d.info(`Creating namespace ${namespace}`) + await k8s.core().createNamespace({ + body: { + metadata: { + name: namespace, + labels: { name: namespace }, + }, + }, + }) + } else { + throw error + } } - - // Create namespace with proper label - d.info(`Creating namespace ${namespace}`) - const nsYaml = `apiVersion: v1 -kind: Namespace -metadata: - name: ${namespace} - labels: - name: ${namespace}` - - await deps.$`echo ${nsYaml} | kubectl apply -f -`.nothrow().quiet() } export async function getSealedSecretsPEM(): Promise { diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index cadefb92cc..7dbd238273 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -19,12 +19,25 @@ jest.mock('@linode/kubeseal-encrypt', () => ({ encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), })) -jest.mock('zx', () => ({ - $: jest.fn().mockReturnValue({ - nothrow: jest.fn().mockReturnValue({ - quiet: jest.fn().mockResolvedValue({ stderr: '', exitCode: 0 }), +jest.mock('src/common/k8s', () => ({ + getK8sSecret: jest.fn().mockResolvedValue(undefined), + ensureNamespaceExists: jest.fn().mockResolvedValue(undefined), + b64enc: jest.fn((v: string) => Buffer.from(v).toString('base64')), + k8s: { + core: jest.fn().mockReturnValue({ + createNamespacedSecret: jest.fn().mockResolvedValue({}), }), - }), + app: jest.fn().mockReturnValue({ + patchNamespacedDeployment: jest.fn().mockResolvedValue({}), + readNamespacedDeployment: jest + .fn() + .mockResolvedValue({ spec: { replicas: 1 }, status: { updatedReplicas: 1, availableReplicas: 1 } }), + }), + custom: jest.fn().mockReturnValue({ + createNamespacedCustomObject: jest.fn().mockResolvedValue({}), + patchNamespacedCustomObject: jest.fn().mockResolvedValue({}), + }), + }, })) jest.mock('src/common/envalid', () => ({ @@ -125,59 +138,27 @@ describe('sealed-secrets', () => { describe('createSealedSecretsKeySecret', () => { it('should create secret if it does not exist', async () => { - const mockQuiet = jest.fn().mockResolvedValue({ stderr: '', exitCode: 0 }) - const mockNothrow = jest.fn().mockReturnValue({ quiet: mockQuiet }) - // First call (namespace): success, Second call (check exists): not found (exitCode 1) - // Third call (create): success, Fourth call (label): success - const mock$ = jest - .fn() - .mockReturnValueOnce({ - nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), - }) // namespace - .mockReturnValueOnce({ - nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 1 }) }), - }) // check exists - NOT FOUND - .mockReturnValueOnce({ - nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), - }) // create - .mockReturnValueOnce({ - nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), - }) // label + const mockGetK8sSecret = jest.fn().mockResolvedValue(undefined) const deps = { - $: mock$ as any, + getK8sSecret: mockGetK8sSecret, terminal, - writeFile: jest.fn(), - mkdir: jest.fn(), } await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) - expect(mock$).toHaveBeenCalledTimes(4) - expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.crt', 'cert-pem') - expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.key', 'key-pem') + expect(mockGetK8sSecret).toHaveBeenCalledWith('sealed-secrets-key', 'sealed-secrets') }) it('should skip creation if secret already exists', async () => { - const mock$ = jest - .fn() - .mockReturnValueOnce({ - nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), - }) // namespace - .mockReturnValueOnce({ - nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), - }) // check exists - FOUND + const mockGetK8sSecret = jest.fn().mockResolvedValue({ 'tls.crt': 'existing-cert' }) const deps = { - $: mock$ as any, + getK8sSecret: mockGetK8sSecret, terminal, - writeFile: jest.fn(), - mkdir: jest.fn(), } await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) - // Should only call namespace and check exists, not create or label - expect(mock$).toHaveBeenCalledTimes(2) - expect(deps.writeFile).not.toHaveBeenCalled() + expect(mockGetK8sSecret).toHaveBeenCalledWith('sealed-secrets-key', 'sealed-secrets') }) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 84d12fd242..271281b76a 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -1,3 +1,4 @@ +import { ApiException, PatchStrategy, setHeaderOptions } from '@kubernetes/client-node' import { encryptSecretItem } from '@linode/kubeseal-encrypt' import { X509Certificate } from 'crypto' import { existsSync } from 'fs' @@ -6,10 +7,10 @@ import { cloneDeep, get, unset } from 'lodash' import { pki } from 'node-forge' import { join } from 'path' import { terminal } from 'src/common/debug' -import { ensureNamespaceExists } from 'src/common/k8s' +import { b64enc, ensureNamespaceExists, getK8sSecret, k8s } from 'src/common/k8s' import { flattenObject, getSchemaSecretsPaths } from 'src/common/utils' import { objectToYaml } from 'src/common/values' -import { $ } from 'zx' +import { parse as parseYaml } from 'yaml' const cmdName = 'sealed-secrets' @@ -136,28 +137,30 @@ export const getPemFromCertificate = (certificate: string): string => { /** * Get the existing sealed-secrets certificate from the cluster if it exists. * Returns the certificate PEM string or undefined if not found. + * Note: Uses k8s client directly instead of getK8sSecret() because PEM certificates + * are corrupted by the YAML parse step in getK8sSecret(). */ -export const getExistingSealedSecretsCert = async (deps = { $, terminal }): Promise => { +export const getExistingSealedSecretsCert = async (deps = { k8s, terminal }): Promise => { const d = deps.terminal(`common:${cmdName}:getExistingSealedSecretsCert`) - const result = - await deps.$`kubectl get secret sealed-secrets-key -n sealed-secrets -o jsonpath='{.data.tls\\.crt}' 2>/dev/null` - .nothrow() - .quiet() - - if (result.exitCode !== 0 || !result.stdout || result.stdout === '') { - d.info('No existing sealed-secrets-key found') - return undefined - } - try { - const certBase64 = result.stdout.replace(/'/g, '') - const cert = Buffer.from(certBase64, 'base64').toString('utf-8') + const secret = await deps.k8s.core().readNamespacedSecret({ + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + }) + if (!secret?.data?.['tls.crt']) { + d.info('No existing sealed-secrets-key found') + return undefined + } + d.info('Found existing sealed-secrets-key certificate') - return cert - } catch { - d.warn('Failed to decode existing certificate') - return undefined + return Buffer.from(secret.data['tls.crt'], 'base64').toString('utf-8') + } catch (error) { + if (error instanceof ApiException && error.code === 404) { + d.info('No existing sealed-secrets-key found') + return undefined + } + throw error } } @@ -169,46 +172,38 @@ export const getExistingSealedSecretsCert = async (deps = { $, terminal }): Prom export const createSealedSecretsKeySecret = async ( certificate: string, privateKey: string, - deps = { $, terminal, writeFile, mkdir }, + deps = { getK8sSecret, terminal }, ): Promise => { const d = deps.terminal(`common:${cmdName}:createSealedSecretsKeySecret`) - // Create namespace if it doesn't exist - await ensureNamespaceExists('sealed-secrets', { $: deps.$, terminal: deps.terminal }) + await ensureNamespaceExists('sealed-secrets') // Check if secret already exists - const existingSecret = await deps.$`kubectl get secret sealed-secrets-key -n sealed-secrets`.nothrow().quiet() - if (existingSecret.exitCode === 0) { + const existing = await deps.getK8sSecret('sealed-secrets-key', 'sealed-secrets') + if (existing) { d.info('sealed-secrets-key already exists, skipping creation') return } d.info('Creating sealed-secrets TLS secret') - // Write temp files for kubectl create secret tls - const tmpDir = '/tmp/sealed-secrets-bootstrap' - await deps.mkdir(tmpDir, { recursive: true }) - const certPath = `${tmpDir}/tls.crt` - const keyPath = `${tmpDir}/tls.key` - await deps.writeFile(certPath, certificate) - await deps.writeFile(keyPath, privateKey) - - // Create the TLS secret (only if it doesn't exist) - const result = - await deps.$`kubectl create secret tls sealed-secrets-key -n sealed-secrets --cert=${certPath} --key=${keyPath}` - .nothrow() - .quiet() - if (result.exitCode !== 0) { - d.error(`Failed to create sealed-secrets-key: ${result.stderr}`) - return - } - - // Label the secret so the controller picks it up - const labelResult = - await deps.$`kubectl label secret sealed-secrets-key -n sealed-secrets sealedsecrets.bitnami.com/sealed-secrets-key=active --overwrite` - .nothrow() - .quiet() - if (labelResult.stderr) d.error(labelResult.stderr) + await k8s.core().createNamespacedSecret({ + namespace: 'sealed-secrets', + body: { + metadata: { + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + labels: { + 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active', + }, + }, + type: 'kubernetes.io/tls', + data: { + 'tls.crt': b64enc(certificate), + 'tls.key': b64enc(privateKey), + }, + }, + }) d.info('Created sealed-secrets TLS secret with key label') } @@ -436,7 +431,7 @@ export const writeSealedSecretManifests = async ( */ export const applySealedSecretManifests = async ( manifests: SealedSecretManifest[], - deps = { $, terminal, objectToYaml }, + deps = { terminal }, ): Promise => { const d = deps.terminal(`common:${cmdName}:applySealedSecretManifests`) @@ -452,14 +447,34 @@ export const applySealedSecretManifests = async ( // Ensure namespaces exist and apply manifests for (const [namespace, nsManifests] of byNamespace) { - await ensureNamespaceExists(namespace, { $: deps.$, terminal: deps.terminal }) + await ensureNamespaceExists(namespace) for (const manifest of nsManifests) { d.info(`Applying SealedSecret ${manifest.metadata.name} to namespace ${namespace}`) - const yaml = deps.objectToYaml(manifest) - const result = await deps.$`echo ${yaml} | kubectl apply -f -`.nothrow().quiet() - if (result.exitCode !== 0) { - d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${result.stderr}`) + try { + await k8s.custom().createNamespacedCustomObject({ + group: 'bitnami.com', + version: 'v1alpha1', + namespace, + plural: 'sealedsecrets', + body: manifest, + }) + } catch (error) { + if (error instanceof ApiException && error.code === 409) { + await k8s.custom().patchNamespacedCustomObject( + { + group: 'bitnami.com', + version: 'v1alpha1', + namespace, + plural: 'sealedsecrets', + name: manifest.metadata.name, + body: manifest, + }, + setHeaderOptions('Content-Type', PatchStrategy.MergePatch), + ) + } else { + d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${error}`) + } } } } @@ -473,7 +488,7 @@ export const applySealedSecretManifests = async ( */ export const applySealedSecretManifestsFromDir = async ( envDir: string, - deps = { $, terminal, readdir, readFile, existsSync }, + deps = { terminal, readdir, readFile, existsSync }, ): Promise => { const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) const manifestsDir = join(envDir, 'env/manifests/ns') @@ -494,8 +509,7 @@ export const applySealedSecretManifestsFromDir = async ( const namespace = nsEntry.name const nsDir = join(manifestsDir, namespace) - // Ensure namespace exists with proper labels - await ensureNamespaceExists(namespace, { $: deps.$, terminal: deps.terminal }) + await ensureNamespaceExists(namespace) // Read all YAML files in the namespace directory const files = await deps.readdir(nsDir) @@ -504,11 +518,39 @@ export const applySealedSecretManifestsFromDir = async ( const filePath = join(nsDir, file) d.info(`Applying SealedSecret from ${filePath}`) - const result = await deps.$`kubectl apply -f ${filePath}`.nothrow().quiet() - if (result.exitCode !== 0) { - d.error(`Failed to apply SealedSecret from ${filePath}: ${result.stderr}`) - } else { - appliedCount += 1 + try { + const content = await deps.readFile(filePath, 'utf-8') + const manifest = parseYaml(content) as SealedSecretManifest + + try { + await k8s.custom().createNamespacedCustomObject({ + group: 'bitnami.com', + version: 'v1alpha1', + namespace, + plural: 'sealedsecrets', + body: manifest, + }) + appliedCount += 1 + } catch (error) { + if (error instanceof ApiException && error.code === 409) { + await k8s.custom().patchNamespacedCustomObject( + { + group: 'bitnami.com', + version: 'v1alpha1', + namespace, + plural: 'sealedsecrets', + name: manifest.metadata.name, + body: manifest, + }, + setHeaderOptions('Content-Type', PatchStrategy.MergePatch), + ) + appliedCount += 1 + } else { + d.error(`Failed to apply SealedSecret from ${filePath}: ${error}`) + } + } + } catch (parseError) { + d.error(`Failed to parse SealedSecret from ${filePath}: ${parseError}`) } } } @@ -521,25 +563,57 @@ export const applySealedSecretManifestsFromDir = async ( * This is needed because if the controller starts before the sealed-secrets-key secret exists, * it will generate its own key. Restarting forces it to pick up the existing key. */ -export const restartSealedSecretsController = async (deps = { $, terminal }): Promise => { +export const restartSealedSecretsController = async (deps = { terminal }): Promise => { const d = deps.terminal(`common:${cmdName}:restartSealedSecretsController`) d.info('Restarting sealed-secrets controller to ensure correct key is used') - const result = await deps.$`kubectl rollout restart deployment/sealed-secrets -n sealed-secrets`.nothrow().quiet() - if (result.exitCode !== 0) { - d.warn(`Failed to restart sealed-secrets controller: ${result.stderr}`) + try { + await k8s.app().patchNamespacedDeployment( + { + name: 'sealed-secrets', + namespace: 'sealed-secrets', + body: { + spec: { + template: { + metadata: { + annotations: { + 'kubectl.kubernetes.io/restartedAt': new Date().toISOString(), + }, + }, + }, + }, + }, + }, + setHeaderOptions('Content-Type', PatchStrategy.StrategicMergePatch), + ) + } catch (error) { + d.warn(`Failed to restart sealed-secrets controller: ${error}`) return } d.info('Waiting for sealed-secrets controller rollout') - const waitResult = await deps.$`kubectl rollout status deployment/sealed-secrets -n sealed-secrets --timeout=120s` - .nothrow() - .quiet() - if (waitResult.exitCode !== 0) { - d.warn(`Rollout status check failed: ${waitResult.stderr}`) - } else { - d.info('Sealed-secrets controller restarted successfully') + const timeoutMs = 120000 + const intervalMs = 3000 + const start = Date.now() + while (Date.now() - start < timeoutMs) { + try { + const deployment = await k8s.app().readNamespacedDeployment({ + name: 'sealed-secrets', + namespace: 'sealed-secrets', + }) + const desired = deployment.spec?.replicas ?? 1 + const updated = deployment.status?.updatedReplicas ?? 0 + const available = deployment.status?.availableReplicas ?? 0 + if (updated >= desired && available >= desired) { + d.info('Sealed-secrets controller restarted successfully') + return + } + } catch { + // Ignore transient read errors during rollout + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)) } + d.warn('Rollout status check timed out') } /** From 16f6b279352deff273deedbd372668e31008b806 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 20 Feb 2026 10:47:23 +0100 Subject: [PATCH 12/66] fix: sealed secret tests --- src/common/sealed-secrets.ts | 61 +++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 271281b76a..8bbe855eca 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -160,7 +160,10 @@ export const getExistingSealedSecretsCert = async (deps = { k8s, terminal }): Pr d.info('No existing sealed-secrets-key found') return undefined } - throw error + // When the cluster is unreachable (e.g., CI environment without a real cluster), + // treat it as no existing cert found and let bootstrap generate a new key pair. + d.info(`Could not reach cluster to check for existing cert: ${error instanceof Error ? error.message : error}`) + return undefined } } @@ -176,36 +179,44 @@ export const createSealedSecretsKeySecret = async ( ): Promise => { const d = deps.terminal(`common:${cmdName}:createSealedSecretsKeySecret`) - await ensureNamespaceExists('sealed-secrets') + try { + await ensureNamespaceExists('sealed-secrets') - // Check if secret already exists - const existing = await deps.getK8sSecret('sealed-secrets-key', 'sealed-secrets') - if (existing) { - d.info('sealed-secrets-key already exists, skipping creation') - return - } + // Check if secret already exists + const existing = await deps.getK8sSecret('sealed-secrets-key', 'sealed-secrets') + if (existing) { + d.info('sealed-secrets-key already exists, skipping creation') + return + } - d.info('Creating sealed-secrets TLS secret') + d.info('Creating sealed-secrets TLS secret') - await k8s.core().createNamespacedSecret({ - namespace: 'sealed-secrets', - body: { - metadata: { - name: 'sealed-secrets-key', - namespace: 'sealed-secrets', - labels: { - 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active', + await k8s.core().createNamespacedSecret({ + namespace: 'sealed-secrets', + body: { + metadata: { + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + labels: { + 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active', + }, + }, + type: 'kubernetes.io/tls', + data: { + 'tls.crt': b64enc(certificate), + 'tls.key': b64enc(privateKey), }, }, - type: 'kubernetes.io/tls', - data: { - 'tls.crt': b64enc(certificate), - 'tls.key': b64enc(privateKey), - }, - }, - }) + }) - d.info('Created sealed-secrets TLS secret with key label') + d.info('Created sealed-secrets TLS secret with key label') + } catch (error) { + // When the cluster is unreachable (e.g., CI/bootstrap without a real cluster), + // skip secret creation. The secret will be created during install when the cluster is available. + d.info( + `Could not create sealed-secrets-key in cluster (will be created during install): ${error instanceof Error ? error.message : error}`, + ) + } } /** From 9f62106fdc92c83abe0f71b06b9ad6012284e6a4 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 20 Feb 2026 14:03:03 +0100 Subject: [PATCH 13/66] feat: remove init and prepare endpoints --- src/server.ts | 43 ------------------------------------------- 1 file changed, 43 deletions(-) diff --git a/src/server.ts b/src/server.ts index d989d8fc1e..65f24d1106 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,12 +1,8 @@ import $RefParser, { JSONSchema } from '@apidevtools/json-schema-ref-parser' import express, { Request, Response } from 'express' -import { copyFile } from 'fs/promises' import { Server } from 'http' -import { bootstrapSops } from 'src/cmd/bootstrap' -import { decrypt, encrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { hfValues } from './common/hf' -import { setValuesFile, unsetValuesFile } from './common/repo' import { loadYaml, rootDir } from './common/utils' import { objectToYaml } from './common/values' @@ -24,47 +20,8 @@ app.get('/', async (req: Request, res: Response): Promise => { type QueryParams = { envDir: string - files?: string[] } -app.get('/init', async (req: Request, res: Response): Promise => { - const { envDir } = req.query as QueryParams - try { - d.log('Request to initialize values repo on', envDir) - await decrypt(envDir) - res.status(200).send('ok') - } catch (error) { - d.error(error) - res.status(500).send(`${error}`) - } -}) - -app.get('/prepare', async (req: Request, res: Response): Promise => { - const { envDir, files } = req.query as QueryParams - try { - d.log('Request to prepare values repo on', envDir) - const file = '.editorconfig' - await copyFile(`${rootDir}/.values/${file}`, `${envDir}/${file}`) - await bootstrapSops(envDir) - await setValuesFile(envDir) - // Encrypt ensures that a brand new secret file is encrypted in place - await encrypt(envDir, ...(files ?? [])) - // Decrypt ensures that a brand new encrypted secret file is decrypted to the .dec file - await decrypt(envDir, ...(files ?? [])) - res.status(200).send('ok') - } catch (error) { - const err = `${error}` - let status = 500 - d.error(`Request to prepare values went wrong: ${err}`) - if (err.includes('Values validation FAILED')) { - status = 422 - } - res.status(status).send(err) - } finally { - await unsetValuesFile(envDir) - } -}) - function parseBoolean(string: any, defaultValue = false): boolean { return string === 'true' ? true : string === 'false' ? false : defaultValue } From ea466f9ea016d9547518e3ad8ee4f8458eacc196 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 20 Feb 2026 15:45:17 +0100 Subject: [PATCH 14/66] fix: harbor secrets --- values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl | 2 +- values/harbor/harbor-raw.gotmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl index 72ad5eb791..0eaf85f5a3 100644 --- a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl +++ b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl @@ -35,7 +35,7 @@ resources: data: - secretKey: harborAdminPassword remoteRef: - key: harbor-secrets + key: otomi-platform-secrets property: adminPassword - secretKey: keycloakClientSecret remoteRef: diff --git a/values/harbor/harbor-raw.gotmpl b/values/harbor/harbor-raw.gotmpl index feb9862641..fde1bcc6c9 100644 --- a/values/harbor/harbor-raw.gotmpl +++ b/values/harbor/harbor-raw.gotmpl @@ -52,7 +52,7 @@ resources: data: - secretKey: adminPassword remoteRef: - key: harbor-secrets + key: otomi-platform-secrets property: adminPassword - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret From db046ba7c37ecbe9c71332aa3e467ec82a98cf7a Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 20 Feb 2026 16:26:53 +0100 Subject: [PATCH 15/66] feat: update tools image and remove /apl/schema endpoint --- .github/workflows/otomi-tools-build-push.yaml | 1 + charts/otomi-api/values.yaml | 7 -- src/server.ts | 9 --- tools/Dockerfile | 65 +------------------ versions.yaml | 4 +- 5 files changed, 4 insertions(+), 82 deletions(-) diff --git a/.github/workflows/otomi-tools-build-push.yaml b/.github/workflows/otomi-tools-build-push.yaml index 050f74b65a..3e5a7e0677 100644 --- a/.github/workflows/otomi-tools-build-push.yaml +++ b/.github/workflows/otomi-tools-build-push.yaml @@ -11,6 +11,7 @@ on: push: branches: - 'main' + - 'APL-523' env: NAMESPACE: linode diff --git a/charts/otomi-api/values.yaml b/charts/otomi-api/values.yaml index f4ddb4695e..280ebff005 100644 --- a/charts/otomi-api/values.yaml +++ b/charts/otomi-api/values.yaml @@ -112,10 +112,3 @@ tools: # requests: # cpu: 100m # memory: 100Mi - - secrets: - SOPS_AGE_KEY: '' - GCLOUD_SERVICE_KEY: '' - AZURE_TENANT_ID: '' - AZURE_CLIENT_ID: '' - AZURE_CLIENT_SECRET: '' diff --git a/src/server.ts b/src/server.ts index 65f24d1106..347cb12e5b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,9 +1,7 @@ -import $RefParser, { JSONSchema } from '@apidevtools/json-schema-ref-parser' import express, { Request, Response } from 'express' import { Server } from 'http' import { terminal } from 'src/common/debug' import { hfValues } from './common/hf' -import { loadYaml, rootDir } from './common/utils' import { objectToYaml } from './common/values' const d = terminal('server') @@ -44,13 +42,6 @@ app.get('/otomi/values', async (req: Request, res: Response): Promise => { } }) -app.get('/apl/schema', async (req: Request, res: Response): Promise => { - const schema = await loadYaml(`${rootDir}/values-schema.yaml`) - const derefSchema = await $RefParser.dereference(schema as JSONSchema) - res.setHeader('Content-type', 'application/json') - res.status(200).send(derefSchema) -}) - export const startServer = (): void => { server = app .listen(17771, '127.0.0.1', () => { diff --git a/tools/Dockerfile b/tools/Dockerfile index af6f1b4836..b957ee09fc 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -3,30 +3,14 @@ FROM ubuntu:24.04 AS builder ARG DEBIAN_FRONTEND=noninteractive ARG TARGETARCH -# https://github.com/kubernetes/kubernetes/releases -ARG KUBECTL_VERSION=1.34.2 # https://github.com/helm/helm/tags ARG HELM_VERSION=3.19.2 # https://github.com/databus23/helm-diff/releases ARG HELM_DIFF_VERSION=3.14.1 -# https://github.com/jkroepke/helm-secrets/releases -ARG HELM_SECRETS_VERSION=4.7.4 -# https://github.com/mozilla/sops/releases -ARG SOPS_VERSION=3.11.0 -# https://github.com/FiloSottile/age/releases -ARG AGE_VERSION=1.2.1 -# https://github.com/noqcks/gucci/releases -ARG GUCCI_VERSION=1.9.0 -# https://github.com/yannh/kubeconform/releases/ -ARG KUBECONFORM_VERSION=0.7.0 # https://github.com/helmfile/helmfile/releases ARG HELMFILE_VERSION=1.2.2 # https://nodejs.org/en/download/ ARG NODE_VERSION=24 -# https://github.com/cloudnative-pg/cloudnative-pg/releases -ARG CNPG_VERSION=1.27.1 -# https://github.com/jqlang/jq/releases/ -ARG JQ_VERSION=1.8.1 ARG HELM_FILE_NAME=helm-v${HELM_VERSION}-linux-${TARGETARCH}.tar.gz WORKDIR / @@ -34,26 +18,12 @@ WORKDIR / # Install all required packages in one layer RUN apt-get update && apt-get install -y \ curl \ - coreutils \ - apache2-utils \ - apt-transport-https \ ca-certificates \ git \ - locales \ - rsync && \ + locales && \ rm -rf /var/lib/apt/lists/* && \ locale-gen en_US.UTF-8 -# jq -RUN curl -LO "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${TARGETARCH}" && \ - curl -L "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/sha256sum.txt" | grep "jq-linux-${TARGETARCH}" > sha256sum.txt && \ - echo sha256sum --check sha256sum.txt && \ - mv jq-linux-${TARGETARCH} /usr/bin/jq && \ - chmod +x /usr/bin/jq - -# yq -COPY --from=mikefarah/yq:4 /usr/bin/yq /usr/bin/yq - RUN mkdir -p /home/app RUN groupadd -r app && \ useradd -r -g app -d /home/app -s /sbin/nologin -c "Docker image user" app @@ -63,48 +33,15 @@ RUN mkdir $APP_HOME WORKDIR $APP_HOME ENV PATH $PATH:$APP_HOME -# kubectl -RUN curl -LO "https://dl.k8s.io/release/v$KUBECTL_VERSION/bin/linux/$TARGETARCH/kubectl" && \ - curl -LO "https://dl.k8s.io/release/v$KUBECTL_VERSION/bin/linux/$TARGETARCH/kubectl.sha256" && \ - echo "$(cat kubectl.sha256) kubectl" | sha256sum --check && \ - chmod +x kubectl - -# cnpg kubectl plugin -RUN CNPG_ARCH=$(if [ "${TARGETARCH}" = "amd64" ]; then echo "x86_64"; else echo "${TARGETARCH}"; fi) && \ - curl -LO "https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v${CNPG_VERSION}/kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz" && \ - tar -zxvf kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz && \ - chmod +x kubectl-cnpg && \ - rm kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz - -# sops -ADD https://github.com/getsops/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux.amd64 sops -RUN chmod +x sops - -# age -ADD https://github.com/FiloSottile/age/releases/download/v${AGE_VERSION}/age-v${AGE_VERSION}-linux-amd64.tar.gz age.tar.gz -RUN tar -zxvf age.tar.gz age/age age/age-keygen --strip-components=1 && \ - chmod +x age age-keygen && \ - rm -rf age.tar.gz - # helm ADD https://get.helm.sh/${HELM_FILE_NAME} /tmp RUN tar -zxvf /tmp/${HELM_FILE_NAME} -C /tmp && mv /tmp/linux-${TARGETARCH}/helm helm && rm -rf /tmp/* RUN helm plugin install https://github.com/databus23/helm-diff --version ${HELM_DIFF_VERSION} -RUN echo "exec \$*" > /usr/bin/sudo && chmod +x /usr/bin/sudo -RUN helm plugin install https://github.com/jkroepke/helm-secrets --version ${HELM_SECRETS_VERSION} # helmfile ADD https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz /tmp RUN tar -zxvf /tmp/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz -C /tmp && mv /tmp/helmfile helmfile -# gucci -ADD https://github.com/noqcks/gucci/releases/download/v${GUCCI_VERSION}/gucci-v${GUCCI_VERSION}-linux-${TARGETARCH} gucci -RUN chmod +x gucci - -# kubeconform -ADD https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-linux-${TARGETARCH}.tar.gz /tmp -RUN tar -zxvf /tmp/kubeconform-linux-${TARGETARCH}.tar.gz -C /tmp && mv /tmp/kubeconform kubeconform - # node # https://github.com/nodesource/distributions RUN set -uex && \ diff --git a/versions.yaml b/versions.yaml index cba4e40597..9ec41ef7f3 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,5 +1,5 @@ -api: main +api: APL-523 console: main consoleLogin: main tasks: APL-1476-1 -tools: main +tools: APL-523 From d48c3bb96c9f9ea5c763bccf12221a50892d15fe Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 20 Feb 2026 16:37:13 +0100 Subject: [PATCH 16/66] fix: versions --- .github/workflows/otomi-tools-build-push.yaml | 1 - versions.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/otomi-tools-build-push.yaml b/.github/workflows/otomi-tools-build-push.yaml index 3e5a7e0677..050f74b65a 100644 --- a/.github/workflows/otomi-tools-build-push.yaml +++ b/.github/workflows/otomi-tools-build-push.yaml @@ -11,7 +11,6 @@ on: push: branches: - 'main' - - 'APL-523' env: NAMESPACE: linode diff --git a/versions.yaml b/versions.yaml index 9ec41ef7f3..29fc9f62a2 100644 --- a/versions.yaml +++ b/versions.yaml @@ -2,4 +2,4 @@ api: APL-523 console: main consoleLogin: main tasks: APL-1476-1 -tools: APL-523 +tools: 2.10.7 From 286faf35c75778cc0fe095372a0341050d70e927 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sat, 21 Feb 2026 12:38:57 +0100 Subject: [PATCH 17/66] test: tools image --- .github/workflows/main.yml | 2 +- Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index feb4919bc9..02a3db4543 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -132,7 +132,7 @@ jobs: if: always() && contains(needs.release.result, 'success') && !github.event.act runs-on: ubuntu-22.04 container: - image: linode/apl-tools:v2.10.6 + image: linode/apl-tools:v2.10.7 options: --user 0 # See https://docs.github.com/en/actions/sharing-automations/creating-actions/dockerfile-support-for-github-actions#user steps: - name: Checkout diff --git a/Dockerfile b/Dockerfile index caa68fc085..58a4f88252 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM linode/apl-tools:v2.10.6 AS ci +FROM linode/apl-tools:v2.10.7 AS ci ENV APP_HOME=/home/app/stack @@ -36,7 +36,7 @@ FROM ci AS clean # below command removes the packages specified in devDependencies and set NODE_ENV to production RUN npm prune --production -FROM linode/apl-tools:v2.10.6 AS prod +FROM linode/apl-tools:v2.10.7 AS prod ARG APPS_REVISION='' ENV APP_HOME=/home/app/stack ENV ENV_DIR=/home/app/stack/env From 41a30f919ed8ffb5a7062c94717239e4f784d497 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sat, 21 Feb 2026 17:54:25 +0100 Subject: [PATCH 18/66] test: tools image --- .github/workflows/otomi-tools-build-push.yaml | 7 ++++++- tools/Dockerfile | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/otomi-tools-build-push.yaml b/.github/workflows/otomi-tools-build-push.yaml index 050f74b65a..3d38c4bc3f 100644 --- a/.github/workflows/otomi-tools-build-push.yaml +++ b/.github/workflows/otomi-tools-build-push.yaml @@ -11,6 +11,7 @@ on: push: branches: - 'main' + - 'APL-523' env: NAMESPACE: linode @@ -70,7 +71,11 @@ jobs: echo OLD_VERSION = ${OLD_VERSION} echo NEW_VERSION = ${NEW_VERSION} fi - echo "No need to bump the version. Will skip next steps." + + # Override: rebuild v2.10.7 with gucci/kubeconform/htpasswd restored + NEW_VERSION="v2.10.7" + echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV + echo "Overriding NEW_VERSION=${NEW_VERSION}" - name: Login to GitHub Container Registry if: ${{ env.NEW_VERSION != null }} diff --git a/tools/Dockerfile b/tools/Dockerfile index b957ee09fc..559b5f30a9 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -7,6 +7,10 @@ ARG TARGETARCH ARG HELM_VERSION=3.19.2 # https://github.com/databus23/helm-diff/releases ARG HELM_DIFF_VERSION=3.14.1 +# https://github.com/noqcks/gucci/releases +ARG GUCCI_VERSION=1.9.0 +# https://github.com/yannh/kubeconform/releases/ +ARG KUBECONFORM_VERSION=0.7.0 # https://github.com/helmfile/helmfile/releases ARG HELMFILE_VERSION=1.2.2 # https://nodejs.org/en/download/ @@ -18,6 +22,7 @@ WORKDIR / # Install all required packages in one layer RUN apt-get update && apt-get install -y \ curl \ + apache2-utils \ ca-certificates \ git \ locales && \ @@ -42,6 +47,14 @@ RUN helm plugin install https://github.com/databus23/helm-diff --version ${HELM_ ADD https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz /tmp RUN tar -zxvf /tmp/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz -C /tmp && mv /tmp/helmfile helmfile +# gucci +ADD https://github.com/noqcks/gucci/releases/download/v${GUCCI_VERSION}/gucci-v${GUCCI_VERSION}-linux-${TARGETARCH} gucci +RUN chmod +x gucci + +# kubeconform +ADD https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-linux-${TARGETARCH}.tar.gz /tmp +RUN tar -zxvf /tmp/kubeconform-linux-${TARGETARCH}.tar.gz -C /tmp && mv /tmp/kubeconform kubeconform + # node # https://github.com/nodesource/distributions RUN set -uex && \ From 1644634f29a67cb31c326602ca419cb8af8b7904 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sat, 21 Feb 2026 18:30:18 +0100 Subject: [PATCH 19/66] feat: remove kms from bootstrap files --- tests/bootstrap/input-local-dev.yaml | 3 --- tests/bootstrap/input.yaml | 3 --- 2 files changed, 6 deletions(-) diff --git a/tests/bootstrap/input-local-dev.yaml b/tests/bootstrap/input-local-dev.yaml index ccaaafd9eb..1f7cdfd0e5 100644 --- a/tests/bootstrap/input-local-dev.yaml +++ b/tests/bootstrap/input-local-dev.yaml @@ -5,9 +5,6 @@ cluster: domainSuffix: local.host otomi: version: 'main' -kms: - sops: - provider: age apps: metrics-server: enabled: true diff --git a/tests/bootstrap/input.yaml b/tests/bootstrap/input.yaml index 0cb49d23af..64d426b912 100644 --- a/tests/bootstrap/input.yaml +++ b/tests/bootstrap/input.yaml @@ -5,9 +5,6 @@ cluster: domainSuffix: local.host otomi: version: 'main' -kms: - sops: - provider: age apps: metrics-server: enabled: false From f3755a640d04fb5bc2163b4828e6053cb19c97f7 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sat, 21 Feb 2026 18:30:59 +0100 Subject: [PATCH 20/66] test: tools image --- .github/workflows/otomi-tools-build-push.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/otomi-tools-build-push.yaml b/.github/workflows/otomi-tools-build-push.yaml index 3d38c4bc3f..050f74b65a 100644 --- a/.github/workflows/otomi-tools-build-push.yaml +++ b/.github/workflows/otomi-tools-build-push.yaml @@ -11,7 +11,6 @@ on: push: branches: - 'main' - - 'APL-523' env: NAMESPACE: linode @@ -71,11 +70,7 @@ jobs: echo OLD_VERSION = ${OLD_VERSION} echo NEW_VERSION = ${NEW_VERSION} fi - - # Override: rebuild v2.10.7 with gucci/kubeconform/htpasswd restored - NEW_VERSION="v2.10.7" - echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV - echo "Overriding NEW_VERSION=${NEW_VERSION}" + echo "No need to bump the version. Will skip next steps." - name: Login to GitHub Container Registry if: ${{ env.NEW_VERSION != null }} From 18d84accf4720f02479863c3bf8956184d7dc632 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sat, 21 Feb 2026 20:19:21 +0100 Subject: [PATCH 21/66] feat: remove kms and sops related code --- .github/workflows/integration.yml | 13 - src/cmd/bootstrap.test.ts | 624 ++++++++++++------------------ src/cmd/bootstrap.ts | 170 +------- src/common/bootstrap.ts | 3 - src/common/envalid.ts | 1 - src/operator/installer.test.ts | 19 +- src/operator/installer.ts | 24 +- 7 files changed, 247 insertions(+), 607 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b273c7964f..d3b5bf0f7a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -22,10 +22,6 @@ on: description: 'Select Domain Zone' type: string default: DNS-Integration - kms: - description: 'Should APL encrypt secrets in values repo (DNS or KMS is turned on)?' - type: string - default: age certificate: description: 'Select certificate issuer' type: string @@ -85,13 +81,6 @@ on: - Zone-2 - Random - DNS-Integration - kms: - type: choice - description: Should APL encrypt secrets in values repo (DNS or KMS is turned on)? - options: - - age - - no_kms - default: age certificate: type: choice description: Select certificate issuer @@ -142,7 +131,6 @@ jobs: echo 'install_profile: ${{ inputs.install_profile }}' echo 'lke_tier: ${{ inputs.lke_tier }}' echo 'kubernetes_version: ${{ inputs.kubernetes_version }}' - echo 'kms: ${{ inputs.kms }}' echo 'domain_zone: ${{ inputs.domain_zone }}' echo 'certificate: ${{ inputs.certificate }}' echo 'is_pre_installed: ${{ inputs.is_pre_installed }}' @@ -314,7 +302,6 @@ jobs: additional_args="" [[ '${{ inputs.certificate }}' == 'letsencrypt_staging' ]] && echo "$LETSENCRYPT_STAGING" >> values.yaml [[ '${{ inputs.certificate }}' == 'letsencrypt_production' ]] && echo "$LETSENCRYPT_PRODUCTION" >> values.yaml - [[ '${{ inputs.kms }}' == 'age' ]] && additional_args+=" --set kms.sops.provider=age" [[ '${{ inputs.is_pre_installed }}' == 'true' ]] && additional_args+=" --set otomi.isPreInstalled=true" if [[ '${{ inputs.disableORCS }}' == 'true' ]]; then additional_args+=" --set otomi.useORCS=false" diff --git a/src/cmd/bootstrap.test.ts b/src/cmd/bootstrap.test.ts index 0a1b004fc7..44e2684cda 100644 --- a/src/cmd/bootstrap.test.ts +++ b/src/cmd/bootstrap.test.ts @@ -3,10 +3,8 @@ import { pki } from 'node-forge' import stubs from 'src/test-stubs' import { bootstrap, - bootstrapSops, copyBasicFiles, createCustomCA, - getKmsValues, getStoredClusterSecrets, handleFileEntry, processValues, @@ -38,23 +36,13 @@ describe('Bootstrapping values', () => { quiet: jest.fn(), }), }), - bootstrapSops: jest.fn(), bootstrapSealedSecrets: jest.fn(), copyBasicFiles: jest.fn(), copyFile: jest.fn(), createCustomCA: jest.fn(), handleFileEntry: jest.fn(), - decrypt: jest.fn(), - encrypt: jest.fn(), - existsSync: jest.fn(), - genSops: jest.fn(), - getDeploymentState: jest.fn().mockReturnValue({}), - getImageTagFromValues: jest.fn(), getK8sSecret: jest.fn(), - hfValues: jest.fn(), - isCli: true, migrate: jest.fn(), - nothrow: jest.fn(), pathExists: jest.fn(), processValues: jest.fn(), terminal, @@ -63,7 +51,6 @@ describe('Bootstrapping values', () => { }) it('should call relevant sub routines', async () => { deps.processValues.mockReturnValue({ originalInput: values, allSecrets: {} }) - deps.hfValues.mockReturnValue(values) await bootstrap(deps) expect(deps.copyBasicFiles).toHaveBeenCalled() expect(deps.bootstrapSealedSecrets).toHaveBeenCalled() @@ -71,7 +58,6 @@ describe('Bootstrapping values', () => { it('should copy only skeleton files to env dir if it is empty or nonexisting', async () => { deps.processValues.mockReturnValue({ originalInput: undefined, allSecrets: {} }) await bootstrap(deps) - expect(deps.hfValues).toHaveBeenCalledTimes(0) }) it('should get stored cluster secrets if those exist', async () => { deps.getK8sSecret.mockReturnValue({ 'otomi-generated-passwords': secrets }) @@ -84,402 +70,274 @@ describe('Bootstrapping values', () => { expect(res).toEqual(undefined) }) - describe('getKmsValues', () => { - let kmsValuesDeps: any - const ageKeys = { publicKey: 'agePublicKey', privateKey: 'agePrivateKey' } - const values = { someKey: 'someValue' } - beforeEach(() => { - kmsValuesDeps = { - generateAgeKeys: jest.fn().mockResolvedValue(ageKeys), - hfValues: jest.fn(), - } - }) - it('should not get kms values if those do not exist', async () => { - kmsValuesDeps.hfValues.mockReturnValue(values) - const res = await getKmsValues(kmsValuesDeps) - expect(res).toBeUndefined() + describe('Copying basic files', () => { + const deps = { + copy: jest.fn(), + copyFile: jest.fn(), + copySchema: jest.fn(), + mkdir: jest.fn(), + pathExists: jest.fn(), + terminal, + } + it('should not throw any exception', async () => { + const res = await copyBasicFiles(deps) + expect(res).toBe(undefined) }) - it('should get kms values if those exist', async () => { - const deps = { - generateAgeKeys: jest.fn(), - } - const values = { - kms: { - sops: { - provider: 'azure', - }, - }, - } - deps.generateAgeKeys.mockResolvedValue({ publicKey: 'key1', privateKey: 'key2' }) - - const res = await getKmsValues(values, deps) - expect(res).toEqual({ - kms: { - sops: { - provider: 'azure', - }, + }) + describe('Creating folders and files for workload', () => { + const values = { + values: { + image: { + repository: 'linode/apl-nodejs-helloworld', + tag: 'v1.5.1', }, - }) + }, + } + const workload = { + files: { + 'env/teams/workloads/demo/values.yaml': JSON.stringify(values), + }, + } + const deps = { + loadYaml: jest.fn().mockReturnValue(workload), + mkdir: jest.fn(), + terminal, + writeFile: jest.fn(), + } + it('should create folders and files based on file entry in yaml', async () => { + await handleFileEntry(deps) + expect(deps.mkdir).toHaveBeenCalledWith('/test/env/teams/workloads/demo', { recursive: true }) + expect(deps.writeFile).toHaveBeenCalledWith('/test/env/teams/workloads/demo/values.yaml', JSON.stringify(values)) }) - it('should generate and return new age keys if provider is age and keys are missing', async () => { - const deps = { - generateAgeKeys: jest.fn(), - } - const values = { - kms: { - sops: { - provider: 'age', - }, + }) + describe('Checking for a custom CA', () => { + const deps = { + pki: { + rsa: { + generateKeyPair: jest.fn().mockReturnValue({ + publicKey: { n: {}, e: {} }, + privateKey: { d: {}, p: {}, q: {} }, + }), }, - } - deps.generateAgeKeys.mockResolvedValue({ publicKey: 'key1', privateKey: 'key2' }) - - const res = await getKmsValues(values, deps) - expect(res).toEqual({ - kms: { - sops: { - provider: 'age', - age: { publicKey: 'key1', privateKey: 'key2' }, + createCertificate: jest.fn().mockReturnValue({ + publicKey: {}, + serialNumber: '01', + validity: {}, + sign: jest.fn(), + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + }), + certificateToPem: jest.fn(), + privateKeyToPem: jest.fn(), + } as unknown as typeof pki, + writeValues: jest.fn(), + terminal, + } + deps.pki.certificateToPem = jest.fn().mockReturnValue('certpem') + deps.pki.privateKeyToPem = jest.fn().mockReturnValue('keypem') + it('should create a new key pair when none exist', () => { + const res = createCustomCA(deps) + expect(res).toMatchObject({ + apps: { + 'cert-manager': { + customRootCA: 'certpem', + customRootCAKey: 'keypem', }, }, }) }) - describe('Copying basic files', () => { - const deps = { - copy: jest.fn(), - copyFile: jest.fn(), - copySchema: jest.fn(), - mkdir: jest.fn(), - pathExists: jest.fn(), + }) + describe('processing values', () => { + const generatedSecrets = { gen: 'x' } + const generatedPassword = 'generated-password' + const usersWithPasswords = [ + { id: 'user1', initialPassword: 'existing-password' }, + { id: 'user2', initialPassword: generatedPassword }, + ] + // Pre-processed users (as stored in allSecrets for sealed secret generation) + const processedUsers = usersWithPasswords.map((u: any) => ({ + email: u.email, + firstName: u.firstName, + lastName: u.lastName, + initialPassword: u.initialPassword, + groups: [ + ...(u.isPlatformAdmin ? ['platform-admin'] : []), + ...(u.isTeamAdmin ? ['team-admin'] : []), + ...(u.teams || []).map((t: string) => `team-${t}`), + ], + })) + const ca = { a: 'cert' } + const mergedSecretsWithCa = merge(cloneDeep(secrets), cloneDeep(ca)) + const mergedSecretsWithGen = merge(cloneDeep(secrets), cloneDeep(generatedSecrets)) + const mergedSecretsWithGenAndCa = merge(cloneDeep(mergedSecretsWithGen), cloneDeep(ca)) + let deps + beforeEach(() => { + deps = { + createCustomCA: jest.fn().mockReturnValue(ca), + createK8sSecret: jest.fn(), + generateSecrets: jest.fn().mockReturnValue(generatedSecrets), + getStoredClusterSecrets: jest.fn().mockReturnValue(secrets), + loadYaml: jest.fn(), terminal, + writeValues: jest.fn(), + getUsers: jest.fn().mockReturnValue(usersWithPasswords), + getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), + stripAllSecrets: jest.fn().mockImplementation((v) => v), } - it('should not throw any exception', async () => { - const res = await copyBasicFiles(deps) - expect(res).toBe(undefined) - }) }) - describe('Creating folders and files for workload', () => { - const values = { - values: { - image: { - repository: 'linode/apl-nodejs-helloworld', - tag: 'v1.5.1', - }, - }, - } - const workload = { - files: { - 'env/teams/workloads/demo/values.yaml': JSON.stringify(values), - }, - } - const deps = { - loadYaml: jest.fn().mockReturnValue(workload), - mkdir: jest.fn(), - terminal, - writeFile: jest.fn(), - } - it('should create folders and files based on file entry in yaml', async () => { - await handleFileEntry(deps) - expect(deps.mkdir).toHaveBeenCalledWith('/test/env/teams/workloads/demo', { recursive: true }) - expect(deps.writeFile).toHaveBeenCalledWith( - '/test/env/teams/workloads/demo/values.yaml', - JSON.stringify(values), - ) + describe('Creating CA', () => { + it('should ask to create a CA if issuer is custom-ca', async () => { + await processValues(deps) + expect(deps.createCustomCA).toHaveBeenCalledTimes(1) }) }) - describe('Generating sops related files', () => { - const settings = { - kms: { - sops: { - provider: 'aws', - aws: { - keys: 'key1,key2', - }, - }, - }, - } - const deps = { - copyFile: jest.fn(), - decrypt: jest.fn(), - encrypt: jest.fn(), - gucci: jest.fn().mockReturnValue('ok'), - hfValues: jest.fn(), - loadYaml: jest.fn().mockReturnValue(settings), - pathExists: jest.fn(), - readFile: jest.fn(), - getKmsSettings: jest.fn(), - terminal, - writeFile: jest.fn(), - createUpdateGenericSecret: jest.fn(), - } - it('should create files on first run and en/de-crypt', async () => { - deps.pathExists.mockReturnValue(false) - deps.getKmsSettings.mockReturnValue({ - kms: { - sops: { - provider: 'age', - age: { publicKey: 'key1', privateKey: 'key2' }, - }, - }, - }) - - await bootstrapSops(undefined, deps) - expect(deps.encrypt).toHaveBeenCalled() - expect(deps.decrypt).toHaveBeenCalled() + describe('processing app values', () => { + it('should generate secrets by taking values and previously generated secrets as input', async () => { + deps.loadYaml.mockReturnValue(values) + await processValues(deps) + expect(deps.generateSecrets).toHaveBeenCalledWith(merge(cloneDeep(secrets), cloneDeep(values))) + expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) }) - it('should just create files on next runs', async () => { - deps.pathExists.mockReturnValue(true) - deps.hfValues.mockReturnValue(settings) - deps.decrypt = jest.fn() - deps.encrypt = jest.fn() - const res = await bootstrapSops(undefined, deps) - expect(res).toBe(undefined) - expect(deps.encrypt).not.toHaveBeenCalled() - expect(deps.decrypt).not.toHaveBeenCalled() + it('should overwrite a stored secret with one that was provided in values', async () => { + const newSecret = { secret: 'new' } + const valuesWithSecrets = merge(cloneDeep(values), newSecret) + const allSecrets = merge(cloneDeep(mergedSecretsWithCa), newSecret) + deps.loadYaml.mockReturnValue(valuesWithSecrets) + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(allSecrets) + await processValues(deps) + const expected = { ...allSecrets, users: processedUsers } + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', expected) + expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) }) - }) - describe('Checking for a custom CA', () => { - const deps = { - pki: { - rsa: { - generateKeyPair: jest.fn().mockReturnValue({ - publicKey: { n: {}, e: {} }, - privateKey: { d: {}, p: {}, q: {} }, - }), - }, - createCertificate: jest.fn().mockReturnValue({ - publicKey: {}, - serialNumber: '01', - validity: {}, - sign: jest.fn(), - setSubject: jest.fn(), - setIssuer: jest.fn(), - setExtensions: jest.fn(), - }), - certificateToPem: jest.fn(), - privateKeyToPem: jest.fn(), - } as unknown as typeof pki, - writeValues: jest.fn(), - terminal, - } - deps.pki.certificateToPem = jest.fn().mockReturnValue('certpem') - deps.pki.privateKeyToPem = jest.fn().mockReturnValue('keypem') - it('should create a new key pair when none exist', () => { - const res = createCustomCA(deps) - expect(res).toMatchObject({ - apps: { - 'cert-manager': { - customRootCA: 'certpem', - customRootCAKey: 'keypem', - }, - }, - }) - }) - }) - describe('processing values', () => { - const generatedSecrets = { gen: 'x' } - const generatedPassword = 'generated-password' - const usersWithPasswords = [ - { id: 'user1', initialPassword: 'existing-password' }, - { id: 'user2', initialPassword: generatedPassword }, - ] - // Pre-processed users (as stored in allSecrets for sealed secret generation) - const processedUsers = usersWithPasswords.map((u: any) => ({ - email: u.email, - firstName: u.firstName, - lastName: u.lastName, - initialPassword: u.initialPassword, - groups: [ - ...(u.isPlatformAdmin ? ['platform-admin'] : []), - ...(u.isTeamAdmin ? ['team-admin'] : []), - ...(u.teams || []).map((t: string) => `team-${t}`), - ], - })) - const ca = { a: 'cert' } - const mergedSecretsWithCa = merge(cloneDeep(secrets), cloneDeep(ca)) - const mergedSecretsWithGen = merge(cloneDeep(secrets), cloneDeep(generatedSecrets)) - const mergedSecretsWithGenAndCa = merge(cloneDeep(mergedSecretsWithGen), cloneDeep(ca)) - let deps - beforeEach(() => { - deps = { - createCustomCA: jest.fn().mockReturnValue(ca), - createK8sSecret: jest.fn(), - decrypt: jest.fn(), - existsSync: jest.fn(), - generateSecrets: jest.fn().mockReturnValue(generatedSecrets), - getStoredClusterSecrets: jest.fn().mockReturnValue(secrets), - getKmsValues: jest.fn().mockReturnValue({}), - hfValues: jest.fn().mockReturnValue(values), - loadYaml: jest.fn(), - terminal, - validateValues: jest.fn().mockReturnValue(true), - writeValues: jest.fn(), - getUsers: jest.fn().mockReturnValue(usersWithPasswords), - generatePassword: jest.fn().mockReturnValue(generatedPassword), - addInitialPasswords: jest.fn().mockReturnValue(usersWithPasswords), - addPlatformAdmin: jest.fn().mockReturnValue(usersWithPasswords), - pathExists: jest.fn().mockReturnValue(true), - getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), - stripAllSecrets: jest.fn().mockImplementation((v) => v), - } + it('should create a custom ca if issuer is custom-ca or undefined and no CA yet exists', async () => { + deps.loadYaml.mockReturnValue({ apps: { 'cert-manager': { issuer: 'custom-ca' } } }) + await processValues(deps) + expect(deps.createCustomCA).toHaveBeenCalled() }) - describe('Creating CA', () => { - it('should ask to create a CA if issuer is custom-ca', async () => { - await processValues(deps) - expect(deps.createCustomCA).toHaveBeenCalledTimes(1) + it('should not re-create a custom ca if issuer is custom-ca or undefined and a CA already exists', async () => { + deps.loadYaml.mockReturnValue({ + apps: { 'cert-manager': { issuer: 'custom-ca', customRootCA: 'certpem', customRootCAKey: 'keypem' } }, }) + await processValues(deps) + expect(deps.createCustomCA).not.toHaveBeenCalled() }) - describe('processing app values', () => { - it('should not retrieve values from env dir', async () => { - await processValues(deps) - expect(deps.hfValues).toHaveBeenCalledTimes(0) - }) - it('should generate secrets by taking values and previously generated secrets as input', async () => { - deps.loadYaml.mockReturnValue(values) - await processValues(deps) - expect(deps.generateSecrets).toHaveBeenCalledWith(merge(cloneDeep(secrets), cloneDeep(values))) - expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) - }) - it('should overwrite a stored secret with one that was provided in values', async () => { - const newSecret = { secret: 'new' } - const valuesWithSecrets = merge(cloneDeep(values), newSecret) - const allSecrets = merge(cloneDeep(mergedSecretsWithCa), newSecret) - deps.loadYaml.mockReturnValue(valuesWithSecrets) - deps.getStoredClusterSecrets.mockReturnValue(secrets) - deps.generateSecrets.mockReturnValue(allSecrets) - await processValues(deps) - const expected = { ...allSecrets, users: processedUsers } - expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', expected) - expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) + it('should only store secrets', async () => { + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(generatedSecrets) + deps.createCustomCA.mockReturnValue(ca) + await processValues(deps) + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { + ...mergedSecretsWithGenAndCa, + users: processedUsers, }) - it('should create a custom ca if issuer is custom-ca or undefined and no CA yet exists', async () => { - deps.loadYaml.mockReturnValue({ apps: { 'cert-manager': { issuer: 'custom-ca' } } }) - await processValues(deps) - expect(deps.createCustomCA).toHaveBeenCalled() - }) - it('should not re-create a custom ca if issuer is custom-ca or undefined and a CA already exists', async () => { - deps.loadYaml.mockReturnValue({ - apps: { 'cert-manager': { issuer: 'custom-ca', customRootCA: 'certpem', customRootCAKey: 'keypem' } }, - }) - await processValues(deps) - expect(deps.createCustomCA).not.toHaveBeenCalled() - }) - it('should only store secrets', async () => { - deps.getStoredClusterSecrets.mockReturnValue(secrets) - deps.generateSecrets.mockReturnValue(generatedSecrets) - deps.createCustomCA.mockReturnValue(ca) - await processValues(deps) - expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { - ...mergedSecretsWithGenAndCa, - users: processedUsers, - }) + }) + it('should not overwrite stored secrets', async () => { + deps.loadYaml.mockReturnValue({}) + deps.getStoredClusterSecrets.mockReturnValue(generatedSecrets) + deps.createCustomCA.mockReturnValue({}) + deps.generateSecrets.mockReturnValue(generatedSecrets) + await processValues(deps) + expect(deps.generateSecrets).toHaveBeenCalledWith(generatedSecrets) + expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { + ...generatedSecrets, + users: processedUsers, }) - it('should not overwrite stored secrets', async () => { - deps.loadYaml.mockReturnValue({}) - deps.getStoredClusterSecrets.mockReturnValue(generatedSecrets) - deps.createCustomCA.mockReturnValue({}) - deps.generateSecrets.mockReturnValue(generatedSecrets) - await processValues(deps) - expect(deps.generateSecrets).toHaveBeenCalledWith(generatedSecrets) - expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { - ...generatedSecrets, - users: processedUsers, - }) + }) + it('should merge allSecrets into disk values so non-secret fields like customRootCA are preserved', async () => { + deps.loadYaml.mockReturnValue({ + cluster: { name: 'bla', provider: 'dida' }, }) - it('should merge allSecrets into disk values so non-secret fields like customRootCA are preserved', async () => { - deps.loadYaml.mockReturnValue({ - cluster: { name: 'bla', provider: 'dida' }, - }) - deps.getStoredClusterSecrets.mockReturnValue({ - users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], - }) - deps.generateSecrets.mockReturnValue({ gen: 'x' }) - deps.createCustomCA.mockReturnValue(ca) - const res = await processValues(deps) - // mergedForDisk includes allSecrets (stripAllSecrets mock is identity, real impl strips x-secret paths) - // processedUsers adds groups:[] to each user via element-wise lodash merge - expect(deps.writeValues).toHaveBeenNthCalledWith(1, { - a: 'cert', - gen: 'x', - cluster: { name: 'bla', provider: 'dida' }, - users: [ - { id: 'user1', initialPassword: 'existing-password', groups: [] }, - { id: 'user2', initialPassword: 'generated-password', groups: [] }, - ], - }) - expect(res.originalInput).toEqual({ - cluster: { name: 'bla', provider: 'dida' }, - users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], - }) + deps.getStoredClusterSecrets.mockReturnValue({ + users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], }) - it('should merge originalInput + allSecrets + users for disk (stripAllSecrets removes x-secret paths)', async () => { - // mergedForDisk = merge(originalInput, allSecrets, { users }) - // allSecrets = merge(ca, storedSecrets, generatedSecrets, kmsValues) + users: processedUsers - const allSecretsExpected = merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { - users: processedUsers, - }) - const expectedDiskValues = merge( - cloneDeep(secrets), - cloneDeep(values), - cloneDeep(allSecretsExpected), - cloneDeep({ users: usersWithPasswords }), - ) - deps.loadYaml.mockReturnValue({ ...values, users }) - deps.getStoredClusterSecrets.mockReturnValue(secrets) - deps.generateSecrets.mockReturnValue(generatedSecrets) - deps.getUsers.mockReturnValue(usersWithPasswords) - await processValues(deps) - expect(deps.writeValues).toHaveBeenNthCalledWith(1, expectedDiskValues) + deps.generateSecrets.mockReturnValue({ gen: 'x' }) + deps.createCustomCA.mockReturnValue(ca) + const res = await processValues(deps) + // mergedForDisk includes allSecrets (stripAllSecrets mock is identity, real impl strips x-secret paths) + // processedUsers adds groups:[] to each user via element-wise lodash merge + expect(deps.writeValues).toHaveBeenNthCalledWith(1, { + a: 'cert', + gen: 'x', + cluster: { name: 'bla', provider: 'dida' }, + users: [ + { id: 'user1', initialPassword: 'existing-password', groups: [] }, + { id: 'user2', initialPassword: 'generated-password', groups: [] }, + ], }) - it('should call stripAllSecrets before writing values to disk', async () => { - deps.loadYaml.mockReturnValue(values) - deps.getSchemaSecretsPaths.mockResolvedValue(['apps.gitea.adminPassword', 'apps.harbor.adminPassword']) - await processValues(deps) - expect(deps.stripAllSecrets).toHaveBeenCalledTimes(1) - expect(deps.getSchemaSecretsPaths).toHaveBeenCalledTimes(1) + expect(res.originalInput).toEqual({ + cluster: { name: 'bla', provider: 'dida' }, + users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], }) - it('should still return full allSecrets for bootstrapSealedSecrets', async () => { - deps.loadYaml.mockReturnValue(values) - deps.getStoredClusterSecrets.mockReturnValue(secrets) - deps.generateSecrets.mockReturnValue(generatedSecrets) - deps.createCustomCA.mockReturnValue(ca) - const result = await processValues(deps) - // allSecrets should contain full unstripped secrets including pre-processed users - expect(result.allSecrets).toEqual( - merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { users: processedUsers }), - ) + }) + it('should merge originalInput + allSecrets + users for disk (stripAllSecrets removes x-secret paths)', async () => { + // mergedForDisk = merge(originalInput, allSecrets, { users }) + // allSecrets = merge(ca, storedSecrets, generatedSecrets, kmsValues) + users: processedUsers + const allSecretsExpected = merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { + users: processedUsers, }) - it('should preserve existing groups when users are recovered from stored secrets (bootstrap retry)', async () => { - // Simulate bootstrap retry: stored secrets contain processed users (with groups, without isPlatformAdmin) - const storedProcessedUsers = [ - { - email: 'platform-admin@example.com', - firstName: 'platform', - lastName: 'admin', - initialPassword: 'existing-pass', - groups: ['platform-admin'], - }, - ] - deps.loadYaml.mockReturnValue({}) - deps.getStoredClusterSecrets.mockReturnValue({ users: storedProcessedUsers }) - deps.generateSecrets.mockReturnValue({}) - deps.createCustomCA.mockReturnValue({}) - // getUsers returns the stored processed users (no isPlatformAdmin flag) - deps.getUsers.mockReturnValue(storedProcessedUsers) + const expectedDiskValues = merge( + cloneDeep(secrets), + cloneDeep(values), + cloneDeep(allSecretsExpected), + cloneDeep({ users: usersWithPasswords }), + ) + deps.loadYaml.mockReturnValue({ ...values, users }) + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(generatedSecrets) + deps.getUsers.mockReturnValue(usersWithPasswords) + await processValues(deps) + expect(deps.writeValues).toHaveBeenNthCalledWith(1, expectedDiskValues) + }) + it('should call stripAllSecrets before writing values to disk', async () => { + deps.loadYaml.mockReturnValue(values) + deps.getSchemaSecretsPaths.mockResolvedValue(['apps.gitea.adminPassword', 'apps.harbor.adminPassword']) + await processValues(deps) + expect(deps.stripAllSecrets).toHaveBeenCalledTimes(1) + expect(deps.getSchemaSecretsPaths).toHaveBeenCalledTimes(1) + }) + it('should still return full allSecrets for bootstrapSealedSecrets', async () => { + deps.loadYaml.mockReturnValue(values) + deps.getStoredClusterSecrets.mockReturnValue(secrets) + deps.generateSecrets.mockReturnValue(generatedSecrets) + deps.createCustomCA.mockReturnValue(ca) + const result = await processValues(deps) + // allSecrets should contain full unstripped secrets including pre-processed users + expect(result.allSecrets).toEqual( + merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { users: processedUsers }), + ) + }) + it('should preserve existing groups when users are recovered from stored secrets (bootstrap retry)', async () => { + // Simulate bootstrap retry: stored secrets contain processed users (with groups, without isPlatformAdmin) + const storedProcessedUsers = [ + { + email: 'platform-admin@example.com', + firstName: 'platform', + lastName: 'admin', + initialPassword: 'existing-pass', + groups: ['platform-admin'], + }, + ] + deps.loadYaml.mockReturnValue({}) + deps.getStoredClusterSecrets.mockReturnValue({ users: storedProcessedUsers }) + deps.generateSecrets.mockReturnValue({}) + deps.createCustomCA.mockReturnValue({}) + // getUsers returns the stored processed users (no isPlatformAdmin flag) + deps.getUsers.mockReturnValue(storedProcessedUsers) - const result = await processValues(deps) + const result = await processValues(deps) - // Groups should be preserved from existing data, not reset to [] - expect(result.allSecrets.users).toEqual([ - { - email: 'platform-admin@example.com', - firstName: 'platform', - lastName: 'admin', - initialPassword: 'existing-pass', - groups: ['platform-admin'], - }, - ]) - }) + // Groups should be preserved from existing data, not reset to [] + expect(result.allSecrets.users).toEqual([ + { + email: 'platform-admin@example.com', + firstName: 'platform', + lastName: 'admin', + initialPassword: 'existing-pass', + groups: ['platform-admin'], + }, + ]) }) }) }) diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index 99da59dac5..06842b9486 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'crypto' import { existsSync } from 'fs' -import { copyFile, cp, mkdir, readFile, writeFile } from 'fs/promises' +import { copyFile, cp, mkdir, writeFile } from 'fs/promises' import { generate as generatePassword } from 'generate-password' import { cloneDeep, get, merge, set } from 'lodash' import { pki } from 'node-forge' @@ -8,25 +8,14 @@ import path from 'path' import { bootstrapGit } from 'src/common/bootstrap' import { prepareEnvironment } from 'src/common/cli' import { DEPLOYMENT_PASSWORDS_SECRET } from 'src/common/constants' -import { decrypt, encrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { env, isCli } from 'src/common/envalid' -import { hfValues } from 'src/common/hf' -import { - createK8sSecret, - createUpdateGenericSecret, - getDeploymentState, - getK8sSecret, - k8s, - secretId, -} from 'src/common/k8s' -import { getKmsSettings } from 'src/common/repo' +import { createK8sSecret, getK8sSecret, secretId } from 'src/common/k8s' import { bootstrapSealedSecrets, stripAllSecrets } from 'src/common/sealed-secrets' import { ensureTeamGitOpsDirectories, getFilename, getSchemaSecretsPaths, - gucci, isCore, loadYaml, rootDir, @@ -36,111 +25,9 @@ import { BasicArguments, setParsedArgs } from 'src/common/yargs' import { Argv } from 'yargs' import { $ } from 'zx' import { migrate } from './migrate' -import { validateValues } from './validate-values' const cmdName = getFilename(__filename) -const kmsMap = { - aws: 'kms', - azure: 'azure_keyvault', - google: 'gcp_kms', - age: 'age', -} - -export const bootstrapSops = async ( - envDir = env.ENV_DIR, - deps = { - copyFile, - decrypt, - encrypt, - gucci, - loadYaml, - pathExists: existsSync, - getKmsSettings, - readFile, - terminal, - writeFile, - createUpdateGenericSecret, - }, -): Promise => { - const d = deps.terminal(`cmd:${cmdName}:genSops`) - const targetPath = `${envDir}/.sops.yaml` - const values = await deps.getKmsSettings(envDir) - - const provider: string | undefined = values?.kms?.sops?.provider - if (!provider) { - d.warn('No sops information given. Assuming no sops enc/decryption needed. Be careful!') - return - } - - const templatePath = `${rootDir}/tpl/.sops.yaml.gotmpl` - const kmsProvider = kmsMap[provider] as string - const kmsKeys = values.kms.sops[provider]?.keys as string - - const obj = { - provider: kmsProvider, - keys: kmsKeys, - } - - if (provider === 'age') { - const { publicKey } = values?.kms?.sops?.age ?? {} - let privateKey = values.kms?.sops?.age?.privateKey - if (privateKey?.startsWith('ENC')) { - privateKey = '' - } - obj.keys = publicKey - if (privateKey && !process.env.SOPS_AGE_KEY) { - process.env.SOPS_AGE_KEY = privateKey - await deps.writeFile(`${envDir}/.secrets`, `SOPS_AGE_KEY=${privateKey}`) - try { - await deps.createUpdateGenericSecret(k8s.core(), 'apl-sops-secrets', 'apl-operator', { - SOPS_AGE_KEY: privateKey, - }) - } catch (e) { - d.warn('Failed to create or update apl-sops-secrets secret with SOPS_AGE_KEY, this might come later') - } - } - } - - const exists = deps.pathExists(targetPath) - d.log(`Creating sops file for provider ${provider}`) - const output = (await deps.gucci(templatePath, obj, true)) as string - await deps.writeFile(targetPath, output) - d.log(`Ready generating sops files. The configuration is written to: ${targetPath}`) - - // prepare some credential files the first time and crypt some - if (!exists) { - if (isCli || env.OTOMI_DEV) { - // first time so we know we have values - const secretsFile = `${envDir}/.secrets` - d.log(`Creating secrets file: ${secretsFile}`) - if (provider === 'google') { - // and we also assume the correct values are given by using '!' (we want to err when not set) - const serviceKeyJson = JSON.parse(values.kms.sops!.google!.accountJson as string) - // and set it in env for later decryption - process.env.GCLOUD_SERVICE_KEY = values.kms.sops!.google!.accountJson - d.log('Creating gcp-key.json for vscode.') - await deps.writeFile(`${envDir}/gcp-key.json`, JSON.stringify(serviceKeyJson)) - d.log(`Creating credentials file: ${secretsFile}`) - await deps.writeFile(secretsFile, `GCLOUD_SERVICE_KEY=${JSON.stringify(JSON.stringify(serviceKeyJson))}`) - } else if (provider === 'aws') { - const v = values.kms.sops!.aws! - await deps.writeFile(secretsFile, `AWS_ACCESS_KEY_ID='${v.accessKey}'\nAWS_ACCESS_KEY_SECRET=${v.secretKey}`) - } else if (provider === 'azure') { - const v = values.kms.sops!.azure! - await deps.writeFile(secretsFile, `AZURE_CLIENT_ID='${v.clientId}'\nAZURE_CLIENT_SECRET=${v.clientSecret}`) - } else if (provider === 'age') { - const { privateKey } = values.kms.sops!.age! - process.env.SOPS_AGE_KEY = privateKey - await deps.writeFile(secretsFile, `SOPS_AGE_KEY=${privateKey}`) - } - } - // now do a round of encryption and decryption to make sure we have all the files in place for later - await deps.encrypt(envDir) - await deps.decrypt(envDir) - } -} - export const copySchema = async (deps = { terminal, rootDir, env, isCore, loadYaml, copyFile }): Promise => { const d = deps.terminal(`cmd:${cmdName}:copySchema`) const { ENV_DIR } = env @@ -179,36 +66,6 @@ export const getStoredClusterSecrets = async ( return undefined } -export const generateAgeKeys = async (deps = { $, terminal }) => { - const d = deps.terminal(`cmd:${cmdName}:generateAgeKeys`) - try { - d.info('Generating age keys') - const result = await deps.$`age-keygen` - const { stdout } = result - const matchPublic = stdout?.match(/age[0-9a-z]+/) - const publicKey = matchPublic ? matchPublic[0] : '' - const matchPrivate = stdout?.match(/AGE-SECRET-KEY-[0-9A-Z]+/) - const privateKey = matchPrivate ? matchPrivate[0] : '' - const ageKeys = { publicKey, privateKey } - return ageKeys - } catch (error) { - d.log('Error generating age keys:', error) - throw error - } -} - -export const getKmsValues = async (values: Record, deps = { generateAgeKeys }) => { - const kms = values?.kms - if (!kms) return undefined - const provider = kms?.sops?.provider - if (!provider) return {} - if (provider !== 'age') return { kms } - const age = kms?.sops?.age - if (age?.publicKey && age?.privateKey) return { kms } - const ageKeys = await deps.generateAgeKeys() - return { kms: { sops: { provider: 'age', age: ageKeys } } } -} - export const addPlatformAdmin = (users: any[], domainSuffix: string) => { const defaultPlatformAdminEmail = `platform-admin@${domainSuffix}` const platformAdminExists = users.find((user) => user.email === defaultPlatformAdminEmail) @@ -295,20 +152,12 @@ export const processValues = async ( deps = { terminal, loadYaml, - decrypt, getStoredClusterSecrets, - getKmsValues, writeValues, - pathExists: existsSync, - hfValues, - validateValues, generateSecrets, createK8sSecret, createCustomCA, getUsers, - generatePassword, - addInitialPasswords, - addPlatformAdmin, getSchemaSecretsPaths, stripAllSecrets, }, @@ -329,15 +178,8 @@ export const processValues = async ( } else { caSecrets = deps.createCustomCA() } - // get any kms values & generate age keys if needed - const kmsValues = (await deps.getKmsValues(originalInput)) || {} // merge existing secrets over newly generated ones to keep them - const allSecrets = merge( - cloneDeep(caSecrets), - cloneDeep(storedSecrets), - cloneDeep(generatedSecrets), - cloneDeep(kmsValues), - ) + const allSecrets = merge(cloneDeep(caSecrets), cloneDeep(storedSecrets), cloneDeep(generatedSecrets)) // add default platform admin & generate initial passwords for users if they don't have one const users = deps.getUsers(originalInput) // Pre-process users into keycloak-operator format (with groups resolved) for sealed secret storage @@ -458,17 +300,11 @@ export const createCustomCA = (deps = { terminal, pki, writeValues }): Record => { diff --git a/src/common/bootstrap.ts b/src/common/bootstrap.ts index 7f1ce9d1d7..6e493353eb 100644 --- a/src/common/bootstrap.ts +++ b/src/common/bootstrap.ts @@ -90,8 +90,5 @@ export const bootstrapGit = async (inValues?: Record): Promise { jest.clearAllMocks() jest.useFakeTimers() - // Save original environment variables - process.env.SOPS_AGE_KEY = '' - mockCoreApi = {} ;(k8s.k8s.core as jest.Mock).mockReturnValue(mockCoreApi) @@ -336,22 +333,8 @@ describe('Installer', () => { }) describe('setEnvAndCreateSecrets', () => { - test('should use existing SOPS key from secret', async () => { - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValueOnce({ SOPS_AGE_KEY: 'existing-sops-key' }) - - await installer.setEnvAndCreateSecrets() - - expect(k8s.getK8sSecret).toHaveBeenCalledWith('apl-sops-secrets', 'apl-operator') - expect(process.env.SOPS_AGE_KEY).toBe('existing-sops-key') - }) - - test('should skip gracefully when SOPS key not found in secret (SealedSecrets in use)', async () => { - ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(null) - + test('should complete without errors', async () => { await installer.setEnvAndCreateSecrets() - - // Should not throw — SOPS is no longer required (replaced by SealedSecrets + ESO) - expect(process.env.SOPS_AGE_KEY).toBe('') }) }) }) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index e55f28edfd..c4616cebd9 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -1,8 +1,7 @@ -import * as process from 'node:process' import { $ } from 'zx' import { terminal } from '../common/debug' import { getStoredGitRepoConfig } from '../common/git-config' -import { createUpdateConfigMap, deletePendingHelmReleases, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' +import { createUpdateConfigMap, deletePendingHelmReleases, getK8sConfigMap, k8s } from '../common/k8s' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' @@ -123,25 +122,6 @@ export class Installer { } public async setEnvAndCreateSecrets(): Promise { - this.d.debug('Retrieving or creating git credentials') - await this.setupSopsEnvironment() - } - - private async setupSopsEnvironment() { - try { - const aplSopsSecret = await getK8sSecret('apl-sops-secrets', 'apl-operator') - - if (aplSopsSecret?.SOPS_AGE_KEY) { - process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY - this.d.debug('Using existing sops credentials from secret') - } else { - // SOPS is no longer used (replaced by SealedSecrets + ESO). - // Skip hfValues() call which requires the git repo that may not exist yet. - this.d.debug('SOPS Age key not found in secret, skipping (SealedSecrets in use)') - } - } catch (error) { - this.d.error('Failed to retrieve sops credentials:', getErrorMessage(error)) - throw error - } + this.d.debug('Environment setup complete (SOPS removed, using SealedSecrets + ESO)') } } From 8cd84f744c23cbeba6ad2821d9a88497c208506e Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sat, 21 Feb 2026 20:48:36 +0100 Subject: [PATCH 22/66] test: tools image --- .github/workflows/otomi-tools-build-push.yaml | 7 ++++- tools/Dockerfile | 29 ++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/otomi-tools-build-push.yaml b/.github/workflows/otomi-tools-build-push.yaml index 050f74b65a..3d38c4bc3f 100644 --- a/.github/workflows/otomi-tools-build-push.yaml +++ b/.github/workflows/otomi-tools-build-push.yaml @@ -11,6 +11,7 @@ on: push: branches: - 'main' + - 'APL-523' env: NAMESPACE: linode @@ -70,7 +71,11 @@ jobs: echo OLD_VERSION = ${OLD_VERSION} echo NEW_VERSION = ${NEW_VERSION} fi - echo "No need to bump the version. Will skip next steps." + + # Override: rebuild v2.10.7 with gucci/kubeconform/htpasswd restored + NEW_VERSION="v2.10.7" + echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV + echo "Overriding NEW_VERSION=${NEW_VERSION}" - name: Login to GitHub Container Registry if: ${{ env.NEW_VERSION != null }} diff --git a/tools/Dockerfile b/tools/Dockerfile index 559b5f30a9..4f22ba1053 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -3,6 +3,8 @@ FROM ubuntu:24.04 AS builder ARG DEBIAN_FRONTEND=noninteractive ARG TARGETARCH +# https://github.com/kubernetes/kubernetes/releases +ARG KUBECTL_VERSION=1.34.2 # https://github.com/helm/helm/tags ARG HELM_VERSION=3.19.2 # https://github.com/databus23/helm-diff/releases @@ -15,6 +17,10 @@ ARG KUBECONFORM_VERSION=0.7.0 ARG HELMFILE_VERSION=1.2.2 # https://nodejs.org/en/download/ ARG NODE_VERSION=24 +# https://github.com/cloudnative-pg/cloudnative-pg/releases +ARG CNPG_VERSION=1.27.1 +# https://github.com/jqlang/jq/releases/ +ARG JQ_VERSION=1.8.1 ARG HELM_FILE_NAME=helm-v${HELM_VERSION}-linux-${TARGETARCH}.tar.gz WORKDIR / @@ -25,10 +31,18 @@ RUN apt-get update && apt-get install -y \ apache2-utils \ ca-certificates \ git \ - locales && \ + locales \ + rsync && \ rm -rf /var/lib/apt/lists/* && \ locale-gen en_US.UTF-8 +# jq +RUN curl -LO "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${TARGETARCH}" && \ + curl -L "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/sha256sum.txt" | grep "jq-linux-${TARGETARCH}" > sha256sum.txt && \ + echo sha256sum --check sha256sum.txt && \ + mv jq-linux-${TARGETARCH} /usr/bin/jq && \ + chmod +x /usr/bin/jq + RUN mkdir -p /home/app RUN groupadd -r app && \ useradd -r -g app -d /home/app -s /sbin/nologin -c "Docker image user" app @@ -38,6 +52,19 @@ RUN mkdir $APP_HOME WORKDIR $APP_HOME ENV PATH $PATH:$APP_HOME +# kubectl +RUN curl -LO "https://dl.k8s.io/release/v$KUBECTL_VERSION/bin/linux/$TARGETARCH/kubectl" && \ + curl -LO "https://dl.k8s.io/release/v$KUBECTL_VERSION/bin/linux/$TARGETARCH/kubectl.sha256" && \ + echo "$(cat kubectl.sha256) kubectl" | sha256sum --check && \ + chmod +x kubectl + +# cnpg kubectl plugin +RUN CNPG_ARCH=$(if [ "${TARGETARCH}" = "amd64" ]; then echo "x86_64"; else echo "${TARGETARCH}"; fi) && \ + curl -LO "https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v${CNPG_VERSION}/kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz" && \ + tar -zxvf kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz && \ + chmod +x kubectl-cnpg && \ + rm kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz + # helm ADD https://get.helm.sh/${HELM_FILE_NAME} /tmp RUN tar -zxvf /tmp/${HELM_FILE_NAME} -C /tmp && mv /tmp/linux-${TARGETARCH}/helm helm && rm -rf /tmp/* From 13f73c41bb8bd2cf1aaf733bc02db1f5dd9d983f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sun, 22 Feb 2026 12:13:06 +0100 Subject: [PATCH 23/66] feat: update user management --- src/common/sealed-secrets.test.ts | 3 + src/common/sealed-secrets.ts | 174 +++++++++++++++++++++++++++++- src/operator/apl-operator.test.ts | 3 + src/operator/apl-operator.ts | 7 +- 4 files changed, 185 insertions(+), 2 deletions(-) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 7dbd238273..01d610f144 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -426,6 +426,7 @@ describe('sealed-secrets', () => { buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), createSealedSecretManifest: jest.fn().mockResolvedValue(mockManifest), writeSealedSecretManifests: jest.fn(), + createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), } @@ -457,6 +458,7 @@ describe('sealed-secrets', () => { buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), createSealedSecretManifest: jest.fn().mockResolvedValue({}), writeSealedSecretManifests: jest.fn(), + createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), encryptSecretItem: jest.fn(), } @@ -488,6 +490,7 @@ describe('sealed-secrets', () => { buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), createSealedSecretManifest: jest.fn(), writeSealedSecretManifests: jest.fn(), + createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), encryptSecretItem: jest.fn(), } diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 8bbe855eca..a454c2f7ca 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -631,6 +631,153 @@ export const restartSealedSecretsController = async (deps = { terminal }): Promi * Orchestrator: bootstrap sealed secrets for the platform. * Replaces bootstrapSops(). */ +/** + * Aggregate individual user K8s secrets from apl-users namespace into a single + * users-secrets Secret in apl-secrets namespace (usersJson key). + * This feeds the Keycloak Helm chart configuration. + * Called during the operator reconcile loop. + */ +export const aggregateUserSecrets = async (deps = { terminal, k8s, b64enc }): Promise => { + const d = deps.terminal(`common:${cmdName}:aggregateUserSecrets`) + const namespace = 'apl-users' + const targetNamespace = 'apl-secrets' + const targetSecretName = 'users-secrets' + + try { + const res: any = await deps.k8s.core().listNamespacedSecret({ namespace }) + const users: any[] = [] + + for (const item of res.items || []) { + if (item.type !== 'Opaque') continue + if (!item.data?.email) continue + + const decoded: Record = {} + for (const [key, value] of Object.entries(item.data as Record)) { + decoded[key] = Buffer.from(value, 'base64').toString('utf-8') + } + + // Build user in keycloak-operator format (with groups) + const groups: string[] = [] + if (decoded.isPlatformAdmin === 'true') groups.push('platform-admin') + if (decoded.isTeamAdmin === 'true') groups.push('team-admin') + const teams = decoded.teams ? JSON.parse(decoded.teams) : [] + for (const team of teams) groups.push(`team-${team}`) + + users.push({ + email: decoded.email, + firstName: decoded.firstName || '', + lastName: decoded.lastName || '', + initialPassword: decoded.initialPassword || '', + groups, + }) + } + + if (users.length === 0) { + d.info('No user secrets found in apl-users namespace, skipping aggregation') + return + } + + const usersJson = JSON.stringify(users) + + // Create or update the users-secrets Secret in apl-secrets + await ensureNamespaceExists(targetNamespace) + + try { + await deps.k8s.core().readNamespacedSecret({ name: targetSecretName, namespace: targetNamespace }) + // Secret exists, patch it + await deps.k8s.core().patchNamespacedSecret( + { + name: targetSecretName, + namespace: targetNamespace, + body: { + data: { usersJson: deps.b64enc(usersJson) }, + }, + }, + setHeaderOptions('Content-Type', PatchStrategy.StrategicMergePatch), + ) + } catch (error) { + if (error instanceof ApiException && error.code === 404) { + // Secret doesn't exist, create it + await deps.k8s.core().createNamespacedSecret({ + namespace: targetNamespace, + body: { + metadata: { + name: targetSecretName, + namespace: targetNamespace, + }, + type: 'Opaque', + data: { usersJson: deps.b64enc(usersJson) }, + }, + }) + } else { + throw error + } + } + + d.info(`Aggregated ${users.length} user(s) into ${targetNamespace}/${targetSecretName}`) + } catch (error) { + d.warn(`Failed to aggregate user secrets: ${error instanceof Error ? error.message : error}`) + } +} + +/** + * Create individual SealedSecret manifests for each user in the apl-users namespace. + * Each user gets their own SealedSecret with all fields encrypted. + */ +export const createUserSealedSecretManifests = async ( + users: any[], + pem: string, + deps = { encryptSecretItem, terminal }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:createUserSealedSecretManifests`) + const namespace = 'apl-users' + const manifests: SealedSecretManifest[] = [] + + for (const user of users) { + const userId = user.name || user.id + if (!userId) { + d.warn('Skipping user without id/name') + continue + } + + const data: Record = { + email: user.email || '', + firstName: user.firstName || '', + lastName: user.lastName || '', + initialPassword: user.initialPassword || '', + isPlatformAdmin: String(user.isPlatformAdmin ?? false), + isTeamAdmin: String(user.isTeamAdmin ?? false), + teams: JSON.stringify(user.teams || []), + } + + const encryptedData: Record = {} + for (const [key, value] of Object.entries(data)) { + encryptedData[key] = await deps.encryptSecretItem(pem, namespace, value) + } + + manifests.push({ + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: userId, + namespace, + }, + spec: { + encryptedData, + template: { + immutable: false, + metadata: { name: userId, namespace }, + type: 'Opaque', + }, + }, + }) + } + + d.info(`Created ${manifests.length} individual user SealedSecret manifests`) + return manifests +} + export const bootstrapSealedSecrets = async ( secrets: Record, envDir: string, @@ -644,6 +791,7 @@ export const bootstrapSealedSecrets = async ( buildSecretToNamespaceMap, createSealedSecretManifest, writeSealedSecretManifests, + createUserSealedSecretManifests, encryptSecretItem, }, ): Promise => { @@ -679,7 +827,31 @@ export const bootstrapSealedSecrets = async ( manifests.push(manifest) } - // 7. Write SealedSecret manifests to disk + // 7. Create individual user SealedSecrets in apl-users namespace + const { users } = secrets + if (Array.isArray(users) && users.length > 0) { + // The users in allSecrets are in processed format (with groups). + // We also need original user data (isPlatformAdmin, isTeamAdmin, teams) from allValues. + const originalUsers = get(allValues, 'users', []) as any[] + // Merge original user fields with processed users for complete SealedSecret data + const usersForSecrets = users.map((processedUser: any) => { + const originalUser = originalUsers.find((u: any) => u.email === processedUser.email) + return { + ...processedUser, + name: originalUser?.name || processedUser.name, + isPlatformAdmin: originalUser?.isPlatformAdmin ?? false, + isTeamAdmin: originalUser?.isTeamAdmin ?? false, + teams: originalUser?.teams || [], + } + }) + const userManifests = await deps.createUserSealedSecretManifests(usersForSecrets, pem, { + encryptSecretItem: deps.encryptSecretItem, + terminal: deps.terminal, + }) + manifests.push(...userManifests) + } + + // 8. Write SealedSecret manifests to disk // Note: These manifests are applied later during install, after the sealed-secrets // controller is deployed and the SealedSecret CRD is available. await deps.writeSealedSecretManifests(manifests, envDir) diff --git a/src/operator/apl-operator.test.ts b/src/operator/apl-operator.test.ts index 671fc88d87..b8df5ae921 100644 --- a/src/operator/apl-operator.test.ts +++ b/src/operator/apl-operator.test.ts @@ -53,6 +53,9 @@ jest.mock('../common/utils', () => ({ jest.mock('../cmd/commit', () => ({ commit: jest.fn().mockResolvedValue(undefined), })) +jest.mock('../common/sealed-secrets', () => ({ + aggregateUserSecrets: jest.fn().mockResolvedValue(undefined), +})) jest.mock('./k8s', () => ({ updateApplyState: jest.fn().mockResolvedValue(undefined), diff --git a/src/operator/apl-operator.ts b/src/operator/apl-operator.ts index 8618955f3b..b631bb0fae 100644 --- a/src/operator/apl-operator.ts +++ b/src/operator/apl-operator.ts @@ -3,8 +3,9 @@ import { commit } from '../cmd/commit' import { terminal } from '../common/debug' import { env } from '../common/envalid' import { GitRepoConfig } from '../common/git-config' -import { hfValues } from '../common/hf' import { waitTillGitRepoAvailable } from '../common/gitea' +import { hfValues } from '../common/hf' +import { aggregateUserSecrets } from '../common/sealed-secrets' import { ensureTeamGitOpsDirectories } from '../common/utils' import { writeValues } from '../common/values' import { HelmArguments } from '../common/yargs' @@ -99,6 +100,10 @@ export class AplOperator { } else { await this.aplOps.apply() } + + // Aggregate individual user secrets into users-secrets for Keycloak + await aggregateUserSecrets() + this.d.info(`[${trigger}] Apply process completed`) await updateApplyState({ From 5dcded550dafa9dc043cb87f408ae5e03454006b Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sun, 22 Feb 2026 13:59:23 +0100 Subject: [PATCH 24/66] feat: update user management --- src/common/sealed-secrets.test.ts | 9 +- src/common/sealed-secrets.ts | 115 +----------------- src/operator/apl-operator.test.ts | 3 - src/operator/apl-operator.ts | 4 - .../apl-keycloak-operator-raw.gotmpl | 5 - versions.yaml | 2 +- 6 files changed, 7 insertions(+), 131 deletions(-) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 01d610f144..9deedf76c1 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -200,7 +200,7 @@ describe('sealed-secrets', () => { expect(result[0].namespace).toBe('apl-secrets') }) - it('should serialize users array as single JSON value in users-secrets', async () => { + it('should skip users path (managed individually in apl-users namespace)', async () => { const secrets = { users: [ { @@ -219,11 +219,8 @@ describe('sealed-secrets', () => { const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) - expect(result).toHaveLength(2) - const usersMapping = result.find((m) => m.secretName === 'users-secrets') - expect(usersMapping).toBeDefined() - expect(usersMapping!.namespace).toBe('apl-secrets') - expect(usersMapping!.data.usersJson).toBe(JSON.stringify(secrets.users)) + expect(result).toHaveLength(1) + expect(result.find((m) => m.secretName === 'users-secrets')).toBeUndefined() }) it('should handle teamConfig dynamic paths in apl-secrets namespace', async () => { diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index a454c2f7ca..2df49ff667 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -73,7 +73,6 @@ export const APP_NAMESPACE_MAP: Record = { dns: 'external-dns', obj: 'otomi', license: 'otomi', - users: 'keycloak', alerts: 'monitoring', cluster: 'cert-manager', } @@ -261,7 +260,6 @@ export const SECRET_NAME_MAP: Record = { dns: 'dns-secrets', obj: 'obj-storage-secrets', license: 'license-secrets', - users: 'users-secrets', alerts: 'alerts-secrets', cluster: 'cluster-secrets', } @@ -319,12 +317,11 @@ const deriveSecretName = (secretPath: string): string => { export const buildSecretToNamespaceMap = async ( secrets: Record, teams: string[], - allValues?: Record, + _allValues?: Record, deps = { getSchemaSecretsPaths }, ): Promise => { const secretPaths = await deps.getSchemaSecretsPaths(teams) const flat = flattenObject(secrets) - const allFlat = allValues ? flattenObject(allValues) : flat // Group by namespace + secretName const groupMap = new Map() @@ -332,21 +329,8 @@ export const buildSecretToNamespaceMap = async ( for (const secretPath of secretPaths) { // Skip SOPS-related paths if (secretPath.startsWith('kms.sops')) continue - // Handle 'users' path specially — serialize pre-processed users array as single JSON value - if (secretPath === 'users') { - const usersData = secrets.users - if (Array.isArray(usersData) && usersData.length > 0) { - const namespace = 'apl-secrets' - const secretName = 'users-secrets' - const groupKey = `${namespace}/${secretName}` - if (!groupMap.has(groupKey)) { - groupMap.set(groupKey, { namespace, secretName, data: {} }) - } - const mapping = groupMap.get(groupKey)! - mapping.data.usersJson = JSON.stringify(usersData) - } - continue - } + // Skip users path — user secrets are managed individually in apl-users namespace + if (secretPath === 'users') continue const namespace = resolveNamespace(secretPath) if (!namespace) continue @@ -627,99 +611,6 @@ export const restartSealedSecretsController = async (deps = { terminal }): Promi d.warn('Rollout status check timed out') } -/** - * Orchestrator: bootstrap sealed secrets for the platform. - * Replaces bootstrapSops(). - */ -/** - * Aggregate individual user K8s secrets from apl-users namespace into a single - * users-secrets Secret in apl-secrets namespace (usersJson key). - * This feeds the Keycloak Helm chart configuration. - * Called during the operator reconcile loop. - */ -export const aggregateUserSecrets = async (deps = { terminal, k8s, b64enc }): Promise => { - const d = deps.terminal(`common:${cmdName}:aggregateUserSecrets`) - const namespace = 'apl-users' - const targetNamespace = 'apl-secrets' - const targetSecretName = 'users-secrets' - - try { - const res: any = await deps.k8s.core().listNamespacedSecret({ namespace }) - const users: any[] = [] - - for (const item of res.items || []) { - if (item.type !== 'Opaque') continue - if (!item.data?.email) continue - - const decoded: Record = {} - for (const [key, value] of Object.entries(item.data as Record)) { - decoded[key] = Buffer.from(value, 'base64').toString('utf-8') - } - - // Build user in keycloak-operator format (with groups) - const groups: string[] = [] - if (decoded.isPlatformAdmin === 'true') groups.push('platform-admin') - if (decoded.isTeamAdmin === 'true') groups.push('team-admin') - const teams = decoded.teams ? JSON.parse(decoded.teams) : [] - for (const team of teams) groups.push(`team-${team}`) - - users.push({ - email: decoded.email, - firstName: decoded.firstName || '', - lastName: decoded.lastName || '', - initialPassword: decoded.initialPassword || '', - groups, - }) - } - - if (users.length === 0) { - d.info('No user secrets found in apl-users namespace, skipping aggregation') - return - } - - const usersJson = JSON.stringify(users) - - // Create or update the users-secrets Secret in apl-secrets - await ensureNamespaceExists(targetNamespace) - - try { - await deps.k8s.core().readNamespacedSecret({ name: targetSecretName, namespace: targetNamespace }) - // Secret exists, patch it - await deps.k8s.core().patchNamespacedSecret( - { - name: targetSecretName, - namespace: targetNamespace, - body: { - data: { usersJson: deps.b64enc(usersJson) }, - }, - }, - setHeaderOptions('Content-Type', PatchStrategy.StrategicMergePatch), - ) - } catch (error) { - if (error instanceof ApiException && error.code === 404) { - // Secret doesn't exist, create it - await deps.k8s.core().createNamespacedSecret({ - namespace: targetNamespace, - body: { - metadata: { - name: targetSecretName, - namespace: targetNamespace, - }, - type: 'Opaque', - data: { usersJson: deps.b64enc(usersJson) }, - }, - }) - } else { - throw error - } - } - - d.info(`Aggregated ${users.length} user(s) into ${targetNamespace}/${targetSecretName}`) - } catch (error) { - d.warn(`Failed to aggregate user secrets: ${error instanceof Error ? error.message : error}`) - } -} - /** * Create individual SealedSecret manifests for each user in the apl-users namespace. * Each user gets their own SealedSecret with all fields encrypted. diff --git a/src/operator/apl-operator.test.ts b/src/operator/apl-operator.test.ts index b8df5ae921..671fc88d87 100644 --- a/src/operator/apl-operator.test.ts +++ b/src/operator/apl-operator.test.ts @@ -53,9 +53,6 @@ jest.mock('../common/utils', () => ({ jest.mock('../cmd/commit', () => ({ commit: jest.fn().mockResolvedValue(undefined), })) -jest.mock('../common/sealed-secrets', () => ({ - aggregateUserSecrets: jest.fn().mockResolvedValue(undefined), -})) jest.mock('./k8s', () => ({ updateApplyState: jest.fn().mockResolvedValue(undefined), diff --git a/src/operator/apl-operator.ts b/src/operator/apl-operator.ts index b631bb0fae..c9a0675169 100644 --- a/src/operator/apl-operator.ts +++ b/src/operator/apl-operator.ts @@ -5,7 +5,6 @@ import { env } from '../common/envalid' import { GitRepoConfig } from '../common/git-config' import { waitTillGitRepoAvailable } from '../common/gitea' import { hfValues } from '../common/hf' -import { aggregateUserSecrets } from '../common/sealed-secrets' import { ensureTeamGitOpsDirectories } from '../common/utils' import { writeValues } from '../common/values' import { HelmArguments } from '../common/yargs' @@ -101,9 +100,6 @@ export class AplOperator { await this.aplOps.apply() } - // Aggregate individual user secrets into users-secrets for Keycloak - await aggregateUserSecrets() - this.d.info(`[${trigger}] Apply process completed`) await updateApplyState({ diff --git a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl index 2516d98edf..43763a08fa 100644 --- a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl +++ b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl @@ -41,7 +41,6 @@ resources: KEYCLOAK_ADMIN: {{ $k.adminUsername }} KEYCLOAK_ADMIN_PASSWORD: '{{ "{{ .adminPassword | toString }}" }}' KEYCLOAK_CLIENT_SECRET: '{{ "{{ .idpClientSecret | toString }}" }}' - USERS: '{{ "{{ .usersJson | toString }}" }}' {{- if $v.otomi.hasExternalIDP }} IDP_CLIENT_ID: '{{ "{{ .oidcClientID | toString }}" }}' IDP_CLIENT_SECRET: '{{ "{{ .oidcClientSecret | toString }}" }}' @@ -55,10 +54,6 @@ resources: remoteRef: key: keycloak-secrets property: idp_clientSecret - - secretKey: usersJson - remoteRef: - key: users-secrets - property: usersJson {{- if $v.otomi.hasExternalIDP }} - secretKey: oidcClientID remoteRef: diff --git a/versions.yaml b/versions.yaml index 29fc9f62a2..419c83400d 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,5 +1,5 @@ api: APL-523 console: main consoleLogin: main -tasks: APL-1476-1 +tasks: APL-523 tools: 2.10.7 From 85d4a32038d3416d772677ba49080a14677d1492 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sun, 22 Feb 2026 20:17:39 +0100 Subject: [PATCH 25/66] fix: create initial platform admin user --- src/cmd/bootstrap.test.ts | 5 ++++- src/cmd/bootstrap.ts | 4 +++- src/cmd/commit.ts | 25 ++++++++++++++++--------- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/cmd/bootstrap.test.ts b/src/cmd/bootstrap.test.ts index 44e2684cda..2c68b1097e 100644 --- a/src/cmd/bootstrap.test.ts +++ b/src/cmd/bootstrap.test.ts @@ -268,7 +268,10 @@ describe('Bootstrapping values', () => { }) expect(res.originalInput).toEqual({ cluster: { name: 'bla', provider: 'dida' }, - users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], + users: [ + { id: 'user1', initialPassword: 'existing-password' }, + { id: 'user2', initialPassword: 'generated-password' }, + ], }) }) it('should merge originalInput + allSecrets + users for disk (stripAllSecrets removes x-secret paths)', async () => { diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index 06842b9486..608627e43b 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -212,7 +212,9 @@ export const processValues = async ( // and do some context dependent post processing: // to support potential failing chart install we store secrets on cluster if (!(env.isDev && env.DISABLE_SYNC)) await deps.createK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, 'otomi', allSecrets) - return { originalInput, allSecrets } + // Include users (with name/UUID) on originalInput for bootstrapSealedSecrets to find them. + // getUsers() may return a detached array when originalInput had no 'users' key initially. + return { originalInput: { ...originalInput, users }, allSecrets } } // create file structure based on file entry diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index a497aa8290..cebef6c6ec 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -169,23 +169,30 @@ export const commit = async ( } export async function initialSetupData(): Promise { + const d = terminal(`cmd:${cmdName}:initialSetupData`) const values = (await hfValues()) as Record const { domainSuffix } = values.cluster const { hasExternalIDP } = values.otomi const secretName = hasExternalIDP ? 'root-credentials' : 'platform-admin-initial-credentials' if (!hasExternalIDP) { - // Read the platform admin's initialPassword from users-secrets (set by keycloak-operator) - const usersSecret = await getK8sSecret('users-secrets', 'apl-secrets') + // Read the platform admin's initialPassword from individual user secrets in apl-users namespace let platformAdminPassword = '' - if (usersSecret?.usersJson) { - // getK8sSecret already parses JSON/YAML values, so usersJson may be an array or a string - const users = Array.isArray(usersSecret.usersJson) - ? usersSecret.usersJson - : JSON.parse(String(usersSecret.usersJson)) + try { + const res: any = await k8s.core().listNamespacedSecret({ namespace: 'apl-users' }) const defaultEmail = `platform-admin@${domainSuffix}` - const platformAdmin = users.find((u: any) => u.email === defaultEmail) - platformAdminPassword = platformAdmin?.initialPassword ?? '' + for (const item of res.items || []) { + if (item.type !== 'Opaque' || !item.data?.email) continue + const email = Buffer.from(item.data.email, 'base64').toString('utf-8') + if (email === defaultEmail) { + platformAdminPassword = item.data.initialPassword + ? Buffer.from(item.data.initialPassword, 'base64').toString('utf-8') + : '' + break + } + } + } catch (error) { + d.warn(`Failed to read user secrets from apl-users: ${error instanceof Error ? error.message : error}`) } return { domainSuffix, From 4951838c27832034c56e37c5833da8c1831eafbc Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Sun, 22 Feb 2026 20:48:43 +0100 Subject: [PATCH 26/66] fix: create initial platform admin user --- src/cmd/commit.ts | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index cebef6c6ec..eaa1b85cb0 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -1,6 +1,7 @@ import retry from 'async-retry' import { bootstrapGit, setIdentity } from 'src/common/bootstrap' import { prepareEnvironment } from 'src/common/cli' +import { DEPLOYMENT_PASSWORDS_SECRET } from 'src/common/constants' import { encrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { env } from 'src/common/envalid' @@ -176,23 +177,17 @@ export async function initialSetupData(): Promise { const secretName = hasExternalIDP ? 'root-credentials' : 'platform-admin-initial-credentials' if (!hasExternalIDP) { - // Read the platform admin's initialPassword from individual user secrets in apl-users namespace + // Read the platform admin's initialPassword from the generated passwords secret let platformAdminPassword = '' try { - const res: any = await k8s.core().listNamespacedSecret({ namespace: 'apl-users' }) + const secretData = await getK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, 'otomi') + const allSecrets = secretData?.[DEPLOYMENT_PASSWORDS_SECRET] + const users = allSecrets?.users || [] const defaultEmail = `platform-admin@${domainSuffix}` - for (const item of res.items || []) { - if (item.type !== 'Opaque' || !item.data?.email) continue - const email = Buffer.from(item.data.email, 'base64').toString('utf-8') - if (email === defaultEmail) { - platformAdminPassword = item.data.initialPassword - ? Buffer.from(item.data.initialPassword, 'base64').toString('utf-8') - : '' - break - } - } + const platformAdmin = users.find((u: any) => u.email === defaultEmail) + platformAdminPassword = platformAdmin?.initialPassword || '' } catch (error) { - d.warn(`Failed to read user secrets from apl-users: ${error instanceof Error ? error.message : error}`) + d.warn(`Failed to read platform admin credentials: ${error instanceof Error ? error.message : error}`) } return { domainSuffix, From 1fa5be8be6ddb1ce6dbb4b54f688602efc4d1dd0 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Feb 2026 14:38:38 +0100 Subject: [PATCH 27/66] revert: sops changes for the migration --- .github/workflows/integration.yml | 13 ++++++++++ .github/workflows/main.yml | 2 +- .github/workflows/otomi-tools-build-push.yaml | 7 +----- Dockerfile | 4 +-- helmfile.d/snippets/sops-env.gotmpl | 24 ++++++++++++++++++ src/common/bootstrap.ts | 5 ++++ src/common/envalid.ts | 1 + src/operator/installer.test.ts | 25 ++++++++++++++++++- src/operator/installer.ts | 19 ++++++++++++-- tools/Dockerfile | 22 ++++++++++++++++ versions.yaml | 2 +- 11 files changed, 111 insertions(+), 13 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d3b5bf0f7a..f1cb7989a9 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -22,6 +22,10 @@ on: description: 'Select Domain Zone' type: string default: DNS-Integration + kms: + description: 'Should APL encrypt secrets in values repo (DNS or KMS is turned on)?' + type: string + default: age certificate: description: 'Select certificate issuer' type: string @@ -81,6 +85,13 @@ on: - Zone-2 - Random - DNS-Integration + kms: + type: choice + description: Should APL encrypt secrets in values repo (DNS or KMS is turned on)? + options: + - age + - no_kms + default: no_kms certificate: type: choice description: Select certificate issuer @@ -131,6 +142,7 @@ jobs: echo 'install_profile: ${{ inputs.install_profile }}' echo 'lke_tier: ${{ inputs.lke_tier }}' echo 'kubernetes_version: ${{ inputs.kubernetes_version }}' + echo 'kms: ${{ inputs.kms }}' echo 'domain_zone: ${{ inputs.domain_zone }}' echo 'certificate: ${{ inputs.certificate }}' echo 'is_pre_installed: ${{ inputs.is_pre_installed }}' @@ -302,6 +314,7 @@ jobs: additional_args="" [[ '${{ inputs.certificate }}' == 'letsencrypt_staging' ]] && echo "$LETSENCRYPT_STAGING" >> values.yaml [[ '${{ inputs.certificate }}' == 'letsencrypt_production' ]] && echo "$LETSENCRYPT_PRODUCTION" >> values.yaml + [[ '${{ inputs.kms }}' == 'age' ]] && additional_args+=" --set kms.sops.provider=age" [[ '${{ inputs.is_pre_installed }}' == 'true' ]] && additional_args+=" --set otomi.isPreInstalled=true" if [[ '${{ inputs.disableORCS }}' == 'true' ]]; then additional_args+=" --set otomi.useORCS=false" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 02a3db4543..feb4919bc9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -132,7 +132,7 @@ jobs: if: always() && contains(needs.release.result, 'success') && !github.event.act runs-on: ubuntu-22.04 container: - image: linode/apl-tools:v2.10.7 + image: linode/apl-tools:v2.10.6 options: --user 0 # See https://docs.github.com/en/actions/sharing-automations/creating-actions/dockerfile-support-for-github-actions#user steps: - name: Checkout diff --git a/.github/workflows/otomi-tools-build-push.yaml b/.github/workflows/otomi-tools-build-push.yaml index 3d38c4bc3f..050f74b65a 100644 --- a/.github/workflows/otomi-tools-build-push.yaml +++ b/.github/workflows/otomi-tools-build-push.yaml @@ -11,7 +11,6 @@ on: push: branches: - 'main' - - 'APL-523' env: NAMESPACE: linode @@ -71,11 +70,7 @@ jobs: echo OLD_VERSION = ${OLD_VERSION} echo NEW_VERSION = ${NEW_VERSION} fi - - # Override: rebuild v2.10.7 with gucci/kubeconform/htpasswd restored - NEW_VERSION="v2.10.7" - echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV - echo "Overriding NEW_VERSION=${NEW_VERSION}" + echo "No need to bump the version. Will skip next steps." - name: Login to GitHub Container Registry if: ${{ env.NEW_VERSION != null }} diff --git a/Dockerfile b/Dockerfile index 58a4f88252..caa68fc085 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM linode/apl-tools:v2.10.7 AS ci +FROM linode/apl-tools:v2.10.6 AS ci ENV APP_HOME=/home/app/stack @@ -36,7 +36,7 @@ FROM ci AS clean # below command removes the packages specified in devDependencies and set NODE_ENV to production RUN npm prune --production -FROM linode/apl-tools:v2.10.7 AS prod +FROM linode/apl-tools:v2.10.6 AS prod ARG APPS_REVISION='' ENV APP_HOME=/home/app/stack ENV ENV_DIR=/home/app/stack/env diff --git a/helmfile.d/snippets/sops-env.gotmpl b/helmfile.d/snippets/sops-env.gotmpl index e69de29bb2..aa9e2ff866 100644 --- a/helmfile.d/snippets/sops-env.gotmpl +++ b/helmfile.d/snippets/sops-env.gotmpl @@ -0,0 +1,24 @@ +{{- with . | get "azure" nil }} +AZURE_CLIENT_ID: {{ .clientId }} +AZURE_CLIENT_SECRET: {{ .clientSecret }} +{{- with . | get "tenantId" nil }} +AZURE_TENANT_ID: {{ . }}{{ end }} +{{- with . | get "environment" nil }} +AZURE_ENVIRONMENT: {{ . }}{{ end }} +{{- end }} +{{- with . | get "aws" nil }} +AWS_ACCESS_KEY_ID: {{ .accessKey }} +AWS_SECRET_ACCESS_KEY: {{ .secretKey }} +{{- with . | get "region" nil }} +AWS_REGION: {{ . }}{{ end }} +{{- end }} +{{- with . | get "age" nil }} +SOPS_AGE_KEY: {{ .privateKey }} +{{- end }} +{{- with . | get "google" nil }} +GCLOUD_SERVICE_KEY: '{{ .accountJson | replace "\n" "" }}' +{{- with . | get "project" nil }} +GOOGLE_PROJECT: {{ . }}{{ end }} +{{- with . | get "region" nil }} +GOOGLE_REGION: {{ . }}{{ end }} +{{- end }} diff --git a/src/common/bootstrap.ts b/src/common/bootstrap.ts index 6e493353eb..a897d1d976 100644 --- a/src/common/bootstrap.ts +++ b/src/common/bootstrap.ts @@ -90,5 +90,10 @@ export const bootstrapGit = async (inValues?: Record): Promise { }) describe('setEnvAndCreateSecrets', () => { - test('should complete without errors', async () => { + test('should set SOPS_AGE_KEY when secret exists', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'AGE-SECRET-KEY-1ABC' }) + await installer.setEnvAndCreateSecrets() + + expect(k8s.getK8sSecret).toHaveBeenCalledWith('apl-sops-secrets', 'apl-operator') + expect(process.env.SOPS_AGE_KEY).toBe('AGE-SECRET-KEY-1ABC') + }) + + test('should not throw when secret does not exist', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) + + await expect(installer.setEnvAndCreateSecrets()).resolves.not.toThrow() + }) + + test('should not throw when getK8sSecret fails', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockRejectedValue(new Error('Not found')) + + await expect(installer.setEnvAndCreateSecrets()).resolves.not.toThrow() + }) + + test('should handle secret without SOPS_AGE_KEY field', async () => { + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ OTHER_KEY: 'value' }) + + await expect(installer.setEnvAndCreateSecrets()).resolves.not.toThrow() }) }) }) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index c4616cebd9..df2d950b3d 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -1,7 +1,7 @@ import { $ } from 'zx' import { terminal } from '../common/debug' import { getStoredGitRepoConfig } from '../common/git-config' -import { createUpdateConfigMap, deletePendingHelmReleases, getK8sConfigMap, k8s } from '../common/k8s' +import { createUpdateConfigMap, deletePendingHelmReleases, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' @@ -122,6 +122,21 @@ export class Installer { } public async setEnvAndCreateSecrets(): Promise { - this.d.debug('Environment setup complete (SOPS removed, using SealedSecrets + ESO)') + this.d.debug('Setting up environment') + await this.setupSopsEnvironment() + } + + private async setupSopsEnvironment(): Promise { + try { + const aplSopsSecret = await getK8sSecret('apl-sops-secrets', 'apl-operator') + if (!aplSopsSecret?.SOPS_AGE_KEY) { + this.d.info('SOPS_AGE_KEY not found — cluster may already use SealedSecrets') + return + } + process.env.SOPS_AGE_KEY = aplSopsSecret.SOPS_AGE_KEY + this.d.info('SOPS environment configured') + } catch (error) { + this.d.info('Could not read apl-sops-secrets — cluster may already use SealedSecrets') + } } } diff --git a/tools/Dockerfile b/tools/Dockerfile index 4f22ba1053..4fad532e80 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -9,6 +9,12 @@ ARG KUBECTL_VERSION=1.34.2 ARG HELM_VERSION=3.19.2 # https://github.com/databus23/helm-diff/releases ARG HELM_DIFF_VERSION=3.14.1 +# https://github.com/jkroepke/helm-secrets/releases +ARG HELM_SECRETS_VERSION=4.7.4 +# https://github.com/getsops/sops/releases +ARG SOPS_VERSION=3.11.0 +# https://github.com/FiloSottile/age/releases +ARG AGE_VERSION=1.2.1 # https://github.com/noqcks/gucci/releases ARG GUCCI_VERSION=1.9.0 # https://github.com/yannh/kubeconform/releases/ @@ -30,6 +36,7 @@ RUN apt-get update && apt-get install -y \ curl \ apache2-utils \ ca-certificates \ + coreutils \ git \ locales \ rsync && \ @@ -43,6 +50,9 @@ RUN curl -LO "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq mv jq-linux-${TARGETARCH} /usr/bin/jq && \ chmod +x /usr/bin/jq +# yq +COPY --from=mikefarah/yq:4 /usr/bin/yq /usr/bin/yq + RUN mkdir -p /home/app RUN groupadd -r app && \ useradd -r -g app -d /home/app -s /sbin/nologin -c "Docker image user" app @@ -65,10 +75,22 @@ RUN CNPG_ARCH=$(if [ "${TARGETARCH}" = "amd64" ]; then echo "x86_64"; else echo chmod +x kubectl-cnpg && \ rm kubectl-cnpg_${CNPG_VERSION}_linux_${CNPG_ARCH}.tar.gz +# sops +ADD https://github.com/getsops/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux.amd64 sops +RUN chmod +x sops + +# age +ADD https://github.com/FiloSottile/age/releases/download/v${AGE_VERSION}/age-v${AGE_VERSION}-linux-amd64.tar.gz age.tar.gz +RUN tar -zxvf age.tar.gz age/age age/age-keygen --strip-components=1 && \ + chmod +x age age-keygen && \ + rm -rf age.tar.gz + # helm ADD https://get.helm.sh/${HELM_FILE_NAME} /tmp RUN tar -zxvf /tmp/${HELM_FILE_NAME} -C /tmp && mv /tmp/linux-${TARGETARCH}/helm helm && rm -rf /tmp/* RUN helm plugin install https://github.com/databus23/helm-diff --version ${HELM_DIFF_VERSION} +RUN echo "exec \$*" > /usr/bin/sudo && chmod +x /usr/bin/sudo +RUN helm plugin install https://github.com/jkroepke/helm-secrets --version ${HELM_SECRETS_VERSION} # helmfile ADD https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz /tmp diff --git a/versions.yaml b/versions.yaml index 419c83400d..d722673428 100644 --- a/versions.yaml +++ b/versions.yaml @@ -2,4 +2,4 @@ api: APL-523 console: main consoleLogin: main tasks: APL-523 -tools: 2.10.7 +tools: main From bbdeaff4542fb403d0afce8f1e3758aed7b855e2 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Feb 2026 16:20:13 +0100 Subject: [PATCH 28/66] test: platform secrets migration --- src/cmd/migrate.test.ts | 205 +++++++++++++++++++++++++++++++++++++++- src/cmd/migrate.ts | 119 +++++++++++++++++++++++ values-changes.yaml | 5 + 3 files changed, 328 insertions(+), 1 deletion(-) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 3d33a34e13..07e2123055 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -1,5 +1,14 @@ +import { existsSync, rmSync } from 'fs' import { globSync } from 'glob' -import { applyChanges, Changes, filterChanges, getBuildName, policiesMigration } from 'src/cmd/migrate' +import { + applyChanges, + Changes, + filterChanges, + getBuildName, + policiesMigration, + removeSopsArtifacts, + sopsMigration, +} from 'src/cmd/migrate' import { terminal } from '../common/debug' import { env } from '../common/envalid' import { getFileMap } from '../common/repo' @@ -13,6 +22,7 @@ jest.mock('../common/k8s') jest.mock('../common/values') jest.mock('../common/yargs') jest.mock('../common/utils') +jest.mock('../common/sealed-secrets') jest.mock('zx') jest.mock('@linode/kubeseal-encrypt') jest.mock('fs', () => ({ @@ -904,3 +914,196 @@ describe('setDefaultAplCatalog migration', () => { ) }, 20000) }) + +describe('sopsMigration', () => { + const mockTerminal = terminal + const mockExistsSync = jest.fn() + const mockGlobSync = jest.fn() + const mockGetExistingSealedSecretsCert = jest.fn() + const mockGetPemFromCertificate = jest.fn() + const mockGenerateSealedSecretsKeyPair = jest.fn() + const mockCreateSealedSecretsKeySecret = jest.fn() + const mockBuildSecretToNamespaceMap = jest.fn() + const mockCreateSealedSecretManifest = jest.fn() + const mockCreateUserSealedSecretManifests = jest.fn() + const mockWriteSealedSecretManifests = jest.fn() + const mockGetSchemaSecretsPaths = jest.fn() + const mockRemoveSopsArtifacts = jest.fn() + + const makeDeps = () => ({ + existsSync: mockExistsSync, + globSync: mockGlobSync, + terminal: mockTerminal, + getExistingSealedSecretsCert: mockGetExistingSealedSecretsCert, + getPemFromCertificate: mockGetPemFromCertificate, + generateSealedSecretsKeyPair: mockGenerateSealedSecretsKeyPair, + createSealedSecretsKeySecret: mockCreateSealedSecretsKeySecret, + buildSecretToNamespaceMap: mockBuildSecretToNamespaceMap, + createSealedSecretManifest: mockCreateSealedSecretManifest, + createUserSealedSecretManifests: mockCreateUserSealedSecretManifests, + writeSealedSecretManifests: mockWriteSealedSecretManifests, + getSchemaSecretsPaths: mockGetSchemaSecretsPaths, + removeSopsArtifacts: mockRemoveSopsArtifacts, + }) + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('should skip when no .sops.yaml exists', async () => { + mockExistsSync.mockReturnValue(false) + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() + expect(mockRemoveSopsArtifacts).not.toHaveBeenCalled() + }) + + it('should only clean up when manifests already exist', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue(['/some/manifest.yaml']) + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockRemoveSopsArtifacts).toHaveBeenCalled() + expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() + }) + + it('should run full migration path', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue([]) + mockGetExistingSealedSecretsCert.mockResolvedValue(undefined) + mockGenerateSealedSecretsKeyPair.mockReturnValue({ certificate: 'cert-pem', privateKey: 'key-pem' }) + mockCreateSealedSecretsKeySecret.mockResolvedValue(undefined) + mockGetPemFromCertificate.mockReturnValue('spki-pem') + mockBuildSecretToNamespaceMap.mockResolvedValue([ + { namespace: 'apl-secrets', secretName: 'gitea-secrets', data: { adminPassword: 'pass' } }, + ]) + mockCreateSealedSecretManifest.mockResolvedValue({ + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { name: 'gitea-secrets', namespace: 'apl-secrets', annotations: {} }, + spec: { encryptedData: { adminPassword: 'encrypted' }, template: {} }, + }) + mockCreateUserSealedSecretManifests.mockResolvedValue([]) + mockWriteSealedSecretManifests.mockResolvedValue(undefined) + mockGetSchemaSecretsPaths.mockResolvedValue(['apps.gitea.adminPassword']) + + const values = { + teamConfig: {}, + versions: { specVersion: 55 }, + apps: { gitea: { adminPassword: 'pass' } }, + } + + await sopsMigration(values, makeDeps()) + + expect(mockGenerateSealedSecretsKeyPair).toHaveBeenCalled() + expect(mockCreateSealedSecretsKeySecret).toHaveBeenCalledWith('cert-pem', 'key-pem') + expect(mockBuildSecretToNamespaceMap).toHaveBeenCalled() + expect(mockCreateSealedSecretManifest).toHaveBeenCalledWith('spki-pem', expect.any(Object)) + expect(mockWriteSealedSecretManifests).toHaveBeenCalled() + expect(mockGetSchemaSecretsPaths).toHaveBeenCalled() + expect(mockRemoveSopsArtifacts).toHaveBeenCalled() + // Secrets should be stripped from values (in-place mutation) + expect(values.apps.gitea.adminPassword).toBeUndefined() + }) + + it('should use existing certificate when available', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue([]) + mockGetExistingSealedSecretsCert.mockResolvedValue('existing-cert-pem') + mockGetPemFromCertificate.mockReturnValue('existing-spki-pem') + mockBuildSecretToNamespaceMap.mockResolvedValue([]) + mockWriteSealedSecretManifests.mockResolvedValue(undefined) + mockGetSchemaSecretsPaths.mockResolvedValue([]) + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockGenerateSealedSecretsKeyPair).not.toHaveBeenCalled() + expect(mockCreateSealedSecretsKeySecret).not.toHaveBeenCalled() + expect(mockGetPemFromCertificate).toHaveBeenCalledWith('existing-cert-pem') + }) + + it('should handle users', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue([]) + mockGetExistingSealedSecretsCert.mockResolvedValue('cert') + mockGetPemFromCertificate.mockReturnValue('pem') + mockBuildSecretToNamespaceMap.mockResolvedValue([]) + mockCreateUserSealedSecretManifests.mockResolvedValue([ + { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { name: 'user1', namespace: 'apl-users' }, + }, + ]) + mockWriteSealedSecretManifests.mockResolvedValue(undefined) + mockGetSchemaSecretsPaths.mockResolvedValue([]) + + const values = { + teamConfig: {}, + versions: { specVersion: 55 }, + users: [{ name: 'user1', email: 'user1@example.com' }], + } + + await sopsMigration(values, makeDeps()) + + expect(mockCreateUserSealedSecretManifests).toHaveBeenCalledWith( + [{ name: 'user1', email: 'user1@example.com' }], + 'pem', + ) + }) + + it('should handle empty secrets gracefully', async () => { + mockExistsSync.mockReturnValue(true) + mockGlobSync.mockReturnValue([]) + mockGetExistingSealedSecretsCert.mockResolvedValue('cert') + mockGetPemFromCertificate.mockReturnValue('pem') + mockBuildSecretToNamespaceMap.mockResolvedValue([]) + mockWriteSealedSecretManifests.mockResolvedValue(undefined) + mockGetSchemaSecretsPaths.mockResolvedValue([]) + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockCreateSealedSecretManifest).not.toHaveBeenCalled() + expect(mockWriteSealedSecretManifests).toHaveBeenCalledWith([], env.ENV_DIR) + expect(mockRemoveSopsArtifacts).toHaveBeenCalled() + }) +}) + +describe('removeSopsArtifacts', () => { + it('should remove .sops.yaml and all secrets files', () => { + const mockExistsSync = jest.fn().mockReturnValue(true) + const mockRmSync = jest.fn() + const mockGlobSync = jest + .fn() + .mockReturnValueOnce([`${env.ENV_DIR}/env/apps/secrets.gitea.yaml`]) + .mockReturnValueOnce([`${env.ENV_DIR}/env/apps/secrets.gitea.yaml.dec`]) + + removeSopsArtifacts({ + existsSync: mockExistsSync, + rmSync: mockRmSync, + globSync: mockGlobSync, + terminal, + }) + + expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/.sops.yaml`) + expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/env/apps/secrets.gitea.yaml`) + expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/env/apps/secrets.gitea.yaml.dec`) + }) + + it('should skip .sops.yaml removal when it does not exist', () => { + const mockExistsSync = jest.fn().mockReturnValue(false) + const mockRmSync = jest.fn() + const mockGlobSync = jest.fn().mockReturnValue([]) + + removeSopsArtifacts({ + existsSync: mockExistsSync, + rmSync: mockRmSync, + globSync: mockGlobSync, + terminal, + }) + + expect(mockRmSync).not.toHaveBeenCalledWith(`${env.ENV_DIR}/.sops.yaml`) + }) +}) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index f5c9cf5121..58783d3c17 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -20,6 +20,17 @@ import { v4 as uuidv4 } from 'uuid' import { parse } from 'yaml' import { Argv } from 'yargs' import { $, cd } from 'zx' +import { + buildSecretToNamespaceMap, + createSealedSecretManifest, + createSealedSecretsKeySecret, + createUserSealedSecretManifests, + generateSealedSecretsKeyPair, + getExistingSealedSecretsCert, + getPemFromCertificate, + SealedSecretManifest, + writeSealedSecretManifests, +} from '../common/sealed-secrets' import { ARGOCD_APP_PARAMS } from '../common/constants' import { getK8sSecret, getSealedSecretsPEM, k8s } from '../common/k8s' @@ -740,6 +751,113 @@ const setDefaultAplCatalog = async (values: Record): Promise set(values, 'catalogs.default', defaultCatalog) } +export const removeSopsArtifacts = (deps = { existsSync, rmSync, globSync, terminal }): void => { + const d = deps.terminal(`cmd:${cmdName}:removeSopsArtifacts`) + + // Remove .sops.yaml — makes encrypt()/decrypt() no-ops + const sopsConfigPath = `${env.ENV_DIR}/.sops.yaml` + if (deps.existsSync(sopsConfigPath)) { + deps.rmSync(sopsConfigPath) + d.info('Removed .sops.yaml') + } + + // Remove SOPS-encrypted files + const sopsEncrypted = deps.globSync(`${env.ENV_DIR}/env/**/secrets.*.yaml`, { dot: false }) + for (const f of sopsEncrypted) { + deps.rmSync(f) + d.info(`Removed ${f}`) + } + + // Remove SOPS-decrypted files + const sopsDecrypted = deps.globSync(`${env.ENV_DIR}/env/**/secrets.*.yaml.dec`, { dot: false }) + for (const f of sopsDecrypted) { + deps.rmSync(f) + d.info(`Removed ${f}`) + } +} + +export const sopsMigration = async ( + values: Record, + deps = { + existsSync, + globSync, + terminal, + getExistingSealedSecretsCert, + getPemFromCertificate, + generateSealedSecretsKeyPair, + createSealedSecretsKeySecret, + buildSecretToNamespaceMap, + createSealedSecretManifest, + createUserSealedSecretManifests, + writeSealedSecretManifests, + getSchemaSecretsPaths, + removeSopsArtifacts, + }, +): Promise => { + const d = deps.terminal(`cmd:${cmdName}:sopsMigration`) + + // Idempotency guard: no SOPS config means nothing to migrate + if (!deps.existsSync(`${env.ENV_DIR}/.sops.yaml`)) { + d.info('No .sops.yaml found, skipping SOPS migration') + return + } + + // Secondary guard: if manifests already exist, just clean up SOPS artifacts + const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/ns/**/*.yaml`, { dot: false }) + if (existingManifests.length > 0) { + d.info('SealedSecret manifests already exist, only cleaning up SOPS artifacts') + deps.removeSopsArtifacts() + return + } + + d.info('Starting SOPS to SealedSecrets migration') + + // Get or generate sealed-secrets key + let pem: string + const existingCert = await deps.getExistingSealedSecretsCert() + if (existingCert) { + d.info('Using existing sealed-secrets certificate') + pem = deps.getPemFromCertificate(existingCert) + } else { + d.info('Generating new sealed-secrets key pair') + const { certificate, privateKey } = deps.generateSealedSecretsKeyPair() + await deps.createSealedSecretsKeySecret(certificate, privateKey) + pem = deps.getPemFromCertificate(certificate) + } + + // Build secret-to-namespace mappings + const teams = Object.keys((values.teamConfig as Record) || {}) + const mappings = await deps.buildSecretToNamespaceMap(values, teams, values) + + // Create core SealedSecret manifests + const manifests: SealedSecretManifest[] = [] + for (const mapping of mappings) { + const manifest = await deps.createSealedSecretManifest(pem, mapping) + manifests.push(manifest) + } + + // Create user SealedSecret manifests + const users = values.users as any[] | undefined + if (Array.isArray(users) && users.length > 0) { + const userManifests = await deps.createUserSealedSecretManifests(users, pem) + manifests.push(...userManifests) + } + + // Write manifests to disk + await deps.writeSealedSecretManifests(manifests, env.ENV_DIR) + d.info(`Wrote ${manifests.length} SealedSecret manifests`) + + // Strip secrets from values (in-place mutation — writeValues() persists after return) + const secretPaths = await deps.getSchemaSecretsPaths(teams) + for (const path of secretPaths) { + unset(values, path) + } + + // Remove SOPS artifacts + deps.removeSopsArtifacts() + d.info('SOPS to SealedSecrets migration complete') +} + const customMigrationFunctions: Record = { networkPoliciesMigration, teamSettingsMigration, @@ -751,6 +869,7 @@ const customMigrationFunctions: Record = { workloadValuesMigration, setLokiStorageSchemaMigration, setDefaultAplCatalog, + sopsMigration, } /** diff --git a/values-changes.yaml b/values-changes.yaml index 52ee7013e5..54b771a686 100644 --- a/values-changes.yaml +++ b/values-changes.yaml @@ -446,3 +446,8 @@ changes: relocations: - apps.gitea.adminPassword: otomi.git.password - apps.gitea.adminUsername: otomi.git.username + - version: 56 + deletions: + - 'kms.sops' + customFunctions: + - sopsMigration From 44781ee97e092f78df9bec1676d735292ca00e9f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Feb 2026 17:05:55 +0100 Subject: [PATCH 29/66] fix: platform secrets migration --- src/cmd/migrate.test.ts | 4 +++- src/cmd/migrate.ts | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 07e2123055..b76d50c3a5 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -1072,13 +1072,14 @@ describe('sopsMigration', () => { }) describe('removeSopsArtifacts', () => { - it('should remove .sops.yaml and all secrets files', () => { + it('should remove .sops.yaml, secrets files, and user files', () => { const mockExistsSync = jest.fn().mockReturnValue(true) const mockRmSync = jest.fn() const mockGlobSync = jest .fn() .mockReturnValueOnce([`${env.ENV_DIR}/env/apps/secrets.gitea.yaml`]) .mockReturnValueOnce([`${env.ENV_DIR}/env/apps/secrets.gitea.yaml.dec`]) + .mockReturnValueOnce([`${env.ENV_DIR}/env/users/some-uuid.yaml`]) removeSopsArtifacts({ existsSync: mockExistsSync, @@ -1090,6 +1091,7 @@ describe('removeSopsArtifacts', () => { expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/.sops.yaml`) expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/env/apps/secrets.gitea.yaml`) expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/env/apps/secrets.gitea.yaml.dec`) + expect(mockRmSync).toHaveBeenCalledWith(`${env.ENV_DIR}/env/users/some-uuid.yaml`) }) it('should skip .sops.yaml removal when it does not exist', () => { diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 58783d3c17..d83297039c 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -774,6 +774,15 @@ export const removeSopsArtifacts = (deps = { existsSync, rmSync, globSync, termi deps.rmSync(f) d.info(`Removed ${f}`) } + + // Remove user YAML files — users are now managed via SealedSecrets in env/manifests/ns/apl-users/. + // These files may contain SOPS-encrypted data that was written by the "Write default values" step + // before SOPS decryption ran, contaminating the public YAML files with ENC[...] strings. + const userFiles = deps.globSync(`${env.ENV_DIR}/env/users/*.yaml`, { dot: false }) + for (const f of userFiles) { + deps.rmSync(f) + d.info(`Removed ${f}`) + } } export const sopsMigration = async ( From d9a168acde412d674e588a4dcdd3272c9f509717 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Feb 2026 21:00:18 +0100 Subject: [PATCH 30/66] fix: platform secrets migration --- src/cmd/migrate.test.ts | 3 +++ src/cmd/migrate.ts | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index b76d50c3a5..8387f31457 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -927,6 +927,7 @@ describe('sopsMigration', () => { const mockCreateSealedSecretManifest = jest.fn() const mockCreateUserSealedSecretManifests = jest.fn() const mockWriteSealedSecretManifests = jest.fn() + const mockApplySealedSecretManifestsFromDir = jest.fn().mockResolvedValue(undefined) const mockGetSchemaSecretsPaths = jest.fn() const mockRemoveSopsArtifacts = jest.fn() @@ -942,6 +943,7 @@ describe('sopsMigration', () => { createSealedSecretManifest: mockCreateSealedSecretManifest, createUserSealedSecretManifests: mockCreateUserSealedSecretManifests, writeSealedSecretManifests: mockWriteSealedSecretManifests, + applySealedSecretManifestsFromDir: mockApplySealedSecretManifestsFromDir, getSchemaSecretsPaths: mockGetSchemaSecretsPaths, removeSopsArtifacts: mockRemoveSopsArtifacts, }) @@ -1002,6 +1004,7 @@ describe('sopsMigration', () => { expect(mockBuildSecretToNamespaceMap).toHaveBeenCalled() expect(mockCreateSealedSecretManifest).toHaveBeenCalledWith('spki-pem', expect.any(Object)) expect(mockWriteSealedSecretManifests).toHaveBeenCalled() + expect(mockApplySealedSecretManifestsFromDir).toHaveBeenCalledWith(env.ENV_DIR) expect(mockGetSchemaSecretsPaths).toHaveBeenCalled() expect(mockRemoveSopsArtifacts).toHaveBeenCalled() // Secrets should be stripped from values (in-place mutation) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index d83297039c..b5b9c93361 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -21,6 +21,7 @@ import { parse } from 'yaml' import { Argv } from 'yargs' import { $, cd } from 'zx' import { + applySealedSecretManifestsFromDir, buildSecretToNamespaceMap, createSealedSecretManifest, createSealedSecretsKeySecret, @@ -799,6 +800,7 @@ export const sopsMigration = async ( createSealedSecretManifest, createUserSealedSecretManifests, writeSealedSecretManifests, + applySealedSecretManifestsFromDir, getSchemaSecretsPaths, removeSopsArtifacts, }, @@ -856,6 +858,11 @@ export const sopsMigration = async ( await deps.writeSealedSecretManifests(manifests, env.ENV_DIR) d.info(`Wrote ${manifests.length} SealedSecret manifests`) + // Apply SealedSecret manifests to the cluster so the sealed-secrets controller + // can decrypt them into K8s Secrets before the apply step needs them. + d.info('Applying SealedSecret manifests to cluster') + await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) + // Strip secrets from values (in-place mutation — writeValues() persists after return) const secretPaths = await deps.getSchemaSecretsPaths(teams) for (const path of secretPaths) { From c28e7d8ecb8bb4fcd7239b2fc9e814cac3edf867 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Feb 2026 22:18:15 +0100 Subject: [PATCH 31/66] fix: platform secrets migration --- src/cmd/migrate.test.ts | 3 +++ src/cmd/migrate.ts | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 8387f31457..560a18ed58 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -928,6 +928,7 @@ describe('sopsMigration', () => { const mockCreateUserSealedSecretManifests = jest.fn() const mockWriteSealedSecretManifests = jest.fn() const mockApplySealedSecretManifestsFromDir = jest.fn().mockResolvedValue(undefined) + const mockRestartSealedSecretsController = jest.fn().mockResolvedValue(undefined) const mockGetSchemaSecretsPaths = jest.fn() const mockRemoveSopsArtifacts = jest.fn() @@ -944,6 +945,7 @@ describe('sopsMigration', () => { createUserSealedSecretManifests: mockCreateUserSealedSecretManifests, writeSealedSecretManifests: mockWriteSealedSecretManifests, applySealedSecretManifestsFromDir: mockApplySealedSecretManifestsFromDir, + restartSealedSecretsController: mockRestartSealedSecretsController, getSchemaSecretsPaths: mockGetSchemaSecretsPaths, removeSopsArtifacts: mockRemoveSopsArtifacts, }) @@ -1005,6 +1007,7 @@ describe('sopsMigration', () => { expect(mockCreateSealedSecretManifest).toHaveBeenCalledWith('spki-pem', expect.any(Object)) expect(mockWriteSealedSecretManifests).toHaveBeenCalled() expect(mockApplySealedSecretManifestsFromDir).toHaveBeenCalledWith(env.ENV_DIR) + expect(mockRestartSealedSecretsController).toHaveBeenCalled() expect(mockGetSchemaSecretsPaths).toHaveBeenCalled() expect(mockRemoveSopsArtifacts).toHaveBeenCalled() // Secrets should be stripped from values (in-place mutation) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index b5b9c93361..ee0f34ddd3 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -29,6 +29,7 @@ import { generateSealedSecretsKeyPair, getExistingSealedSecretsCert, getPemFromCertificate, + restartSealedSecretsController, SealedSecretManifest, writeSealedSecretManifests, } from '../common/sealed-secrets' @@ -801,6 +802,7 @@ export const sopsMigration = async ( createUserSealedSecretManifests, writeSealedSecretManifests, applySealedSecretManifestsFromDir, + restartSealedSecretsController, getSchemaSecretsPaths, removeSopsArtifacts, }, @@ -863,6 +865,11 @@ export const sopsMigration = async ( d.info('Applying SealedSecret manifests to cluster') await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) + // Restart the sealed-secrets controller so it picks up the migration-generated key. + // Without this, the controller uses its auto-generated key and cannot decrypt. + d.info('Restarting sealed-secrets controller to use migration key') + await deps.restartSealedSecretsController() + // Strip secrets from values (in-place mutation — writeValues() persists after return) const secretPaths = await deps.getSchemaSecretsPaths(teams) for (const path of secretPaths) { From 7b4574c05cb3913d5927652f2c19a963c606751c Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Feb 2026 22:37:35 +0100 Subject: [PATCH 32/66] fix: platform secrets migration --- src/cmd/migrate.test.ts | 31 ++++++++++++++++++++++++++++++- src/cmd/migrate.ts | 16 +++++++++++++++- src/common/constants.ts | 2 +- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 560a18ed58..001ee67523 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -929,6 +929,7 @@ describe('sopsMigration', () => { const mockWriteSealedSecretManifests = jest.fn() const mockApplySealedSecretManifestsFromDir = jest.fn().mockResolvedValue(undefined) const mockRestartSealedSecretsController = jest.fn().mockResolvedValue(undefined) + const mockGetK8sSecret = jest.fn().mockResolvedValue(undefined) const mockGetSchemaSecretsPaths = jest.fn() const mockRemoveSopsArtifacts = jest.fn() @@ -946,6 +947,7 @@ describe('sopsMigration', () => { writeSealedSecretManifests: mockWriteSealedSecretManifests, applySealedSecretManifestsFromDir: mockApplySealedSecretManifestsFromDir, restartSealedSecretsController: mockRestartSealedSecretsController, + getK8sSecret: mockGetK8sSecret, getSchemaSecretsPaths: mockGetSchemaSecretsPaths, removeSopsArtifacts: mockRemoveSopsArtifacts, }) @@ -954,15 +956,42 @@ describe('sopsMigration', () => { jest.clearAllMocks() }) - it('should skip when no .sops.yaml exists', async () => { + it('should skip when no .sops.yaml exists and no manifests on disk', async () => { mockExistsSync.mockReturnValue(false) + mockGlobSync.mockReturnValue([]) await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() + expect(mockApplySealedSecretManifestsFromDir).not.toHaveBeenCalled() + expect(mockRestartSealedSecretsController).not.toHaveBeenCalled() expect(mockRemoveSopsArtifacts).not.toHaveBeenCalled() }) + it('should re-apply and restart controller when manifests exist but K8s Secrets are missing', async () => { + mockExistsSync.mockReturnValue(false) + mockGlobSync.mockReturnValue(['/env/manifests/ns/apl-secrets/otomi-platform-secrets.yaml']) + mockGetK8sSecret.mockResolvedValue(undefined) // Secret doesn't exist yet + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockApplySealedSecretManifestsFromDir).toHaveBeenCalledWith(env.ENV_DIR) + expect(mockRestartSealedSecretsController).toHaveBeenCalled() + expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() + }) + + it('should skip re-apply when manifests exist and K8s Secrets already exist', async () => { + mockExistsSync.mockReturnValue(false) + mockGlobSync.mockReturnValue(['/env/manifests/ns/apl-secrets/otomi-platform-secrets.yaml']) + mockGetK8sSecret.mockResolvedValue({ git_password: 'somepassword' }) // Secret exists + + await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) + + expect(mockApplySealedSecretManifestsFromDir).not.toHaveBeenCalled() + expect(mockRestartSealedSecretsController).not.toHaveBeenCalled() + expect(mockBuildSecretToNamespaceMap).not.toHaveBeenCalled() + }) + it('should only clean up when manifests already exist', async () => { mockExistsSync.mockReturnValue(true) mockGlobSync.mockReturnValue(['/some/manifest.yaml']) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index ee0f34ddd3..f6a8d5ac64 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -803,14 +803,28 @@ export const sopsMigration = async ( writeSealedSecretManifests, applySealedSecretManifestsFromDir, restartSealedSecretsController, + getK8sSecret, getSchemaSecretsPaths, removeSopsArtifacts, }, ): Promise => { const d = deps.terminal(`cmd:${cmdName}:sopsMigration`) - // Idempotency guard: no SOPS config means nothing to migrate + // Idempotency guard: no SOPS config means migration already ran. + // However, if SealedSecret manifests exist on disk but the K8s Secrets are not yet + // decrypted (e.g. the operator was killed after writing manifests but before applying + // them, or the controller used its auto-generated key), re-apply them and restart + // the controller so subsequent steps can resolve the git password. if (!deps.existsSync(`${env.ENV_DIR}/.sops.yaml`)) { + const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/ns/**/*.yaml`, { dot: false }) + if (existingManifests.length > 0) { + const platformSecret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') + if (!platformSecret) { + d.info('SealedSecret manifests exist but K8s Secrets are missing — re-applying and restarting controller') + await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) + await deps.restartSealedSecretsController() + } + } d.info('No .sops.yaml found, skipping SOPS migration') return } diff --git a/src/common/constants.ts b/src/common/constants.ts index 43feae7157..f6252f3985 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -12,7 +12,7 @@ export const ARGOCD_APP_DEFAULT_SYNC_POLICY = { allowEmpty: false, selfHeal: true, }, - syncOptions: ['ServerSideApply=true'], + syncOptions: ['ServerSideApply=true', 'CreateNamespace=true'], } export interface ObjectMetadata { From 23b900f5608bbc3242909c72ede32e0c46cbff4d Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 26 Feb 2026 00:55:17 +0100 Subject: [PATCH 33/66] test: versions --- versions.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions.yaml b/versions.yaml index d722673428..0fa5b279e6 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,5 +1,5 @@ api: APL-523 -console: main -consoleLogin: main +console: APL-523 +consoleLogin: APL-523 tasks: APL-523 tools: main From 1199673933dbe493f961bd9018ad87e9293813e2 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 26 Feb 2026 00:58:41 +0100 Subject: [PATCH 34/66] fix: sealed secrets opaque type --- src/common/sealed-secrets.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 2df49ff667..40503068ed 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -394,7 +394,7 @@ export const createSealedSecretManifest = async ( template: { immutable: false, metadata: { name: mapping.secretName, namespace: mapping.namespace }, - type: 'Opaque', + type: 'kubernetes.io/opaque', }, }, } @@ -659,7 +659,7 @@ export const createUserSealedSecretManifests = async ( template: { immutable: false, metadata: { name: userId, namespace }, - type: 'Opaque', + type: 'kubernetes.io/opaque', }, }, }) From 2c6bc1f206a84c0cdd86a9a125ce7c5e252d5637 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 26 Feb 2026 11:48:08 +0100 Subject: [PATCH 35/66] fix: sealed secrets opaque type test --- src/common/sealed-secrets.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 9deedf76c1..fc5e1a5052 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -321,7 +321,7 @@ describe('sealed-secrets', () => { expect(result.metadata.annotations['sealedsecrets.bitnami.com/namespace-wide']).toBe('true') expect(result.spec.encryptedData.adminPassword).toBe('encrypted-value') expect(result.spec.encryptedData.secretKey).toBe('encrypted-value') - expect(result.spec.template.type).toBe('Opaque') + expect(result.spec.template.type).toBe('kubernetes.io/opaque') expect(result.spec.template.metadata.name).toBe('harbor-secrets') expect(result.spec.template.metadata.namespace).toBe('apl-secrets') }) From a0ed5251e6ae2acba8aab273ff5229a92901778f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:22:00 +0100 Subject: [PATCH 36/66] fix: installer tests --- src/operator/installer.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index 2f1a1c189a..12d59e7ccb 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -37,6 +37,10 @@ jest.mock('../common/git-config', () => ({ getStoredGitRepoConfig: jest.fn(), })) +jest.mock('src/common/bootstrap', () => ({ + recoverFromGit: jest.fn().mockResolvedValue(undefined), +})) + jest.mock('src/cmd/traces', () => ({ runTraceCollectionLoop: jest.fn().mockResolvedValue(undefined), })) From aa3e3a1dad324934bd2df3da0e8b39b687d8c4df Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:56:13 +0100 Subject: [PATCH 37/66] feat: sealed secrets disaster recovery --- .cspell.json | 1 + src/operator/installer.test.ts | 205 ++++++++++++++++++++++++++++++++- src/operator/installer.ts | 55 ++++++++- src/operator/main.ts | 3 +- values-schema.yaml | 25 ++++ 5 files changed, 284 insertions(+), 5 deletions(-) diff --git a/.cspell.json b/.cspell.json index db8f8e9472..60b53049a1 100644 --- a/.cspell.json +++ b/.cspell.json @@ -103,6 +103,7 @@ "backoff", "basepath", "binzx", + "bitnami", "blackbox", "bootstrapper", "calico", diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index 12d59e7ccb..e12f58e6d6 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -1,5 +1,7 @@ +import { ApiException } from '@kubernetes/client-node' import * as gitConfig from '../common/git-config' import * as k8s from '../common/k8s' +import * as utils from '../common/utils' import { AplOperations } from './apl-operations' import { Installer } from './installer' @@ -28,11 +30,21 @@ jest.mock('../common/k8s', () => ({ createUpdateConfigMap: jest.fn(), createUpdateGenericSecret: jest.fn(), deletePendingHelmReleases: jest.fn().mockResolvedValue(undefined), + ensureNamespaceExists: jest.fn().mockResolvedValue(undefined), k8s: { core: jest.fn(), }, })) +jest.mock('../common/utils', () => ({ + ...jest.requireActual('../common/utils'), + loadYaml: jest.fn(), +})) + +jest.mock('../common/envalid', () => ({ + env: { VALUES_INPUT: '/tmp/test-values.yaml' }, +})) + jest.mock('../common/git-config', () => ({ getStoredGitRepoConfig: jest.fn(), })) @@ -58,7 +70,9 @@ describe('Installer', () => { jest.clearAllMocks() jest.useFakeTimers() - mockCoreApi = {} + mockCoreApi = { + createNamespacedSecret: jest.fn().mockResolvedValue(undefined), + } ;(k8s.k8s.core as jest.Mock).mockReturnValue(mockCoreApi) mockAplOps = { @@ -372,4 +386,193 @@ describe('Installer', () => { await expect(installer.setEnvAndCreateSecrets()).resolves.not.toThrow() }) }) + + describe('applyRecoveryManifests', () => { + test('should create secrets from manifest items', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { + recovery: { + manifests: { + apiVersion: 'v1', + kind: 'List', + items: [ + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + labels: { 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active' }, + }, + type: 'kubernetes.io/tls', + data: { 'tls.crt': 'Y2VydA==', 'tls.key': 'a2V5' }, + }, + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: 'sealed-secrets-key2', + namespace: 'sealed-secrets', + labels: { 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active' }, + }, + type: 'kubernetes.io/tls', + data: { 'tls.crt': 'Y2VydDI=', 'tls.key': 'a2V5Mg==' }, + }, + ], + }, + }, + }, + }) + + await installer.applyRecoveryManifests() + + expect(k8s.ensureNamespaceExists).toHaveBeenCalledWith('sealed-secrets') + expect(k8s.ensureNamespaceExists).toHaveBeenCalledTimes(2) + expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledTimes(2) + expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledWith({ + namespace: 'sealed-secrets', + body: { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: 'sealed-secrets-key', + namespace: 'sealed-secrets', + labels: { 'sealedsecrets.bitnami.com/sealed-secrets-key': 'active' }, + }, + type: 'kubernetes.io/tls', + data: { 'tls.crt': 'Y2VydA==', 'tls.key': 'a2V5' }, + }, + }) + }) + + test('should handle 409 conflict (secret already exists)', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { + recovery: { + manifests: { + items: [ + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { name: 'sealed-secrets-key', namespace: 'sealed-secrets' }, + type: 'kubernetes.io/tls', + data: { 'tls.crt': 'Y2VydA==' }, + }, + ], + }, + }, + }, + }) + + mockCoreApi.createNamespacedSecret.mockRejectedValue(new ApiException(409, 'Conflict', {}, {})) + + await expect(installer.applyRecoveryManifests()).resolves.not.toThrow() + }) + + test('should skip when no manifests present', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { mode: 'recovery' }, + }) + + await installer.applyRecoveryManifests() + + expect(mockCoreApi.createNamespacedSecret).not.toHaveBeenCalled() + }) + + test('should skip when items array is empty', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { recovery: { manifests: { items: [] } } }, + }) + + await installer.applyRecoveryManifests() + + expect(mockCoreApi.createNamespacedSecret).not.toHaveBeenCalled() + }) + + test('should rethrow non-409 errors', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { + recovery: { + manifests: { + items: [ + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { name: 'sealed-secrets-key', namespace: 'sealed-secrets' }, + data: {}, + }, + ], + }, + }, + }, + }) + + mockCoreApi.createNamespacedSecret.mockRejectedValue(new ApiException(500, 'Internal Server Error', {}, {})) + + await expect(installer.applyRecoveryManifests()).rejects.toThrow() + }) + + test('should use default namespace when metadata.namespace is not set', async () => { + ;(utils.loadYaml as jest.Mock).mockResolvedValue({ + installation: { + recovery: { + manifests: { + items: [ + { + apiVersion: 'v1', + kind: 'Secret', + metadata: { name: 'my-secret' }, + data: { key: 'val' }, + }, + ], + }, + }, + }, + }) + + await installer.applyRecoveryManifests() + + expect(k8s.ensureNamespaceExists).toHaveBeenCalledWith('default') + expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledWith( + expect.objectContaining({ + namespace: 'default', + }), + ) + }) + }) + + describe('ensureRecoveryPrerequisites', () => { + test('should succeed without SOPS secret', async () => { + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://user:pass@github.com/org/repo.git', + }) + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue(undefined) + + await expect(installer.ensureRecoveryPrerequisites()).resolves.not.toThrow() + }) + + test('should succeed with SOPS secret', async () => { + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://user:pass@github.com/org/repo.git', + }) + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ SOPS_AGE_KEY: 'AGE-SECRET-KEY-1ABC' }) + + await expect(installer.ensureRecoveryPrerequisites()).resolves.not.toThrow() + }) + + test('should succeed with empty SOPS secret', async () => { + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockResolvedValue({ + authenticatedUrl: 'https://user:pass@github.com/org/repo.git', + }) + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({}) + + await expect(installer.ensureRecoveryPrerequisites()).resolves.not.toThrow() + }) + + test('should throw when git config is missing', async () => { + ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockRejectedValue(new Error('Git config not found')) + + await expect(installer.ensureRecoveryPrerequisites()).rejects.toThrow('Git config not found') + }) + }) }) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 48b98cf4a3..03835a3faf 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -1,11 +1,21 @@ +import { ApiException } from '@kubernetes/client-node' import * as process from 'node:process' import { runTraceCollectionLoop } from 'src/cmd/traces' import { recoverFromGit } from 'src/common/bootstrap' import { APL_OPERATOR_NS, APL_OPERATOR_STATUS_CM } from 'src/common/constants' import { $ } from 'zx' import { terminal } from '../common/debug' +import { env } from '../common/envalid' import { getStoredGitRepoConfig, GIT_CONFIG_NAMESPACE } from '../common/git-config' -import { createUpdateConfigMap, deletePendingHelmReleases, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' +import { + createUpdateConfigMap, + deletePendingHelmReleases, + ensureNamespaceExists, + getK8sConfigMap, + getK8sSecret, + k8s, +} from '../common/k8s' +import { loadYaml } from '../common/utils' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' @@ -84,9 +94,48 @@ export class Installer { public async ensureRecoveryPrerequisites(): Promise { await getStoredGitRepoConfig() + // SOPS is optional — sealed-secrets clusters don't have it const sopsSecret = await getK8sSecret('apl-sops-secrets', GIT_CONFIG_NAMESPACE) - if (!sopsSecret || Object.keys(sopsSecret).length === 0) { - throw new Error('KMS/SOPS config not found in apl-sops-secrets secret') + if (sopsSecret && Object.keys(sopsSecret).length > 0) { + this.d.info('SOPS configuration found for recovery') + } else { + this.d.info('No SOPS configuration — sealed-secrets mode recovery') + } + } + + public async applyRecoveryManifests(): Promise { + const values = (await loadYaml(env.VALUES_INPUT)) as Record + const items = values?.installation?.recovery?.manifests?.items + if (!Array.isArray(items) || items.length === 0) { + this.d.info('No recovery manifests to apply') + return + } + + this.d.info(`Applying ${items.length} recovery manifest(s)`) + for (const item of items) { + const namespace = item.metadata?.namespace || 'default' + const name = item.metadata?.name + await ensureNamespaceExists(namespace) + + try { + await k8s.core().createNamespacedSecret({ + namespace, + body: { + apiVersion: item.apiVersion, + kind: item.kind, + metadata: { name, namespace, labels: item.metadata?.labels }, + type: item.type, + data: item.data, + }, + }) + this.d.info(`Created recovery secret ${namespace}/${name}`) + } catch (error) { + if (error instanceof ApiException && error.code === 409) { + this.d.info(`Recovery secret ${namespace}/${name} already exists, skipping`) + } else { + throw error + } + } } } diff --git a/src/operator/main.ts b/src/operator/main.ts index 8dd20baf31..b76747770b 100644 --- a/src/operator/main.ts +++ b/src/operator/main.ts @@ -76,8 +76,9 @@ async function main(): Promise { if (isInstalled) { d.info('Installation already completed, skipping install steps') } else if (isRecoveryMode) { - d.info('Recovery mode enabled, checking external git and kms prerequisites') + d.info('Recovery mode enabled, checking prerequisites') await installer.ensureRecoveryPrerequisites() + await installer.applyRecoveryManifests() await installer.recoverFromGit() d.info('Recovery installation completed, switching installation mode to standard') await installer.resetRecoveryModeToStandard() diff --git a/values-schema.yaml b/values-schema.yaml index ce24bd18b5..45c5dfbafd 100644 --- a/values-schema.yaml +++ b/values-schema.yaml @@ -1565,6 +1565,31 @@ properties: - recovery - standard default: standard + recovery: + type: object + description: Recovery settings for disaster recovery scenarios. Only used when mode=recovery. + properties: + manifests: + type: object + description: > + K8s resource List containing recovery manifests (e.g., sealed-secrets TLS key pairs) + to apply during installation. Not stored in the values repository. + Export from old cluster: + kubectl get secret -n sealed-secrets -l sealedsecrets.bitnami.com/sealed-secrets-key=active -o yaml + properties: + apiVersion: + type: string + kind: + type: string + items: + type: array + items: + type: object + additionalProperties: true + metadata: + type: object + additionalProperties: true + additionalProperties: true azure: description: Azure specific configuration. properties: From beffad6d4e48c24205a877ec15d8cc6da688f350 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 2 Mar 2026 20:58:21 +0100 Subject: [PATCH 38/66] feat: improve users during bootstrap --- src/cmd/bootstrap.test.ts | 56 ++++++++++++------------------------ src/cmd/bootstrap.ts | 24 ++-------------- src/common/sealed-secrets.ts | 16 +---------- 3 files changed, 22 insertions(+), 74 deletions(-) diff --git a/src/cmd/bootstrap.test.ts b/src/cmd/bootstrap.test.ts index 2c68b1097e..21f8b043af 100644 --- a/src/cmd/bootstrap.test.ts +++ b/src/cmd/bootstrap.test.ts @@ -155,18 +155,8 @@ describe('Bootstrapping values', () => { { id: 'user1', initialPassword: 'existing-password' }, { id: 'user2', initialPassword: generatedPassword }, ] - // Pre-processed users (as stored in allSecrets for sealed secret generation) - const processedUsers = usersWithPasswords.map((u: any) => ({ - email: u.email, - firstName: u.firstName, - lastName: u.lastName, - initialPassword: u.initialPassword, - groups: [ - ...(u.isPlatformAdmin ? ['platform-admin'] : []), - ...(u.isTeamAdmin ? ['team-admin'] : []), - ...(u.teams || []).map((t: string) => `team-${t}`), - ], - })) + // Users stored directly in allSecrets (keycloak-operator derives groups from raw fields) + const ca = { a: 'cert' } const mergedSecretsWithCa = merge(cloneDeep(secrets), cloneDeep(ca)) const mergedSecretsWithGen = merge(cloneDeep(secrets), cloneDeep(generatedSecrets)) @@ -207,7 +197,7 @@ describe('Bootstrapping values', () => { deps.getStoredClusterSecrets.mockReturnValue(secrets) deps.generateSecrets.mockReturnValue(allSecrets) await processValues(deps) - const expected = { ...allSecrets, users: processedUsers } + const expected = { ...allSecrets, users: usersWithPasswords } expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', expected) expect(deps.createK8sSecret).toHaveBeenCalledTimes(1) }) @@ -230,7 +220,7 @@ describe('Bootstrapping values', () => { await processValues(deps) expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { ...mergedSecretsWithGenAndCa, - users: processedUsers, + users: usersWithPasswords, }) }) it('should not overwrite stored secrets', async () => { @@ -242,7 +232,7 @@ describe('Bootstrapping values', () => { expect(deps.generateSecrets).toHaveBeenCalledWith(generatedSecrets) expect(deps.createK8sSecret).toHaveBeenCalledWith('otomi-generated-passwords', 'otomi', { ...generatedSecrets, - users: processedUsers, + users: usersWithPasswords, }) }) it('should merge allSecrets into disk values so non-secret fields like customRootCA are preserved', async () => { @@ -256,14 +246,13 @@ describe('Bootstrapping values', () => { deps.createCustomCA.mockReturnValue(ca) const res = await processValues(deps) // mergedForDisk includes allSecrets (stripAllSecrets mock is identity, real impl strips x-secret paths) - // processedUsers adds groups:[] to each user via element-wise lodash merge expect(deps.writeValues).toHaveBeenNthCalledWith(1, { a: 'cert', gen: 'x', cluster: { name: 'bla', provider: 'dida' }, users: [ - { id: 'user1', initialPassword: 'existing-password', groups: [] }, - { id: 'user2', initialPassword: 'generated-password', groups: [] }, + { id: 'user1', initialPassword: 'existing-password' }, + { id: 'user2', initialPassword: 'generated-password' }, ], }) expect(res.originalInput).toEqual({ @@ -276,9 +265,9 @@ describe('Bootstrapping values', () => { }) it('should merge originalInput + allSecrets + users for disk (stripAllSecrets removes x-secret paths)', async () => { // mergedForDisk = merge(originalInput, allSecrets, { users }) - // allSecrets = merge(ca, storedSecrets, generatedSecrets, kmsValues) + users: processedUsers + // allSecrets = merge(ca, storedSecrets, generatedSecrets) + users: usersWithPasswords const allSecretsExpected = merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { - users: processedUsers, + users: usersWithPasswords, }) const expectedDiskValues = merge( cloneDeep(secrets), @@ -308,39 +297,30 @@ describe('Bootstrapping values', () => { const result = await processValues(deps) // allSecrets should contain full unstripped secrets including pre-processed users expect(result.allSecrets).toEqual( - merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { users: processedUsers }), + merge(cloneDeep(ca), cloneDeep(secrets), cloneDeep(generatedSecrets), { users: usersWithPasswords }), ) }) - it('should preserve existing groups when users are recovered from stored secrets (bootstrap retry)', async () => { - // Simulate bootstrap retry: stored secrets contain processed users (with groups, without isPlatformAdmin) - const storedProcessedUsers = [ + it('should store users as-is in allSecrets (keycloak-operator derives groups)', async () => { + const storedUsers = [ { email: 'platform-admin@example.com', firstName: 'platform', lastName: 'admin', initialPassword: 'existing-pass', - groups: ['platform-admin'], + isPlatformAdmin: true, + teams: ['dev'], }, ] deps.loadYaml.mockReturnValue({}) - deps.getStoredClusterSecrets.mockReturnValue({ users: storedProcessedUsers }) + deps.getStoredClusterSecrets.mockReturnValue({ users: storedUsers }) deps.generateSecrets.mockReturnValue({}) deps.createCustomCA.mockReturnValue({}) - // getUsers returns the stored processed users (no isPlatformAdmin flag) - deps.getUsers.mockReturnValue(storedProcessedUsers) + deps.getUsers.mockReturnValue(storedUsers) const result = await processValues(deps) - // Groups should be preserved from existing data, not reset to [] - expect(result.allSecrets.users).toEqual([ - { - email: 'platform-admin@example.com', - firstName: 'platform', - lastName: 'admin', - initialPassword: 'existing-pass', - groups: ['platform-admin'], - }, - ]) + // Users stored directly — no groups transformation + expect(result.allSecrets.users).toEqual(storedUsers) }) }) }) diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index 608627e43b..64e8447129 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -182,27 +182,9 @@ export const processValues = async ( const allSecrets = merge(cloneDeep(caSecrets), cloneDeep(storedSecrets), cloneDeep(generatedSecrets)) // add default platform admin & generate initial passwords for users if they don't have one const users = deps.getUsers(originalInput) - // Pre-process users into keycloak-operator format (with groups resolved) for sealed secret storage - const processedUsers = users.map((user: any) => { - const groups: string[] = [] - if (user.isPlatformAdmin) groups.push('platform-admin') - if (user.isTeamAdmin) groups.push('team-admin') - for (const team of user.teams || []) groups.push(`team-${team}`) - // Preserve existing groups when boolean flags are absent (e.g., user recovered - // from stored secrets which uses the processed format without isPlatformAdmin/isTeamAdmin) - if (groups.length === 0 && Array.isArray(user.groups) && user.groups.length > 0) { - groups.push(...(user.groups as string[])) - } - return { - email: user.email, - firstName: user.firstName, - lastName: user.lastName, - initialPassword: user.initialPassword, - groups, - } - }) - // Store processed users in allSecrets so they flow into sealed secret generation - allSecrets.users = processedUsers + // Store users in allSecrets for sealed secret generation + // The keycloak-operator derives groups from isPlatformAdmin/isTeamAdmin/teams directly + allSecrets.users = users // Write only non-secret values to disk — secrets are stored exclusively in SealedSecrets // Include allSecrets so non-secret fields like customRootCA are preserved (stripAllSecrets removes only x-secret paths) const mergedForDisk = merge(cloneDeep(originalInput), cloneDeep(allSecrets), cloneDeep({ users })) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 40503068ed..c63af2fc6b 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -721,21 +721,7 @@ export const bootstrapSealedSecrets = async ( // 7. Create individual user SealedSecrets in apl-users namespace const { users } = secrets if (Array.isArray(users) && users.length > 0) { - // The users in allSecrets are in processed format (with groups). - // We also need original user data (isPlatformAdmin, isTeamAdmin, teams) from allValues. - const originalUsers = get(allValues, 'users', []) as any[] - // Merge original user fields with processed users for complete SealedSecret data - const usersForSecrets = users.map((processedUser: any) => { - const originalUser = originalUsers.find((u: any) => u.email === processedUser.email) - return { - ...processedUser, - name: originalUser?.name || processedUser.name, - isPlatformAdmin: originalUser?.isPlatformAdmin ?? false, - isTeamAdmin: originalUser?.isTeamAdmin ?? false, - teams: originalUser?.teams || [], - } - }) - const userManifests = await deps.createUserSealedSecretManifests(usersForSecrets, pem, { + const userManifests = await deps.createUserSealedSecretManifests(users, pem, { encryptSecretItem: deps.encryptSecretItem, terminal: deps.terminal, }) From c16577efe9911a290fbc8d974e58cffa36195c25 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 2 Mar 2026 21:25:07 +0100 Subject: [PATCH 39/66] fix: update sealed secret manifests path --- src/cmd/migrate.test.ts | 5 ++--- src/cmd/migrate.ts | 12 ++++++++---- src/common/sealed-secrets.test.ts | 6 ++++-- src/common/sealed-secrets.ts | 9 ++++----- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 001ee67523..26cfcf0b26 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -1,4 +1,3 @@ -import { existsSync, rmSync } from 'fs' import { globSync } from 'glob' import { applyChanges, @@ -970,7 +969,7 @@ describe('sopsMigration', () => { it('should re-apply and restart controller when manifests exist but K8s Secrets are missing', async () => { mockExistsSync.mockReturnValue(false) - mockGlobSync.mockReturnValue(['/env/manifests/ns/apl-secrets/otomi-platform-secrets.yaml']) + mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-platform-secrets.yaml']) mockGetK8sSecret.mockResolvedValue(undefined) // Secret doesn't exist yet await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) @@ -982,7 +981,7 @@ describe('sopsMigration', () => { it('should skip re-apply when manifests exist and K8s Secrets already exist', async () => { mockExistsSync.mockReturnValue(false) - mockGlobSync.mockReturnValue(['/env/manifests/ns/apl-secrets/otomi-platform-secrets.yaml']) + mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-platform-secrets.yaml']) mockGetK8sSecret.mockResolvedValue({ git_password: 'somepassword' }) // Secret exists await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 322d44381b..9b344e78ae 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -709,7 +709,7 @@ const createCatalogSealedSecret = async ( }, }, } - const sealedSecretPath = `${env.ENV_DIR}/env/manifests/namespaces/argocd/${SEALED_SECRET_NAME}.yaml` + const sealedSecretPath = `${env.ENV_DIR}/env/manifests/namespaces/argocd/sealedsecrets/${SEALED_SECRET_NAME}.yaml` mkdirSync(dirname(sealedSecretPath), { recursive: true }) d.info(`Writing sealed secret to ${sealedSecretPath}`) writeFileSync(sealedSecretPath, objectToYaml(sealedSecret)) @@ -777,7 +777,7 @@ export const removeSopsArtifacts = (deps = { existsSync, rmSync, globSync, termi d.info(`Removed ${f}`) } - // Remove user YAML files — users are now managed via SealedSecrets in env/manifests/ns/apl-users/. + // Remove user YAML files — users are now managed via SealedSecrets in env/manifests/namespaces/apl-users/sealedsecrets. // These files may contain SOPS-encrypted data that was written by the "Write default values" step // before SOPS decryption ran, contaminating the public YAML files with ENC[...] strings. const userFiles = deps.globSync(`${env.ENV_DIR}/env/users/*.yaml`, { dot: false }) @@ -816,7 +816,9 @@ export const sopsMigration = async ( // them, or the controller used its auto-generated key), re-apply them and restart // the controller so subsequent steps can resolve the git password. if (!deps.existsSync(`${env.ENV_DIR}/.sops.yaml`)) { - const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/ns/**/*.yaml`, { dot: false }) + const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/namespaces/**/sealedsecrets/*.yaml`, { + dot: false, + }) if (existingManifests.length > 0) { const platformSecret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') if (!platformSecret) { @@ -830,7 +832,9 @@ export const sopsMigration = async ( } // Secondary guard: if manifests already exist, just clean up SOPS artifacts - const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/ns/**/*.yaml`, { dot: false }) + const existingManifests = deps.globSync(`${env.ENV_DIR}/env/manifests/namespaces/**/sealedsecrets/*.yaml`, { + dot: false, + }) if (existingManifests.length > 0) { d.info('SealedSecret manifests already exist, only cleaning up SOPS artifacts') deps.removeSopsArtifacts() diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index fc5e1a5052..69dbc56f6c 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -375,9 +375,11 @@ describe('sealed-secrets', () => { await writeSealedSecretManifests(manifests, '/test', deps) - expect(deps.mkdir).toHaveBeenCalledWith('/test/env/manifests/ns/apl-secrets', { recursive: true }) + expect(deps.mkdir).toHaveBeenCalledWith('/test/env/manifests/namespaces/apl-secrets/sealedsecrets', { + recursive: true, + }) expect(deps.writeFile).toHaveBeenCalledWith( - '/test/env/manifests/ns/apl-secrets/harbor-secrets.yaml', + '/test/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml', 'yaml-content', ) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index c63af2fc6b..84b6e8d3d1 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -401,7 +401,7 @@ export const createSealedSecretManifest = async ( } /** - * Write SealedSecret manifests to the env/manifests/ns directory. + * Write SealedSecret manifests to the env/manifests/namespaces directory. */ export const writeSealedSecretManifests = async ( manifests: SealedSecretManifest[], @@ -411,8 +411,7 @@ export const writeSealedSecretManifests = async ( const d = deps.terminal(`common:${cmdName}:writeSealedSecretManifests`) for (const manifest of manifests) { - // /env/manifests/ns/argocd/ - const dir = `${envDir}/env/manifests/ns/${manifest.metadata.namespace}` + const dir = `${envDir}/env/manifests/namespaces/${manifest.metadata.namespace}/sealedsecrets` await deps.mkdir(dir, { recursive: true }) const filePath = `${dir}/${manifest.metadata.name}.yaml` d.info(`Writing sealed secret to ${filePath}`) @@ -478,7 +477,7 @@ export const applySealedSecretManifests = async ( } /** - * Read and apply all SealedSecret manifests from the env/manifests/ns directory. + * Read and apply all SealedSecret manifests from the env/manifests/namespaces directory. * This should be called during install, after the sealed-secrets controller is deployed. */ export const applySealedSecretManifestsFromDir = async ( @@ -486,7 +485,7 @@ export const applySealedSecretManifestsFromDir = async ( deps = { terminal, readdir, readFile, existsSync }, ): Promise => { const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) - const manifestsDir = join(envDir, 'env/manifests/ns') + const manifestsDir = join(envDir, 'env/manifests/namespaces') if (!deps.existsSync(manifestsDir)) { d.info(`No SealedSecret manifests directory found at ${manifestsDir}`) From 0f4d223b94088739c5bafa69a624b3d257408b51 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 2 Mar 2026 22:16:48 +0100 Subject: [PATCH 40/66] feat: update tests/fixtures for local dev env users --- src/common/sealed-secrets.ts | 10 +++++---- .../23d63558-49ed-48ba-bc28-8037a7236ddf.yaml | 22 +++++++++++++++++++ .../9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml | 22 +++++++++++++++++++ .../a83e20b7-474a-4262-a3ad-b09813364ece.yaml | 22 +++++++++++++++++++ .../bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml | 22 +++++++++++++++++++ ....23d63558-49ed-48ba-bc28-8037a7236ddf.yaml | 12 ---------- ....9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml | 12 ---------- ....a83e20b7-474a-4262-a3ad-b09813364ece.yaml | 10 --------- ....bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml | 10 --------- 9 files changed, 94 insertions(+), 48 deletions(-) create mode 100644 tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/23d63558-49ed-48ba-bc28-8037a7236ddf.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/a83e20b7-474a-4262-a3ad-b09813364ece.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml delete mode 100644 tests/fixtures/env/users/secrets.23d63558-49ed-48ba-bc28-8037a7236ddf.yaml delete mode 100644 tests/fixtures/env/users/secrets.9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml delete mode 100644 tests/fixtures/env/users/secrets.a83e20b7-474a-4262-a3ad-b09813364ece.yaml delete mode 100644 tests/fixtures/env/users/secrets.bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 84b6e8d3d1..ebfe22d2c0 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -501,15 +501,17 @@ export const applySealedSecretManifestsFromDir = async ( for (const nsEntry of namespaces) { if (!nsEntry.isDirectory()) continue const namespace = nsEntry.name - const nsDir = join(manifestsDir, namespace) + const sealedSecretsDir = join(manifestsDir, namespace, 'sealedsecrets') + + if (!deps.existsSync(sealedSecretsDir)) continue await ensureNamespaceExists(namespace) - // Read all YAML files in the namespace directory - const files = await deps.readdir(nsDir) + // Read all YAML files in the sealedsecrets subdirectory + const files = await deps.readdir(sealedSecretsDir) for (const file of files) { if (!file.endsWith('.yaml') && !file.endsWith('.yml')) continue - const filePath = join(nsDir, file) + const filePath = join(sealedSecretsDir, file) d.info(`Applying SealedSecret from ${filePath}`) try { diff --git a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/23d63558-49ed-48ba-bc28-8037a7236ddf.yaml b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/23d63558-49ed-48ba-bc28-8037a7236ddf.yaml new file mode 100644 index 0000000000..cd965eb983 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/23d63558-49ed-48ba-bc28-8037a7236ddf.yaml @@ -0,0 +1,22 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: 'true' + name: 23d63558-49ed-48ba-bc28-8037a7236ddf + namespace: apl-users +spec: + encryptedData: + email: team@admin.com + firstName: team + lastName: admin + initialPassword: team-admin-password + isPlatformAdmin: 'false' + isTeamAdmin: 'true' + teams: '["demo"]' + template: + immutable: false + metadata: + name: 23d63558-49ed-48ba-bc28-8037a7236ddf + namespace: apl-users + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml new file mode 100644 index 0000000000..a9cabecfdb --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml @@ -0,0 +1,22 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: 'true' + name: 9a3a478b-a747-4b4a-be69-a9abf1979df2 + namespace: apl-users +spec: + encryptedData: + email: team@member.com + firstName: team + lastName: member + initialPassword: team-member-password + isPlatformAdmin: 'false' + isTeamAdmin: 'false' + teams: '["demo"]' + template: + immutable: false + metadata: + name: 9a3a478b-a747-4b4a-be69-a9abf1979df2 + namespace: apl-users + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/a83e20b7-474a-4262-a3ad-b09813364ece.yaml b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/a83e20b7-474a-4262-a3ad-b09813364ece.yaml new file mode 100644 index 0000000000..1ae170984a --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/a83e20b7-474a-4262-a3ad-b09813364ece.yaml @@ -0,0 +1,22 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: 'true' + name: a83e20b7-474a-4262-a3ad-b09813364ece + namespace: apl-users +spec: + encryptedData: + email: platform@admin.com + firstName: platform + lastName: admin + initialPassword: platform-admin-password + isPlatformAdmin: 'true' + isTeamAdmin: 'true' + teams: '[]' + template: + immutable: false + metadata: + name: a83e20b7-474a-4262-a3ad-b09813364ece + namespace: apl-users + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml new file mode 100644 index 0000000000..358ce1b2fe --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml @@ -0,0 +1,22 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: 'true' + name: bc2fe5b1-835c-4998-ad64-e15d90062b16 + namespace: apl-users +spec: + encryptedData: + email: platform-admin@dev.linode-apl.net + firstName: platform + lastName: admin + initialPassword: 02LDWB#qzknkeF8f*m%% + isPlatformAdmin: 'true' + isTeamAdmin: 'false' + teams: '[]' + template: + immutable: false + metadata: + name: bc2fe5b1-835c-4998-ad64-e15d90062b16 + namespace: apl-users + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/users/secrets.23d63558-49ed-48ba-bc28-8037a7236ddf.yaml b/tests/fixtures/env/users/secrets.23d63558-49ed-48ba-bc28-8037a7236ddf.yaml deleted file mode 100644 index 6576468c59..0000000000 --- a/tests/fixtures/env/users/secrets.23d63558-49ed-48ba-bc28-8037a7236ddf.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: AplUser -metadata: - name: 23d63558-49ed-48ba-bc28-8037a7236ddf -spec: - email: team@admin.com - firstName: team - initialPassword: team-admin-password - isPlatformAdmin: false - isTeamAdmin: true - lastName: admin - teams: - - demo diff --git a/tests/fixtures/env/users/secrets.9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml b/tests/fixtures/env/users/secrets.9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml deleted file mode 100644 index 839049b33f..0000000000 --- a/tests/fixtures/env/users/secrets.9a3a478b-a747-4b4a-be69-a9abf1979df2.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: AplUser -metadata: - name: 9a3a478b-a747-4b4a-be69-a9abf1979df2 -spec: - email: team@member.com - firstName: team - initialPassword: team-member-password - isPlatformAdmin: false - isTeamAdmin: false - lastName: member - teams: - - demo diff --git a/tests/fixtures/env/users/secrets.a83e20b7-474a-4262-a3ad-b09813364ece.yaml b/tests/fixtures/env/users/secrets.a83e20b7-474a-4262-a3ad-b09813364ece.yaml deleted file mode 100644 index a489101d4f..0000000000 --- a/tests/fixtures/env/users/secrets.a83e20b7-474a-4262-a3ad-b09813364ece.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: AplUser -metadata: - name: a83e20b7-474a-4262-a3ad-b09813364ece -spec: - email: platform@admin.com - firstName: platform - initialPassword: platform-admin-password - isPlatformAdmin: true - isTeamAdmin: true - lastName: admin diff --git a/tests/fixtures/env/users/secrets.bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml b/tests/fixtures/env/users/secrets.bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml deleted file mode 100644 index e0fc2601d6..0000000000 --- a/tests/fixtures/env/users/secrets.bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: AplUser -metadata: - name: bc2fe5b1-835c-4998-ad64-e15d90062b16 -spec: - email: platform-admin@dev.linode-apl.net - firstName: platform - lastName: admin - isPlatformAdmin: true - isTeamAdmin: false - initialPassword: 02LDWB#qzknkeF8f*m%% From 6aa05e68d0395a3015304e53a131b8252efe4c10 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 2 Mar 2026 23:37:23 +0100 Subject: [PATCH 41/66] fix: update tests/fixtures for local dev env users --- .../bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml diff --git a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml b/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml deleted file mode 100644 index 358ce1b2fe..0000000000 --- a/tests/fixtures/env/manifests/namespaces/apl-users/sealedsecrets/bc2fe5b1-835c-4998-ad64-e15d90062b16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: bitnami.com/v1alpha1 -kind: SealedSecret -metadata: - annotations: - sealedsecrets.bitnami.com/namespace-wide: 'true' - name: bc2fe5b1-835c-4998-ad64-e15d90062b16 - namespace: apl-users -spec: - encryptedData: - email: platform-admin@dev.linode-apl.net - firstName: platform - lastName: admin - initialPassword: 02LDWB#qzknkeF8f*m%% - isPlatformAdmin: 'true' - isTeamAdmin: 'false' - teams: '[]' - template: - immutable: false - metadata: - name: bc2fe5b1-835c-4998-ad64-e15d90062b16 - namespace: apl-users - type: kubernetes.io/opaque From 1f5e7e00193f5bbbe270de8b20be406bb69b289b Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Tue, 3 Mar 2026 10:14:16 +0100 Subject: [PATCH 42/66] fix: ci error --- src/cmd/migrate.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 9b344e78ae..fd993ff289 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -820,11 +820,15 @@ export const sopsMigration = async ( dot: false, }) if (existingManifests.length > 0) { - const platformSecret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') - if (!platformSecret) { - d.info('SealedSecret manifests exist but K8s Secrets are missing — re-applying and restarting controller') - await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) - await deps.restartSealedSecretsController() + try { + const platformSecret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') + if (!platformSecret) { + d.info('SealedSecret manifests exist but K8s Secrets are missing — re-applying and restarting controller') + await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) + await deps.restartSealedSecretsController() + } + } catch { + d.info('Could not reach K8s API to check secrets, skipping re-apply') } } d.info('No .sops.yaml found, skipping SOPS migration') From e5d7fedf878bc5a9a26c478e209db6706fbc858f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 4 Mar 2026 23:14:31 +0100 Subject: [PATCH 43/66] feat: enhance sealed secrets management and update dependencies --- helmfile.d/helmfile-60.teams.yaml.gotmpl | 57 ++++++++++ helmfile.d/snippets/defaults.yaml | 2 +- src/cmd/install.test.ts | 30 +++--- src/cmd/install.ts | 40 ++++--- src/common/sealed-secrets.test.ts | 101 +++++++++++++++--- src/common/sealed-secrets.ts | 96 ++++++++--------- tools/Dockerfile | 5 +- values/apl-operator/apl-operator-raw.gotmpl | 4 +- values/apl-operator/apl-operator.gotmpl | 3 +- values/argocd/argocd-raw.gotmpl | 18 ++++ values/cert-manager/cert-manager-raw.gotmpl | 15 +-- values/external-dns/external-dns.gotmpl | 2 +- .../external-secrets/external-secrets.gotmpl | 2 +- values/oauth2-proxy/oauth2-proxy-raw.gotmpl | 6 +- .../prometheus-operator-raw.gotmpl | 6 ++ .../prometheus-operator.gotmpl | 2 + 16 files changed, 268 insertions(+), 121 deletions(-) diff --git a/helmfile.d/helmfile-60.teams.yaml.gotmpl b/helmfile.d/helmfile-60.teams.yaml.gotmpl index ce769d6040..646f90166e 100644 --- a/helmfile.d/helmfile-60.teams.yaml.gotmpl +++ b/helmfile.d/helmfile-60.teams.yaml.gotmpl @@ -121,6 +121,8 @@ releases: url: http://loki-query-frontend-headless.monitoring:3101 basicAuth: true basicAuthUser: {{ $teamId }} + secureJsonData: + basicAuthPassword: $__env{GF_LOKI_BASIC_AUTH_PASSWORD} {{- if has "msteams" ($teamSettings | get "alerts.receivers" list) }} - name: prometheus-msteams-{{ $teamId }} installed: {{ $teamSettings | get "managedMonitoring.alertmanager" false }} @@ -179,6 +181,7 @@ releases: pipeline: otomi-task-teams values: - resources: + {{- if $teamSettings | get "managedMonitoring.grafana" false }} - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: @@ -202,6 +205,54 @@ releases: remoteRef: key: team-{{ $teamId }}-settings-secrets property: settings_password + {{- end }} + {{- if $teamSettings | get "managedMonitoring.grafana" false }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-oidc-secret + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-oidc-secret + creationPolicy: Owner + template: + type: Opaque + data: + client_id: {{ $v.apps.keycloak.idp.clientID }} + client_secret: '{{ "{{ .clientSecret | toString }}" }}' + data: + - secretKey: clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: grafana-loki-datasource-secret + namespace: team-{{ $teamId }} + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: grafana-loki-datasource-secret + creationPolicy: Owner + template: + type: Opaque + data: + password: '{{ "{{ .adminPassword | toString }}" }}' + data: + - secretKey: adminPassword + remoteRef: + key: loki-secrets + property: adminPassword + {{- end }} {{- if $teamSettings | get "managedMonitoring.alertmanager" false }} - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret @@ -238,5 +289,11 @@ releases: key: smtp-secrets property: auth_secret {{- end }} + {{- if has "opsgenie" $teamReceivers }} + - secretKey: opsgenieApiKey + remoteRef: + key: alerts-secrets + property: opsgenie_apiKey + {{- end }} {{- end }} {{- end }} diff --git a/helmfile.d/snippets/defaults.yaml b/helmfile.d/snippets/defaults.yaml index 1f60127045..0c1431b7be 100644 --- a/helmfile.d/snippets/defaults.yaml +++ b/helmfile.d/snippets/defaults.yaml @@ -1174,4 +1174,4 @@ environments: branch: main enabled: true versions: - specVersion: 55 + specVersion: 56 diff --git a/src/cmd/install.test.ts b/src/cmd/install.test.ts index 5803bda66b..16704a0499 100644 --- a/src/cmd/install.test.ts +++ b/src/cmd/install.test.ts @@ -61,7 +61,7 @@ jest.mock('zx', () => { jest.mock('src/common/sealed-secrets', () => ({ applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue(undefined), restartSealedSecretsController: jest.fn().mockResolvedValue(undefined), - buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), + SECRET_NAME_MAP: {}, })) jest.mock('src/common/utils', () => ({ @@ -281,32 +281,28 @@ describe('Install command', () => { }) describe('error handling', () => { - test('should handle deployment state errors', async () => { - const error = new Error('Failed to get deployment state') - mockDeps.getDeploymentState.mockRejectedValueOnce(error) + test('should throw on deployment state errors', async () => { + mockDeps.getDeploymentState.mockRejectedValueOnce(new Error('Failed to get deployment state')) - expect(mockDeps.getDeploymentState).toBeDefined() + await expect(installAll()).rejects.toThrow('Failed to get deployment state') }) - test('should handle image tag retrieval errors', async () => { - const error = new Error('Failed to get image tag') - mockDeps.getImageTagFromValues.mockRejectedValueOnce(error) + test('should throw on image tag retrieval errors', async () => { + mockDeps.getImageTagFromValues.mockRejectedValueOnce(new Error('Failed to get image tag')) - expect(mockDeps.getImageTagFromValues).toBeDefined() + await expect(installAll()).rejects.toThrow('Failed to get image tag') }) - test('should handle helmfile errors', async () => { - const error = new Error('Helmfile execution failed') - mockDeps.hf.mockRejectedValueOnce(error) + test('should throw on helmfile errors during sealed-secrets deploy', async () => { + mockDeps.hf.mockRejectedValueOnce(new Error('Helmfile execution failed')) - expect(mockDeps.hf).toBeDefined() + await expect(installAll()).rejects.toThrow('Helmfile execution failed') }) - test('should handle CRDs deployment errors', async () => { - const error = new Error('CRDs deployment failed') - mockDeps.applyServerSide.mockRejectedValueOnce(error) + test('should throw on essential deployment failure', async () => { + mockDeps.deployEssential.mockResolvedValueOnce(false) - expect(mockDeps.applyServerSide).toBeDefined() + await expect(installAll()).rejects.toThrow('Failed to deploy essential manifests') }) }) }) diff --git a/src/cmd/install.ts b/src/cmd/install.ts index edf04db93b..0e0a4c713a 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -19,8 +19,8 @@ import { } from 'src/common/k8s' import { applySealedSecretManifestsFromDir, - buildSecretToNamespaceMap, restartSealedSecretsController, + SECRET_NAME_MAP, } from 'src/common/sealed-secrets' import { getFilename, getSchemaSecretsPaths, rootDir } from 'src/common/utils' import { getImageTagFromValues, getPackageVersion, writeValuesToFile } from 'src/common/values' @@ -63,48 +63,46 @@ const retryInstallStep = async ( /** * Wait for SealedSecrets controller to decrypt SealedSecret resources into K8s Secrets. - * Derives the list of secrets to wait for from schema x-secret fields via buildSecretToNamespaceMap(). + * Derives the expected secret names from schema x-secret fields + SECRET_NAME_MAP. */ const waitForSealedSecrets = async ( timeoutMs = 120000, intervalMs = 3000, - deps = { getK8sSecret, terminal, buildSecretToNamespaceMap, getSchemaSecretsPaths }, + deps = { getK8sSecret, terminal, getSchemaSecretsPaths }, ): Promise => { const d = deps.terminal(`cmd:${cmdName}:waitForSealedSecrets`) - // Build list of secrets to wait for from schema-driven mappings - // We pass empty secrets/teams since we just need the secret names and namespaces - const mappings = await deps.buildSecretToNamespaceMap({}, [], undefined, { - getSchemaSecretsPaths: deps.getSchemaSecretsPaths, - }) - - // Deduplicate by namespace/secretName - const secretsToWait = new Map() - for (const mapping of mappings) { - const key = `${mapping.namespace}/${mapping.secretName}` - if (!secretsToWait.has(key)) { - secretsToWait.set(key, { namespace: mapping.namespace, secretName: mapping.secretName }) + // Get all x-secret paths from schema and map them to K8s secret names + const secretPaths = await deps.getSchemaSecretsPaths([]) + const sortedPrefixes = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) + const secretNames = new Set() + for (const path of secretPaths) { + for (const prefix of sortedPrefixes) { + if (path === prefix || path.startsWith(`${prefix}.`)) { + secretNames.add(SECRET_NAME_MAP[prefix]) + break + } } } - if (secretsToWait.size === 0) { + if (secretNames.size === 0) { d.info('No sealed secrets to wait for') return } - d.info(`Waiting for ${secretsToWait.size} sealed secrets to be decrypted`) + d.info(`Waiting for ${secretNames.size} sealed secrets to be decrypted: ${[...secretNames].join(', ')}`) await retry( async () => { const pending: string[] = [] - for (const { namespace, secretName } of secretsToWait.values()) { + for (const secretName of secretNames) { try { - const secret = await deps.getK8sSecret(secretName, namespace) + const secret = await deps.getK8sSecret(secretName, 'apl-secrets') if (!secret) { - pending.push(`${namespace}/${secretName}`) + pending.push(secretName) } } catch { - pending.push(`${namespace}/${secretName}`) + pending.push(secretName) } } diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 69dbc56f6c..25835a38a9 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -1,14 +1,17 @@ import { pki } from 'node-forge' import stubs from 'src/test-stubs' import { - APP_NAMESPACE_MAP, + applySealedSecretManifests, bootstrapSealedSecrets, buildSecretToNamespaceMap, createSealedSecretManifest, createSealedSecretsKeySecret, + createUserSealedSecretManifests, generateSealedSecretsKeyPair, getPemFromCertificate, + restartSealedSecretsController, SECRET_NAME_MAP, + SealedSecretManifest, stripAllSecrets, writeSealedSecretManifests, } from './sealed-secrets' @@ -499,18 +502,6 @@ describe('sealed-secrets', () => { }) }) - describe('APP_NAMESPACE_MAP', () => { - it('should have expected mappings', () => { - expect(APP_NAMESPACE_MAP['apps.harbor']).toBe('harbor') - expect(APP_NAMESPACE_MAP['apps.gitea']).toBe('gitea') - expect(APP_NAMESPACE_MAP['apps.oauth2-proxy']).toBe('istio-system') - expect(APP_NAMESPACE_MAP['apps.loki']).toBe('monitoring') - expect(APP_NAMESPACE_MAP['otomi']).toBe('otomi') - expect(APP_NAMESPACE_MAP['dns']).toBe('external-dns') - expect(APP_NAMESPACE_MAP['cluster']).toBe('cert-manager') - }) - }) - describe('SECRET_NAME_MAP', () => { it('should have expected secret name mappings', () => { expect(SECRET_NAME_MAP['apps.harbor']).toBe('harbor-secrets') @@ -555,4 +546,88 @@ describe('sealed-secrets', () => { expect(values.apps.gitea.adminPassword).toBe('secret') }) }) + + describe('applySealedSecretManifests', () => { + const makeMockManifest = (name: string, namespace: string): SealedSecretManifest => ({ + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name, + namespace, + }, + spec: { + encryptedData: { key: 'encrypted-value' }, + template: { + immutable: false, + metadata: { name, namespace }, + type: 'kubernetes.io/opaque', + }, + }, + }) + + it('should apply manifests successfully', async () => { + const manifests = [makeMockManifest('test-secret', 'apl-secrets')] + + await applySealedSecretManifests(manifests, { terminal }) + + const { k8s: mockK8s } = require('src/common/k8s') + expect(mockK8s.custom().createNamespacedCustomObject).toHaveBeenCalled() + }) + + it('should log error when manifests fail to apply', async () => { + const { k8s: mockK8s } = require('src/common/k8s') + mockK8s.custom().createNamespacedCustomObject.mockRejectedValueOnce(new Error('apply failed')) + + const manifests = [makeMockManifest('test-secret', 'apl-secrets')] + + await applySealedSecretManifests(manifests, { terminal }) + // Should not throw, just log the error + }) + }) + + describe('restartSealedSecretsController', () => { + it('should succeed when deployment rolls out quickly', async () => { + const { k8s: mockK8s } = require('src/common/k8s') + // Mock deployment that is already ready + mockK8s.app().readNamespacedDeployment.mockResolvedValue({ + spec: { replicas: 1 }, + status: { updatedReplicas: 1, availableReplicas: 1 }, + }) + + await restartSealedSecretsController({ terminal }) + + expect(mockK8s.app().patchNamespacedDeployment).toHaveBeenCalled() + }) + }) + + describe('createUserSealedSecretManifests', () => { + it('should create individual SealedSecret for each user', async () => { + const users = [ + { name: 'user1', email: 'user1@test.com', firstName: 'User', lastName: 'One', initialPassword: 'pass1' }, + { name: 'user2', email: 'user2@test.com', firstName: 'User', lastName: 'Two', initialPassword: 'pass2' }, + ] + + const manifests = await createUserSealedSecretManifests(users, 'pem-data', { + encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), + terminal, + }) + + expect(manifests).toHaveLength(2) + expect(manifests[0].metadata.name).toBe('user1') + expect(manifests[0].metadata.namespace).toBe('apl-users') + expect(manifests[1].metadata.name).toBe('user2') + }) + + it('should skip users without id/name', async () => { + const users = [{ email: 'noname@test.com' }] + + const manifests = await createUserSealedSecretManifests(users, 'pem-data', { + encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), + terminal, + }) + + expect(manifests).toHaveLength(0) + }) + }) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index ebfe22d2c0..cd764bafda 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -52,30 +52,10 @@ export interface SealedSecretManifest { } /** - * Mapping from secret path prefix to target Kubernetes namespace. - * Dynamic entries like `teamConfig.{teamId}` are handled separately. + * All SealedSecrets are placed in the 'apl-secrets' namespace. + * ESO ClusterSecretStore reads from this namespace and distributes secrets to target namespaces. */ -export const APP_NAMESPACE_MAP: Record = { - 'apps.harbor': 'harbor', - 'apps.gitea': 'gitea', - 'apps.keycloak': 'keycloak', - 'apps.grafana': 'grafana', - 'apps.loki': 'monitoring', - 'apps.oauth2-proxy': 'istio-system', - 'apps.oauth2-proxy-redis': 'istio-system', - 'apps.prometheus': 'monitoring', - 'apps.otomi-api': 'otomi', - 'apps.cert-manager': 'cert-manager', - 'apps.kubeflow-pipelines': 'kfp', - otomi: 'otomi', - oidc: 'otomi', - smtp: 'otomi', - dns: 'external-dns', - obj: 'otomi', - license: 'otomi', - alerts: 'monitoring', - cluster: 'cert-manager', -} +const SEALED_SECRETS_NAMESPACE = 'apl-secrets' /** * Generate an RSA 4096-bit key pair and self-signed X.509 certificate for Sealed Secrets. @@ -220,21 +200,19 @@ export const createSealedSecretsKeySecret = async ( /** * Resolve the namespace for a given secret path. - * All core secrets go to 'apl-secrets' namespace for ESO access. - * APP_NAMESPACE_MAP is kept for reference but not used for SealedSecret placement. + * All secrets go to 'apl-secrets' namespace for ESO ClusterSecretStore access. */ const resolveNamespace = (secretPath: string): string | undefined => { // Check for teamConfig dynamic paths - const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) - if (teamMatch) { - return 'apl-secrets' + if (secretPath.match(/^teamConfig\.[^.]+/)) { + return SEALED_SECRETS_NAMESPACE } - // Check if this path matches any known prefix - const sortedKeys = Object.keys(APP_NAMESPACE_MAP).sort((a, b) => b.length - a.length) + // Check if this path matches any known secret name prefix + const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) for (const prefix of sortedKeys) { if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { - return 'apl-secrets' + return SEALED_SECRETS_NAMESPACE } } @@ -355,6 +333,8 @@ export const buildSecretToNamespaceMap = async ( groupPrefix && (flatKey === groupPrefix || flatKey.startsWith(`${groupPrefix}.`)) ? flatKey.slice(groupPrefix.length + 1) : flatKey + // Skip empty relative paths (happens when flatKey === groupPrefix) + if (!relativePath) continue const dataKey = relativePath.replace(/\./g, '_') if (value !== undefined && value !== null && value !== '') { mapping.data[dataKey] = String(value) @@ -455,17 +435,21 @@ export const applySealedSecretManifests = async ( }) } catch (error) { if (error instanceof ApiException && error.code === 409) { - await k8s.custom().patchNamespacedCustomObject( - { - group: 'bitnami.com', - version: 'v1alpha1', - namespace, - plural: 'sealedsecrets', - name: manifest.metadata.name, - body: manifest, - }, - setHeaderOptions('Content-Type', PatchStrategy.MergePatch), - ) + try { + await k8s.custom().patchNamespacedCustomObject( + { + group: 'bitnami.com', + version: 'v1alpha1', + namespace, + plural: 'sealedsecrets', + name: manifest.metadata.name, + body: manifest, + }, + setHeaderOptions('Content-Type', PatchStrategy.MergePatch), + ) + } catch (patchError) { + d.error(`Failed to patch SealedSecret ${manifest.metadata.name}: ${patchError}`) + } } else { d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${error}`) } @@ -529,18 +513,22 @@ export const applySealedSecretManifestsFromDir = async ( appliedCount += 1 } catch (error) { if (error instanceof ApiException && error.code === 409) { - await k8s.custom().patchNamespacedCustomObject( - { - group: 'bitnami.com', - version: 'v1alpha1', - namespace, - plural: 'sealedsecrets', - name: manifest.metadata.name, - body: manifest, - }, - setHeaderOptions('Content-Type', PatchStrategy.MergePatch), - ) - appliedCount += 1 + try { + await k8s.custom().patchNamespacedCustomObject( + { + group: 'bitnami.com', + version: 'v1alpha1', + namespace, + plural: 'sealedsecrets', + name: manifest.metadata.name, + body: manifest, + }, + setHeaderOptions('Content-Type', PatchStrategy.MergePatch), + ) + appliedCount += 1 + } catch (patchError) { + d.error(`Failed to patch SealedSecret from ${filePath}: ${patchError}`) + } } else { d.error(`Failed to apply SealedSecret from ${filePath}: ${error}`) } diff --git a/tools/Dockerfile b/tools/Dockerfile index 4fad532e80..af6f1b4836 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -11,7 +11,7 @@ ARG HELM_VERSION=3.19.2 ARG HELM_DIFF_VERSION=3.14.1 # https://github.com/jkroepke/helm-secrets/releases ARG HELM_SECRETS_VERSION=4.7.4 -# https://github.com/getsops/sops/releases +# https://github.com/mozilla/sops/releases ARG SOPS_VERSION=3.11.0 # https://github.com/FiloSottile/age/releases ARG AGE_VERSION=1.2.1 @@ -34,9 +34,10 @@ WORKDIR / # Install all required packages in one layer RUN apt-get update && apt-get install -y \ curl \ + coreutils \ apache2-utils \ + apt-transport-https \ ca-certificates \ - coreutils \ git \ locales \ rsync && \ diff --git a/values/apl-operator/apl-operator-raw.gotmpl b/values/apl-operator/apl-operator-raw.gotmpl index bd18d18d0e..9b6652744f 100644 --- a/values/apl-operator/apl-operator-raw.gotmpl +++ b/values/apl-operator/apl-operator-raw.gotmpl @@ -4,14 +4,14 @@ resources: - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: - name: gitea-credentials + name: apl-git-credentials spec: refreshInterval: 1h secretStoreRef: name: core-secrets-store kind: ClusterSecretStore target: - name: gitea-credentials + name: apl-git-credentials creationPolicy: Owner template: type: Opaque diff --git a/values/apl-operator/apl-operator.gotmpl b/values/apl-operator/apl-operator.gotmpl index bfe276488b..a54c848fa6 100644 --- a/values/apl-operator/apl-operator.gotmpl +++ b/values/apl-operator/apl-operator.gotmpl @@ -24,4 +24,5 @@ git: branch: {{ $v.otomi.git.branch | default "main" | quote }} email: {{ $v.otomi.git.email | default "pipeline@cluster.local" | quote }} username: {{ $v.otomi.git.username | default "otomi-admin" | quote }} - password: {{ $v.otomi.git | get "password" "" | quote }} + # Password intentionally empty — operator reads git credentials from K8s secret (apl-git-credentials) + password: "" diff --git a/values/argocd/argocd-raw.gotmpl b/values/argocd/argocd-raw.gotmpl index 3d71ec6385..8c9966a6f9 100644 --- a/values/argocd/argocd-raw.gotmpl +++ b/values/argocd/argocd-raw.gotmpl @@ -96,6 +96,24 @@ resources: key: otomi-platform-secrets property: git_password {{- end }} + - apiVersion: external-secrets.io/v1beta1 + kind: ExternalSecret + metadata: + name: argocd-oidc-secret + namespace: argocd + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: argocd-secret + creationPolicy: Merge + data: + - secretKey: oidc.clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret - apiVersion: v1 kind: Secret metadata: diff --git a/values/cert-manager/cert-manager-raw.gotmpl b/values/cert-manager/cert-manager-raw.gotmpl index e7f0df883f..e9fc23ce19 100644 --- a/values/cert-manager/cert-manager-raw.gotmpl +++ b/values/cert-manager/cert-manager-raw.gotmpl @@ -41,13 +41,7 @@ resources: secret: '{{ "{{ .secret | toString }}" }}' {{- end }} data: - - secretKey: secret - remoteRef: - key: dns-secrets - {{- if hasKey $p "google" }} - property: provider_google_serviceAccountKey - {{- else if hasKey $p "akamai" }} - property: provider_akamai_clientSecret + {{- if hasKey $p "akamai" }} - secretKey: access_token remoteRef: key: dns-secrets @@ -60,6 +54,12 @@ resources: remoteRef: key: dns-secrets property: provider_akamai_clientSecret + {{- else }} + - secretKey: secret + remoteRef: + key: dns-secrets + {{- if hasKey $p "google" }} + property: provider_google_serviceAccountKey {{- else if hasKey $p "azure-private-dns" }} property: provider_azure-private-dns_aadClientSecret {{- else if hasKey $p "azure" }} @@ -73,6 +73,7 @@ resources: {{- else if hasKey $p "linode" }} property: provider_linode_apiToken {{- end }} + {{- end }} {{- end }} - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret diff --git a/values/external-dns/external-dns.gotmpl b/values/external-dns/external-dns.gotmpl index e5f1d5e83c..88f1c64e75 100644 --- a/values/external-dns/external-dns.gotmpl +++ b/values/external-dns/external-dns.gotmpl @@ -214,7 +214,7 @@ env: extraVolumes: - name: google-service-account secret: - secretName: GOOGLE-DNS + secretName: google-dns extraVolumeMounts: - name: google-service-account mountPath: /etc/secrets/service-account diff --git a/values/external-secrets/external-secrets.gotmpl b/values/external-secrets/external-secrets.gotmpl index 30ba6ada8c..0e16bcfaaf 100644 --- a/values/external-secrets/external-secrets.gotmpl +++ b/values/external-secrets/external-secrets.gotmpl @@ -1,5 +1,5 @@ {{- $v := .Values }} -{{- $app := $v.apps | get "sealed-secrets" }} +{{- $app := $v.apps | get "external-secrets" }} replicaCount: 1 diff --git a/values/oauth2-proxy/oauth2-proxy-raw.gotmpl b/values/oauth2-proxy/oauth2-proxy-raw.gotmpl index 17696000bf..61450b8510 100644 --- a/values/oauth2-proxy/oauth2-proxy-raw.gotmpl +++ b/values/oauth2-proxy/oauth2-proxy-raw.gotmpl @@ -45,12 +45,16 @@ resources: data: client-id: {{ $k.idp.clientID }} client-secret: '{{ "{{ .clientSecret | toString }}" }}' - cookie-secret: {{ $oauth2 | get "config.cookieSecret" (randAlpha 32) }} + cookie-secret: '{{ "{{ .cookieSecret | toString }}" }}' data: - secretKey: clientSecret remoteRef: key: keycloak-secrets property: idp_clientSecret + - secretKey: cookieSecret + remoteRef: + key: oauth2-proxy-secrets + property: config_cookieSecret - apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/values/prometheus-operator/prometheus-operator-raw.gotmpl b/values/prometheus-operator/prometheus-operator-raw.gotmpl index b74bf5873b..b6ef5709b9 100644 --- a/values/prometheus-operator/prometheus-operator-raw.gotmpl +++ b/values/prometheus-operator/prometheus-operator-raw.gotmpl @@ -138,4 +138,10 @@ resources: key: smtp-secrets property: auth_secret {{- end }} + {{- if has "opsgenie" $receivers }} + - secretKey: opsgenieApiKey + remoteRef: + key: alerts-secrets + property: opsgenie_apiKey + {{- end }} {{- end }} diff --git a/values/prometheus-operator/prometheus-operator.gotmpl b/values/prometheus-operator/prometheus-operator.gotmpl index bc7a7199e5..6a92d8326d 100644 --- a/values/prometheus-operator/prometheus-operator.gotmpl +++ b/values/prometheus-operator/prometheus-operator.gotmpl @@ -247,6 +247,8 @@ grafana: {{- end }} basicAuth: true basicAuthUser: otomi-admin + secureJsonData: + basicAuthPassword: $__env{GF_LOKI_BASIC_AUTH_PASSWORD} {{- end }} {{- end }} admin: From be165b247307465f5d5ff86bdeac1db682e99bab Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 4 Mar 2026 23:58:27 +0100 Subject: [PATCH 44/66] feat: update sealed secrets handling to return applied secrets list --- src/cmd/install.test.ts | 4 +--- src/cmd/install.ts | 44 ++++++++++++------------------------ src/common/sealed-secrets.ts | 14 +++++++----- 3 files changed, 24 insertions(+), 38 deletions(-) diff --git a/src/cmd/install.test.ts b/src/cmd/install.test.ts index 16704a0499..ba2c8ce625 100644 --- a/src/cmd/install.test.ts +++ b/src/cmd/install.test.ts @@ -59,15 +59,13 @@ jest.mock('zx', () => { }) jest.mock('src/common/sealed-secrets', () => ({ - applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue(undefined), + applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue([]), restartSealedSecretsController: jest.fn().mockResolvedValue(undefined), - SECRET_NAME_MAP: {}, })) jest.mock('src/common/utils', () => ({ ...jest.requireActual('src/common/utils'), rootDir: '/test/root', - getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), })) jest.mock('./commit', () => ({ diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 0e0a4c713a..9bc108f0cc 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -17,12 +17,8 @@ import { setDeploymentState, waitForCRD, } from 'src/common/k8s' -import { - applySealedSecretManifestsFromDir, - restartSealedSecretsController, - SECRET_NAME_MAP, -} from 'src/common/sealed-secrets' -import { getFilename, getSchemaSecretsPaths, rootDir } from 'src/common/utils' +import { applySealedSecretManifestsFromDir, restartSealedSecretsController } from 'src/common/sealed-secrets' +import { getFilename, rootDir } from 'src/common/utils' import { getImageTagFromValues, getPackageVersion, writeValuesToFile } from 'src/common/values' import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs' import { Argv, CommandModule } from 'yargs' @@ -63,46 +59,36 @@ const retryInstallStep = async ( /** * Wait for SealedSecrets controller to decrypt SealedSecret resources into K8s Secrets. - * Derives the expected secret names from schema x-secret fields + SECRET_NAME_MAP. + * Takes the list of applied secrets from applySealedSecretManifestsFromDir. */ const waitForSealedSecrets = async ( + appliedSecrets: { namespace: string; secretName: string }[], timeoutMs = 120000, intervalMs = 3000, - deps = { getK8sSecret, terminal, getSchemaSecretsPaths }, + deps = { getK8sSecret, terminal }, ): Promise => { const d = deps.terminal(`cmd:${cmdName}:waitForSealedSecrets`) - // Get all x-secret paths from schema and map them to K8s secret names - const secretPaths = await deps.getSchemaSecretsPaths([]) - const sortedPrefixes = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) - const secretNames = new Set() - for (const path of secretPaths) { - for (const prefix of sortedPrefixes) { - if (path === prefix || path.startsWith(`${prefix}.`)) { - secretNames.add(SECRET_NAME_MAP[prefix]) - break - } - } - } - - if (secretNames.size === 0) { + if (appliedSecrets.length === 0) { d.info('No sealed secrets to wait for') return } - d.info(`Waiting for ${secretNames.size} sealed secrets to be decrypted: ${[...secretNames].join(', ')}`) + d.info( + `Waiting for ${appliedSecrets.length} sealed secrets to be decrypted: ${appliedSecrets.map((s) => s.secretName).join(', ')}`, + ) await retry( async () => { const pending: string[] = [] - for (const secretName of secretNames) { + for (const { namespace, secretName } of appliedSecrets) { try { - const secret = await deps.getK8sSecret(secretName, 'apl-secrets') + const secret = await deps.getK8sSecret(secretName, namespace) if (!secret) { - pending.push(secretName) + pending.push(`${namespace}/${secretName}`) } } catch { - pending.push(secretName) + pending.push(`${namespace}/${secretName}`) } } @@ -172,13 +158,13 @@ export const installAll = async () => { await retryInstallStep(waitForCRD, 'sealedsecrets.bitnami.com') d.info('Applying SealedSecret manifests') - await applySealedSecretManifestsFromDir(env.ENV_DIR) + const appliedSecrets = await applySealedSecretManifestsFromDir(env.ENV_DIR) d.info('Restarting sealed-secrets controller') await restartSealedSecretsController() d.info('Waiting for sealed secrets to be decrypted into K8s Secrets') - await waitForSealedSecrets() + await waitForSealedSecrets(appliedSecrets) // Deploy ESO (External Secrets Operator) d.info('Deploying external-secrets operator') diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index cd764bafda..0d4510d6c8 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -463,24 +463,25 @@ export const applySealedSecretManifests = async ( /** * Read and apply all SealedSecret manifests from the env/manifests/namespaces directory. * This should be called during install, after the sealed-secrets controller is deployed. + * Returns the list of applied secrets (namespace + secretName) so callers can wait for them. */ export const applySealedSecretManifestsFromDir = async ( envDir: string, deps = { terminal, readdir, readFile, existsSync }, -): Promise => { +): Promise<{ namespace: string; secretName: string }[]> => { const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) const manifestsDir = join(envDir, 'env/manifests/namespaces') if (!deps.existsSync(manifestsDir)) { d.info(`No SealedSecret manifests directory found at ${manifestsDir}`) - return + return [] } d.info(`Applying SealedSecret manifests from ${manifestsDir}`) // Read all namespace directories const namespaces = await deps.readdir(manifestsDir, { withFileTypes: true }) - let appliedCount = 0 + const appliedSecrets: { namespace: string; secretName: string }[] = [] for (const nsEntry of namespaces) { if (!nsEntry.isDirectory()) continue @@ -510,7 +511,7 @@ export const applySealedSecretManifestsFromDir = async ( plural: 'sealedsecrets', body: manifest, }) - appliedCount += 1 + appliedSecrets.push({ namespace, secretName: manifest.metadata.name }) } catch (error) { if (error instanceof ApiException && error.code === 409) { try { @@ -525,7 +526,7 @@ export const applySealedSecretManifestsFromDir = async ( }, setHeaderOptions('Content-Type', PatchStrategy.MergePatch), ) - appliedCount += 1 + appliedSecrets.push({ namespace, secretName: manifest.metadata.name }) } catch (patchError) { d.error(`Failed to patch SealedSecret from ${filePath}: ${patchError}`) } @@ -539,7 +540,8 @@ export const applySealedSecretManifestsFromDir = async ( } } - d.info(`Applied ${appliedCount} SealedSecret manifests from directory`) + d.info(`Applied ${appliedSecrets.length} SealedSecret manifests from directory`) + return appliedSecrets } /** From 526ee121b7043b34987cfc313b49dfadd63eadda Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Mar 2026 00:23:32 +0100 Subject: [PATCH 45/66] fix: secret data keys --- values/apl-operator/apl-operator-raw.gotmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/values/apl-operator/apl-operator-raw.gotmpl b/values/apl-operator/apl-operator-raw.gotmpl index 9b6652744f..7d1853716a 100644 --- a/values/apl-operator/apl-operator-raw.gotmpl +++ b/values/apl-operator/apl-operator-raw.gotmpl @@ -16,8 +16,8 @@ resources: template: type: Opaque data: - GIT_USERNAME: {{ $v.otomi.git | get "username" "otomi-admin" }} - GIT_PASSWORD: '{{ "{{ .git_password | toString }}" }}' + username: {{ $v.otomi.git | get "username" "otomi-admin" }} + password: '{{ "{{ .git_password | toString }}" }}' data: - secretKey: git_password remoteRef: From 22143df9a9e7be9e914863fdcff2bb3438d48034 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Mar 2026 09:08:54 +0100 Subject: [PATCH 46/66] fix: values-schema x-secret fields --- values-schema.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/values-schema.yaml b/values-schema.yaml index 45c5dfbafd..584243e4c8 100644 --- a/values-schema.yaml +++ b/values-schema.yaml @@ -907,7 +907,7 @@ definitions: secretTemplates: definitions: otomiAdminUsername: - default: 'admin' + x-secret: 'admin' securityContext: additionalProperties: uniqueItems: true @@ -1690,6 +1690,7 @@ properties: To be used with issuer externally-managed-tls-secret. $ref: '#/definitions/idName' customRootCA: + x-secret: '' type: string description: CA that is used to create and verify self-signed certificates. Leave it empty to generate one automatically. customRootCAKey: From b10418ad718212dd16f9c9f11d0925d9ee8ab409 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Mar 2026 10:57:57 +0100 Subject: [PATCH 47/66] fix: restart sealed secrets controller --- src/cmd/install.ts | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 9bc108f0cc..451feee015 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -61,6 +61,21 @@ const retryInstallStep = async ( * Wait for SealedSecrets controller to decrypt SealedSecret resources into K8s Secrets. * Takes the list of applied secrets from applySealedSecretManifestsFromDir. */ +const allSecretsExist = async ( + secrets: { namespace: string; secretName: string }[], + deps = { getK8sSecret }, +): Promise => { + for (const { namespace, secretName } of secrets) { + try { + const secret = await deps.getK8sSecret(secretName, namespace) + if (!secret) return false + } catch { + return false + } + } + return true +} + const waitForSealedSecrets = async ( appliedSecrets: { namespace: string; secretName: string }[], timeoutMs = 120000, @@ -160,11 +175,21 @@ export const installAll = async () => { d.info('Applying SealedSecret manifests') const appliedSecrets = await applySealedSecretManifestsFromDir(env.ENV_DIR) - d.info('Restarting sealed-secrets controller') - await restartSealedSecretsController() - - d.info('Waiting for sealed secrets to be decrypted into K8s Secrets') - await waitForSealedSecrets(appliedSecrets) + if (appliedSecrets.length > 0) { + // Check if all secrets are already decrypted (e.g. on retry after a previous successful run) + const allExist = await allSecretsExist(appliedSecrets, { getK8sSecret }) + if (allExist) { + d.info('All sealed secrets already decrypted, skipping controller restart') + } else { + d.info('Restarting sealed-secrets controller to pick up new manifests') + await restartSealedSecretsController() + + d.info('Waiting for sealed secrets to be decrypted into K8s Secrets') + await waitForSealedSecrets(appliedSecrets) + } + } else { + d.info('No sealed secret manifests found, skipping controller restart') + } // Deploy ESO (External Secrets Operator) d.info('Deploying external-secrets operator') From 20eb516a4ab93f2b7c01506b1c7328019edbc6e9 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Mar 2026 11:53:14 +0100 Subject: [PATCH 48/66] fix: remove x-secret field from customRootCA --- values-schema.yaml | 1 - values/cert-manager/cert-manager-raw.gotmpl | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/values-schema.yaml b/values-schema.yaml index 584243e4c8..1c0400be78 100644 --- a/values-schema.yaml +++ b/values-schema.yaml @@ -1690,7 +1690,6 @@ properties: To be used with issuer externally-managed-tls-secret. $ref: '#/definitions/idName' customRootCA: - x-secret: '' type: string description: CA that is used to create and verify self-signed certificates. Leave it empty to generate one automatically. customRootCAKey: diff --git a/values/cert-manager/cert-manager-raw.gotmpl b/values/cert-manager/cert-manager-raw.gotmpl index e9fc23ce19..af97af4995 100644 --- a/values/cert-manager/cert-manager-raw.gotmpl +++ b/values/cert-manager/cert-manager-raw.gotmpl @@ -75,6 +75,7 @@ resources: {{- end }} {{- end }} {{- end }} +{{- if $cm | get "customRootCA" "" }} - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: @@ -97,6 +98,7 @@ resources: remoteRef: key: cert-manager-secrets property: customRootCAKey +{{- end }} - apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: From 7f0a422b97d9b2a1ab7b8cd53d9ce18ac62347aa Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Mar 2026 17:27:42 +0100 Subject: [PATCH 49/66] fix: create team settings secrets --- src/common/values.test.ts | 23 +++++++++++++++++++++++ src/common/values.ts | 24 ++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/common/values.test.ts b/src/common/values.test.ts index 06f76fa4e6..3af2dead81 100644 --- a/src/common/values.test.ts +++ b/src/common/values.test.ts @@ -60,6 +60,7 @@ describe('generateSecrets', () => { deps = { terminal, getValuesSchema: jest.fn().mockReturnValue(schema), + getSchemaSecretsPaths: jest.fn().mockResolvedValue([]), } }) it('should generate new secrets and return only secrets', async () => { @@ -72,4 +73,26 @@ describe('generateSecrets', () => { const res = await generateSecrets(valuesWithExisting, deps) expect(res.nested.twoStage).toBe('exists') }) + it('should include team secrets with expanded paths', async () => { + const teamValues = cloneDeep(values) + set(teamValues, 'teamConfig.demo.settings.password', 'team-secret-pw') + + deps.getSchemaSecretsPaths.mockResolvedValue(['teamConfig.demo.settings.password']) + + const res = await generateSecrets(teamValues, deps) + expect(deps.getSchemaSecretsPaths).toHaveBeenCalledWith(['demo']) + expect(res.teamConfig.demo.settings.password).toBe('team-secret-pw') + }) + it('should not call getSchemaSecretsPaths when no dynamic teams exist', async () => { + const res = await generateSecrets(values, deps) + expect(deps.getSchemaSecretsPaths).not.toHaveBeenCalled() + expect(res).toEqual(expected) + }) + it('should exclude admin team from dynamic team expansion', async () => { + const teamValues = cloneDeep(values) + set(teamValues, 'teamConfig.admin.settings.password', 'admin-pw') + + const res = await generateSecrets(teamValues, deps) + expect(deps.getSchemaSecretsPaths).not.toHaveBeenCalled() + }) }) diff --git a/src/common/values.ts b/src/common/values.ts index de44be2dbf..e8270d034a 100644 --- a/src/common/values.ts +++ b/src/common/values.ts @@ -1,6 +1,6 @@ import { existsSync } from 'fs' import { mkdir, unlink, writeFile } from 'fs/promises' -import { cloneDeep, isEmpty, isEqual, merge, mergeWith, pick, set } from 'lodash' +import { cloneDeep, get, isEmpty, isEqual, merge, mergeWith, pick, set } from 'lodash' import path from 'path' import { supportedK8sVersions } from 'src/supportedK8sVersions.json' import { stringify } from 'yaml' @@ -10,7 +10,16 @@ import { terminal } from './debug' import { env } from './envalid' import { hfValues } from './hf' import { saveValues } from './repo' -import { extract, flattenObject, getValuesSchema, gucci, loadYaml, pkg, removeBlankAttributes } from './utils' +import { + extract, + flattenObject, + getSchemaSecretsPaths, + getValuesSchema, + gucci, + loadYaml, + pkg, + removeBlankAttributes, +} from './utils' import { HelmArguments } from './yargs' export const objectToYaml = (obj: Record, indent = 4, lineWidth = 200): string => { @@ -135,6 +144,7 @@ export const generateSecrets = async ( deps = { terminal, getValuesSchema, + getSchemaSecretsPaths, }, ): Promise> => { const d = deps.terminal('common:values:generateSecrets') @@ -158,6 +168,16 @@ export const generateSecrets = async ( // Only return values that have x-secrets prop and are now fully templated: const templatePaths = Object.keys(flattenObject(schemaSecrets)) const res = pick(allSecrets, templatePaths) + + // Template paths use schema patternProperties regex keys which don't match concrete team names. + // Expand team paths so team secrets are included in the result. + const teamNames = Object.keys(get(values, 'teamConfig', {})).filter((t) => t !== 'admin') + if (teamNames.length > 0) { + const expandedPaths = await deps.getSchemaSecretsPaths(teamNames) + const teamSecrets = pick(allSecrets, expandedPaths) + merge(res, teamSecrets) + } + d.debug('generateSecrets result: ', res) return res } From f333c235b0c8bf15ed07818887cdf5c69c8d06ca Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:33:04 +0100 Subject: [PATCH 50/66] fix: harbor push issues --- src/cmd/pull.ts | 2 +- values-schema.yaml | 2 +- values/k8s/k8s-raw.gotmpl | 13 +++++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/cmd/pull.ts b/src/cmd/pull.ts index 550773d67a..28b7f6edb7 100644 --- a/src/cmd/pull.ts +++ b/src/cmd/pull.ts @@ -17,7 +17,7 @@ export const pull = async (): Promise => { d.error('No values found, skipping git pull') return } - const gitRepo = getRepo(allValues) + const gitRepo = await getRepo(allValues) const { branch } = gitRepo d.info('Pulling latest values') cd(env.ENV_DIR) diff --git a/values-schema.yaml b/values-schema.yaml index 1c0400be78..f1b0c184fa 100644 --- a/values-schema.yaml +++ b/values-schema.yaml @@ -907,7 +907,7 @@ definitions: secretTemplates: definitions: otomiAdminUsername: - x-secret: 'admin' + x-secret: 'otomi-admin' securityContext: additionalProperties: uniqueItems: true diff --git a/values/k8s/k8s-raw.gotmpl b/values/k8s/k8s-raw.gotmpl index 65631e6f26..1eec03a63e 100644 --- a/values/k8s/k8s-raw.gotmpl +++ b/values/k8s/k8s-raw.gotmpl @@ -72,3 +72,16 @@ resources: value: 1000000 globalDefault: false description: "This priority class should be used for Otomi High priority service pods only." + {{- if $v.cluster.domainSuffix }} + # CoreDNS custom config to resolve platform domains to the ingress controller ClusterIP. + # This avoids hairpin NAT issues where pods cannot reach services via the external LoadBalancer IP. + - apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns-custom + namespace: kube-system + data: + otomi-hairpin.include: | + {{- $escapedDomain := $v.cluster.domainSuffix | replace "." "\\." }} + rewrite name regex (.+)\.{{ $escapedDomain }} ingress-nginx-platform-controller.ingress.svc.cluster.local answer auto + {{- end }} From f49a5db3613238c69b9b7070cbd2ba866fe6f3e0 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Mar 2026 19:27:49 +0100 Subject: [PATCH 51/66] feat: use commands with cwd instead of cd --- src/cmd/commit.ts | 42 +++++++++++++++++++---------------------- src/cmd/pull.ts | 8 ++++---- src/common/bootstrap.ts | 7 ++++--- 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 89faf6e757..00f381848d 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -12,12 +12,12 @@ import { createUpdateConfigMap, createUpdateGenericSecret, getK8sSecret, k8s } f import { getFilename } from 'src/common/utils' import { HelmArguments, setParsedArgs } from 'src/common/yargs' import { Argv } from 'yargs' -import { $, cd } from 'zx' +import { $ } from 'zx' import { validateValues } from './validate-values' const cmdName = getFilename(__filename) -export const rootDir = process.cwd() === '/home/app/stack/env' ? '/home/app/stack' : process.cwd() +const $git = $({ cwd: env.ENV_DIR }) interface Arguments extends HelmArguments { m?: string @@ -43,12 +43,11 @@ const isConflictError = (error: any): boolean => { const cleanupGitState = async (d: any): Promise => { try { - cd(env.ENV_DIR) // Try to abort any ongoing merge or rebase - await $`git merge --abort`.nothrow().quiet() - await $`git rebase --abort`.nothrow().quiet() + await $git`git merge --abort`.nothrow().quiet() + await $git`git rebase --abort`.nothrow().quiet() // Reset to the commit before our failed commit to discard local changes - await $`git reset --hard HEAD~1`.quiet() + await $git`git reset --hard HEAD~1`.quiet() d.info('Git state cleaned up after conflict - local commit discarded, reconciliation will retry') } catch (cleanupError) { d.warn('Error during git cleanup:', cleanupError?.message) @@ -65,25 +64,25 @@ const commitAndPush = async ( d.info('Committing values') const message = initialInstall ? 'otomi commit' : 'updated values [ci skip]' const { password } = gitConfig ?? (await getRepo(values)) - cd(env.ENV_DIR) try { try { - await $`git rev-list HEAD --count`.quiet() + await $git`git rev-list HEAD --count`.quiet() } catch { d.log('Very first commit') // We need at least two commits in repo, so git diff in Tekton pipeline always works. This is why the very first time we commit twice. - await $`git add README.md`.quiet() - await $`git commit -m ${message} --no-verify`.quiet() + await $git`git add README.md`.quiet() + await $git`git commit -m ${message} --no-verify`.quiet() } - await $`git add -A` + await $git`git add -A` // The below 'git status' command will always return at least single new line - const filesChangedCount = (await $`git status --untracked-files=no --porcelain`).toString().split('\n').length - 1 + const statusOutput = (await $git`git status --untracked-files=no --porcelain`).toString() + const filesChangedCount = statusOutput.split('\n').length - 1 if (filesChangedCount === 0) { d.log('Nothing to commit') return } - await $`git commit -m ${message} --no-verify`.quiet() + await $git`git commit -m ${message} --no-verify`.quiet() } catch (e) { const errorMsg = `commitAndPush error: ${e?.message?.replace(password, '****')}` d.error(errorMsg) @@ -93,24 +92,23 @@ const commitAndPush = async ( await retry( async () => { try { - cd(env.ENV_DIR) // Check if remote branch exists let remoteBranchExists = true try { - await $`git ls-remote --exit-code --heads origin ${branch}`.quiet() + await $git`git ls-remote --exit-code --heads origin ${branch}`.quiet() } catch { remoteBranchExists = false } // We're not always sure that we are on the correct branch, // so we checkout the branch and create it if it does not exist - await $`git checkout -B ${branch}`.quiet() + await $git`git checkout -B ${branch}`.quiet() if (remoteBranchExists) { - await $`git pull --rebase origin ${branch}`.quiet() + await $git`git pull --rebase origin ${branch}`.quiet() } else { d.log(`Remote branch '${branch}' does not exist. Skipping pull.`) } - await $`git push -u origin ${branch}`.quiet() + await $git`git push -u origin ${branch}`.quiet() } catch (pullPushError) { // Check if this is a merge conflict - if so, skip the commit if (isConflictError(pullPushError)) { @@ -152,13 +150,11 @@ export const commit = async ( // the bootstrap phase before install) may have set the URL with unresolved placeholder // passwords because K8s secrets didn't exist yet. Now that secrets are decrypted, // we need to update the URL with the real credentials. - cd(env.ENV_DIR) - await $`git remote set-url origin ${remote}`.nothrow().quiet() + await $git`git remote set-url origin ${remote}`.nothrow().quiet() } else { - cd(env.ENV_DIR) - await setIdentity(username, email) + await setIdentity(username, email, env.ENV_DIR) // the url might need updating (e.g. if credentials changed) - await $`git remote set-url origin ${remote}` + await $git`git remote set-url origin ${remote}` } // let's wait until the remote is ready if (values?.apps!.gitea!.enabled ?? true) { diff --git a/src/cmd/pull.ts b/src/cmd/pull.ts index 28b7f6edb7..b56ab13262 100644 --- a/src/cmd/pull.ts +++ b/src/cmd/pull.ts @@ -5,7 +5,7 @@ import { hfValues } from 'src/common/hf' import { getFilename } from 'src/common/utils' import { HelmArguments, setParsedArgs } from 'src/common/yargs' import { Argv } from 'yargs' -import { $, cd } from 'zx' +import { $ } from 'zx' import { getRepo } from '../common/git-config' const cmdName = getFilename(__filename) @@ -20,10 +20,10 @@ export const pull = async (): Promise => { const gitRepo = await getRepo(allValues) const { branch } = gitRepo d.info('Pulling latest values') - cd(env.ENV_DIR) + const $git = $({ cwd: env.ENV_DIR }) try { - await $`git fetch` - await $`if git --no-pager log --decorate=short --pretty=oneline -n1; then git merge origin/${branch}; fi` + await $git`git fetch` + await $git`if git --no-pager log --decorate=short --pretty=oneline -n1; then git merge origin/${branch}; fi` } catch (error) { d.warn( `An error occured when trying to pull (maybe not problematic).\nIf you see merge conflicts then please resolve these and run \`otomi commit\` again.`, diff --git a/src/common/bootstrap.ts b/src/common/bootstrap.ts index fe82d245a6..587a5a4031 100644 --- a/src/common/bootstrap.ts +++ b/src/common/bootstrap.ts @@ -12,9 +12,10 @@ import { $, cd } from 'zx' const cmdName = getFilename(__filename) -export const setIdentity = async (username, email) => { - await $`git config --local user.name ${username}`.nothrow().quiet() - await $`git config --local user.email ${email}`.nothrow().quiet() +export const setIdentity = async (username, email, cwd?: string) => { + const $run = cwd ? $({ cwd }) : $ + await $run`git config --local user.name ${username}`.nothrow().quiet() + await $run`git config --local user.email ${email}`.nothrow().quiet() } export const recoverFromGit = async (gitConfig: GitRepoConfig): Promise => { From db4524b1e982ada6b2882c9a954011777359537d Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Mar 2026 21:14:45 +0100 Subject: [PATCH 52/66] fix: use commands with cwd instead of cd --- src/common/gitea.ts | 8 ++++---- src/common/hf.ts | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/common/gitea.ts b/src/common/gitea.ts index ab6268419b..5d46cbdafd 100644 --- a/src/common/gitea.ts +++ b/src/common/gitea.ts @@ -1,5 +1,5 @@ import retry from 'async-retry' -import { $, cd } from 'zx' +import { $ } from 'zx' import { APL_OPERATOR_NS } from './constants' import { terminal } from './debug' import { env } from './envalid' @@ -29,14 +29,14 @@ export const waitTillGitRepoAvailable = async (repoUrl: string): Promise = const d = terminal('common:gitea:waitTillGitRepoAvailable') await retry( async () => { + const $git = $({ cwd: env.ENV_DIR }) try { - cd(env.ENV_DIR) // the ls-remote exists with zero even if repo is empty - await $`git ls-remote ${repoUrl}` + await $git`git ls-remote ${repoUrl}` } catch (e) { if (e.stderr && e.stderr.includes('remote: Update your password')) { await resetGiteaPasswordValidity() - await $`git ls-remote ${repoUrl}` + await $git`git ls-remote ${repoUrl}` } else { d.warn(`The values repository is not yet reachable. Retrying in ${env.MIN_TIMEOUT} ms`) throw e diff --git a/src/common/hf.ts b/src/common/hf.ts index 53f41bf450..4068fef4bc 100644 --- a/src/common/hf.ts +++ b/src/common/hf.ts @@ -64,10 +64,12 @@ const hfCore = (args: HFParams, envDir = env.ENV_DIR): ProcessPromise => { stringArray.push(`--log-level=${paramsCopy.logLevel.toLowerCase()}`) process.env.HELM_DIFF_COLOR = 'true' process.env.HELM_DIFF_USE_UPGRADE_DRY_RUN = 'true' + // Always run helmfile from rootDir to ensure helmfile.d/ is found regardless of process cwd + const $hf = $({ cwd: rootDir }) if ((parsedArgs?.dryRun || parsedArgs?.local) && paramsCopy.args.includes('sync')) { - return $`echo ENV_DIR=${envDir} helmfile ${stringArray} ${paramsCopy.args}`.quiet() + return $hf`echo ENV_DIR=${envDir} helmfile ${stringArray} ${paramsCopy.args}`.quiet() } else { - return $`ENV_DIR=${envDir} helmfile ${stringArray} ${paramsCopy.args}`.quiet() + return $hf`ENV_DIR=${envDir} helmfile ${stringArray} ${paramsCopy.args}`.quiet() } } From dd8419efe8233f1cbda4be6e67b7239eb4e9bd1f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 13 Mar 2026 08:13:09 +0100 Subject: [PATCH 53/66] fix: update sealed secrets handling and improve deployment configurations --- chart/apl/templates/sops-secrets.yaml | 35 ------ chart/chart-index/Chart.yaml | 3 + charts/apl-operator/templates/deployment.yaml | 1 - .../ingress-nginx/templates/clusterrole.yaml | 1 - src/cmd/commit.ts | 12 +- src/cmd/install.ts | 18 +-- src/cmd/migrate.ts | 11 +- src/common/constants.ts | 3 + src/common/git-config.ts | 4 +- src/common/sealed-secrets.test.ts | 6 +- src/common/sealed-secrets.ts | 114 +++++++----------- 11 files changed, 81 insertions(+), 127 deletions(-) delete mode 100644 chart/apl/templates/sops-secrets.yaml diff --git a/chart/apl/templates/sops-secrets.yaml b/chart/apl/templates/sops-secrets.yaml deleted file mode 100644 index 7544e0ec6c..0000000000 --- a/chart/apl/templates/sops-secrets.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{- $kms := .Values.kms | default dict }} -{{- if hasKey $kms "sops" }} -{{- $v := $kms.sops }} -apiVersion: v1 -kind: Secret -metadata: - name: apl-sops-secrets - namespace: apl-operator -type: Opaque -data: -{{- with $v.azure }} - AZURE_CLIENT_ID: {{ .clientId | b64enc }} - AZURE_CLIENT_SECRET: {{ .clientSecret | b64enc }} -{{- with .tenantId }} - AZURE_TENANT_ID: {{ . | b64enc }}{{ end }} -{{- with .environment }} - AZURE_ENVIRONMENT: {{ . | b64enc }}{{ end }} -{{- end }} -{{- with $v.aws }} - AWS_ACCESS_KEY_ID: {{ .accessKey | b64enc }} - AWS_SECRET_ACCESS_KEY: {{ .secretKey | b64enc }} -{{- with .region }} - AWS_REGION: {{ . | b64enc }}{{ end }} -{{- end }} -{{- with $v.age }} - SOPS_AGE_KEY: {{ .privateKey | b64enc }} -{{- end }} -{{- with $v.google }} - GCLOUD_SERVICE_KEY: {{ .accountJson | b64enc }} -{{- with .project }} - GOOGLE_PROJECT: {{ . | b64enc }}{{ end }} -{{- with .region }} - GOOGLE_REGION: {{ . | b64enc }}{{ end }} -{{- end }} -{{- end }} diff --git a/chart/chart-index/Chart.yaml b/chart/chart-index/Chart.yaml index 908ec6f1d0..c4eea630db 100644 --- a/chart/chart-index/Chart.yaml +++ b/chart/chart-index/Chart.yaml @@ -31,6 +31,9 @@ dependencies: - name: external-dns version: 1.20.0 repository: https://kubernetes-sigs.github.io/external-dns + - name: external-secrets + version: 0.14.3 + repository: https://charts.external-secrets.io - name: gitea version: 12.5.0 repository: https://dl.gitea.io/charts diff --git a/charts/apl-operator/templates/deployment.yaml b/charts/apl-operator/templates/deployment.yaml index 5f28093cc1..0991e2e09b 100644 --- a/charts/apl-operator/templates/deployment.yaml +++ b/charts/apl-operator/templates/deployment.yaml @@ -47,7 +47,6 @@ spec: optional: true - secretRef: name: apl-git-credentials - optional: true livenessProbe: exec: command: diff --git a/charts/ingress-nginx/templates/clusterrole.yaml b/charts/ingress-nginx/templates/clusterrole.yaml index 15b2ac4b55..51bc5002cc 100644 --- a/charts/ingress-nginx/templates/clusterrole.yaml +++ b/charts/ingress-nginx/templates/clusterrole.yaml @@ -27,7 +27,6 @@ rules: - namespaces {{- end}} verbs: - - get - list - watch - apiGroups: diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 00f381848d..0b8b5e08b3 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -1,7 +1,13 @@ import retry from 'async-retry' import { bootstrapGit, setIdentity } from 'src/common/bootstrap' import { prepareEnvironment } from 'src/common/cli' -import { APL_OPERATOR_NS, DEPLOYMENT_PASSWORDS_SECRET } from 'src/common/constants' +import { + APL_OPERATOR_NS, + DEPLOYMENT_PASSWORDS_SECRET, + OTOMI_NAMESPACE, + OTOMI_PLATFORM_SECRETS, + SEALED_SECRETS_NAMESPACE, +} from 'src/common/constants' import { encrypt } from 'src/common/crypt' import { terminal } from 'src/common/debug' import { env } from 'src/common/envalid' @@ -176,7 +182,7 @@ export async function initialSetupData(): Promise { // Read the platform admin's initialPassword from the generated passwords secret let platformAdminPassword = '' try { - const secretData = await getK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, 'otomi') + const secretData = await getK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, OTOMI_NAMESPACE) const allSecrets = secretData?.[DEPLOYMENT_PASSWORDS_SECRET] const users = allSecrets?.users || [] const defaultEmail = `platform-admin@${domainSuffix}` @@ -193,7 +199,7 @@ export async function initialSetupData(): Promise { } } else { // External IDP: show Keycloak admin credentials (keycloak-initial-admin uses otomi-platform-secrets.adminPassword) - const otomiSecret = await getK8sSecret('otomi-platform-secrets', 'apl-secrets') + const otomiSecret = await getK8sSecret(OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE) const adminPassword = otomiSecret?.adminPassword ? String(otomiSecret.adminPassword) : '' return { domainSuffix, diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 451feee015..a15ab01ef5 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -17,7 +17,11 @@ import { setDeploymentState, waitForCRD, } from 'src/common/k8s' -import { applySealedSecretManifestsFromDir, restartSealedSecretsController } from 'src/common/sealed-secrets' +import { + AppliedSecret, + applySealedSecretManifestsFromDir, + restartSealedSecretsController, +} from 'src/common/sealed-secrets' import { getFilename, rootDir } from 'src/common/utils' import { getImageTagFromValues, getPackageVersion, writeValuesToFile } from 'src/common/values' import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs' @@ -61,10 +65,7 @@ const retryInstallStep = async ( * Wait for SealedSecrets controller to decrypt SealedSecret resources into K8s Secrets. * Takes the list of applied secrets from applySealedSecretManifestsFromDir. */ -const allSecretsExist = async ( - secrets: { namespace: string; secretName: string }[], - deps = { getK8sSecret }, -): Promise => { +const allSecretsExist = async (secrets: AppliedSecret[], deps = { getK8sSecret }): Promise => { for (const { namespace, secretName } of secrets) { try { const secret = await deps.getK8sSecret(secretName, namespace) @@ -77,7 +78,7 @@ const allSecretsExist = async ( } const waitForSealedSecrets = async ( - appliedSecrets: { namespace: string; secretName: string }[], + appliedSecrets: AppliedSecret[], timeoutMs = 120000, intervalMs = 3000, deps = { getK8sSecret, terminal }, @@ -108,7 +109,6 @@ const waitForSealedSecrets = async ( } if (pending.length > 0) { - d.info(`Still waiting for sealed secrets: ${pending.join(', ')}`) throw new Error(`Sealed secrets not yet decrypted: ${pending.join(', ')}`) } @@ -181,7 +181,9 @@ export const installAll = async () => { if (allExist) { d.info('All sealed secrets already decrypted, skipping controller restart') } else { - d.info('Restarting sealed-secrets controller to pick up new manifests') + // The controller may have started before the sealed-secrets-key TLS secret existed, + // causing it to generate its own key. Restarting forces it to pick up the pre-created key. + d.info('Restarting sealed-secrets controller to ensure correct key is used') await restartSealedSecretsController() d.info('Waiting for sealed secrets to be decrypted into K8s Secrets') diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 830ca62fd4..46e5952f33 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -20,7 +20,12 @@ import { v4 as uuidv4 } from 'uuid' import { parse } from 'yaml' import { Argv } from 'yargs' import { $, cd } from 'zx' -import { APL_OPERATOR_NS, ARGOCD_APP_PARAMS } from '../common/constants' +import { + APL_OPERATOR_NS, + ARGOCD_APP_PARAMS, + OTOMI_PLATFORM_SECRETS, + SEALED_SECRETS_NAMESPACE, +} from '../common/constants' import { getK8sSecret, getSealedSecretsPEM, k8s } from '../common/k8s' import { applySealedSecretManifestsFromDir, @@ -725,7 +730,7 @@ const setDefaultAplCatalog = async (values: Record): Promise let secretCreated = false if (useGiteaCatalog) { try { - const giteaSecrets = await getK8sSecret('gitea-secrets', 'apl-secrets') + const giteaSecrets = await getK8sSecret('gitea-secrets', SEALED_SECRETS_NAMESPACE) const resolvedGitea = { adminUsername: giteaSecrets?.adminUsername ? String(giteaSecrets.adminUsername) : String(gitea!.adminUsername), adminPassword: giteaSecrets?.adminPassword ? String(giteaSecrets.adminPassword) : String(gitea!.adminPassword), @@ -895,7 +900,7 @@ export const sopsMigration = async ( }) if (existingManifests.length > 0) { try { - const platformSecret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') + const platformSecret = await deps.getK8sSecret(OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE) if (!platformSecret) { d.info('SealedSecret manifests exist but K8s Secrets are missing — re-applying and restarting controller') await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) diff --git a/src/common/constants.ts b/src/common/constants.ts index 283c14d69a..bb67c8919f 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -2,6 +2,9 @@ export const DEPLOYMENT_PASSWORDS_SECRET = 'otomi-generated-passwords' export const DEPLOYMENT_STATUS_CONFIGMAP = 'otomi-status' export const APL_OPERATOR_NS = 'apl-operator' export const APL_OPERATOR_STATUS_CM = 'apl-installation-status' +export const OTOMI_NAMESPACE = 'otomi' +export const SEALED_SECRETS_NAMESPACE = 'apl-secrets' +export const OTOMI_PLATFORM_SECRETS = 'otomi-platform-secrets' export const ARGOCD_APP_PARAMS = { group: 'argoproj.io', version: 'v1alpha1', diff --git a/src/common/git-config.ts b/src/common/git-config.ts index 9498ee6fe5..662df6fb0b 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -1,5 +1,5 @@ import type { CoreV1Api } from '@kubernetes/client-node' -import { APL_OPERATOR_NS } from './constants' +import { APL_OPERATOR_NS, OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE } from './constants' import { terminal } from './debug' import { createUpdateConfigMap, getK8sConfigMap, getK8sSecret, k8s } from './k8s' @@ -160,7 +160,7 @@ export const getRepo = async (values: Record, deps = { getK8sSecret // try reading the real password from the K8s secret (populated by ESO from SealedSecrets) if (!password || (typeof password === 'string' && password.startsWith('sealed:'))) { try { - const secret = await deps.getK8sSecret('otomi-platform-secrets', 'apl-secrets') + const secret = await deps.getK8sSecret(OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE) if (secret?.git_password) { password = String(secret.git_password) d.debug('Read git password from K8s secret (ESO)') diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 25835a38a9..8874db2784 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -10,8 +10,8 @@ import { generateSealedSecretsKeyPair, getPemFromCertificate, restartSealedSecretsController, - SECRET_NAME_MAP, SealedSecretManifest, + SECRET_NAME_MAP, stripAllSecrets, writeSealedSecretManifests, } from './sealed-secrets' @@ -572,12 +572,12 @@ describe('sealed-secrets', () => { await applySealedSecretManifests(manifests, { terminal }) const { k8s: mockK8s } = require('src/common/k8s') - expect(mockK8s.custom().createNamespacedCustomObject).toHaveBeenCalled() + expect(mockK8s.custom().patchNamespacedCustomObject).toHaveBeenCalled() }) it('should log error when manifests fail to apply', async () => { const { k8s: mockK8s } = require('src/common/k8s') - mockK8s.custom().createNamespacedCustomObject.mockRejectedValueOnce(new Error('apply failed')) + mockK8s.custom().patchNamespacedCustomObject.mockRejectedValueOnce(new Error('apply failed')) const manifests = [makeMockManifest('test-secret', 'apl-secrets')] diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 0d4510d6c8..b04882892b 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -6,6 +6,7 @@ import { mkdir, readdir, readFile, writeFile } from 'fs/promises' import { cloneDeep, get, unset } from 'lodash' import { pki } from 'node-forge' import { join } from 'path' +import { SEALED_SECRETS_NAMESPACE } from 'src/common/constants' import { terminal } from 'src/common/debug' import { b64enc, ensureNamespaceExists, getK8sSecret, k8s } from 'src/common/k8s' import { flattenObject, getSchemaSecretsPaths } from 'src/common/utils' @@ -13,6 +14,8 @@ import { objectToYaml } from 'src/common/values' import { parse as parseYaml } from 'yaml' const cmdName = 'sealed-secrets' +const ROLLOUT_TIMEOUT_MS = 120000 +const ROLLOUT_INTERVAL_MS = 3000 /** * Strip ALL x-secret fields from values before writing to disk. @@ -51,17 +54,21 @@ export interface SealedSecretManifest { } } -/** - * All SealedSecrets are placed in the 'apl-secrets' namespace. - * ESO ClusterSecretStore reads from this namespace and distributes secrets to target namespaces. - */ -const SEALED_SECRETS_NAMESPACE = 'apl-secrets' +export interface SealedSecretsKeyPair { + certificate: string + privateKey: string +} + +export interface AppliedSecret { + namespace: string + secretName: string +} /** * Generate an RSA 4096-bit key pair and self-signed X.509 certificate for Sealed Secrets. * Follows the pattern from createCustomCA() in bootstrap.ts. */ -export const generateSealedSecretsKeyPair = (deps = { terminal, pki }): { certificate: string; privateKey: string } => { +export const generateSealedSecretsKeyPair = (deps = { terminal, pki }): SealedSecretsKeyPair => { const d = deps.terminal(`common:${cmdName}:generateSealedSecretsKeyPair`) d.info('Generating sealed-secrets RSA key pair') @@ -399,6 +406,26 @@ export const writeSealedSecretManifests = async ( } } +/** + * Apply a single SealedSecret manifest using server-side apply (create-or-update). + * Uses the same pattern as applyArgocdApp() in apply-as-apps.ts. + */ +const applySealedSecretResource = async (manifest: SealedSecretManifest): Promise => { + await k8s.custom().patchNamespacedCustomObject( + { + group: 'bitnami.com', + version: 'v1alpha1', + namespace: manifest.metadata.namespace, + plural: 'sealedsecrets', + name: manifest.metadata.name, + body: manifest, + fieldManager: 'apl-operator', + force: true, + }, + setHeaderOptions('Content-Type', PatchStrategy.ServerSideApply), + ) +} + /** * Apply SealedSecret manifests to the Kubernetes cluster. * Creates namespaces if needed and applies the SealedSecret resources. @@ -426,33 +453,9 @@ export const applySealedSecretManifests = async ( for (const manifest of nsManifests) { d.info(`Applying SealedSecret ${manifest.metadata.name} to namespace ${namespace}`) try { - await k8s.custom().createNamespacedCustomObject({ - group: 'bitnami.com', - version: 'v1alpha1', - namespace, - plural: 'sealedsecrets', - body: manifest, - }) + await applySealedSecretResource(manifest) } catch (error) { - if (error instanceof ApiException && error.code === 409) { - try { - await k8s.custom().patchNamespacedCustomObject( - { - group: 'bitnami.com', - version: 'v1alpha1', - namespace, - plural: 'sealedsecrets', - name: manifest.metadata.name, - body: manifest, - }, - setHeaderOptions('Content-Type', PatchStrategy.MergePatch), - ) - } catch (patchError) { - d.error(`Failed to patch SealedSecret ${manifest.metadata.name}: ${patchError}`) - } - } else { - d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${error}`) - } + d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${error}`) } } } @@ -468,7 +471,7 @@ export const applySealedSecretManifests = async ( export const applySealedSecretManifestsFromDir = async ( envDir: string, deps = { terminal, readdir, readFile, existsSync }, -): Promise<{ namespace: string; secretName: string }[]> => { +): Promise => { const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) const manifestsDir = join(envDir, 'env/manifests/namespaces') @@ -481,7 +484,7 @@ export const applySealedSecretManifestsFromDir = async ( // Read all namespace directories const namespaces = await deps.readdir(manifestsDir, { withFileTypes: true }) - const appliedSecrets: { namespace: string; secretName: string }[] = [] + const appliedSecrets: AppliedSecret[] = [] for (const nsEntry of namespaces) { if (!nsEntry.isDirectory()) continue @@ -503,39 +506,10 @@ export const applySealedSecretManifestsFromDir = async ( const content = await deps.readFile(filePath, 'utf-8') const manifest = parseYaml(content) as SealedSecretManifest - try { - await k8s.custom().createNamespacedCustomObject({ - group: 'bitnami.com', - version: 'v1alpha1', - namespace, - plural: 'sealedsecrets', - body: manifest, - }) - appliedSecrets.push({ namespace, secretName: manifest.metadata.name }) - } catch (error) { - if (error instanceof ApiException && error.code === 409) { - try { - await k8s.custom().patchNamespacedCustomObject( - { - group: 'bitnami.com', - version: 'v1alpha1', - namespace, - plural: 'sealedsecrets', - name: manifest.metadata.name, - body: manifest, - }, - setHeaderOptions('Content-Type', PatchStrategy.MergePatch), - ) - appliedSecrets.push({ namespace, secretName: manifest.metadata.name }) - } catch (patchError) { - d.error(`Failed to patch SealedSecret from ${filePath}: ${patchError}`) - } - } else { - d.error(`Failed to apply SealedSecret from ${filePath}: ${error}`) - } - } - } catch (parseError) { - d.error(`Failed to parse SealedSecret from ${filePath}: ${parseError}`) + await applySealedSecretResource(manifest) + appliedSecrets.push({ namespace, secretName: manifest.metadata.name }) + } catch (error) { + d.error(`Failed to apply SealedSecret from ${filePath}: ${error}`) } } } @@ -578,10 +552,8 @@ export const restartSealedSecretsController = async (deps = { terminal }): Promi } d.info('Waiting for sealed-secrets controller rollout') - const timeoutMs = 120000 - const intervalMs = 3000 const start = Date.now() - while (Date.now() - start < timeoutMs) { + while (Date.now() - start < ROLLOUT_TIMEOUT_MS) { try { const deployment = await k8s.app().readNamespacedDeployment({ name: 'sealed-secrets', @@ -597,7 +569,7 @@ export const restartSealedSecretsController = async (deps = { terminal }): Promi } catch { // Ignore transient read errors during rollout } - await new Promise((resolve) => setTimeout(resolve, intervalMs)) + await new Promise((resolve) => setTimeout(resolve, ROLLOUT_INTERVAL_MS)) } d.warn('Rollout status check timed out') } From ace06e202d50fd783de213623cb3ea4b7bc2183b Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 13 Mar 2026 08:13:52 +0100 Subject: [PATCH 54/66] fix: update sealed secrets handling and rename secrets --- src/cmd/commit.ts | 9 +- src/cmd/migrate.test.ts | 4 +- src/cmd/migrate.ts | 9 +- src/common/constants.ts | 4 +- src/common/git-config.test.ts | 6 +- src/common/git-config.ts | 4 +- src/common/sealed-secrets.test.ts | 29 ++++-- src/common/sealed-secrets.ts | 97 ++++--------------- src/operator/installer.test.ts | 3 +- src/operator/installer.ts | 6 +- src/operator/main.ts | 6 -- .../apl-gitea-operator-raw.gotmpl | 2 +- .../apl-harbor-operator-raw.gotmpl | 2 +- .../apl-keycloak-operator-raw.gotmpl | 2 +- values/apl-operator/apl-operator-raw.gotmpl | 2 +- values/argocd/argocd-raw.gotmpl | 6 +- values/gitea/gitea-raw.gotmpl | 4 +- values/harbor/harbor-raw.gotmpl | 6 +- values/keycloak/keycloak-raw.gotmpl | 4 +- values/loki/loki-raw.gotmpl | 2 +- values/otomi-api/otomi-api-raw.gotmpl | 2 +- .../prometheus-operator-raw.gotmpl | 2 +- 22 files changed, 77 insertions(+), 134 deletions(-) diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 0b8b5e08b3..4a4d9417a7 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -5,7 +5,7 @@ import { APL_OPERATOR_NS, DEPLOYMENT_PASSWORDS_SECRET, OTOMI_NAMESPACE, - OTOMI_PLATFORM_SECRETS, + OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE, } from 'src/common/constants' import { encrypt } from 'src/common/crypt' @@ -198,12 +198,13 @@ export async function initialSetupData(): Promise { secretName, } } else { - // External IDP: show Keycloak admin credentials (keycloak-initial-admin uses otomi-platform-secrets.adminPassword) - const otomiSecret = await getK8sSecret(OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE) + // External IDP: show Keycloak admin credentials + const adminUsername = values?.apps?.keycloak?.adminUsername || 'otomi-admin' + const otomiSecret = await getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) const adminPassword = otomiSecret?.adminPassword ? String(otomiSecret.adminPassword) : '' return { domainSuffix, - username: 'otomi-admin', + username: adminUsername, password: adminPassword, secretName, } diff --git a/src/cmd/migrate.test.ts b/src/cmd/migrate.test.ts index 26cfcf0b26..e2f9a895fb 100644 --- a/src/cmd/migrate.test.ts +++ b/src/cmd/migrate.test.ts @@ -969,7 +969,7 @@ describe('sopsMigration', () => { it('should re-apply and restart controller when manifests exist but K8s Secrets are missing', async () => { mockExistsSync.mockReturnValue(false) - mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-platform-secrets.yaml']) + mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml']) mockGetK8sSecret.mockResolvedValue(undefined) // Secret doesn't exist yet await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) @@ -981,7 +981,7 @@ describe('sopsMigration', () => { it('should skip re-apply when manifests exist and K8s Secrets already exist', async () => { mockExistsSync.mockReturnValue(false) - mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-platform-secrets.yaml']) + mockGlobSync.mockReturnValue(['/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml']) mockGetK8sSecret.mockResolvedValue({ git_password: 'somepassword' }) // Secret exists await sopsMigration({ teamConfig: {}, versions: { specVersion: 55 } }, makeDeps()) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 46e5952f33..c49b85841b 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -20,12 +20,7 @@ import { v4 as uuidv4 } from 'uuid' import { parse } from 'yaml' import { Argv } from 'yargs' import { $, cd } from 'zx' -import { - APL_OPERATOR_NS, - ARGOCD_APP_PARAMS, - OTOMI_PLATFORM_SECRETS, - SEALED_SECRETS_NAMESPACE, -} from '../common/constants' +import { APL_OPERATOR_NS, ARGOCD_APP_PARAMS, OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE } from '../common/constants' import { getK8sSecret, getSealedSecretsPEM, k8s } from '../common/k8s' import { applySealedSecretManifestsFromDir, @@ -900,7 +895,7 @@ export const sopsMigration = async ( }) if (existingManifests.length > 0) { try { - const platformSecret = await deps.getK8sSecret(OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE) + const platformSecret = await deps.getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) if (!platformSecret) { d.info('SealedSecret manifests exist but K8s Secrets are missing — re-applying and restarting controller') await deps.applySealedSecretManifestsFromDir(env.ENV_DIR) diff --git a/src/common/constants.ts b/src/common/constants.ts index bb67c8919f..b48480b27f 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -4,7 +4,7 @@ export const APL_OPERATOR_NS = 'apl-operator' export const APL_OPERATOR_STATUS_CM = 'apl-installation-status' export const OTOMI_NAMESPACE = 'otomi' export const SEALED_SECRETS_NAMESPACE = 'apl-secrets' -export const OTOMI_PLATFORM_SECRETS = 'otomi-platform-secrets' +export const OTOMI_SECRETS = 'otomi-secrets' export const ARGOCD_APP_PARAMS = { group: 'argoproj.io', version: 'v1alpha1', @@ -17,7 +17,7 @@ export const ARGOCD_APP_DEFAULT_SYNC_POLICY = { allowEmpty: false, selfHeal: true, }, - syncOptions: ['ServerSideApply=true', 'CreateNamespace=true'], + syncOptions: ['ServerSideApply=true'], } export interface ObjectMetadata { diff --git a/src/common/git-config.test.ts b/src/common/git-config.test.ts index ef0c5a2ab2..c68a6a1c37 100644 --- a/src/common/git-config.test.ts +++ b/src/common/git-config.test.ts @@ -71,7 +71,7 @@ describe('git-config', () => { it('should return undefined when password is a sealed-secret placeholder', async () => { mockGetK8sSecret.mockResolvedValue({ username: 'admin', - password: 'sealed:apl-secrets/otomi-platform-secrets/git_password', + password: 'sealed:apl-secrets/otomi-secrets/git_password', }) const result = await getGitCredentials() expect(result).toBeUndefined() @@ -362,7 +362,7 @@ describe('git-config', () => { git: { repoUrl: 'https://github.com/org/repo.git', username: 'admin', - password: 'sealed:apl-secrets/otomi-platform-secrets/git_password', + password: 'sealed:apl-secrets/otomi-secrets/git_password', branch: 'main', email: 'pipeline@cluster.local', }, @@ -370,7 +370,7 @@ describe('git-config', () => { } const result = await getRepo(values, { getK8sSecret: secretMock }) - expect(secretMock).toHaveBeenCalledWith('otomi-platform-secrets', 'apl-secrets') + expect(secretMock).toHaveBeenCalledWith('otomi-secrets', 'apl-secrets') expect(result.password).toBe('real-password') expect(result.authenticatedUrl).toContain('real-password') }) diff --git a/src/common/git-config.ts b/src/common/git-config.ts index 662df6fb0b..bf15462522 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -1,5 +1,5 @@ import type { CoreV1Api } from '@kubernetes/client-node' -import { APL_OPERATOR_NS, OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE } from './constants' +import { APL_OPERATOR_NS, OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE } from './constants' import { terminal } from './debug' import { createUpdateConfigMap, getK8sConfigMap, getK8sSecret, k8s } from './k8s' @@ -160,7 +160,7 @@ export const getRepo = async (values: Record, deps = { getK8sSecret // try reading the real password from the K8s secret (populated by ESO from SealedSecrets) if (!password || (typeof password === 'string' && password.startsWith('sealed:'))) { try { - const secret = await deps.getK8sSecret(OTOMI_PLATFORM_SECRETS, SEALED_SECRETS_NAMESPACE) + const secret = await deps.getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) if (secret?.git_password) { password = String(secret.git_password) d.debug('Read git password from K8s secret (ESO)') diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 8874db2784..cee5b3ab0e 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -11,7 +11,6 @@ import { getPemFromCertificate, restartSealedSecretsController, SealedSecretManifest, - SECRET_NAME_MAP, stripAllSecrets, writeSealedSecretManifests, } from './sealed-secrets' @@ -502,14 +501,26 @@ describe('sealed-secrets', () => { }) }) - describe('SECRET_NAME_MAP', () => { - it('should have expected secret name mappings', () => { - expect(SECRET_NAME_MAP['apps.harbor']).toBe('harbor-secrets') - expect(SECRET_NAME_MAP['apps.gitea']).toBe('gitea-secrets') - expect(SECRET_NAME_MAP['apps.keycloak']).toBe('keycloak-secrets') - expect(SECRET_NAME_MAP['otomi']).toBe('otomi-platform-secrets') - expect(SECRET_NAME_MAP['oidc']).toBe('oidc-secrets') - expect(SECRET_NAME_MAP['dns']).toBe('dns-secrets') + describe('secret name derivation', () => { + it('should derive correct secret names via buildSecretToNamespaceMap', async () => { + const secrets = { + apps: { harbor: { adminPassword: 'pass' } }, + otomi: { adminPassword: 'pass' }, + obj: { provider: { linode: { secretAccessKey: 'key' } } }, + dns: { provider: { linode: { apiToken: 'token' } } }, + } + const result = await buildSecretToNamespaceMap(secrets, [], undefined, { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue([ + 'apps.harbor.adminPassword', + 'otomi.adminPassword', + 'obj.provider.linode.secretAccessKey', + 'dns.provider.linode.apiToken', + ]), + }) + const names = result.map((m) => m.secretName).sort() + expect(names).toEqual(['dns-secrets', 'harbor-secrets', 'obj-secrets', 'otomi-secrets']) }) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index b04882892b..1f92b9e4e1 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -205,94 +205,40 @@ export const createSealedSecretsKeySecret = async ( } } -/** - * Resolve the namespace for a given secret path. - * All secrets go to 'apl-secrets' namespace for ESO ClusterSecretStore access. - */ -const resolveNamespace = (secretPath: string): string | undefined => { - // Check for teamConfig dynamic paths - if (secretPath.match(/^teamConfig\.[^.]+/)) { - return SEALED_SECRETS_NAMESPACE - } - - // Check if this path matches any known secret name prefix - const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) - for (const prefix of sortedKeys) { - if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { - return SEALED_SECRETS_NAMESPACE - } - } - - return undefined -} - -// Map specific path prefixes to secret names -export const SECRET_NAME_MAP: Record = { - 'apps.harbor': 'harbor-secrets', - 'apps.gitea': 'gitea-secrets', - 'apps.keycloak': 'keycloak-secrets', - 'apps.grafana': 'grafana-secrets', - 'apps.loki': 'loki-secrets', - 'apps.oauth2-proxy': 'oauth2-proxy-secrets', - 'apps.oauth2-proxy-redis': 'oauth2-proxy-redis-secrets', - 'apps.prometheus': 'prometheus-secrets', - 'apps.otomi-api': 'otomi-api-secrets', - 'apps.cert-manager': 'cert-manager-secrets', - 'apps.kubeflow-pipelines': 'kubeflow-pipelines-secrets', - otomi: 'otomi-platform-secrets', - oidc: 'oidc-secrets', - smtp: 'smtp-secrets', - dns: 'dns-secrets', - obj: 'obj-storage-secrets', - license: 'license-secrets', - alerts: 'alerts-secrets', - cluster: 'cluster-secrets', -} - /** * Find the group prefix for a secret path. - * Returns the prefix that maps to the secret name (e.g., 'apps.harbor' for 'apps.harbor.adminPassword'). + * Groups: teamConfig.X, apps.X, or a single top-level key (e.g., 'otomi', 'dns'). */ const findGroupPrefix = (secretPath: string): string | undefined => { const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) - if (teamMatch) { - return `teamConfig.${teamMatch[1]}` - } + if (teamMatch) return `teamConfig.${teamMatch[1]}` - const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) - for (const prefix of sortedKeys) { - if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { - return prefix - } - } + const appsMatch = secretPath.match(/^apps\.([^.]+)/) + if (appsMatch) return `apps.${appsMatch[1]}` + + // Top-level paths: use the first segment as the group prefix + const [firstSegment] = secretPath.split('.') + // Skip paths like 'kms' and 'users' which are handled separately + if (firstSegment && firstSegment !== 'kms' && firstSegment !== 'users') return firstSegment - // Fallback: use first two path segments - const parts = secretPath.split('.') - if (parts.length >= 2) { - return parts.slice(0, 2).join('.') - } return undefined } /** - * Derive a K8s secret name from the secret path prefix. + * Derive a K8s secret name from a secret path. + * Convention: all secrets follow {name}-secrets pattern. + * - teamConfig.X -> team-X-settings-secrets + * - apps.X -> X-secrets + * - topLevel -> topLevel-secrets */ const deriveSecretName = (secretPath: string): string => { const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) - if (teamMatch) { - return `team-${teamMatch[1]}-settings-secrets` - } + if (teamMatch) return `team-${teamMatch[1]}-settings-secrets` - const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) - for (const prefix of sortedKeys) { - if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { - return SECRET_NAME_MAP[prefix] - } - } + const appsMatch = secretPath.match(/^apps\.([^.]+)/) + if (appsMatch) return `${appsMatch[1]}-secrets` - // Fallback: derive from first two path segments - const parts = secretPath.split('.') - return `${parts.slice(0, 2).join('-')}-secrets` + return `${secretPath.split('.')[0]}-secrets` } /** @@ -317,14 +263,13 @@ export const buildSecretToNamespaceMap = async ( // Skip users path — user secrets are managed individually in apl-users namespace if (secretPath === 'users') continue - const namespace = resolveNamespace(secretPath) - if (!namespace) continue + if (!findGroupPrefix(secretPath)) continue const secretName = deriveSecretName(secretPath) - const groupKey = `${namespace}/${secretName}` + const groupKey = `${SEALED_SECRETS_NAMESPACE}/${secretName}` if (!groupMap.has(groupKey)) { - groupMap.set(groupKey, { namespace, secretName, data: {} }) + groupMap.set(groupKey, { namespace: SEALED_SECRETS_NAMESPACE, secretName, data: {} }) } const mapping = groupMap.get(groupKey)! diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index e12f58e6d6..400d069f62 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -169,7 +169,7 @@ describe('Installer', () => { }), ) - // Verify failed status was recorded with error message + // Verify failed status was recorded expect(k8s.createUpdateConfigMap).toHaveBeenCalledWith( mockCoreApi, 'apl-installation-status', @@ -177,7 +177,6 @@ describe('Installer', () => { expect.objectContaining({ status: 'failed', attempt: '1', - error: 'Install failed', }), ) diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 03835a3faf..8deb485858 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -170,7 +170,7 @@ export class Installer { } catch (error) { const errorMessage = getErrorMessage(error) this.d.error(`Installation attempt ${attemptNumber} failed:`, errorMessage) - await this.updateInstallationStatus('failed', attemptNumber, errorMessage) + await this.updateInstallationStatus('failed', attemptNumber) // Clean up stuck Helm releases (e.g. pending-install, pending-upgrade) // so the next retry can proceed without "another operation is in progress" errors @@ -194,14 +194,12 @@ export class Installer { return status } - private async updateInstallationStatus(status: string, attempt: number, error?: string): Promise { + private async updateInstallationStatus(status: string, attempt: number): Promise { try { const data = { status, attempt: attempt.toString(), timestamp: new Date().toISOString(), - // Always include error field to prevent stale values from StrategicMergePatch - error: error ?? '', } await createUpdateConfigMap(k8s.core(), APL_OPERATOR_STATUS_CM, APL_OPERATOR_NS, data) diff --git a/src/operator/main.ts b/src/operator/main.ts index 38d45eb407..26c50691e4 100644 --- a/src/operator/main.ts +++ b/src/operator/main.ts @@ -2,7 +2,6 @@ import * as dotenv from 'dotenv' import fs from 'fs' import process from 'node:process' import path from 'path' -import { runTraceCollectionLoop } from '../cmd/traces' import { terminal } from '../common/debug' import { env } from '../common/envalid' import { getStoredGitRepoConfig } from '../common/git-config' @@ -93,11 +92,6 @@ async function main(): Promise { // Set up SOPS environment if applicable (no-op when SealedSecrets + ESO is in use) await installer.setEnvAndCreateSecrets() - // Start trace collection in background (runs for 30 minutes from ConfigMap creation) - runTraceCollectionLoop().catch((error) => { - d.warn('Trace collection loop failed:', getErrorMessage(error)) - }) - // Phase 2: Set environment variables and start operator for GitOps operations const config = await loadConfig(aplOps) const operator = new AplOperator(config) diff --git a/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl b/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl index a89537c3ca..d6b6c4c167 100644 --- a/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl +++ b/values/apl-gitea-operator/apl-gitea-operator-raw.gotmpl @@ -42,7 +42,7 @@ resources: data: - secretKey: gitPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password - secretKey: keycloakClientSecret remoteRef: diff --git a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl index 0eaf85f5a3..265e5058e0 100644 --- a/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl +++ b/values/apl-harbor-operator/apl-harbor-operator-raw.gotmpl @@ -35,7 +35,7 @@ resources: data: - secretKey: harborAdminPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: adminPassword - secretKey: keycloakClientSecret remoteRef: diff --git a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl index 43763a08fa..b07da67e9b 100644 --- a/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl +++ b/values/apl-keycloak-operator/apl-keycloak-operator-raw.gotmpl @@ -48,7 +48,7 @@ resources: data: - secretKey: adminPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: adminPassword - secretKey: idpClientSecret remoteRef: diff --git a/values/apl-operator/apl-operator-raw.gotmpl b/values/apl-operator/apl-operator-raw.gotmpl index 7d1853716a..41228f7257 100644 --- a/values/apl-operator/apl-operator-raw.gotmpl +++ b/values/apl-operator/apl-operator-raw.gotmpl @@ -21,5 +21,5 @@ resources: data: - secretKey: git_password remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password diff --git a/values/argocd/argocd-raw.gotmpl b/values/argocd/argocd-raw.gotmpl index 8c9966a6f9..d400ea4a16 100644 --- a/values/argocd/argocd-raw.gotmpl +++ b/values/argocd/argocd-raw.gotmpl @@ -36,7 +36,7 @@ resources: data: - secretKey: gitPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret @@ -64,7 +64,7 @@ resources: data: - secretKey: gitPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password {{- else }} - apiVersion: external-secrets.io/v1beta1 @@ -93,7 +93,7 @@ resources: data: - secretKey: gitPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password {{- end }} - apiVersion: external-secrets.io/v1beta1 diff --git a/values/gitea/gitea-raw.gotmpl b/values/gitea/gitea-raw.gotmpl index 2098889f8d..d7d84c04a4 100644 --- a/values/gitea/gitea-raw.gotmpl +++ b/values/gitea/gitea-raw.gotmpl @@ -33,7 +33,7 @@ resources: data: - secretKey: git_password remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password {{- if $v._derived.untrustedCA }} - apiVersion: v1 @@ -65,7 +65,7 @@ resources: data: - secretKey: secretAccessKey remoteRef: - key: obj-storage-secrets + key: obj-secrets property: provider_linode_secretAccessKey {{- end }} {{- with $v | get "smtp" nil }} diff --git a/values/harbor/harbor-raw.gotmpl b/values/harbor/harbor-raw.gotmpl index fde1bcc6c9..4f1f61ffe6 100644 --- a/values/harbor/harbor-raw.gotmpl +++ b/values/harbor/harbor-raw.gotmpl @@ -52,7 +52,7 @@ resources: data: - secretKey: adminPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: adminPassword - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret @@ -206,7 +206,7 @@ resources: data: - secretKey: secretAccessKey remoteRef: - key: obj-storage-secrets + key: obj-secrets property: provider_linode_secretAccessKey - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret @@ -228,6 +228,6 @@ resources: data: - secretKey: secretAccessKey remoteRef: - key: obj-storage-secrets + key: obj-secrets property: provider_linode_secretAccessKey {{- end }} diff --git a/values/keycloak/keycloak-raw.gotmpl b/values/keycloak/keycloak-raw.gotmpl index dcf7214cc4..ef85209a5b 100644 --- a/values/keycloak/keycloak-raw.gotmpl +++ b/values/keycloak/keycloak-raw.gotmpl @@ -30,7 +30,7 @@ resources: data: - secretKey: adminPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: adminPassword {{- if eq $obj.type "linode" }} - apiVersion: external-secrets.io/v1beta1 @@ -53,6 +53,6 @@ resources: data: - secretKey: secretAccessKey remoteRef: - key: obj-storage-secrets + key: obj-secrets property: provider_linode_secretAccessKey {{- end }} diff --git a/values/loki/loki-raw.gotmpl b/values/loki/loki-raw.gotmpl index 3ca9d69809..bbebacc435 100644 --- a/values/loki/loki-raw.gotmpl +++ b/values/loki/loki-raw.gotmpl @@ -66,7 +66,7 @@ resources: data: - secretKey: secretAccessKey remoteRef: - key: obj-storage-secrets + key: obj-secrets property: provider_linode_secretAccessKey {{- end }} {{- end }} diff --git a/values/otomi-api/otomi-api-raw.gotmpl b/values/otomi-api/otomi-api-raw.gotmpl index 51e4f6023b..a3169981dc 100644 --- a/values/otomi-api/otomi-api-raw.gotmpl +++ b/values/otomi-api/otomi-api-raw.gotmpl @@ -20,5 +20,5 @@ resources: data: - secretKey: gitPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: git_password diff --git a/values/prometheus-operator/prometheus-operator-raw.gotmpl b/values/prometheus-operator/prometheus-operator-raw.gotmpl index b6ef5709b9..0805b2c827 100644 --- a/values/prometheus-operator/prometheus-operator-raw.gotmpl +++ b/values/prometheus-operator/prometheus-operator-raw.gotmpl @@ -29,7 +29,7 @@ resources: data: - secretKey: adminPassword remoteRef: - key: otomi-platform-secrets + key: otomi-secrets property: adminPassword - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret From c7dc6e3e2733790ec7cfb38cc5167695d2f26ede Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 13 Mar 2026 14:36:22 +0100 Subject: [PATCH 55/66] feat: remove deprecated secret files from tests/fixtures and replace them with SealedSecrets --- src/common/repo.test.ts | 68 +-------- src/common/repo.ts | 136 +++++++++++++----- tests/fixtures/env/apps/cert-manager.yaml | 22 +++ .../env/apps/secrets.cert-manager.yaml | 55 ------- tests/fixtures/env/apps/secrets.gitea.yaml | 6 - tests/fixtures/env/apps/secrets.grafana.yaml | 6 - tests/fixtures/env/apps/secrets.harbor.yaml | 18 --- tests/fixtures/env/apps/secrets.keycloak.yaml | 7 - .../env/apps/secrets.kubeflow-pipelines.yaml | 6 - tests/fixtures/env/apps/secrets.loki.yaml | 6 - .../env/apps/secrets.oauth2-proxy-redis.yaml | 6 - .../env/apps/secrets.oauth2-proxy.yaml | 7 - .../fixtures/env/apps/secrets.prometheus.yaml | 9 -- .../sealedsecrets/alerts-secrets.yaml | 18 +++ .../sealedsecrets/cert-manager-secrets.yaml | 43 ++++++ .../sealedsecrets/dns-secrets.yaml | 16 +++ .../sealedsecrets/gitea-secrets.yaml | 16 +++ .../sealedsecrets/grafana-secrets.yaml | 16 +++ .../sealedsecrets/harbor-secrets.yaml | 24 ++++ .../sealedsecrets/keycloak-secrets.yaml | 16 +++ .../kubeflow-pipelines-secrets.yaml | 16 +++ .../sealedsecrets/loki-secrets.yaml | 16 +++ .../oauth2-proxy-redis-secrets.yaml | 16 +++ .../sealedsecrets/oauth2-proxy-secrets.yaml | 16 +++ .../sealedsecrets/obj-secrets.yaml | 16 +++ .../sealedsecrets/oidc-secrets.yaml | 16 +++ .../sealedsecrets/otomi-secrets.yaml | 18 +++ .../sealedsecrets/prometheus-secrets.yaml | 16 +++ .../sealedsecrets/smtp-secrets.yaml | 16 +++ .../team-admin-settings-secrets.yaml | 16 +++ .../team-demo-settings-secrets.yaml | 16 +++ .../team-dev-settings-secrets.yaml | 16 +++ tests/fixtures/env/settings/kms.yaml | 1 + .../fixtures/env/settings/secrets.alerts.yaml | 10 -- tests/fixtures/env/settings/secrets.dns.yaml | 8 -- tests/fixtures/env/settings/secrets.kms.yaml | 8 -- tests/fixtures/env/settings/secrets.obj.yaml | 8 -- tests/fixtures/env/settings/secrets.oidc.yaml | 6 - .../fixtures/env/settings/secrets.otomi.yaml | 10 -- tests/fixtures/env/settings/secrets.smtp.yaml | 6 - .../env/teams/admin/secrets.settings.yaml | 6 - .../env/teams/demo/secrets.settings.yaml | 9 -- .../env/teams/dev/secrets.settings.yaml | 6 - 43 files changed, 471 insertions(+), 302 deletions(-) delete mode 100644 tests/fixtures/env/apps/secrets.cert-manager.yaml delete mode 100644 tests/fixtures/env/apps/secrets.gitea.yaml delete mode 100644 tests/fixtures/env/apps/secrets.grafana.yaml delete mode 100644 tests/fixtures/env/apps/secrets.harbor.yaml delete mode 100644 tests/fixtures/env/apps/secrets.keycloak.yaml delete mode 100644 tests/fixtures/env/apps/secrets.kubeflow-pipelines.yaml delete mode 100644 tests/fixtures/env/apps/secrets.loki.yaml delete mode 100644 tests/fixtures/env/apps/secrets.oauth2-proxy-redis.yaml delete mode 100644 tests/fixtures/env/apps/secrets.oauth2-proxy.yaml delete mode 100644 tests/fixtures/env/apps/secrets.prometheus.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/alerts-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/cert-manager-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/dns-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/gitea-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/grafana-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/keycloak-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/kubeflow-pipelines-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/loki-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-redis-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/obj-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oidc-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/prometheus-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/smtp-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-admin-settings-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-demo-settings-secrets.yaml create mode 100644 tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-dev-settings-secrets.yaml delete mode 100644 tests/fixtures/env/settings/secrets.alerts.yaml delete mode 100644 tests/fixtures/env/settings/secrets.dns.yaml delete mode 100644 tests/fixtures/env/settings/secrets.kms.yaml delete mode 100644 tests/fixtures/env/settings/secrets.obj.yaml delete mode 100644 tests/fixtures/env/settings/secrets.oidc.yaml delete mode 100644 tests/fixtures/env/settings/secrets.otomi.yaml delete mode 100644 tests/fixtures/env/settings/secrets.smtp.yaml delete mode 100644 tests/fixtures/env/teams/admin/secrets.settings.yaml delete mode 100644 tests/fixtures/env/teams/demo/secrets.settings.yaml delete mode 100644 tests/fixtures/env/teams/dev/secrets.settings.yaml diff --git a/src/common/repo.test.ts b/src/common/repo.test.ts index de6c443315..5c950477b9 100644 --- a/src/common/repo.test.ts +++ b/src/common/repo.test.ts @@ -10,7 +10,6 @@ import { getUniqueIdentifierFromFilePath, hasCorrespondingDecryptedFile, renderManifest, - renderManifestForSecrets, saveResourceGroupToFiles, sortTeamConfigArraysByName, sortUserArraysByName, @@ -47,7 +46,6 @@ describe('getFilePath', () => { const data = {} const jsonPath = ['$', 'apps', 'grafana'] expect(getFilePath(fileMap, jsonPath, data, '')).toEqual('/tmp/values/env/apps/grafana.yaml') - expect(getFilePath(fileMap, jsonPath, data, 'secrets.')).toEqual('/tmp/values/env/apps/secrets.grafana.yaml') }) it('should get path for teamA', () => { const fileMap: FileMap = { @@ -234,11 +232,8 @@ describe('getFilePath', () => { } const jsonPath = ['$', 'teamConfig', 'demo', 'netpols', '[1]'] const data = { name: 'a' } - let filePath = getFilePath(fileMap, jsonPath, data, '') + const filePath = getFilePath(fileMap, jsonPath, data, '') expect(filePath).toBe('/tmp/env/teams/demo/netpols/a.yaml') - - filePath = getFilePath(fileMap, jsonPath, data, 'secrets.') - expect(filePath).toBe('/tmp/env/teams/demo/netpols/secrets.a.yaml') }) it('should return file path for platfrom dns', () => { @@ -254,11 +249,8 @@ describe('getFilePath', () => { } const jsonPath = ['$', 'dns'] const data = { name: 'a' } - let filePath = getFilePath(fileMap, jsonPath, data, '') + const filePath = getFilePath(fileMap, jsonPath, data, '') expect(filePath).toBe('/tmp/env/settings/dns.yaml') - - filePath = getFilePath(fileMap, jsonPath, data, 'secrets.') - expect(filePath).toBe('/tmp/env/settings/secrets.dns.yaml') }) it('should return file path for user', () => { const fileMap: FileMap = { @@ -273,11 +265,8 @@ describe('getFilePath', () => { } const jsonPath = ['$', 'dns'] const data = { id: 'a' } - let filePath = getFilePath(fileMap, jsonPath, data, '') + const filePath = getFilePath(fileMap, jsonPath, data, '') expect(filePath).toBe('/tmp/env/users/a.yaml') - - filePath = getFilePath(fileMap, jsonPath, data, 'secrets.') - expect(filePath).toBe('/tmp/env/users/secrets.a.yaml') }) }) @@ -692,13 +681,6 @@ describe('AplCatalog', () => { expect(filePath).toBe('/tmp/values/env/catalogs/default.yaml') }) - it('should return the correct secrets file path for a catalog', () => { - const data = { name: 'default', repositoryUrl: 'https://example.com/charts.git' } - const jsonPath = ['$', 'catalogs', 'default'] - const filePath = getFilePath(catalogFileMap, jsonPath, data, 'secrets.') - expect(filePath).toBe('/tmp/values/env/catalogs/secrets.default.yaml') - }) - it('should use the map key for the file name, not data.name', () => { const data = { name: 'production-charts', repositoryUrl: 'https://example.com/charts.git' } const jsonPath = ['$', 'catalogs', 'prod'] @@ -779,17 +761,6 @@ describe('AplCatalog', () => { }) }) - describe('renderManifestForSecrets', () => { - it('should render a secrets manifest for a catalog', () => { - const data = { secretName: 'git-credentials' } - const manifest = renderManifestForSecrets(catalogFileMap, 'default', data) - - expect(manifest.kind).toBe('AplCatalog') - expect(manifest.metadata.name).toBe('default') - expect(manifest.spec).toEqual(data) - }) - }) - describe('saveResourceGroupToFiles', () => { it('should save a single catalog to a file', async () => { const writeValuesToFile = jest.fn() @@ -856,39 +827,6 @@ describe('AplCatalog', () => { ) }) - it('should save catalog secrets to files with secrets prefix', async () => { - const writeValuesToFile = jest.fn() - const valuesPublic = { - catalogs: { - default: { - name: 'default', - repositoryUrl: 'https://github.com/linode/apl-charts.git', - branch: 'main', - enabled: true, - }, - }, - } - const valuesSecrets = { - catalogs: { - default: { - secretName: 'git-credentials', - }, - }, - } - - await saveResourceGroupToFiles(catalogFileMap, valuesPublic, valuesSecrets, { writeValuesToFile }) - - expect(writeValuesToFile).toHaveBeenCalledTimes(2) - expect(writeValuesToFile).toHaveBeenCalledWith( - '/tmp/values/env/catalogs/secrets.default.yaml', - expect.objectContaining({ - kind: 'AplCatalog', - metadata: { name: 'default' }, - spec: valuesSecrets.catalogs.default, - }), - ) - }) - it('should not write anything when there are no catalogs', async () => { const writeValuesToFile = jest.fn() diff --git a/src/common/repo.ts b/src/common/repo.ts index 6257c89f60..5f7177b514 100644 --- a/src/common/repo.ts +++ b/src/common/repo.ts @@ -4,7 +4,7 @@ import { globSync } from 'glob' import jsonpath from 'jsonpath' import { cloneDeep, get, merge, omit, set } from 'lodash' import path from 'path' -import { getDirNames, loadYaml } from './utils' +import { getDirNames, getSchemaSecretsPaths, loadYaml } from './utils' import { objectToYaml, writeValuesToFile } from './values' export async function getTeamNames(envDir: string): Promise> { @@ -462,28 +462,13 @@ export function renderManifest( return manifest } -export function renderManifestForSecrets(fileMap: FileMap, resourceName: string, data: Record) { - let spec = data - if (fileMap.resourceGroup === 'users') { - spec = omit(data, ['id', 'name']) - } - return { - kind: fileMap.kind, - metadata: { - name: resourceName, - }, - spec, - } -} - export async function saveResourceGroupToFiles( fileMap: FileMap, valuesPublic: Record, - valuesSecrets: Record, + _valuesSecrets: Record, deps = { writeValuesToFile }, ): Promise { const jsonPathsValuesPublic = jsonpath.nodes(valuesPublic, fileMap.jsonPathExpression) - const jsonPathsvaluesSecrets = jsonpath.nodes(valuesSecrets, fileMap.jsonPathExpression) await Promise.all( jsonPathsValuesPublic.map(async (node) => { @@ -506,22 +491,8 @@ export async function saveResourceGroupToFiles( }), ) - await Promise.all( - jsonPathsvaluesSecrets.map(async (node) => { - const nodePath = node.path - const nodeValue = node.value - try { - const filePath = getFilePath(fileMap, nodePath, nodeValue, 'secrets.') - const resourceName = getResourceName(fileMap, nodePath, nodeValue) - const manifest = renderManifestForSecrets(fileMap, resourceName, nodeValue) - await deps.writeValuesToFile(filePath, manifest) - } catch (e) { - console.log(nodePath) - console.log(fileMap) - throw e - } - }), - ) + // Secrets are now stored as SealedSecret manifests via buildSecretToNamespaceMap() + writeSealedSecretManifests() + // No longer writing secrets.*.yaml files } export function getUniqueIdentifierFromFilePath(filePath: string): string { @@ -588,7 +559,10 @@ export function unsetValuesFileSync(envDir: string): string { return valuesPath } -export async function loadValues(envDir: string, deps = { loadToSpec }): Promise> { +export async function loadValues( + envDir: string, + deps = { loadToSpec, loadSealedSecretsToSpec }, +): Promise> { const fileMaps = getFileMaps(envDir).filter((map) => map.loadToSpec === true) const spec = {} @@ -597,11 +571,105 @@ export async function loadValues(envDir: string, deps = { loadToSpec }): Promise await deps.loadToSpec(spec, fileMap) }), ) + await deps.loadSealedSecretsToSpec(spec, envDir) sortTeamConfigArraysByName(spec) sortUserArraysByName(spec) return spec } +/** + * Read sealed secret manifests and merge their encryptedData back into the values spec. + * This restores secret values that helmfile templates need at render time. + * + * Uses the values schema (x-secret paths) to correctly map sealed secret data keys + * back to their original dot-paths, since some property names contain underscores + * (e.g., smtp.auth_password) that should NOT be converted to nested paths. + */ +export async function loadSealedSecretsToSpec( + spec: Record, + envDir: string, + deps = { loadYaml, getSchemaSecretsPaths }, +): Promise { + const sealedSecretsGlob = `${envDir}/env/manifests/namespaces/*/sealedsecrets/*.yaml` + const files = globSync(sealedSecretsGlob, { nodir: true }) + if (files.length === 0) return + + // Get team names from spec to expand teamConfig.* paths + const teams = Object.keys(get(spec, 'teamConfig', {})) + const secretPaths = await deps.getSchemaSecretsPaths(teams) + + // Build a lookup: for each group prefix + dataKey → full schema path + // e.g., "apps.harbor" + "core_secret" → "apps.harbor.core.secret" + const dataKeyToPath = buildDataKeyToPathMap(secretPaths) + + for (const filePath of files) { + const manifest = (await deps.loadYaml(filePath)) as Record | undefined + if (!manifest?.spec?.encryptedData) continue + + const { name: secretName = '' } = (manifest.metadata ?? {}) as { name?: string } + const { encryptedData } = manifest.spec as { encryptedData: Record } + + // Determine target path in spec from the secret name + const targetPath = resolveSecretTargetPath(secretName, spec) + if (!targetPath) continue + + for (const [dataKey, value] of Object.entries(encryptedData)) { + const lookupKey = `${targetPath}/${dataKey}` + const fullPath = dataKeyToPath.get(lookupKey) + if (fullPath) { + set(spec, fullPath, value) + } + } + } +} + +/** + * Build a map from "groupPrefix/dataKey" → "full.schema.path". + * The dataKey is derived from the relative path by replacing dots with underscores, + * matching the convention used in buildSecretToNamespaceMap (sealed-secrets.ts). + */ +function buildDataKeyToPathMap(secretPaths: string[]): Map { + const result = new Map() + for (const secretPath of secretPaths) { + const groupPrefix = findGroupPrefix(secretPath) + if (!groupPrefix) continue + const relativePath = secretPath.slice(groupPrefix.length + 1) + if (!relativePath) continue + const dataKey = relativePath.replace(/\./g, '_') + result.set(`${groupPrefix}/${dataKey}`, secretPath) + } + return result +} + +/** + * Find the group prefix for a secret path — mirrors the logic in sealed-secrets.ts. + */ +function findGroupPrefix(secretPath: string): string | undefined { + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) return `teamConfig.${teamMatch[1]}` + const appsMatch = secretPath.match(/^apps\.([^.]+)/) + if (appsMatch) return `apps.${appsMatch[1]}` + const [firstSegment] = secretPath.split('.') + if (firstSegment && firstSegment !== 'kms' && firstSegment !== 'users') return firstSegment + return undefined +} + +function resolveSecretTargetPath(secretName: string, spec: Record): string | undefined { + // team-{name}-settings-secrets → teamConfig.{name} + const teamMatch = secretName.match(/^team-(.+)-settings-secrets$/) + if (teamMatch) return `teamConfig.${teamMatch[1]}` + + // {name}-secrets → apps.{name} or {name} + const nameMatch = secretName.match(/^(.+)-secrets$/) + if (!nameMatch) return undefined + + const [, name] = nameMatch + if (get(spec, `apps.${name}`) !== undefined) return `apps.${name}` + if (get(spec, name) !== undefined) return name + + return undefined +} + export function extractTeamDirectory(filePath: string): string { const match = filePath.match(/\/teams\/([^/]+)/) if (match === null) throw new Error(`Cannot extract team name from ${filePath} string`) diff --git a/tests/fixtures/env/apps/cert-manager.yaml b/tests/fixtures/env/apps/cert-manager.yaml index be2c7211a3..d484facf18 100644 --- a/tests/fixtures/env/apps/cert-manager.yaml +++ b/tests/fixtures/env/apps/cert-manager.yaml @@ -3,6 +3,28 @@ metadata: name: cert-manager labels: {} spec: + customRootCA: | + -----BEGIN CERTIFICATE----- + MIIDdDCCAlygAwIBAgIBATANBgkqhkiG9w0BAQUFADBuMRUwEwYDVQQDEwxyZWRr + dWJlcy5jb20xCzAJBgNVBAYTAk5MMRAwDgYDVQQIEwdVdHJlY2h0MRAwDgYDVQQH + EwdVdHJlY2h0MQ4wDAYDVQQKEwVPdG9taTEUMBIGA1UECxMLU2VsZi1TaWduZWQw + HhcNMjExMTAzMTAxOTAyWhcNMzExMTAzMTAxOTAyWjBuMRUwEwYDVQQDEwxyZWRr + dWJlcy5jb20xCzAJBgNVBAYTAk5MMRAwDgYDVQQIEwdVdHJlY2h0MRAwDgYDVQQH + EwdVdHJlY2h0MQ4wDAYDVQQKEwVPdG9taTEUMBIGA1UECxMLU2VsZi1TaWduZWQw + ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQD4quPwHrharZhmqVQx/75N + M7Vp3ZmSd3gR2u8Dc1PkmEa6W9CiheVAB5KCzdN5sWaOlFKTy5sHg/zvyYZjvNGX + xaHCa4i6OyRgiTOC4NCrxuN5010G0vAxYaM1aIFcqObXuLcaK6miOybDLRfDxoHl + g/TKqdiPOMEb2ZgphFxL7oYXjkobOggH+wzwwMIc/1nA3eBjEPsIkQehmb0R0Kxw + K5VHPCvbPQb3USVqUs+NmsuCxmqkTtI32WqR0IuNAVqjaD9oNqcsKBgUOPYLYXM8 + xsTzIn0QPysJIKUCRn1quHwvCQc1RnQBB8UG6iJboVdRe0GNS13vu5ikhoCb0oyV + AgMBAAGjHTAbMAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgL0MA0GCSqGSIb3DQEB + BQUAA4IBAQBJWHPGnTqXME/MGwG2nAG/JqiCQ0ZOOyKgwN97wrQIlbra2BaUT1K4 + tMDOjZlft1Luipg/IkzzMXt4eAmqGMxLIweqbve6aLm8KTpHkLdxLm3VPnhK8zzg + ysRRRjtkMo9KUOSvrS2dFsY+fQnbGUzpRcK8RrzM6CpgIaf29neP1xLUWQuUsy5y + yKCb6OQ9vaJBf/uvz73rQq0ym4Kx0FCFssshaja6lbz/jqCJmppdZE5pe5jvMVVv + ae5UQLbva0JyLY8Rc1vSY/epIHMLrV90GEagSkF/ejgF3uh8cliLuUYFAFyU8TnN + FWG+enMJfR04aWjp8M3MQ1IoCPVxoXxI + -----END CERTIFICATE----- externallyManagedTlsSecretName: mysecret issuer: externally-managed-tls-secret _rawValues: {} diff --git a/tests/fixtures/env/apps/secrets.cert-manager.yaml b/tests/fixtures/env/apps/secrets.cert-manager.yaml deleted file mode 100644 index 671916d60e..0000000000 --- a/tests/fixtures/env/apps/secrets.cert-manager.yaml +++ /dev/null @@ -1,55 +0,0 @@ -kind: AplApp -spec: - customRootCA: | - -----BEGIN CERTIFICATE----- - MIIDdDCCAlygAwIBAgIBATANBgkqhkiG9w0BAQUFADBuMRUwEwYDVQQDEwxyZWRr - dWJlcy5jb20xCzAJBgNVBAYTAk5MMRAwDgYDVQQIEwdVdHJlY2h0MRAwDgYDVQQH - EwdVdHJlY2h0MQ4wDAYDVQQKEwVPdG9taTEUMBIGA1UECxMLU2VsZi1TaWduZWQw - HhcNMjExMTAzMTAxOTAyWhcNMzExMTAzMTAxOTAyWjBuMRUwEwYDVQQDEwxyZWRr - dWJlcy5jb20xCzAJBgNVBAYTAk5MMRAwDgYDVQQIEwdVdHJlY2h0MRAwDgYDVQQH - EwdVdHJlY2h0MQ4wDAYDVQQKEwVPdG9taTEUMBIGA1UECxMLU2VsZi1TaWduZWQw - ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQD4quPwHrharZhmqVQx/75N - M7Vp3ZmSd3gR2u8Dc1PkmEa6W9CiheVAB5KCzdN5sWaOlFKTy5sHg/zvyYZjvNGX - xaHCa4i6OyRgiTOC4NCrxuN5010G0vAxYaM1aIFcqObXuLcaK6miOybDLRfDxoHl - g/TKqdiPOMEb2ZgphFxL7oYXjkobOggH+wzwwMIc/1nA3eBjEPsIkQehmb0R0Kxw - K5VHPCvbPQb3USVqUs+NmsuCxmqkTtI32WqR0IuNAVqjaD9oNqcsKBgUOPYLYXM8 - xsTzIn0QPysJIKUCRn1quHwvCQc1RnQBB8UG6iJboVdRe0GNS13vu5ikhoCb0oyV - AgMBAAGjHTAbMAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgL0MA0GCSqGSIb3DQEB - BQUAA4IBAQBJWHPGnTqXME/MGwG2nAG/JqiCQ0ZOOyKgwN97wrQIlbra2BaUT1K4 - tMDOjZlft1Luipg/IkzzMXt4eAmqGMxLIweqbve6aLm8KTpHkLdxLm3VPnhK8zzg - ysRRRjtkMo9KUOSvrS2dFsY+fQnbGUzpRcK8RrzM6CpgIaf29neP1xLUWQuUsy5y - yKCb6OQ9vaJBf/uvz73rQq0ym4Kx0FCFssshaja6lbz/jqCJmppdZE5pe5jvMVVv - ae5UQLbva0JyLY8Rc1vSY/epIHMLrV90GEagSkF/ejgF3uh8cliLuUYFAFyU8TnN - FWG+enMJfR04aWjp8M3MQ1IoCPVxoXxI - -----END CERTIFICATE----- - customRootCAKey: | - -----BEGIN RSA PRIVATE KEY----- - MIIEpQIBAAKCAQEA+Krj8B64Wq2YZqlUMf++TTO1ad2Zknd4EdrvA3NT5JhGulvQ - ooXlQAeSgs3TebFmjpRSk8ubB4P878mGY7zRl8WhwmuIujskYIkzguDQq8bjedNd - BtLwMWGjNWiBXKjm17i3Giupojsmwy0Xw8aB5YP0yqnYjzjBG9mYKYRcS+6GF45K - GzoIB/sM8MDCHP9ZwN3gYxD7CJEHoZm9EdCscCuVRzwr2z0G91ElalLPjZrLgsZq - pE7SN9lqkdCLjQFao2g/aDanLCgYFDj2C2FzPMbE8yJ9ED8rCSClAkZ9arh8LwkH - NUZ0AQfFBuoiW6FXUXtBjUtd77uYpIaAm9KMlQIDAQABAoIBAQCTsIuotdYwpSH6 - 9172Qzq3h5qbwe3QO/yoPivvFLQi9P4s+RM1M+kw2k5+Odj8UgzjadyRwz/UeuPj - VwHmguLJDaxBWLTgRvgYDeT2Oqg1He9FD/AUeXwHGEJjGiqa6gYQ4bh+Zqhdnlwr - V8DhmijUNEdThwUEK2UmMVpabi6TOW/dfO6sbnOHYwx326qF3LhYcUrmdeowEGLT - UhxdXJQQUsfD+zft6dcPnqucIxd5OEsn3L8/pcumOxHUGFBHDMuB5nTpcZyDxKaN - joC0zQy0BDQIMN1F7wNRukSiSYqvRvmvztF2Ka0yEWAdvBBXVVN8nKQw69oGoe9B - EQ5HSkKBAoGBAP0KnCE19jW9jq1CWFkFwd1BALWA3GOuxJqSX6M5APM+JRBR4UOZ - AUOogvGlrc1ns58q7oNoc1CHiMHd2lNFgfqWqopfVz1Tt9qHqU6VoJnkmJlJRriE - 57F08RTjslTFzYEsE9zMlL1xa5pq1aGAFB0/mYuxopRw39mS14ugxF1pAoGBAPuT - MFLfp2wttGe2WhVepOnhD13sEMGCS6GE16jinjP3qAWIPM/Wdy+Ab8n+KWQkYPLw - UsQVi+41LWFIexjzdrq9rG5LQbZdjDCyR4eomDGZhc0Vtsu79NbVrnSmjH8w6psa - DXB1uN9/VcCzae/hRpk2Q6zFiMcPE8utUU5RvFRNAoGBAJ19+wsYoPN11dW0k3Rl - BvKEwMI3P/SzFB74t5nJovPCXCM6MzB1jLnlqgppCjHsN3n7qJQVcKBQmye+w2JM - wseK+v5AtPWwo5/aC+CjdGAUTX4qg1/ZKLPkiyBrT9U/f9bD7mDg3DrE2yozEGAC - bYJ+0TyHBR/K2Sh8Irf/CfjxAoGBAM4wuwCRkpUVeLEwQfEV2zBdZ8zg+HLBqd8+ - E8u1wVhyeOHf4YevDYx/RiBWEfKj5ln3Ir7XshKQvxrm3w16Liur3bGgOMGRNp+K - 3xmO0v6EB6gpTeL5sBiMlinBf5GXtBFfbvhnZBi6Mrx30DHtf4F/ekQWup37+4uK - CAOa9jJZAoGAYbU4CoCxktBECxAVAjtpvYW5176cxiitd75s1ANhXGiOH1A6/y6H - rnZ+fMAuvPjrDXbtmqJsq0RXq1E07ng4ZDIjN+0pShVFQdakJRFo1y+d3b82lBYX - EZrfMBCWVj31dXeGEHfVvOpwrQ5ffTzs2lVmTh7Ft61gs4TJ7gNTDbE= - -----END RSA PRIVATE KEY----- -name: cert-manager -metadata: - name: cert-manager diff --git a/tests/fixtures/env/apps/secrets.gitea.yaml b/tests/fixtures/env/apps/secrets.gitea.yaml deleted file mode 100644 index 25e72d933c..0000000000 --- a/tests/fixtures/env/apps/secrets.gitea.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - postgresqlPassword: postgresqlPassword -name: gitea -metadata: - name: gitea diff --git a/tests/fixtures/env/apps/secrets.grafana.yaml b/tests/fixtures/env/apps/secrets.grafana.yaml deleted file mode 100644 index 984e5171de..0000000000 --- a/tests/fixtures/env/apps/secrets.grafana.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - adminPassword: somesecretvalue -name: grafana -metadata: - name: grafana diff --git a/tests/fixtures/env/apps/secrets.harbor.yaml b/tests/fixtures/env/apps/secrets.harbor.yaml deleted file mode 100644 index 84dcfe2783..0000000000 --- a/tests/fixtures/env/apps/secrets.harbor.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kind: AplApp -spec: - adminPassword: harborsomesecretvalue - core: - secret: vQFMm9Qk0pTUF3MK - xsrfKey: txS2sHQGqiJmmhFf15oCUfF5BgbchIsi - jobservice: - secret: CfpanIkcGWz3wGLO - registry: - secret: PjHGEdmPhrmNrEkj - credentials: - htpasswd: admin:$2a$10$zXwH8y4snDAtV4mZmgyylOqfX2AOPNaUL5e6yPm2EqPyy2G2OQsX6 - username: admin - password: TJmTw62K9y4fZ83wgt0xmXzLwxpTHoJ4 - secretKey: somesecretvalue -name: harbor -metadata: - name: harbor diff --git a/tests/fixtures/env/apps/secrets.keycloak.yaml b/tests/fixtures/env/apps/secrets.keycloak.yaml deleted file mode 100644 index 34f328f775..0000000000 --- a/tests/fixtures/env/apps/secrets.keycloak.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: AplApp -spec: - idp: - clientSecret: somsecretvalue -name: keycloak -metadata: - name: keycloak diff --git a/tests/fixtures/env/apps/secrets.kubeflow-pipelines.yaml b/tests/fixtures/env/apps/secrets.kubeflow-pipelines.yaml deleted file mode 100644 index 953af4f5c9..0000000000 --- a/tests/fixtures/env/apps/secrets.kubeflow-pipelines.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - rootPassword: mysqlsomesecretvalue -name: kubeflow-pipelines -metadata: - name: kubeflow-pipelines diff --git a/tests/fixtures/env/apps/secrets.loki.yaml b/tests/fixtures/env/apps/secrets.loki.yaml deleted file mode 100644 index fd1fbcf597..0000000000 --- a/tests/fixtures/env/apps/secrets.loki.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - adminPassword: somesecretvalue -name: loki -metadata: - name: loki diff --git a/tests/fixtures/env/apps/secrets.oauth2-proxy-redis.yaml b/tests/fixtures/env/apps/secrets.oauth2-proxy-redis.yaml deleted file mode 100644 index 56f2d4ce60..0000000000 --- a/tests/fixtures/env/apps/secrets.oauth2-proxy-redis.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplApp -spec: - password: gkhugxJsPjhbCybH -name: oauth2-proxy-redis -metadata: - name: oauth2-proxy-redis diff --git a/tests/fixtures/env/apps/secrets.oauth2-proxy.yaml b/tests/fixtures/env/apps/secrets.oauth2-proxy.yaml deleted file mode 100644 index 9361e610fe..0000000000 --- a/tests/fixtures/env/apps/secrets.oauth2-proxy.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: AplApp -spec: - config: - cookieSecret: gkhugxJsPjhbCybH -name: oauth2-proxy -metadata: - name: oauth2-proxy diff --git a/tests/fixtures/env/apps/secrets.prometheus.yaml b/tests/fixtures/env/apps/secrets.prometheus.yaml deleted file mode 100644 index 74b9528e64..0000000000 --- a/tests/fixtures/env/apps/secrets.prometheus.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: AplApp -spec: - remoteWrite: - rwConfig: - basicAuth: - password: blalalalalal -name: prometheus -metadata: - name: prometheus diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/alerts-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/alerts-secrets.yaml new file mode 100644 index 0000000000..ad4e1b66c5 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/alerts-secrets.yaml @@ -0,0 +1,18 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: alerts-secrets + namespace: apl-secrets +spec: + encryptedData: + slack_url: https://hooks.slack.com/services/id + msteams_highPrio: https://xxxxxxx.com + msteams_lowPrio: https://xxxxxxxx.com + template: + immutable: false + metadata: + name: alerts-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/cert-manager-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/cert-manager-secrets.yaml new file mode 100644 index 0000000000..0dd9c28078 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/cert-manager-secrets.yaml @@ -0,0 +1,43 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: cert-manager-secrets + namespace: apl-secrets +spec: + encryptedData: + customRootCAKey: | + -----BEGIN RSA PRIVATE KEY----- + MIIEpQIBAAKCAQEA+Krj8B64Wq2YZqlUMf++TTO1ad2Zknd4EdrvA3NT5JhGulvQ + ooXlQAeSgs3TebFmjpRSk8ubB4P878mGY7zRl8WhwmuIujskYIkzguDQq8bjedNd + BtLwMWGjNWiBXKjm17i3Giupojsmwy0Xw8aB5YP0yqnYjzjBG9mYKYRcS+6GF45K + GzoIB/sM8MDCHP9ZwN3gYxD7CJEHoZm9EdCscCuVRzwr2z0G91ElalLPjZrLgsZq + pE7SN9lqkdCLjQFao2g/aDanLCgYFDj2C2FzPMbE8yJ9ED8rCSClAkZ9arh8LwkH + NUZ0AQfFBuoiW6FXUXtBjUtd77uYpIaAm9KMlQIDAQABAoIBAQCTsIuotdYwpSH6 + 9172Qzq3h5qbwe3QO/yoPivvFLQi9P4s+RM1M+kw2k5+Odj8UgzjadyRwz/UeuPj + VwHmguLJDaxBWLTgRvgYDeT2Oqg1He9FD/AUeXwHGEJjGiqa6gYQ4bh+Zqhdnlwr + V8DhmijUNEdThwUEK2UmMVpabi6TOW/dfO6sbnOHYwx326qF3LhYcUrmdeowEGLT + UhxdXJQQUsfD+zft6dcPnqucIxd5OEsn3L8/pcumOxHUGFBHDMuB5nTpcZyDxKaN + joC0zQy0BDQIMN1F7wNRukSiSYqvRvmvztF2Ka0yEWAdvBBXVVN8nKQw69oGoe9B + EQ5HSkKBAoGBAP0KnCE19jW9jq1CWFkFwd1BALWA3GOuxJqSX6M5APM+JRBR4UOZ + AUOogvGlrc1ns58q7oNoc1CHiMHd2lNFgfqWqopfVz1Tt9qHqU6VoJnkmJlJRriE + 57F08RTjslTFzYEsE9zMlL1xa5pq1aGAFB0/mYuxopRw39mS14ugxF1pAoGBAPuT + MFLfp2wttGe2WhVepOnhD13sEMGCS6GE16jinjP3qAWIPM/Wdy+Ab8n+KWQkYPLw + UsQVi+41LWFIexjzdrq9rG5LQbZdjDCyR4eomDGZhc0Vtsu79NbVrnSmjH8w6psa + DXB1uN9/VcCzae/hRpk2Q6zFiMcPE8utUU5RvFRNAoGBAJ19+wsYoPN11dW0k3Rl + BvKEwMI3P/SzFB74t5nJovPCXCM6MzB1jLnlqgppCjHsN3n7qJQVcKBQmye+w2JM + wseK+v5AtPWwo5/aC+CjdGAUTX4qg1/ZKLPkiyBrT9U/f9bD7mDg3DrE2yozEGAC + bYJ+0TyHBR/K2Sh8Irf/CfjxAoGBAM4wuwCRkpUVeLEwQfEV2zBdZ8zg+HLBqd8+ + E8u1wVhyeOHf4YevDYx/RiBWEfKj5ln3Ir7XshKQvxrm3w16Liur3bGgOMGRNp+K + 3xmO0v6EB6gpTeL5sBiMlinBf5GXtBFfbvhnZBi6Mrx30DHtf4F/ekQWup37+4uK + CAOa9jJZAoGAYbU4CoCxktBECxAVAjtpvYW5176cxiitd75s1ANhXGiOH1A6/y6H + rnZ+fMAuvPjrDXbtmqJsq0RXq1E07ng4ZDIjN+0pShVFQdakJRFo1y+d3b82lBYX + EZrfMBCWVj31dXeGEHfVvOpwrQ5ffTzs2lVmTh7Ft61gs4TJ7gNTDbE= + -----END RSA PRIVATE KEY----- + template: + immutable: false + metadata: + name: cert-manager-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/dns-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/dns-secrets.yaml new file mode 100644 index 0000000000..fc1b043bc6 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/dns-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: dns-secrets + namespace: apl-secrets +spec: + encryptedData: + provider_linode_apiToken: xvxvxvxvxvxvxvxvxvxvxvxvx + template: + immutable: false + metadata: + name: dns-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/gitea-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/gitea-secrets.yaml new file mode 100644 index 0000000000..f9c85ad94d --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/gitea-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: gitea-secrets + namespace: apl-secrets +spec: + encryptedData: + postgresqlPassword: postgresqlPassword + template: + immutable: false + metadata: + name: gitea-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/grafana-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/grafana-secrets.yaml new file mode 100644 index 0000000000..4138e88e3b --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/grafana-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: grafana-secrets + namespace: apl-secrets +spec: + encryptedData: + adminPassword: somesecretvalue + template: + immutable: false + metadata: + name: grafana-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml new file mode 100644 index 0000000000..60b72c8778 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/harbor-secrets.yaml @@ -0,0 +1,24 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: harbor-secrets + namespace: apl-secrets +spec: + encryptedData: + adminPassword: harborsomesecretvalue + core_secret: vQFMm9Qk0pTUF3MK + core_xsrfKey: txS2sHQGqiJmmhFf15oCUfF5BgbchIsi + jobservice_secret: CfpanIkcGWz3wGLO + registry_secret: PjHGEdmPhrmNrEkj + registry_credentials_htpasswd: "admin:$2a$10$zXwH8y4snDAtV4mZmgyylOqfX2AOPNaUL5e6yPm2EqPyy2G2OQsX6" + registry_credentials_username: admin + registry_credentials_password: TJmTw62K9y4fZ83wgt0xmXzLwxpTHoJ4 + secretKey: somesecretvalue + template: + immutable: false + metadata: + name: harbor-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/keycloak-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/keycloak-secrets.yaml new file mode 100644 index 0000000000..9f1dac4209 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/keycloak-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: keycloak-secrets + namespace: apl-secrets +spec: + encryptedData: + idp_clientSecret: somsecretvalue + template: + immutable: false + metadata: + name: keycloak-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/kubeflow-pipelines-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/kubeflow-pipelines-secrets.yaml new file mode 100644 index 0000000000..949018981b --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/kubeflow-pipelines-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: kubeflow-pipelines-secrets + namespace: apl-secrets +spec: + encryptedData: + rootPassword: mysqlsomesecretvalue + template: + immutable: false + metadata: + name: kubeflow-pipelines-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/loki-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/loki-secrets.yaml new file mode 100644 index 0000000000..713eb67bd7 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/loki-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: loki-secrets + namespace: apl-secrets +spec: + encryptedData: + adminPassword: somesecretvalue + template: + immutable: false + metadata: + name: loki-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-redis-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-redis-secrets.yaml new file mode 100644 index 0000000000..0e0e823755 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-redis-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: oauth2-proxy-redis-secrets + namespace: apl-secrets +spec: + encryptedData: + password: gkhugxJsPjhbCybH + template: + immutable: false + metadata: + name: oauth2-proxy-redis-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-secrets.yaml new file mode 100644 index 0000000000..b47b028e1b --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oauth2-proxy-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: oauth2-proxy-secrets + namespace: apl-secrets +spec: + encryptedData: + config_cookieSecret: gkhugxJsPjhbCybH + template: + immutable: false + metadata: + name: oauth2-proxy-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/obj-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/obj-secrets.yaml new file mode 100644 index 0000000000..e8e1f6336f --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/obj-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: obj-secrets + namespace: apl-secrets +spec: + encryptedData: + provider_linode_secretAccessKey: somesecretvalue + template: + immutable: false + metadata: + name: obj-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oidc-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oidc-secrets.yaml new file mode 100644 index 0000000000..b764c3fd6d --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/oidc-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: oidc-secrets + namespace: apl-secrets +spec: + encryptedData: + clientSecret: somesecretvalue + template: + immutable: false + metadata: + name: oidc-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml new file mode 100644 index 0000000000..2c805cb155 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/otomi-secrets.yaml @@ -0,0 +1,18 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: otomi-secrets + namespace: apl-secrets +spec: + encryptedData: + adminPassword: bladibla + git_password: gitPasswordForTesting + globalPullSecret_password: blablabla + template: + immutable: false + metadata: + name: otomi-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/prometheus-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/prometheus-secrets.yaml new file mode 100644 index 0000000000..d8589306cd --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/prometheus-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: prometheus-secrets + namespace: apl-secrets +spec: + encryptedData: + remoteWrite_rwConfig_basicAuth_password: blalalalalal + template: + immutable: false + metadata: + name: prometheus-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/smtp-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/smtp-secrets.yaml new file mode 100644 index 0000000000..b03b2ab05e --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/smtp-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: smtp-secrets + namespace: apl-secrets +spec: + encryptedData: + auth_password: somesecretvalue + template: + immutable: false + metadata: + name: smtp-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-admin-settings-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-admin-settings-secrets.yaml new file mode 100644 index 0000000000..971824fd93 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-admin-settings-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: team-admin-settings-secrets + namespace: apl-secrets +spec: + encryptedData: + password: YTrnkdUsKPcGATfg + template: + immutable: false + metadata: + name: team-admin-settings-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-demo-settings-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-demo-settings-secrets.yaml new file mode 100644 index 0000000000..214aa453c3 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-demo-settings-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: team-demo-settings-secrets + namespace: apl-secrets +spec: + encryptedData: + password: somesecretvalue + template: + immutable: false + metadata: + name: team-demo-settings-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-dev-settings-secrets.yaml b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-dev-settings-secrets.yaml new file mode 100644 index 0000000000..fd3676c286 --- /dev/null +++ b/tests/fixtures/env/manifests/namespaces/apl-secrets/sealedsecrets/team-dev-settings-secrets.yaml @@ -0,0 +1,16 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/namespace-wide: "true" + name: team-dev-settings-secrets + namespace: apl-secrets +spec: + encryptedData: + password: IkdUsKPcGAdanjas + template: + immutable: false + metadata: + name: team-dev-settings-secrets + namespace: apl-secrets + type: kubernetes.io/opaque diff --git a/tests/fixtures/env/settings/kms.yaml b/tests/fixtures/env/settings/kms.yaml index 70e04a7c17..f3dd212daf 100644 --- a/tests/fixtures/env/settings/kms.yaml +++ b/tests/fixtures/env/settings/kms.yaml @@ -6,5 +6,6 @@ spec: sops: azure: clientId: somesecretvalue + clientSecret: somesecretvalue keys: somesecretvalue tenantId: somesecretvalue diff --git a/tests/fixtures/env/settings/secrets.alerts.yaml b/tests/fixtures/env/settings/secrets.alerts.yaml deleted file mode 100644 index 1bbbfdaf12..0000000000 --- a/tests/fixtures/env/settings/secrets.alerts.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: AplAlertSet -spec: - slack: - url: https://hooks.slack.com/services/id - msteams: - highPrio: https://xxxxxxx.com - lowPrio: https://xxxxxxxx.com -name: alerts -metadata: - name: alerts diff --git a/tests/fixtures/env/settings/secrets.dns.yaml b/tests/fixtures/env/settings/secrets.dns.yaml deleted file mode 100644 index a6024d2e07..0000000000 --- a/tests/fixtures/env/settings/secrets.dns.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: AplDns -spec: - provider: - linode: - apiToken: xvxvxvxvxvxvxvxvxvxvxvxvx -name: dns -metadata: - name: dns diff --git a/tests/fixtures/env/settings/secrets.kms.yaml b/tests/fixtures/env/settings/secrets.kms.yaml deleted file mode 100644 index 74b33f1f1c..0000000000 --- a/tests/fixtures/env/settings/secrets.kms.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: AplKms -spec: - sops: - azure: - clientSecret: somesecretvalue -name: kms -metadata: - name: kms diff --git a/tests/fixtures/env/settings/secrets.obj.yaml b/tests/fixtures/env/settings/secrets.obj.yaml deleted file mode 100644 index 82c77762ff..0000000000 --- a/tests/fixtures/env/settings/secrets.obj.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: AplObjectStorage -spec: - provider: - linode: - secretAccessKey: somesecretvalue -name: obj -metadata: - name: obj diff --git a/tests/fixtures/env/settings/secrets.oidc.yaml b/tests/fixtures/env/settings/secrets.oidc.yaml deleted file mode 100644 index 5f03c2f11d..0000000000 --- a/tests/fixtures/env/settings/secrets.oidc.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplIdentityProvider -spec: - clientSecret: somesecretvalue -name: oidc -metadata: - name: oidc diff --git a/tests/fixtures/env/settings/secrets.otomi.yaml b/tests/fixtures/env/settings/secrets.otomi.yaml deleted file mode 100644 index 1819165b7c..0000000000 --- a/tests/fixtures/env/settings/secrets.otomi.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: AplCapabilitySet -spec: - adminPassword: bladibla - git: - password: gitPasswordForTesting - globalPullSecret: - password: blablabla -name: otomi -metadata: - name: otomi diff --git a/tests/fixtures/env/settings/secrets.smtp.yaml b/tests/fixtures/env/settings/secrets.smtp.yaml deleted file mode 100644 index 62a1d33e38..0000000000 --- a/tests/fixtures/env/settings/secrets.smtp.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplSmtp -spec: - auth_password: somesecretvalue -name: smtp -metadata: - name: smtp diff --git a/tests/fixtures/env/teams/admin/secrets.settings.yaml b/tests/fixtures/env/teams/admin/secrets.settings.yaml deleted file mode 100644 index 595f37ad22..0000000000 --- a/tests/fixtures/env/teams/admin/secrets.settings.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplTeamSettingSet -spec: - password: YTrnkdUsKPcGATfg -name: admin -metadata: - name: admin diff --git a/tests/fixtures/env/teams/demo/secrets.settings.yaml b/tests/fixtures/env/teams/demo/secrets.settings.yaml deleted file mode 100644 index abf4440deb..0000000000 --- a/tests/fixtures/env/teams/demo/secrets.settings.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: AplTeamSettingSet -spec: - password: somesecretvalue - alerts: - slack: - url: https://slack.con -name: demo -metadata: - name: demo diff --git a/tests/fixtures/env/teams/dev/secrets.settings.yaml b/tests/fixtures/env/teams/dev/secrets.settings.yaml deleted file mode 100644 index 41599125ef..0000000000 --- a/tests/fixtures/env/teams/dev/secrets.settings.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: AplTeamSettingSet -spec: - password: IkdUsKPcGAdanjas -name: dev -metadata: - name: dev From c0fea6e795cb736f2e7108c21253985914ec112b Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:29:05 +0100 Subject: [PATCH 56/66] fix: improve password retrieval logic in getRepo function --- src/common/git-config.ts | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/common/git-config.ts b/src/common/git-config.ts index bf15462522..a19130f267 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -152,21 +152,29 @@ export const getRepo = async (values: Record, deps = { getK8sSecret otomiGit.repoUrl = process.env.GIT_REPO_URL } const username = otomiGit?.username - let password = otomiGit?.password ?? '' + let password = '' const email = otomiGit?.email const branch = otomiGit?.branch - // If password is missing or is an unresolved sealed-secret placeholder, - // try reading the real password from the K8s secret (populated by ESO from SealedSecrets) - if (!password || (typeof password === 'string' && password.startsWith('sealed:'))) { - try { - const secret = await deps.getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) - if (secret?.git_password) { - password = String(secret.git_password) - d.debug('Read git password from K8s secret (ESO)') - } - } catch { - d.warn('Could not read git password from K8s secret, using value from config') + // Always try the K8s secret first for the real password. + // Values may contain encrypted sealed-secret ciphertext (from loadSealedSecretsToSpec) + // which must not be used as the actual git password. + try { + const secret = await deps.getK8sSecret(OTOMI_SECRETS, SEALED_SECRETS_NAMESPACE) + if (secret?.git_password) { + password = String(secret.git_password) + d.debug('Read git password from K8s secret (ESO)') + } + } catch { + d.debug('Could not read git password from K8s secret') + } + + // Fall back to values if K8s secret is not available (e.g., during bootstrap) + if (!password) { + const valuesPassword = otomiGit?.password ?? '' + // Only use the values password if it's not an unresolved sealed-secret placeholder + if (valuesPassword && !(typeof valuesPassword === 'string' && valuesPassword.startsWith('sealed:'))) { + password = valuesPassword } } From 4c8ed8234e84104a03adcceb881650eb8b01047a Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 16 Mar 2026 21:46:06 +0100 Subject: [PATCH 57/66] fix: add new namespaces in core.yaml --- core.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core.yaml b/core.yaml index 9023df3cf7..bf5568ad84 100644 --- a/core.yaml +++ b/core.yaml @@ -87,6 +87,11 @@ k8s: disablePolicyChecks: true - name: sealed-secrets app: sealed-secrets + - name: external-secrets + app: external-secrets + disableIstioInjection: true + - name: apl-secrets + disableIstioInjection: true - name: policy-reporter app: policy-reporter disablePolicyChecks: true From 19405755eb00ba4c1a89a55c5735eb74f7e2a0c3 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 1 Apr 2026 14:45:48 +0200 Subject: [PATCH 58/66] fix: conditional rewrite rules for ingress --- values/k8s/k8s-raw.gotmpl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/values/k8s/k8s-raw.gotmpl b/values/k8s/k8s-raw.gotmpl index 1eec03a63e..7ebb4eef16 100644 --- a/values/k8s/k8s-raw.gotmpl +++ b/values/k8s/k8s-raw.gotmpl @@ -83,5 +83,9 @@ resources: data: otomi-hairpin.include: | {{- $escapedDomain := $v.cluster.domainSuffix | replace "." "\\." }} + {{- if $v.apps | get "ingress-nginx-platform.enabled" false }} rewrite name regex (.+)\.{{ $escapedDomain }} ingress-nginx-platform-controller.ingress.svc.cluster.local answer auto + {{- else }} + rewrite name regex (.+)\.{{ $escapedDomain }} {{ $v.ingress.platformClass.className }}-istio.istio-system.svc.cluster.local answer auto + {{- end }} {{- end }} From 8d0a018cd1737842119d509d88cd65b584da9312 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 1 Apr 2026 22:36:15 +0200 Subject: [PATCH 59/66] fix: harbor registry username --- values/harbor/harbor-raw.gotmpl | 5 +++++ values/harbor/harbor.gotmpl | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/values/harbor/harbor-raw.gotmpl b/values/harbor/harbor-raw.gotmpl index 6d36bb30ba..a6d199dc1a 100644 --- a/values/harbor/harbor-raw.gotmpl +++ b/values/harbor/harbor-raw.gotmpl @@ -72,6 +72,7 @@ resources: data: REGISTRY_PASSWD: '{{ "{{ .password | toString }}" }}' REGISTRY_HTPASSWD: '{{ "{{ .htpasswd | toString }}" }}' + REGISTRY_USERNAME: '{{ "{{ .username | toString }}" }}' data: - secretKey: password remoteRef: @@ -81,6 +82,10 @@ resources: remoteRef: key: harbor-secrets property: registry_credentials_htpasswd + - secretKey: username + remoteRef: + key: harbor-secrets + property: registry_credentials_username - apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: diff --git a/values/harbor/harbor.gotmpl b/values/harbor/harbor.gotmpl index 5e1f10a742..c871d9e649 100644 --- a/values/harbor/harbor.gotmpl +++ b/values/harbor/harbor.gotmpl @@ -28,6 +28,12 @@ core: existingSecret: harbor-core-secret existingXsrfSecret: harbor-core-xsrf-secret existingXsrfSecretKey: CSRF_KEY + extraEnvVars: + - name: REGISTRY_CREDENTIAL_USERNAME + valueFrom: + secretKeyRef: + name: harbor-registry-credentials + key: REGISTRY_USERNAME database: maxOpenConns: {{ $h.databaseMaxConnections }} @@ -73,6 +79,12 @@ jobservice: - stdout resources: {{- $h.resources.jobservice | toYaml | nindent 4 }} existingSecret: harbor-jobservice-secret + extraEnvVars: + - name: REGISTRY_CREDENTIAL_USERNAME + valueFrom: + secretKeyRef: + name: harbor-registry-credentials + key: REGISTRY_USERNAME metrics: serviceMonitor: From c256ca738c85a897bf9966d67127b129030792ed Mon Sep 17 00:00:00 2001 From: Jehoszafat Zimnowoda <17126497+j-zimnowoda@users.noreply.github.com> Date: Mon, 13 Apr 2026 16:18:46 +0200 Subject: [PATCH 60/66] chore: add agents configuration --- .ignore | 22 ++ AGENTS.md | 700 ++++++++++++++++++++++++++++++++++ charts/AGENTS.md | 51 +++ charts/team-ns/AGENTS.md | 43 +++ helmfile.d/AGENTS.md | 46 +++ helmfile.d/snippets/AGENTS.md | 50 +++ src/AGENTS.md | 48 +++ src/cmd/AGENTS.md | 55 +++ src/common/AGENTS.md | 37 ++ src/operator/AGENTS.md | 48 +++ 10 files changed, 1100 insertions(+) create mode 100644 .ignore create mode 100644 AGENTS.md create mode 100644 charts/AGENTS.md create mode 100644 charts/team-ns/AGENTS.md create mode 100644 helmfile.d/AGENTS.md create mode 100644 helmfile.d/snippets/AGENTS.md create mode 100644 src/AGENTS.md create mode 100644 src/cmd/AGENTS.md create mode 100644 src/common/AGENTS.md create mode 100644 src/operator/AGENTS.md diff --git a/.ignore b/.ignore new file mode 100644 index 0000000000..d19e7c237b --- /dev/null +++ b/.ignore @@ -0,0 +1,22 @@ +# we don't allow json files in the root except package.json + +.history +.tmp +_.bak +node_modules/ +/coverage/ +/dist/ +/env/ +_.DS*Store +.vscode/values-schema.yaml +*.env +/.secrets +chart/apl/values.schema.json +chart/apl/README.md +workflow/ +\_.new +.envrc +otomi.cpuprofile +/.idea/ +tmp +\*\*values-repo.yaml diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..6e40d764b9 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,700 @@ +# APL Core — AI Agent Index + +> **Akamai App Platform (APL)** is a Kubernetes PaaS that integrates 30+ cloud-native applications into a cohesive, multi-tenant platform. +> This index provides the context AI agents need to perform software development tasks efficiently. + +**Stack:** TypeScript CLI + Kubernetes Operator + Helmfile/Helm + Go Templates +**Scale:** 3655 files, 357K lines, 54 charts, 76 TS source files + +## Subdirectory Knowledge Base + +| Path | Focus | +| ---------------------------------------------------------------- | --------------------------------------------------- | +| [`src/AGENTS.md`](src/AGENTS.md) | TypeScript source structure, conventions, dev setup | +| [`src/cmd/AGENTS.md`](src/cmd/AGENTS.md) | CLI command inventory, patterns | +| [`src/common/AGENTS.md`](src/common/AGENTS.md) | Shared utility modules, dependency graph | +| [`src/operator/AGENTS.md`](src/operator/AGENTS.md) | GitOps operator architecture, execution flow | +| [`helmfile.d/AGENTS.md`](helmfile.d/AGENTS.md) | Helmfile release phases, execution order | +| [`helmfile.d/snippets/AGENTS.md`](helmfile.d/snippets/AGENTS.md) | Critical templates, defaults, derived values | +| [`charts/AGENTS.md`](charts/AGENTS.md) | Custom vs vendored chart inventory | +| [`charts/team-ns/AGENTS.md`](charts/team-ns/AGENTS.md) | Team namespace chart (most complex) | + +--- + +## 1. Architecture Overview + +### Codebase Structure + +``` +/workspace +├── src/ # TypeScript CLI + Operator code +│ ├── cmd/ # CLI command implementations (apply, bootstrap, install, migrate, etc.) +│ ├── common/ # Shared utilities (git, k8s, values, crypto, helmfile wrapper) +│ ├── operator/ # APL Operator (watches CRDs, runs install/apply) +│ └── otomi.ts # CLI entrypoint +├── helmfile.d/ # Helmfile specs (executed alphabetically) +│ ├── helmfile-01..09.init # Core infrastructure (cert-manager, istio, keycloak, etc.) +│ ├── helmfile-03.databases # Platform databases (CloudNativePG) +│ ├── helmfile-10.monitoring # Monitoring stack (prometheus, alertmanager, grafana) +│ ├── helmfile-15.ingress-core# Core ingress + admin team namespace +│ ├── helmfile-20.ingress # External DNS +│ ├── helmfile-50.services # Optional services (trivy, kubeflow, kserve) +│ ├── helmfile-60.teams # Per-team releases (prometheus, grafana, tekton, team-ns) +│ ├── helmfile-70.shared # Shared services (harbor, oauth2-proxy, otomi-api, console) +│ ├── helmfile-90/91.artifacts# Raw K8s manifests (istio artifacts, otel artifacts) +│ └── snippets/ # Reusable templates, defaults, derived values +├── charts/ # Helm charts (custom + vendored) +│ ├── apl-*/ # Custom APL charts (operator, harbor-op, keycloak-op, gitea-op, network-policies) +│ ├── team-ns/ # Team namespace chart (RBAC, quotas, netpols, builds, ArgoCD, Kyverno) +│ ├── raw/ raw-cr/ # Charts for deploying raw K8s resources +│ ├── jobs/ # Chart for K8s Jobs/CronJobs +│ ├── skeleton/ # Template chart for new apps +│ └── / # Vendored upstream charts (ingress-nginx, keycloak, harbor, etc.) +├── values/ # Go template value files per chart +│ ├── /.gotmpl # Primary values template +│ ├── /-raw.gotmpl # Raw K8s manifest templates +│ └── jobs/.gotmpl # Job value templates +├── values-schema.yaml # JSON Schema for ALL user-configurable parameters +├── core.yaml # Namespace definitions + admin/team app ingress config +├── apps.yaml # App metadata (versions, descriptions, dependencies) +├── versions.yaml # Component version tags +├── values-changes.yaml # Values migration definitions (version-to-version) +└── tests/ # Test fixtures + integration tests +``` + +### Values Flow (3-Stage Merge) + +Every helmfile release loads values in this strict order: + +``` +1. DEFAULTS → helmfile.d/snippets/defaults.yaml (static defaults for all apps) +2. USER INPUT → $ENV_DIR/env/**/*.yaml (user-provided configuration) +3. DERIVED → helmfile.d/snippets/derived.gotmpl (computed: URLs, certs, Istio config) +``` + +**Critical Rule:** Never write defaults or derived values to `$ENV_DIR`. That directory holds ONLY user-supplied values. + +### Helmfile Release Anchors + +Defined in `helmfile.d/snippets/templates.gotmpl`: + +| Anchor | Chart Used | Values From | Purpose | +| ---------- | ----------------- | ----------------------------------------------------------- | ------------------------------- | +| `*default` | `charts/` | `values//.gotmpl` + `snippets/common.gotmpl` | Standard chart deployment | +| `*raw` | `charts/raw` | `values//-raw.gotmpl` | Deploy additional K8s manifests | +| `*rawCR` | `charts/raw-cr` | `values//-cr.gotmpl` + `snippets/common.gotmpl` | Deploy custom resources | +| `*jobs` | `charts/jobs` | `values/jobs/.gotmpl` | Jobs in `maintenance` namespace | +| `*otomiDb` | `charts/otomi-db` | `values//-otomi-db.gotmpl` | Platform database charts | + +### Helmfile Spec Pattern + +Every helmfile spec file follows this base loading pattern: + +```yaml +bases: + - snippets/defaults.yaml # Stage 1: defaults +--- +bases: + - snippets/env.gotmpl # Stage 2: user input from $ENV_DIR +--- +bases: + - snippets/derived.gotmpl # Stage 3: computed values +--- +{{ readFile "snippets/templates.gotmpl" }} # Load release anchors +{{- $v := .Values }} +{{- $a := $v.apps }} + +releases: + - name: myapp + installed: {{ $a | get "myapp.enabled" }} + namespace: my-namespace + <<: *default # Use anchor pattern +``` + +--- + +## 2. Key Concepts + +### App Enablement + +Apps are toggled via `apps..enabled` in user config. Defaults are in `helmfile.d/snippets/defaults.yaml`. Some apps are always enabled (derived in `derived.gotmpl`): `argocd`, `cert-manager`, `ingress-nginx`, `istio`, `keycloak`, `sealed-secrets`. + +### Namespace Model + +Namespaces are defined in `core.yaml` under `k8s.namespaces`. Each entry can have: + +- `name` — K8s namespace name +- `app` — The app that owns this namespace (defaults to name) +- `disableIstioInjection` — Skip Istio sidecar +- `disablePolicyChecks` — Skip Kyverno checks +- `labels` — Extra labels + +Team namespaces follow the pattern `team-` and are managed by the `team-ns` chart. + +### Ingress Architecture + +Admin apps are defined in `core.yaml` under `adminApps`. Team apps under `teamApps`. Each entry configures: + +- `ownHost` — Gets its own subdomain (e.g., `grafana.`) +- `ingress[].svc/namespace/port` — Backend service details +- `ingress[].type` — `public` or `private` +- `ingress[].auth` — Enable OAuth2 proxy authentication +- `isShared` — Shared across teams + +Ingress uses Gateway API (`HTTPRoute`) with Istio as the gateway implementation + nginx as the ingress controller. + +### Multi-Tenancy + +Teams are configured under `teamConfig.` in user values. Each team gets: + +- Dedicated namespace (`team-`) +- Network policies (default deny + platform allowlist) +- Resource quotas and limit ranges +- Optional managed monitoring (Grafana, Alertmanager, Prometheus) +- ArgoCD project + GitOps repo in Gitea +- Tekton dashboard for CI/CD +- Kyverno security policies +- Services, workloads, builds, secrets + +### Schema Validation + +All user-configurable parameters must be defined in `values-schema.yaml` (JSON Schema draft-07). The schema uses: + +- `$ref` for reusable definitions (resources, images, networking) +- `x-secret` annotation for sensitive values (triggers secret generation) +- `additionalProperties: false` to prevent typos +- Pattern validation for names, domains, resource quantities + +### Values Migration + +When schema changes between versions, `values-changes.yaml` defines migrations: + +- `deletions` — Remove deprecated keys +- `relocations` — Move keys to new paths +- `additions` — Add new keys with defaults +- `mutations` — Change existing values +- `customFunctions` — Complex migrations in TypeScript (`src/common/runtime-upgrades/`) +- `fileDeletions/fileAdditions` — File-level changes in `$ENV_DIR` + +Current schema version: **60** (see `versions.specVersion` in defaults.yaml) + +--- + +## 3. Agent Definitions + +### Agent: Network Policy Definor + +**Scope:** Create, modify, or troubleshoot Kubernetes NetworkPolicies for platform applications and teams. + +**Key Files:** +| File | Purpose | +|------|---------| +| `charts/apl-network-policies/templates/networkpolicies/*.yaml` | Platform-level network policies (per-app) | +| `charts/apl-network-policies/values.yaml` | Enable/disable flags (`netpols.: true`) | +| `values/apl-network-policies/apl-network-policies.gotmpl` | Values template passing config to chart | +| `charts/team-ns/templates/netpols/default-network-policies.yaml` | Team default policies (deny-all + platform allowlist) | +| `charts/team-ns/templates/netpols/custom-network-policies.yaml` | Team custom ingress policies (AllowAll/AllowOnly) | +| `charts/team-ns/templates/netpols/custom-istio-service-entries.yaml` | Team egress policies via Istio ServiceEntries | +| `values-schema.yaml` → `definitions.netpol` | Schema for user-defined network policies | +| `values-schema.yaml` → `definitions.appNetworkPolicyConfig` | Schema for per-app netpol toggle | +| `core.yaml` → `k8s.namespaces` | Namespace labels used in selectors | +| `helmfile.d/snippets/defaults.yaml` → `apps..networkPolicies.enabled` | Default netpol enablement per app | + +**Patterns:** + +- **Platform policies** (`apl-network-policies` chart): Each app gets a template file in `templates/networkpolicies/.yaml`, gated by `{{ if .Values.netpols. }}`. Policies use namespace selectors referencing labels from `core.yaml`. Common ingress sources: Istio gateway (via `.Values.ingressGatewaySelectors`), monitoring namespace, operators, internal namespace communication. +- **Team default policies** (`team-ns` chart): When `networkPolicy.ingressPrivate` is true, deploy deny-all + allow from platform services (istio-system, knative-serving, monitoring, tekton-pipelines, gitea). +- **Team custom policies**: Users define in `teamConfig..netpols[]` with schema `definitions.netpol`. Types: `ingress` (AllowAll from team namespaces, or AllowOnly from specific namespaces+labels) and `egress` (FQDN + port via Istio ServiceEntry). + +**How to add a network policy for a new platform app:** + +1. Create `charts/apl-network-policies/templates/networkpolicies/.yaml` +2. Gate with `{{- if .Values.netpols. }}` +3. Define podSelector, policyTypes, ingress rules using namespace/pod selectors +4. Add `netpols.` to the values template in `values/apl-network-policies/apl-network-policies.gotmpl` +5. Optionally add `apps..networkPolicies` schema entry in `values-schema.yaml` using `$ref: '#/definitions/appNetworkPolicyConfig'` +6. Set default in `helmfile.d/snippets/defaults.yaml` under `apps..networkPolicies.enabled` + +--- + +### Agent: New Application Integrator + +**Scope:** Add a new core or optional application to the APL platform. + +**Key Files:** +| File | Purpose | +|------|---------| +| `charts//` | Helm chart directory (custom or vendored) | +| `charts/skeleton/` | Template chart to copy from for custom charts | +| `values//.gotmpl` | Primary values template | +| `values//-raw.gotmpl` | Optional: raw K8s resources (RBAC, CRDs, etc.) | +| `helmfile.d/helmfile-*.yaml.gotmpl` | Helmfile release definition | +| `helmfile.d/snippets/defaults.yaml` | Default values for the app | +| `values-schema.yaml` → `properties.apps.` | User-configurable schema | +| `core.yaml` → `k8s.namespaces` | Namespace registration | +| `core.yaml` → `adminApps` or `teamApps` | Ingress configuration (if app has UI) | +| `apps.yaml` → `appsInfo.` | App metadata (version, description, links) | + +**Step-by-step procedure:** + +1. **Chart:** Place Helm chart in `charts//` (copy `charts/skeleton/` for custom, or vendor upstream) +2. **Values template:** Create `values//.gotmpl` — access platform values via `{{ $v := .Values }}`, `{{ $a := $v.apps }}` +3. **Helmfile release:** Add release in appropriate `helmfile.d/helmfile-*.yaml.gotmpl`: + ```yaml + - name: + installed: {{ $a | get ".enabled" }} + namespace: + <<: *default + ``` +4. **Defaults:** Add `apps.` section in `helmfile.d/snippets/defaults.yaml` with `enabled`, `resources`, `_rawValues: {}` +5. **Schema:** Add `apps.` under `properties.apps` in `values-schema.yaml` +6. **Namespace:** Add to `core.yaml` → `k8s.namespaces` (with appropriate flags) +7. **Ingress** (if app has web UI): Add to `core.yaml` → `adminApps` or `teamApps` +8. **Metadata:** Add to `apps.yaml` → `appsInfo.` (title, version, description, etc.) +9. **Raw artifacts** (optional): Create `values//-raw.gotmpl` and add `*raw` release + +**Placement rules for helmfile releases:** +| Phase | File | When to use | +|-------|------|-------------| +| 01-09 | `helmfile-0X.init` | Core infra that other apps depend on | +| 03 | `helmfile-03.databases` | Database releases using `*otomiDb` | +| 10 | `helmfile-10.monitoring` | Monitoring stack components | +| 15 | `helmfile-15.ingress-core` | Ingress-related core components | +| 20 | `helmfile-20.ingress` | DNS-related releases | +| 50 | `helmfile-50.services` | Optional/addon services | +| 60 | `helmfile-60.teams` | Per-team releases (iterated over teams) | +| 70 | `helmfile-70.shared` | Shared services (harbor, console, API) | +| 90-91 | `helmfile-90/91.artifacts` | Raw K8s manifest releases | + +--- + +### Agent: Ingress Configurator + +**Scope:** Configure ingress routes, gateways, HTTPRoutes, OAuth2 authentication, and domain management. + +**Key Files:** +| File | Purpose | +|------|---------| +| `core.yaml` → `adminApps` | Admin app ingress definitions | +| `core.yaml` → `teamApps` | Team app ingress definitions | +| `helmfile.d/snippets/routes.gotmpl` | HTTPRoute template (parentRefs, auth rules) | +| `helmfile.d/snippets/authpolicy-oauth2-ext.gotmpl` | OAuth2 external auth policy template | +| `helmfile.d/snippets/authpolicy-jwt.gotmpl` | JWT authentication policy template | +| `helmfile.d/snippets/serviceentry.gotmpl` | Istio ServiceEntry template for domain routing | +| `helmfile.d/snippets/derived.gotmpl` | Computed domain names, gateway names, TLS | +| `helmfile.d/snippets/domains.gotmpl` | Domain configuration helpers | +| `charts/team-ns/templates/routes.yaml` | Team service route rendering | +| `charts/team-ns/templates/ingress.yaml` | Team ingress resources | +| `charts/kubernetes-gateways/` | Gateway API gateway definitions | +| `charts/istio-gateway/` | Istio-specific gateway chart | +| `charts/ingress-nginx/` | NGINX ingress controller chart | +| `values/ingress-nginx/` | NGINX ingress values | +| `values/istio-gateway/` | Istio gateway values | +| `values/kubernetes-gateways/` | Gateway API values | +| `values-schema.yaml` → `definitions.service` | Service/ingress schema | +| `values-schema.yaml` → `definitions.ingressClassParameters` | Ingress class config | +| `values-schema.yaml` → `properties.ingress` | Platform ingress schema | + +**Concepts:** + +- Domains follow pattern `.` (admin) or `-.` (team) +- Derived values compute: `_derived.consoleDomain`, `_derived.giteaDomain`, `_derived.keycloakDomain`, etc. +- Gateway API `HTTPRoute` resources route to backend services +- OAuth2 proxy handles authentication (`auth: true` in core.yaml) +- Istio `ServiceEntry` resources make domains resolvable from within the mesh +- `ingress.platformClass` configures the main load balancer (IP, autoscaling, resources) +- Additional ingress classes via `ingress.classes[]` + +--- + +### Agent: Schema Manager + +**Scope:** Modify `values-schema.yaml` to add, change, or deprecate user-configurable parameters. + +**Key Files:** +| File | Purpose | +|------|---------| +| `values-schema.yaml` | THE schema file (JSON Schema draft-07 in YAML) | +| `helmfile.d/snippets/defaults.yaml` | Defaults that MUST match schema | +| `values-changes.yaml` | Migration definitions when schema changes | +| `src/common/runtime-upgrades/` | Custom migration functions | +| `src/cmd/validate-values.ts` | Schema validation command | +| `src/cmd/migrate.ts` | Migration execution | + +**Rules:** + +- Every user-configurable parameter MUST have a schema entry +- Use `$ref` to reference reusable definitions (`resources`, `image`, `idName`, etc.) +- Mark secrets with `x-secret: ''` (or `x-secret: '{{ randAlphaNum N }}'` for auto-generation) +- Use `additionalProperties: false` on app schemas to catch typos +- Always include `_rawValues: { $ref: '#/definitions/rawValues' }` for escape-hatch overrides +- When removing/renaming keys, add a migration in `values-changes.yaml` and bump `versions.specVersion` + +**Common schema patterns:** + +```yaml +# App with resources + enabled flag +apps: + myapp: + additionalProperties: false + properties: + _rawValues: + $ref: '#/definitions/rawValues' + enabled: + type: boolean + default: false + resources: + additionalProperties: false + properties: + main: + $ref: '#/definitions/resources' + sidecar: + $ref: '#/definitions/resources' +``` + +**Reusable definitions (most common):** +| Definition | Purpose | +|------------|---------| +| `resources` | CPU/memory requests+limits | +| `rawValues` | Escape-hatch for unschema'd chart values | +| `image` / `imageSimple` | Container image config | +| `idName` | Lowercase DNS-safe name pattern | +| `domain` | Domain pattern | +| `autoscaling` / `autoscalingEnabled` | HPA config | +| `service` | Team service definition | +| `netpol` | Network policy definition | +| `workload` | ArgoCD workload definition | +| `build` | Tekton build definition | +| `secret` | External secret definition | +| `appNetworkPolicyConfig` | Per-app netpol toggle | + +--- + +### Agent: Team Configuration Manager + +**Scope:** Manage team namespaces, RBAC, resource quotas, security policies, and team-level services. + +**Key Files:** +| File | Purpose | +|------|---------| +| `charts/team-ns/` | THE team namespace chart | +| `charts/team-ns/values.yaml` | Team chart values structure | +| `charts/team-ns/templates/rbac.yaml` | Service accounts, roles, role bindings | +| `charts/team-ns/templates/limitrange.yaml` | Default resource limits for containers | +| `charts/team-ns/templates/quota.yaml` | Resource quotas (pods, load balancers) | +| `charts/team-ns/templates/netpols/` | Network policies (default + custom) | +| `charts/team-ns/templates/policies/` | Kyverno security policies | +| `charts/team-ns/templates/argocd/` | ArgoCD application + project templates | +| `charts/team-ns/templates/builds/` | Tekton build configurations (Docker, Buildpacks) | +| `charts/team-ns/templates/tekton-tasks/` | Tekton tasks (kaniko, git-clone, grype, buildpacks) | +| `charts/team-ns/templates/routes.yaml` | Service routing | +| `charts/team-ns/templates/ingress.yaml` | Ingress config | +| `charts/team-ns/templates/_helpers.tpl` | Helper templates | +| `values/team-ns/team-ns.gotmpl` | Values template for team-ns | +| `helmfile.d/helmfile-15.ingress-core.yaml.gotmpl` | Admin team namespace release | +| `helmfile.d/helmfile-60.teams.yaml.gotmpl` | Per-team releases (iterates over `teamConfig`) | +| `values-schema.yaml` → `definitions.team` | Full team configuration schema | +| `values-schema.yaml` → `definitions.teamSelfService` | Team self-service permissions | + +**Team iteration pattern (helmfile-60):** + +```gotmpl +{{- range $teamId, $team := omit $tc "admin" }} + - name: team-ns-{{ $teamId }} + installed: true + namespace: team-{{ $teamId }} + chart: ../charts/team-ns + values: + - ../values/team-ns/team-ns.gotmpl +{{- end }} +``` + +**Per-team releases deployed:** `tekton-dashboard-`, `prometheus-`, `grafana-dashboards-`, `team-ns-`, `team-secrets-`, `prometheus-msteams-` (if msteams alerts enabled). + +--- + +### Agent: Values Template Author + +**Scope:** Create or modify Go template value files in `values/`. + +**Key Files:** +| File | Purpose | +|------|---------| +| `values//.gotmpl` | Primary values template for each app | +| `values//-raw.gotmpl` | Raw K8s manifests template | +| `values/jobs/.gotmpl` | Job values templates | +| `helmfile.d/snippets/common.gotmpl` | Common values included in most releases | +| `helmfile.d/snippets/templates.gotmpl` | How values files are loaded per anchor type | + +**Template conventions:** + +```gotmpl +{{- $v := .Values }} # All merged values +{{- $a := $v.apps }} # All app configs +{{- $app := $a.myapp }} # Specific app config +{{- $tc := $v.teamConfig }} # Team configurations +{{- $d := $v._derived }} # Derived/computed values +{{- $provider := $v.cluster.provider }} # Cloud provider +{{- $domain := $v.cluster.domainSuffix }}# Cluster domain +``` + +**Accessing common derived values:** + +- `$v._derived.oidcBaseUrl` — Keycloak OIDC URL +- `$v._derived.untrustedCA` — Whether CA is untrusted +- `$v._derived.tlsSecretName` — TLS wildcard cert secret name +- `$v._derived.consoleDomain`, `giteaDomain`, `keycloakDomain`, etc. +- `$v._derived.ingressPublicGatewayName` — Istio ingress gateway name + +**Raw template pattern (`*-raw.gotmpl`):** + +```gotmpl +resources: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: my-config + data: + key: value +``` + +--- + +### Agent: Helmfile Release Manager + +**Scope:** Add, modify, or reorder helmfile releases. + +**Key Files:** +| File | Purpose | +|------|---------| +| `helmfile.d/helmfile-*.yaml.gotmpl` | All release definitions (see placement table above) | +| `helmfile.d/snippets/templates.gotmpl` | Release anchor definitions | +| `helmfile.d/snippets/defaults.yaml` | Default values loaded by all specs | +| `helmfile.d/snippets/env.gotmpl` | User environment values | +| `helmfile.d/snippets/derived.gotmpl` | Computed values | + +**Adding a release:** + +```yaml +releases: + - name: my-app # Must match chart dir and values dir + installed: {{ $a | get "my-app.enabled" }} # Toggle via user config + namespace: my-namespace # Target K8s namespace + labels: # Optional labels for filtering + pkg: my-app + <<: *default # Use standard deployment anchor +``` + +**Release with raw artifacts:** + +```yaml + - name: my-app-artifacts + installed: {{ $a | get "my-app.enabled" }} + namespace: my-namespace + <<: *raw +``` + +**Release with labels for `otomi apply -l name=my-app`:** + +```yaml +labels: + tag: teams # Group label + team: { { $teamId } } # Team-specific label + pipeline: otomi-task-teams # Pipeline label +``` + +--- + +### Agent: TypeScript CLI Developer + +**Scope:** Work on CLI commands, operators, and TypeScript utility code. + +**Key Files:** +| File | Purpose | +|------|---------| +| `src/otomi.ts` | CLI entrypoint (yargs-based) | +| `src/cmd/index.ts` | Command registry | +| `src/cmd/*.ts` | Individual CLI commands | +| `src/common/hf.ts` | Helmfile wrapper (invokes helmfile with proper args) | +| `src/common/values.ts` | Values loading/merging logic | +| `src/common/k8s.ts` | Kubernetes API interactions | +| `src/common/bootstrap.ts` | Bootstrap logic (initial setup) | +| `src/common/utils.ts` | Shared utilities | +| `src/common/cli.ts` | CLI helper utilities | +| `src/common/constants.ts` | Constants and paths | +| `src/common/zx-enhance.ts` | Enhanced zx (Google's shell scripting lib) | +| `src/common/runtime-upgrade.ts` | Migration runner | +| `src/common/runtime-upgrades/` | Custom migration functions | +| `src/operator/` | APL Operator (watches CRDs, runs installs) | +| `tsconfig.json` / `tsconfig.build.json` | TypeScript config | +| `jest.config.ts` | Test config | +| `eslint.config.mjs` | Linting config | +| `babel.config.js` | Babel config | + +**Key CLI commands:** +| Command | Source | Purpose | +|---------|--------|---------| +| `otomi apply` | `src/cmd/apply.ts` | Deploy charts via helmfile | +| `otomi bootstrap` | `src/cmd/bootstrap.ts` | Initialize values repo | +| `otomi install` | `src/cmd/install.ts` | Full cluster installation | +| `otomi validate-values` | `src/cmd/validate-values.ts` | Validate user values against schema | +| `otomi validate-templates` | `src/cmd/validate-templates.ts` | Validate rendered K8s manifests | +| `otomi diff` | `src/cmd/diff.ts` | Show diff before applying | +| `otomi migrate` | `src/cmd/migrate.ts` | Run value migrations | +| `otomi values` | `src/cmd/values.ts` | Render merged values | +| `otomi x` | `src/cmd/x.ts` | Execute arbitrary helmfile commands | + +**Test patterns:** Tests are co-located as `*.test.ts` files. Use Jest with `npm test`. Fixtures in `tests/fixtures/`. + +**Dev setup:** + +```bash +export IN_DOCKER=false +export ENV_DIR=$PWD/tests/fixtures +export NODE_ENV=test +npm run compile +npm test +``` + +--- + +### Agent: Migration Author + +**Scope:** Define value migrations when schema changes between versions. + +**Key Files:** +| File | Purpose | +|------|---------| +| `values-changes.yaml` | Migration definitions (version → version) | +| `src/common/runtime-upgrade.ts` | Migration runner | +| `src/common/runtime-upgrades/` | Custom migration TypeScript functions | +| `src/cmd/migrate.ts` | Migration CLI command | +| `helmfile.d/snippets/defaults.yaml` → `versions.specVersion` | Current schema version | + +**Migration types:** + +```yaml +changes: + - version: 61 # Next version number + deletions: + - 'apps.old-app' # Remove deprecated keys + relocations: + - 'apps.foo.old': 'apps.foo.new' # Move keys + additions: + - apps.new-app.enabled: false # Add with default + mutations: + - apps.foo.bar: 'newValue' # Change values + customFunctions: + - myMigrationFunction # TypeScript function in runtime-upgrades/ + fileDeletions: + - env/teams/{team}/old-file.yaml + fileAdditions: + - env/teams/new-file.yaml +``` + +**After adding a migration:** Bump `versions.specVersion` in `helmfile.d/snippets/defaults.yaml`. + +--- + +### Agent: Monitoring Stack Configurator + +**Scope:** Configure Prometheus, Grafana, Alertmanager, Loki, and OpenTelemetry. + +**Key Files:** +| File | Purpose | +|------|---------| +| `helmfile.d/helmfile-10.monitoring.yaml.gotmpl` | Monitoring stack releases | +| `helmfile.d/helmfile-60.teams.yaml.gotmpl` | Per-team Prometheus/Grafana/Alertmanager | +| `charts/kube-prometheus-stack/` | Prometheus operator chart | +| `charts/grafana-dashboards/` | Grafana dashboard definitions | +| `charts/loki/` | Loki log aggregation | +| `charts/otel-operator/` | OpenTelemetry operator | +| `charts/promtail/` | Promtail log shipper | +| `charts/prometheus-blackbox-exporter/` | Blackbox exporter | +| `charts/prometheus-msteams/` | MS Teams alerting bridge | +| `values/prometheus-operator/` | Prometheus operator values | +| `values/loki/` | Loki values | +| `values/otel-operator/` | OTel values | +| `values/grafana-dashboards/` | Dashboard values | +| `helmfile.d/snippets/alertmanager.gotmpl` | Alertmanager config template | +| `helmfile.d/snippets/alertmanager-teams.gotmpl` | Per-team alertmanager config | +| `helmfile.d/snippets/alertmanager/slack.gotmpl` | Slack integration | +| `helmfile.d/snippets/alertmanager/opsgenie.gotmpl` | Opsgenie integration | +| `helmfile.d/snippets/blackbox-targets.gotmpl` | Blackbox probe targets | +| `values-schema.yaml` → `definitions.alerts` | Alert configuration schema | +| `values-schema.yaml` → `properties.apps.prometheus` | Prometheus schema | +| `values-schema.yaml` → `properties.apps.loki` | Loki schema | + +**Team monitoring pattern:** + +- Each team can have managed monitoring (`teamConfig..settings.managedMonitoring.grafana/alertmanager`) +- Per-team Prometheus instance scrapes team namespace +- Per-team Grafana with OIDC auth + team-scoped datasources +- Per-team Alertmanager with configurable receivers (slack, msteams, opsgenie) +- Dashboards auto-provisioned from `grafana-dashboards` chart + +--- + +### Agent: Database Manager + +**Scope:** Manage CloudNativePG PostgreSQL databases for platform applications. + +**Key Files:** +| File | Purpose | +|------|---------| +| `helmfile.d/helmfile-03.databases.yaml.gotmpl` | Database releases | +| `charts/otomi-db/` | Database chart (wraps CloudNativePG) | +| `charts/cloudnative-pg/` | CloudNativePG operator chart | +| `charts/cloudnative-pg-plugin-barman-cloud/` | Backup plugin chart | +| `helmfile.d/snippets/defaults.yaml` → `databases` | Database defaults (keycloak, harbor, gitea) | +| `values-schema.yaml` → `properties.databases` (if exists) | Database schema | +| `values-schema.yaml` → `properties.platformBackups.database` | Backup configuration | + +**Platform databases:** keycloak, harbor, gitea — each has configurable replicas, storage size, resources, PostgreSQL parameters, and backup settings. + +--- + +## 4. Common Go Template Helpers + +Used across values templates: + +| Expression | Purpose | +| -------------------------------------- | ----------------------------------- | +| `$v.cluster.domainSuffix` | Cluster base domain | +| `$v.cluster.provider` | Cloud provider (`linode`, `custom`) | +| `$a \| get "app.key" defaultValue` | Safe key access with fallback | +| `$v \| dig "deep" "path" defaultValue` | Deep path access | +| `hasKey $dict "key"` | Check if key exists | +| `$v.otomi.isMultitenant` | Multi-tenancy flag | +| `$v._derived.*` | All computed values | +| `tpl (readFile "path") $v` | Render a sub-template with values | +| `toYaml \| nindent N` | YAML serialization with indentation | + +--- + +## 5. Testing & Validation + +| Command | Purpose | +| ------------------------------------------- | ------------------------------------- | +| `npm test` | Run Jest unit tests | +| `otomi validate-values` | Validate user config against schema | +| `otomi validate-templates [-l name=app]` | Validate rendered K8s manifests | +| `otomi diff [-l name=app]` | Preview changes before apply | +| `otomi x helmfile -l name=app template` | Render chart templates for inspection | +| `otomi x helmfile -l name=app write-values` | Render values for inspection | +| `npm run test:opa` | Run OPA/Rego policy tests | + +--- + +## 6. Conventions & Gotchas + +- **Helmfile labels:** Use `-l name=myapp` to select releases, not `-l app=myapp` +- **Raw values override:** `apps.._rawValues` overrides chart values not in schema (use sparingly) +- **YAML anchors:** Search for `&anchorname` when you see `<<: *anchorname` +- **Keycloak OIDC:** Use `_derived.oidcBaseUrl`, `apps.keycloak.idp.clientID/clientSecret` +- **Untrusted CA:** Check `_derived.untrustedCA` to conditionally disable cert verification +- **Chart naming:** Release name = chart directory name = values directory name (unless overridden in anchor) +- **App naming in schema vs defaults:** Some apps have different chartName in `apps.yaml` (e.g., `cnpg` → `cloudnative-pg`, `otel` → `otel-operator`, `trivy` → `trivy-operator`) +- **Team admin is special:** `team-admin` namespace is deployed in helmfile-15 (not helmfile-60) and has `networkPolicy: null`, `resourceQuota: null` +- **Alphabetical execution:** Helmfile specs run 01→91. Dependencies must be in earlier-numbered files. +- **.gitignore is minimal:** Only ignores `node_modules`, `package.json`, `bun.lock`, `.gitignore` itself. Most files ARE tracked. diff --git a/charts/AGENTS.md b/charts/AGENTS.md new file mode 100644 index 0000000000..57281c3611 --- /dev/null +++ b/charts/AGENTS.md @@ -0,0 +1,51 @@ +# charts/ — Helm Charts + +## OVERVIEW +Contains APL's custom infrastructure charts and 44 vendored upstream charts. +This directory provides the building blocks for the APL platform. + +## CUSTOM CHARTS +| Chart | Purpose | Complexity | +|-------|---------|------------| +| apl-operator/ | Core platform operator logic | Medium | +| apl-gitea-operator/ | Gitea lifecycle and configuration | Low | +| apl-harbor-operator/ | Harbor lifecycle and configuration | Low | +| apl-keycloak-operator/| Keycloak lifecycle and configuration | Low | +| apl-network-policies/ | Platform-wide default network policies | Medium | +| team-ns/ | Team namespace engine (RBAC, quotas, builds, ArgoCD) | HIGH | +| raw/ | Deploy arbitrary K8s manifests (ConfigMaps, etc.) | Low | +| raw-cr/ | Deploy arbitrary custom resources (CRs) | Low | +| jobs/ | Reusable templates for Kubernetes Jobs/CronJobs | Low | +| skeleton/ | Template chart for creating new custom apps | Starter | + +## VENDORED CHARTS +Upstream charts are mirrored here to ensure stability and local modification capability. +- **Core Platform:** argocd, cert-manager, istio-base, istio-gateway, istiod, keycloak, sealed-secrets, ingress-nginx +- **Monitoring:** kube-prometheus-stack, grafana-dashboards, loki, otel-operator, promtail, prometheus-blackbox-exporter, prometheus-msteams +- **CI/CD:** tekton-pipelines, tekton-triggers, tekton-dashboard +- **Storage/DB:** cloudnative-pg, cloudnative-pg-plugin-barman-cloud, harbor, gitea, valkey, rabbitmq +- **Security:** kyverno, trivy-operator, policy-reporter, oauth2-proxy, external-secrets +- **DNS/Ingress:** external-dns, cert-manager-webhook-linode, kubernetes-gateways +- **ML/Serverless:** knative-operator, kserve, kubeflow-pipelines +- **APL Services:** otomi-api, otomi-console, otomi-operator, otomi-db, linode-cfw +- **Utilities:** base, metrics-server, argocd-image-updater, wait-for + +## PATTERNS +- **Structure:** Every chart follows the standard Helm layout (Chart.yaml, values.yaml, templates/). +- **Helpers:** Critical naming and labeling logic resides in `templates/_helpers.tpl`. +- **Feature Toggles:** Templates use `{{- if }}` for logic (e.g., ingress, RBAC, persistence). +- **Static Defaults:** Chart `values.yaml` holds base defaults for the chart itself. +- **Dynamic Input:** Configuration is injected from `values/*.gotmpl` during Helmfile execution. +- **Escape Hatches:** Use `_rawValues` in user config to override chart values not in schema. +- **Skeleton:** Always use `charts/skeleton/` as the baseline for new custom charts. +- **Labels:** Consistently use `app.kubernetes.io/managed-by: otomi` and APL-specific labels. +- **Selectors:** Cross-app communication relies on stable labels defined in `_helpers.tpl`. + +## WHERE TO LOOK +- **Logic:** `templates/` — Examine for APL-specific logic, annotations, and labels. +- **Dependencies:** `Chart.yaml` — Check for upstream versions and sub-charts. +- **Default Values:** `values.yaml` — View the base chart configuration. +- **Multi-Tenancy:** `charts/team-ns/` — The primary engine for tenant isolation. +- **Manifest Injection:** `charts/raw/` or `charts/raw-cr/` for non-standard resources. +- **Job Templates:** `charts/jobs/` for platform maintenance or migration tasks. +- **Network Isolation:** `charts/apl-network-policies/` for cluster-wide restrictions. diff --git a/charts/team-ns/AGENTS.md b/charts/team-ns/AGENTS.md new file mode 100644 index 0000000000..9252a96cf9 --- /dev/null +++ b/charts/team-ns/AGENTS.md @@ -0,0 +1,43 @@ +# charts/team-ns/ — Team Namespace Chart + +## OVERVIEW +The core multi-tenancy engine of APL. Provisions isolated team environments with RBAC, quotas, +networking, security policies, and CI/CD (ArgoCD + Tekton) integration. + +## TEMPLATE STRUCTURE +- `templates/` + - `_helpers.tpl` — Label logic, domain math, Docker/volume config generation. + - `rbac.yaml` — Massive policy (SAs: team, kubectl, tekton; RoleBindings). + - `routes.yaml` — Gateway API `HTTPRoute` for team services. + - `ingress.yaml` — Legacy/Standard Ingress resources. + - `limitrange.yaml` — Container resource defaults. + - `quota.yaml` — Team resource constraints (skipped for `team-admin`). + - `argocd/` — Team Apps/Projects for GitOps lifecycle. + - `builds/` — Tekton pipeline/build specs (Docker/Buildpacks). + - `netpols/` — Ingress/Egress isolation (Platform allowlist + Custom). + - `policies/` — Kyverno PSS (Baseline/Restricted) enforcement. + - `tekton-tasks/` — Reusable build steps (Kaniko, Grype, GitClone). + - `telemetry/` — Istio/OTel instrumentation. + +## KEY TEMPLATES +- `rbac.yaml`: Manages complex multi-identity RBAC for team users and automation. +- `netpols/default-network-policies.yaml`: Implements platform isolation (deny-all + core allow). +- `argocd/argocd-application-workload.yaml`: Bridges team config to ArgoCD deployments. + +## VALUE SOURCES +- Primary: `values/team-ns/team-ns.gotmpl` (Injected via Helmfile). +- Global: `helmfile.d/snippets/defaults.yaml`. +- Context: Iterated via `helmfile-60.teams.yaml.gotmpl` (except `team-admin`). + +## PATTERNS +- **Identity naming**: Resources suffix/prefix with `{{ $v.teamId }}`. +- **Namespace isolation**: Always targets `team-{{ $v.teamId }}`. +- **Labeling**: Every resource MUST have `otomi.io/team: {{ $v.teamId }}`. +- **Security**: Aggregates secrets for Gitea/Harbor/Internal-Registry. +- **Builds**: Standardized on Tekton with Kaniko for Docker or Buildpacks for source. + +## ANTI-PATTERNS +- **Hardcoded IDs**: Never use static team names; always use `teamId` from values. +- **Missing labels**: Resources without `otomi.io/team` will break platform logic. +- **Manual NS creation**: Let this chart handle namespace-scoped resource lifecycle. +- **Bypassing NetPols**: Adding global allows outside `custom-network-policies.yaml`. diff --git a/helmfile.d/AGENTS.md b/helmfile.d/AGENTS.md new file mode 100644 index 0000000000..31336110dc --- /dev/null +++ b/helmfile.d/AGENTS.md @@ -0,0 +1,46 @@ +# helmfile.d/ — Helmfile Release Definitions + +## OVERVIEW +Orchestrates 30+ cloud-native apps via ordered Helmfile specs using 3-stage value merge (defaults → user → derived). + +## EXECUTION ORDER +Specs execute alphabetically. Dependencies MUST be in earlier-numbered files. +| File | Phase | Key Releases | +|------|-------|--------------| +| `01-09.init` | Core Infra | Kyverno, Cert-Manager, Keycloak, External-Secrets | +| `03.databases`| DBs | CloudNativePG (Gitea, Keycloak, Harbor) using `*otomiDb` | +| `03.init` | Core Infra | Components requiring post-DB setup | +| `10.monitoring`| Observability | Prometheus, Grafana, Loki, OTel, Alertmanager | +| `15.ingress-core`| Ingress | Ingress-Nginx, Istio-Base, Admin Team Namespace | +| `20.ingress` | DNS | External-DNS | +| `50.services` | Optional | Knative, Kubeflow, Trivy, Kserve | +| `60.teams` | Per-Team | Tekton, Team-Prometheus/Grafana/Secrets (iterated) | +| `70.shared` | Platform | Harbor, OAuth2-Proxy, Otomi-API, Otomi-Console | +| `90-91.artifacts`| Manifests | Raw K8s artifacts for Istio and OTel | + +## RELEASE PATTERNS +Standardized loading via anchors in `snippets/templates.gotmpl`: +- `<<: *default`: Standard chart (uses `values//.gotmpl`) +- `<<: *raw`: Additional K8s manifests (uses `values//-raw.gotmpl`) +- `<<: *rawCR`: Custom Resources using `raw-cr` chart +- `<<: *jobs`: Maintenance jobs (uses `values/jobs/.gotmpl`) +- `<<: *otomiDb`: Database releases wrapping CloudNativePG + +## WHERE TO ADD NEW RELEASES +- **Core Infra:** `01-09.init` (ensure alphabetical ordering for dependencies) +- **DB-backed apps:** `03.databases` for the DB, then later for the app +- **Add-on services:** `50.services` +- **Team-scoped resources:** `60.teams` (requires iteration over `teamConfig`) + +## ANTI-PATTERNS +- **Ordering errors:** Placing a dependency in a higher-numbered file than its consumer +- **Hardcoding values:** Use `.Values` or `snippets/derived.gotmpl` instead +- **Direct $ENV_DIR access:** ALWAYS use `snippets/env.gotmpl` base +- **Skipping anchors:** Standard releases MUST use `*default`, `*raw`, etc. +- **Duplicate Prefixes:** `03.databases` vs `03.init` — remember alphabetical full-filename sorting +- **Manual Values Merge:** Use `snippets/common.gotmpl` within chart values for shared state +- **Recursive Bases:** Do NOT include bases that loop back to snippets + +## DIRECTORY STRUCTURE +- `snippets/`: Reusable Go templates, defaults, and derived values. +- `utils/`: Helper scripts for helmfile processing and manifest generation. diff --git a/helmfile.d/snippets/AGENTS.md b/helmfile.d/snippets/AGENTS.md new file mode 100644 index 0000000000..96baddec10 --- /dev/null +++ b/helmfile.d/snippets/AGENTS.md @@ -0,0 +1,50 @@ +# helmfile.d/snippets/ — Core Templates & Values + +## OVERVIEW +Core configuration layer defining the platform's 3-stage values merge and release templates. + +## FILE INVENTORY +| File | Purpose | +|------|---------| +| `defaults.yaml` | Stage 1: Static defaults for ALL apps (1203 lines). | +| `env.gotmpl` | Stage 2: Loads user values from `$ENV_DIR/env/**/*.yaml`. | +| `derived.gotmpl` | Stage 3: Computes `_derived.*` values (URLs, certs, gateways). | +| `templates.gotmpl` | Defines release anchors (`*default`, `*raw`, `*rawCR`, `*jobs`). | +| `common.gotmpl` | Shared values for releases (pull secrets, node selectors). | +| `routes.gotmpl` | HTTPRoute template for Gateway API routing and auth. | +| `domains.gotmpl` | Domain configuration and normalization helpers. | +| `authpolicy-jwt.gotmpl` | JWT authentication policy template. | +| `authpolicy-oauth2-ext.gotmpl` | OAuth2 external auth policy template. | +| `serviceentry.gotmpl` | Istio ServiceEntry for internal-to-external domain routing. | +| `alertmanager.gotmpl` | Platform-level Alertmanager configuration. | +| `alertmanager-teams.gotmpl` | Per-team Alertmanager configuration. | +| `alertmanager/opsgenie.gotmpl` | Opsgenie integration template for Alertmanager. | +| `alertmanager/slack.gotmpl` | Slack integration template for Alertmanager. | +| `blackbox-targets.gotmpl` | Targets for Prometheus blackbox prober. | +| `defaults.gotmpl` | Helper for loading default values in Go templates. | +| `grafana.gotmpl` | Grafana-specific configuration snippets. | +| `sops-env.gotmpl` | SOPS-encrypted environment variable loader. | +| `env.old.gotmpl` | Legacy environment loading (DEPRECATED). | +| `version-tags.gotmpl` | Component version mapping for platform services. | +| `provider-engine-map.gotmpl` | Maps cloud providers to specific engines. | +| `dockercfg.gotmpl` | Docker registry credential configuration. | +| `helmfile-utils.gotmpl` | Common Go template utilities for Helmfile specs. | + +## CRITICAL FILES +1. **`defaults.yaml`**: The source of truth for platform defaults. EVERY new app must register its default state, resources, and configuration here. It is the largest and most foundational file in the snippets directory, defining the initial state for the 3-stage merge. +2. **`derived.gotmpl`**: Computes complex values from raw user input and defaults. Crucial for understanding how OIDC URLs, TLS secrets, and gateway names are formed. It acts as the "logical" layer that transforms user intent into platform-specific configuration. +3. **`templates.gotmpl`**: Central registry of release patterns. Modifying an anchor here affects EVERY helmfile release that references it. It provides the standard `*default`, `*raw`, and `*jobs` building blocks used across all `helmfile-*.yaml` specs. + +## TEMPLATE HELPERS +- **Routing**: `routes.gotmpl` handles the mapping of hostnames to services with integrated Istio AuthPolicy support for Gateway API. +- **Domains**: `domains.gotmpl` provides functions to compute admin and team application domains consistently across the platform using cluster-level settings. +- **Monitoring**: `alertmanager/*.gotmpl` contains integration-specific templates for Slack and Opsgenie, while `blackbox-targets.gotmpl` manages probe targets for health monitoring. +- **Versions**: `version-tags.gotmpl` ensures consistent component versioning across all platform services and Helm charts. +- **Utility**: `helmfile-utils.gotmpl` contains generic helpers for string manipulation and template rendering. + +## ANTI-PATTERNS +- **Hardcoding**: Never hardcode values that belong in `defaults.yaml` or `derived.gotmpl`. +- **Direct Snippet Writes**: AI agents should never write temporary data to this directory. +- **Circular Logic**: Avoid cross-referencing between snippets that causes template recursion. +- **Stage Violation**: Don't put logic in `env.gotmpl` that belongs in `derived.gotmpl`. +- **Legacy Use**: Avoid modifying or using `env.old.gotmpl`; use the modern `env.gotmpl` instead. diff --git a/src/AGENTS.md b/src/AGENTS.md new file mode 100644 index 0000000000..37e08602b2 --- /dev/null +++ b/src/AGENTS.md @@ -0,0 +1,48 @@ +# src/ — TypeScript Source + +## OVERVIEW + +Core logic for APL CLI and GitOps Operator. Handles values merging, +Kubernetes orchestration, and Helmfile execution. + +## STRUCTURE + +- `cmd/`: Yargs command implementations (`{command, describe, builder, handler}`). +- `common/`: Core utilities (k8s, helmfile, values, git, crypto). +- `operator/`: GitOps reconciliation loop (watches Git repo, not CRDs). +- `otomi.ts`: Main entrypoint; registers commands from `cmd/index.ts`. +- `test-init.ts`: Jest global setup (mocks, console silencing). +- `stubs/`: Mock implementations for yargs and uuid used in tests. + +## WHERE TO LOOK + +- **Adding Commands**: New file in `src/cmd/`, register in `src/cmd/index.ts`. +- **K8s Interactions**: `src/common/k8s.ts` (1k+ lines, use caution). +- **Values/Migrations**: `src/common/values.ts` and `src/cmd/migrate.ts` (1.4k+ lines). +- **Git Operations**: `src/common/repo.ts` (700+ lines). +- **Helmfile Wrapper**: `src/common/hf.ts`. + +## CONVENTIONS + +- **Tests**: Co-locate `*.test.ts`. Use `ts-jest` and `test-init.ts`. +- **CLI**: Commands must export `CommandModule`. +- **Style**: No semicolons, single quotes, 120 char width. +- **TS**: Strict mode, ES2022, NodeNext resolution. +- **Lint**: No `++` (use `+= 1`), no `require`, no `param-reassign`. +- **Async**: Await all promises; `no-floating-promises` is enforced. + +## ANTI-PATTERNS + +- **God Files**: Do not grow `k8s.ts` or `migrate.ts`; extract logic to new modules. +- **CommonJS**: Never use `module.exports` or `require()`. +- **Mutation**: Avoid reassigning function parameters (`no-param-reassign`). +- **Mocks**: Avoid inline `jest.mock`; use `test-init.ts` or co-located mocks. + +## DEV SETUP + +```bash +export IN_DOCKER=false +export ENV_DIR=$PWD/tests/fixtures +export NODE_ENV=test +npm run compile && npm test +``` diff --git a/src/cmd/AGENTS.md b/src/cmd/AGENTS.md new file mode 100644 index 0000000000..91bba2b8d5 --- /dev/null +++ b/src/cmd/AGENTS.md @@ -0,0 +1,55 @@ +# src/cmd/ — CLI Commands + +OVERVIEW: Implementation of all `otomi` CLI commands using Yargs. + +| Command | File | Purpose | +| ------------------ | --------------------- | -------------------------------------------- | +| apply | apply.ts | Deploy charts via helmfile | +| apply-as-apps | apply-as-apps.ts | Deploy via ArgoCD (ArgoCD integration) | +| apply-teams | apply-teams.ts | Apply team-specific resources | +| bash | bash.ts | Interactive bash in container | +| bootstrap | bootstrap.ts | Initialize values repo | +| collect | collect.ts | Cluster diagnostics | +| commit | commit.ts | Validate + commit + push values | +| decrypt | decrypt.ts | Decrypt secrets files | +| destroy | destroy.ts | Destroy k8s resources | +| diff | diff.ts | Show diff before applying | +| encrypt | encrypt.ts | Encrypt secrets files | +| files | files.ts | Show values repo files | +| hf | hf.ts | Direct helmfile execution | +| install | install.ts | Full cluster installation | +| lint | lint.ts | Lint manifests via helmfile | +| migrate | migrate.ts | Migrate values between versions (1.4k lines) | +| playground | playground.ts | Dev playground | +| pull | pull.ts | Git pull + bootstrap | +| score-templates | score-templates.ts | Score template quality | +| server | server.ts | Server mode | +| status | status.ts | Show cluster status | +| sync | sync.ts | Sync k8s resources | +| template | template.ts | Export k8s resource templates | +| test | test.ts | Run cluster tests | +| traces | traces.ts | Collect failed resource traces | +| validate-cluster | validate-cluster.ts | Validate k8s version support | +| validate-templates | validate-templates.ts | Validate rendered manifests | +| validate-values | validate-values.ts | Validate user config against schema | +| values | values.ts | Render merged values | +| x | x.ts | Arbitrary helmfile execution | + +PATTERNS: + +- Export `CommandModule`: `{ command, describe, builder, handler }` +- Command registration: All files exported as array in `index.ts` +- Shared utilities: Call `../common/` for `hf.ts`, `values.ts`, `k8s.ts` +- Filtering: Use `-l/--label` for helmfile label selection + +WHERE TO LOOK: + +- `index.ts`: Command registry +- `migrate.ts`: Complex migration logic + functions +- `apply-as-apps.ts`: ArgoCD logic + +ANTI-PATTERNS: + +- Business logic in `handler`: Move to `../common/` +- Direct shell calls: Use `../common/zx-enhance.ts` +- Manual value merging: Use `../common/values.ts:getValues()` diff --git a/src/common/AGENTS.md b/src/common/AGENTS.md new file mode 100644 index 0000000000..b5cc3a5b8d --- /dev/null +++ b/src/common/AGENTS.md @@ -0,0 +1,37 @@ +# src/common/ — Shared Utilities + +## OVERVIEW + +Core logic for values orchestration, K8s/Helm wrappers, and platform state management used by CLI and Operator. + +## MODULE INVENTORY + +| Module | Purpose | +| -------------------- | ----------------------------------------------------------------- | +| `values.ts` | **PLATFORM ENGINE.** Merging, secrets, K8s detection, image tags. | +| `k8s.ts` | **GOD FILE.** K8s API client (Secrets, Apps, Helm, Exec, Patch). | +| `hf.ts` | Helmfile wrapper (`hf()`, `hfValues()`, `hfTemplate()`). | +| `repo.ts` | Values repository & team configuration file management. | +| `sealed-secrets.ts` | Encryption and manifest generation for Sealed Secrets. | +| `bootstrap.ts` | Initial environment/values repository setup logic. | +| `constants.ts` | File paths, environment variables, and platform constants. | +| `zx-enhance.ts` | Enhanced `zx` shell execution with robust error handling. | +| `runtime-upgrade.ts` | Migration runner for schema version upgrades. | +| `git-config.ts` | Git identity and authentication management. | +| `utils.ts` | Shared primitives (retry, sleep, object parsing). | + +## KEY MODULES & PATTERNS + +- **values.ts:** Central orchestrator for the 3-stage merge (Defaults -> User -> Derived). +- **k8s.ts:** Directly interacts with `@kubernetes/client-node`. Handles complex platform state like ArgoCD App reconciliation. +- **hf.ts:** Abstracts `helmfile` execution. Ensure `$ENV_DIR` is set before calling. +- **Dependency Flow:** `cmd/` & `operator/` -> `common/`. `common/` MUST NOT import from callers. +- **Constants:** Always use `constants.ts` for paths instead of hardcoded strings. + +## ANTI-PATTERNS + +- **Bootstrap Guard:** Never call `values.ts` functions within `bootstrap.ts` to avoid circular logic during init. +- **HF Naming:** `hf.ts` contains `withWorkloadValues`; treat as `withFiles` (pending rename). +- **Repo Workarounds:** `repo.ts` contains legacy "workloadValues" logic; avoid extending. +- **Async Safety:** Watch for `no-floating-promises` in entrypoints; ensure all `hf` calls are awaited. +- **Mutation:** Avoid `no-param-reassign` patterns found in `k8s.ts` when adding new methods. diff --git a/src/operator/AGENTS.md b/src/operator/AGENTS.md new file mode 100644 index 0000000000..f4baa454d6 --- /dev/null +++ b/src/operator/AGENTS.md @@ -0,0 +1,48 @@ +# src/operator/ — APL Operator + +## OVERVIEW + +GitOps-style operator managing platform installation and continuous reconciliation. +Drives CLI `apply` and `install` commands based on Git changes and periodic heartbeats. + +## EXECUTION FLOW + +1. **Bootstrap**: `main.ts` → `Installer.reconcileInstall()` (retries until success). +2. **Steady State**: `AplOperator.start()` launches parallel loops: + - **Poll (30s)**: `GitRepository` sync → Detect changes → `applyTeams()` or `apply()`. + - **Reconcile (5m)**: Forced `apply()` to ensure state consistency. +3. **Execution**: `AplOperations` wraps CLI commands (apply, install) using `runApplyIfNotBusy`. +4. **State**: `k8s.ts` updates ConfigMap heartbeats and apply status. +5. **Diagrams**: See `EXECUTION_FLOW.md` for detailed sequence diagrams. + +## KEY FILES + +| File | Role | +| ------------------- | --------------------------------------------------------------------- | +| `main.ts` | Entry point; switches from Install phase to GitOps phase. | +| `apl-operator.ts` | Core logic; manages Poll/Reconcile loops and concurrency locks. | +| `apl-operations.ts` | Integration layer; maps operator intent to CLI command handlers. | +| `installer.ts` | Finite state machine for initial platform deployment. | +| `git-repository.ts` | Git lifecycle; change detection (Teams-only vs. Global). | +| `k8s.ts` | Persistence; tracks operator health and apply results in K8s. | +| `validators.ts` | Bootstrapping; ensures environment variables and paths are valid. | +| `errors.ts` | Error hierarchy; specific types for Install vs. Operational failures. | +| `utils.ts` | Shared logic; logging decorators and async retry wrappers. | + +## PATTERNS + +- **Non-blocking loops**: Poll and Reconcile run independently; both use `isApplying` lock. +- **CLI Re-use**: Operator MUST call CLI logic via `AplOperations` to ensure consistency. +- **Granular Apply**: Differentiate between `applyTeams` (fast) and `apply` (full). +- **Heartbeat/Status**: Use `ConfigMaps` for observability instead of internal state. +- **Finite Retry**: `Installer` uses exponential backoff for the initial cluster setup. +- **Fail-fast Validation**: `validators.ts` checks environment before starting. + +## ANTI-PATTERNS + +- **CRD Watching**: Do NOT implement K8s controllers/watchers for configuration. +- **Stateful Logic**: Avoid keeping source-of-truth in memory; rely on Git/K8s. +- **Concurrent Applies**: Never bypass the `isApplying` lock in `apl-operator.ts`. +- **Direct Helmfile Calls**: Logic must pass through `AplOperations` command wrappers. +- **Silent Failures**: All errors must be wrapped in `OperatorError` or `InstallError`. +- **Mixing Phases**: Keep Installation logic strictly separate from GitOps reconciliation. From fdb0b8c0443065867ebee77afb323bb34cfb06f0 Mon Sep 17 00:00:00 2001 From: Jehoszafat Zimnowoda <17126497+j-zimnowoda@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:58:25 +0200 Subject: [PATCH 61/66] chore: add agents configuration --- values/AGENTS.md | 210 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 values/AGENTS.md diff --git a/values/AGENTS.md b/values/AGENTS.md new file mode 100644 index 0000000000..05d98f6499 --- /dev/null +++ b/values/AGENTS.md @@ -0,0 +1,210 @@ +# Values Directory — AI Agent Guide + +> Go template value files that configure Helm chart releases. Each subdirectory corresponds to a helmfile release. + +## Structure + +``` +values/ +├── / +│ ├── .gotmpl # Primary chart values (required) +│ ├── -raw.gotmpl # Raw K8s manifests via *raw anchor (optional) +│ ├── -otomi-db.gotmpl # Database chart values via *otomiDb anchor (optional) +│ ├── -cr.gotmpl # Custom resources via *rawCR anchor (optional) +│ └── -valkey.gotmpl # Valkey (Redis) sidecar values (optional) +├── jobs/ +│ └── scripts/ # Job script files +├── raw/ +│ └── istio-raw.gotmpl # Shared Istio raw resources +└── tests/ + └── connectivity-raw.gotmpl # Connectivity test manifests +``` + +**54 subdirectories** — one per app/component. + +## File Naming Conventions + +| Suffix | Helmfile Anchor | Chart Used | Purpose | +| ----------------------- | --------------- | ----------------- | ----------------------------- | +| `.gotmpl` | `*default` | `charts/` | Primary Helm values | +| `-raw.gotmpl` | `*raw` | `charts/raw` | Additional raw K8s manifests | +| `-cr.gotmpl` | `*rawCR` | `charts/raw-cr` | Custom resource manifests | +| `-otomi-db.gotmpl` | `*otomiDb` | `charts/otomi-db` | CloudNativePG database config | + +## Template Boilerplate + +Every `.gotmpl` file starts with standard variable bindings: + +```gotmpl +{{- $v := .Values }} # All merged values (defaults + user + derived) +{{- $a := $v.apps. }} # This app's config +{{- $k := $v.apps.keycloak }} # Keycloak config (if OIDC needed) +``` + +### Common Patterns + +```gotmpl +# Access derived values +$v._derived.untrustedCA # Whether CA is self-signed +$v._derived.oidcBaseUrl # Keycloak OIDC endpoint +$v._derived.tlsSecretName # Wildcard TLS secret name +$v._derived.consoleDomain # Console URL + +# Access cluster info +$v.cluster.domainSuffix # Base domain +$v.cluster.provider # Cloud provider + +# Safe key access with fallback +$a | get "some.key" "default" +$v | dig "deep" "path" "default" + +# Include shared templates +$httpRoute := tpl (readFile "../../helmfile.d/snippets/routes.gotmpl") $v | fromYaml + +# Conditional on app enabled +{{- if $v.apps.someApp.enabled }} + +# Image override for Linode LKE +{{- if $v.otomi.linodeLkeImageRepository }} +image: + repository: "{{ $v.otomi.linodeLkeImageRepository }}/registry/image" +{{- end }} + +# Node selector +{{- with $v.otomi | get "nodeSelector" nil }} +nodeSelector: + {{- range $key, $val := . }} + {{ $key }}: {{ $val }} + {{- end }} +{{- end }} +``` + +### Raw Template Pattern (`*-raw.gotmpl`) + +```gotmpl +resources: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: my-config + data: + key: value +``` + +## App Inventory + +### Core Infrastructure + +| Directory | Files | Notes | +| --------------------- | --------------------------------------------------------- | -------------------------- | +| `cert-manager` | `.gotmpl`, `-raw.gotmpl` | TLS certificate management | +| `istio-base` | `.gotmpl` | Istio CRDs | +| `istiod` | `.gotmpl` | Istio control plane | +| `istio-gateway` | `egressgateway.yaml.gotmpl`, `ingressgateway.yaml.gotmpl` | Istio gateways | +| `istio-resources` | `-raw.gotmpl` | Istio policies, peer auth | +| `ingress-nginx` | `.gotmpl`, `-raw.gotmpl` | NGINX ingress controller | +| `kubernetes-gateways` | `.gotmpl`, `-raw.gotmpl` | Gateway API resources | +| `sealed-secrets` | `.gotmpl`, `-raw.gotmpl` | Sealed secrets controller | +| `external-secrets` | `.gotmpl`, `-raw.gotmpl` | External secrets operator | + +### Identity & Auth + +| Directory | Files | Notes | +| -------------- | -------------------------------------------- | --------------------------- | +| `keycloak` | `.gotmpl`, `-raw.gotmpl`, `-otomi-db.gotmpl` | Identity provider | +| `oauth2-proxy` | `.gotmpl`, `-raw.gotmpl` | OAuth2 authentication proxy | + +### GitOps & CI/CD + +| Directory | Files | Notes | +| ---------------------- | -------------------------------------------------------------- | --------------------- | +| `argocd` | `.gotmpl`, `-raw.gotmpl` | GitOps deployment | +| `argocd-image-updater` | `.gotmpl` | Auto image updates | +| `gitea` | `.gotmpl`, `-raw.gotmpl`, `-otomi-db.gotmpl`, `-valkey.gotmpl` | Git server | +| `tekton-pipelines` | `.gotmpl`, `-raw.gotmpl` | CI/CD pipelines | +| `tekton-triggers` | `.gotmpl` | Tekton event triggers | +| `tekton-dashboard` | `.gotmpl`, `-raw.gotmpl`, `-teams.gotmpl` | Tekton UI | + +### Platform Operators + +| Directory | Files | Notes | +| ----------------------- | ------------------------ | ------------------------- | +| `apl-operator` | `.gotmpl`, `-raw.gotmpl` | APL platform operator | +| `apl-gitea-operator` | `.gotmpl`, `-raw.gotmpl` | Gitea repo/org management | +| `apl-harbor-operator` | `.gotmpl`, `-raw.gotmpl` | Harbor project management | +| `apl-keycloak-operator` | `.gotmpl`, `-raw.gotmpl` | Keycloak realm management | +| `apl-network-policies` | `.gotmpl` | Platform network policies | + +### Monitoring & Observability + +| Directory | Files | Notes | +| ------------------------------ | ---------------------------------------------------------------------------------------------------- | ---------------------- | +| `prometheus-operator` | `.gotmpl`, `-raw.gotmpl`, `-team.gotmpl`, `pod-monitors.gotmpl`, `service-monitors.gotmpl`, `rules/` | Full monitoring stack | +| `grafana-dashboards` | `.gotmpl` | Dashboard provisioning | +| `loki` | `.gotmpl`, `-raw.gotmpl`, `auth-config.gotmpl` | Log aggregation | +| `promtail` | `.gotmpl` | Log shipping | +| `otel-operator` | `.gotmpl`, `-raw.gotmpl` | OpenTelemetry | +| `prometheus-blackbox-exporter` | `.gotmpl` | Endpoint probing | +| `prometheus-msteams` | `.gotmpl` | MS Teams alert bridge | + +### Databases + +| Directory | Files | Notes | +| ------------------------------------ | ------------------------ | ---------------------- | +| `cloudnative-pg` | `.gotmpl`, `-raw.gotmpl` | CloudNativePG operator | +| `cloudnative-pg-plugin-barman-cloud` | `.gotmpl` | Backup plugin | + +### Optional Services + +| Directory | Files | Notes | +| -------------------- | -------------------------------------------- | -------------------------- | +| `harbor` | `.gotmpl`, `-raw.gotmpl`, `-otomi-db.gotmpl` | Container registry | +| `trivy-operator` | `.gotmpl` | Vulnerability scanning | +| `kyverno` | `.gotmpl`, `-raw.gotmpl` | Policy engine | +| `policy-reporter` | `.gotmpl` | Policy violation reporting | +| `knative-operator` | `.gotmpl` | Knative operator | +| `knative-serving` | `-cr.gotmpl`, `-raw.gotmpl` | Serverless workloads | +| `kserve` | `.gotmpl` | ML model serving | +| `kubeflow-pipelines` | `.gotmpl`, `-raw.gotmpl` | ML pipelines | +| `rabbitmq` | `.gotmpl` | Message broker | + +### Platform UI & API + +| Directory | Files | Notes | +| ----------------- | ------------------------ | -------------------- | +| `otomi-console` | `.gotmpl` | Platform web console | +| `otomi-api` | `.gotmpl`, `-raw.gotmpl` | Platform API | +| `otomi-pipelines` | `.gotmpl` | Pipeline definitions | + +### Multi-Tenancy + +| Directory | Files | Notes | +| --------- | ---------------------------------------- | --------------------------------------------------------------- | +| `team-ns` | `.gotmpl` | Team namespace config (RBAC, quotas, netpols, builds, services) | +| `k8s` | `k8s-raw.gotmpl`, `k8s-raw-teams.gotmpl` | Raw K8s resources for platform and teams | + +### Other + +| Directory | Files | Notes | +| ----------------------------- | ------------------------ | --------------------- | +| `external-dns` | `.gotmpl`, `-raw.gotmpl` | DNS record management | +| `metrics-server` | `.gotmpl` | K8s metrics API | +| `linode-cfw` | `.gotmpl` | Linode Cloud Firewall | +| `cert-manager-webhook-linode` | `.gotmpl` | Linode DNS01 solver | +| `gitea-db-secret` | `-raw.gotmpl` | Gitea database secret | + +## How to Add Values for a New App + +1. Create `values//.gotmpl` with standard boilerplate +2. Optionally create `-raw.gotmpl` for additional K8s resources +3. Reference in helmfile release using appropriate anchor (`*default`, `*raw`, etc.) +4. Ensure defaults exist in `helmfile.d/snippets/defaults.yaml` +5. Add schema in `values-schema.yaml` + +## Key Rules + +- **Never hardcode secrets** — use `external-secrets` or `x-secret` schema annotation +- **Never write derived values** — they're computed in `helmfile.d/snippets/derived.gotmpl` +- **Match release name** — directory name must match helmfile release name +- **Use `_rawValues`** — for chart values not covered by schema (escape hatch) +- **Relative paths** — snippets are referenced as `../../helmfile.d/snippets/...` From 4564f39f294126336b9cf4671126967bbfdd6677 Mon Sep 17 00:00:00 2001 From: Jehoszafat Zimnowoda <17126497+j-zimnowoda@users.noreply.github.com> Date: Mon, 4 May 2026 14:56:35 +0200 Subject: [PATCH 62/66] chore: improve agents.md --- AGENTS.md | 36 +++++++++++++++++++++++++++--------- package-lock.json | 41 ++++++++++++++++++++++++++++++++++------- 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6e40d764b9..ab1d353100 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,8 +52,7 @@ │ └── / # Vendored upstream charts (ingress-nginx, keycloak, harbor, etc.) ├── values/ # Go template value files per chart │ ├── /.gotmpl # Primary values template -│ ├── /-raw.gotmpl # Raw K8s manifest templates -│ └── jobs/.gotmpl # Job value templates +│ └── /-raw.gotmpl # Raw K8s manifest templates ├── values-schema.yaml # JSON Schema for ALL user-configurable parameters ├── core.yaml # Namespace definitions + admin/team app ingress config ├── apps.yaml # App metadata (versions, descriptions, dependencies) @@ -115,6 +114,30 @@ releases: ## 2. Key Concepts +### APL operator modes + +Operation modes: + +- installer (src/installer.ts) +- operator (src/operator/apl-operator.ts) + +In the installer mode: + +- kubernetes maniefts are rendered and applied by helmfile + +In the operator mode: + +- App deployment is delegated to ArgoCD controller +- The operator only renders the values (using helmfile) and applies ArgoCD application manifests to the Kubernetes cluster +- The operator perform upgrades + +Operation modes are set based on the status stored in the kubernetes config map (see `APL_OPERATOR_STATUS_CM` variable) + +### Secrets + +All secrets are stored as SealedSecrets in git repository. Platform secres are deployed to the apl-secrets namespace. +The External Secrets Operator (ESO) is used to propagate platform secrets to the right namespace and expected format. + ### App Enablement Apps are toggled via `apps..enabled` in user config. Defaults are in `helmfile.d/snippets/defaults.yaml`. Some apps are always enabled (derived in `derived.gotmpl`): `argocd`, `cert-manager`, `ingress-nginx`, `istio`, `keycloak`, `sealed-secrets`. @@ -135,13 +158,8 @@ Team namespaces follow the pattern `team-` and are managed by the `team- Admin apps are defined in `core.yaml` under `adminApps`. Team apps under `teamApps`. Each entry configures: -- `ownHost` — Gets its own subdomain (e.g., `grafana.`) -- `ingress[].svc/namespace/port` — Backend service details -- `ingress[].type` — `public` or `private` -- `ingress[].auth` — Enable OAuth2 proxy authentication -- `isShared` — Shared across teams - -Ingress uses Gateway API (`HTTPRoute`) with Istio as the gateway implementation + nginx as the ingress controller. +Ingress uses Gateway API (`HTTPRoute`) with Istio as the gateway implementation. +HTTPRoute binding to Gateway is set either in the `values//app.gotmpl` (if the corresponding charts/ delivers predefined routes) or oin the `values//-raw.gotmpl`. ### Multi-Tenancy diff --git a/package-lock.json b/package-lock.json index 253dc8873c..ed4ce75909 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "apl-core", - "version": "4.16.0-rc.0", + "version": "6.0.0-rc.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "apl-core", - "version": "4.16.0-rc.0", + "version": "6.0.0-rc.0", "license": "Apache-2.0", "dependencies": { "@apidevtools/json-schema-ref-parser": "15.3.5", @@ -200,6 +200,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -2491,6 +2492,7 @@ "integrity": "sha512-IQA++Idqb8fZzkCbHq3+T+9yG9WpeaBxomOrG2KcR/Pj0CgnovzuApYKL2cc35UWLePboKinMeqEPiweFpHVug==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=22.18.0" } @@ -2572,7 +2574,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-css/-/dict-css-4.1.1.tgz", "integrity": "sha512-y/Vgo6qY08e1t9OqR56qjoFLBCpi4QfWMf2qzD1l9omRZwvSMQGRPz4x0bxkkkU4oocMAeztjzCsmLew//c/8w==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-dart": { "version": "2.3.2", @@ -2712,14 +2715,16 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-html/-/dict-html-4.0.15.tgz", "integrity": "sha512-GJYnYKoD9fmo2OI0aySEGZOjThnx3upSUvV7mmqUu8oG+mGgzqm82P/f7OqsuvTaInZZwZbo+PwJQd/yHcyFIw==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-html-symbol-entities": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@cspell/dict-html-symbol-entities/-/dict-html-symbol-entities-4.0.5.tgz", "integrity": "sha512-429alTD4cE0FIwpMucvSN35Ld87HCyuM8mF731KU5Rm4Je2SG6hmVx7nkBsLyrmH3sQukTcr1GaiZsiEg8svPA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-java": { "version": "5.0.12", @@ -2917,7 +2922,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-typescript/-/dict-typescript-3.2.3.tgz", "integrity": "sha512-zXh1wYsNljQZfWWdSPYwQhpwiuW0KPW1dSd8idjMRvSD0aSvWWHoWlrMsmZeRl4qM4QCEAjua8+cjflm41cQBg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-vue": { "version": "3.0.5", @@ -4923,6 +4929,7 @@ "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -6530,7 +6537,8 @@ "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/json5": { "version": "0.0.29", @@ -6596,6 +6604,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.2.tgz", "integrity": "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -6795,6 +6804,7 @@ "integrity": "sha512-/Zb/xaIDfxeJnvishjGdcR4jmr7S+bda8PKNhRGdljDM+elXhlvN0FyPSsMnLmJUrVG9aPO6dof80wjMawsASg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.58.2", "@typescript-eslint/types": "8.58.2", @@ -7319,6 +7329,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -8200,6 +8211,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -10369,6 +10381,7 @@ "integrity": "sha512-hr4ihw+DBqcvrsEDioRO31Z17x71pUYoNe/4h6Z0wB72p7MU7/9gH8Q3s12NFhHPfYBBOV3qyfUxmr/Yn3shnQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", @@ -11762,6 +11775,7 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -11822,6 +11836,7 @@ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", "dev": true, "license": "MIT", + "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -11956,6 +11971,7 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -15367,6 +15383,7 @@ "integrity": "sha512-AkXIIFcaazymvey2i/+F94XRnM6TsVLZDhBMLsd1Sf/W0wzsvvpjeyUrCZD6HGG4SDYPgDJDBKeiJTBb10WzMg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "30.3.0", "@jest/types": "30.3.0", @@ -17054,6 +17071,7 @@ "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", "license": "MIT", + "peer": true, "engines": { "node": ">= 10.16.0" } @@ -18109,6 +18127,7 @@ "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", "dev": true, "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -21187,6 +21206,7 @@ "dev": true, "inBundle": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -22208,6 +22228,7 @@ "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -23209,6 +23230,7 @@ "integrity": "sha512-WRgl5GcypwramYX4HV+eQGzUbD7UUbljVmS+5G1uMwX/wLgYuJAxGeerXJDMO2xshng4+FXqCgyB5QfClV6WjA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@semantic-release/commit-analyzer": "^13.0.1", "@semantic-release/error": "^4.0.0", @@ -25013,6 +25035,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -25216,6 +25239,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -25513,6 +25537,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -25692,6 +25717,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "napi-postinstall": "^0.2.4" }, @@ -26194,6 +26220,7 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz", "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=10.0.0" }, From e282b875ba2675843005399c3614f9140ca59d7f Mon Sep 17 00:00:00 2001 From: Jehoszafat Zimnowoda <17126497+j-zimnowoda@users.noreply.github.com> Date: Mon, 4 May 2026 15:00:05 +0200 Subject: [PATCH 63/66] fix: restore files --- helmfile.d/helmfile-60.teams.yaml.gotmpl | 2 -- values/ingress-nginx/ingress-nginx-raw.gotmpl | 32 ------------------- 2 files changed, 34 deletions(-) diff --git a/helmfile.d/helmfile-60.teams.yaml.gotmpl b/helmfile.d/helmfile-60.teams.yaml.gotmpl index e081c6ae5d..767d1a3ce6 100644 --- a/helmfile.d/helmfile-60.teams.yaml.gotmpl +++ b/helmfile.d/helmfile-60.teams.yaml.gotmpl @@ -24,8 +24,6 @@ releases: {{- $grafanaHostname := printf "grafana-%s.%s" $teamId $domain }} {{- $tektonHostname := printf "tekton-%s.%s" $teamId $domain }} {{- $teamApps := index $tc $teamId "apps" | default dict }} - {{- $teamReceivers := $teamSettings | get "alerts.receivers" ($v | get "alerts.receivers" (list "slack")) }} - {{- $teamAlertmanagerConfig := tpl (readFile "../helmfile.d/snippets/alertmanager-teams.gotmpl") (dict "instance" $teamSettings "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | toString }} - name: tekton-dashboard-{{ $teamId }} installed: {{ $a | get "tekton.enabled" }} namespace: team-{{ $teamId }} diff --git a/values/ingress-nginx/ingress-nginx-raw.gotmpl b/values/ingress-nginx/ingress-nginx-raw.gotmpl index 366b4d6389..7ee4eef902 100644 --- a/values/ingress-nginx/ingress-nginx-raw.gotmpl +++ b/values/ingress-nginx/ingress-nginx-raw.gotmpl @@ -13,36 +13,4 @@ resources: {{- end }} spec: controller: "k8s.io/{{ $ingress.className }}" -{{- end }} -# ClusterRole to allow ingress controller to read TLS secrets from all namespaces -{{- range $ingress := $v.ingress.classes }} -- apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: ingress-nginx-{{ $ingress.className }}-secrets-reader - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx-{{ $ingress.className }} - rules: - - apiGroups: - - "" - resources: - - secrets - verbs: - - get -- apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: ingress-nginx-{{ $ingress.className }}-secrets-reader - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx-{{ $ingress.className }} - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx-{{ $ingress.className }}-secrets-reader - subjects: - - kind: ServiceAccount - name: ingress-nginx-{{ $ingress.className }} - namespace: ingress {{- end }} \ No newline at end of file From d8a09b0a268f82d32603d1feadcb8a7a03be23a6 Mon Sep 17 00:00:00 2001 From: Jehoszafat Zimnowoda <17126497+j-zimnowoda@users.noreply.github.com> Date: Mon, 4 May 2026 15:02:53 +0200 Subject: [PATCH 64/66] chore: improve agents.md --- src/common/git-config.ts | 5 -- src/common/repo.ts | 101 +-------------------------------- src/operator/installer.test.ts | 13 ----- versions.yaml | 8 +-- 4 files changed, 6 insertions(+), 121 deletions(-) diff --git a/src/common/git-config.ts b/src/common/git-config.ts index 2ec014d884..812fafcb43 100644 --- a/src/common/git-config.ts +++ b/src/common/git-config.ts @@ -48,11 +48,6 @@ export async function getGitCredentials(): Promise { return undefined } - // Reject unresolved sealed-secret placeholders (e.g. during first deploy before secrets are decrypted) - if (typeof secretData.password === 'string' && secretData.password.startsWith('sealed:')) { - return undefined - } - return { username: secretData.username, password: secretData.password, diff --git a/src/common/repo.ts b/src/common/repo.ts index 5f7177b514..ab89e6060b 100644 --- a/src/common/repo.ts +++ b/src/common/repo.ts @@ -4,7 +4,7 @@ import { globSync } from 'glob' import jsonpath from 'jsonpath' import { cloneDeep, get, merge, omit, set } from 'lodash' import path from 'path' -import { getDirNames, getSchemaSecretsPaths, loadYaml } from './utils' +import { getDirNames, loadYaml } from './utils' import { objectToYaml, writeValuesToFile } from './values' export async function getTeamNames(envDir: string): Promise> { @@ -559,10 +559,7 @@ export function unsetValuesFileSync(envDir: string): string { return valuesPath } -export async function loadValues( - envDir: string, - deps = { loadToSpec, loadSealedSecretsToSpec }, -): Promise> { +export async function loadValues(envDir: string, deps = { loadToSpec }): Promise> { const fileMaps = getFileMaps(envDir).filter((map) => map.loadToSpec === true) const spec = {} @@ -571,105 +568,11 @@ export async function loadValues( await deps.loadToSpec(spec, fileMap) }), ) - await deps.loadSealedSecretsToSpec(spec, envDir) sortTeamConfigArraysByName(spec) sortUserArraysByName(spec) return spec } -/** - * Read sealed secret manifests and merge their encryptedData back into the values spec. - * This restores secret values that helmfile templates need at render time. - * - * Uses the values schema (x-secret paths) to correctly map sealed secret data keys - * back to their original dot-paths, since some property names contain underscores - * (e.g., smtp.auth_password) that should NOT be converted to nested paths. - */ -export async function loadSealedSecretsToSpec( - spec: Record, - envDir: string, - deps = { loadYaml, getSchemaSecretsPaths }, -): Promise { - const sealedSecretsGlob = `${envDir}/env/manifests/namespaces/*/sealedsecrets/*.yaml` - const files = globSync(sealedSecretsGlob, { nodir: true }) - if (files.length === 0) return - - // Get team names from spec to expand teamConfig.* paths - const teams = Object.keys(get(spec, 'teamConfig', {})) - const secretPaths = await deps.getSchemaSecretsPaths(teams) - - // Build a lookup: for each group prefix + dataKey → full schema path - // e.g., "apps.harbor" + "core_secret" → "apps.harbor.core.secret" - const dataKeyToPath = buildDataKeyToPathMap(secretPaths) - - for (const filePath of files) { - const manifest = (await deps.loadYaml(filePath)) as Record | undefined - if (!manifest?.spec?.encryptedData) continue - - const { name: secretName = '' } = (manifest.metadata ?? {}) as { name?: string } - const { encryptedData } = manifest.spec as { encryptedData: Record } - - // Determine target path in spec from the secret name - const targetPath = resolveSecretTargetPath(secretName, spec) - if (!targetPath) continue - - for (const [dataKey, value] of Object.entries(encryptedData)) { - const lookupKey = `${targetPath}/${dataKey}` - const fullPath = dataKeyToPath.get(lookupKey) - if (fullPath) { - set(spec, fullPath, value) - } - } - } -} - -/** - * Build a map from "groupPrefix/dataKey" → "full.schema.path". - * The dataKey is derived from the relative path by replacing dots with underscores, - * matching the convention used in buildSecretToNamespaceMap (sealed-secrets.ts). - */ -function buildDataKeyToPathMap(secretPaths: string[]): Map { - const result = new Map() - for (const secretPath of secretPaths) { - const groupPrefix = findGroupPrefix(secretPath) - if (!groupPrefix) continue - const relativePath = secretPath.slice(groupPrefix.length + 1) - if (!relativePath) continue - const dataKey = relativePath.replace(/\./g, '_') - result.set(`${groupPrefix}/${dataKey}`, secretPath) - } - return result -} - -/** - * Find the group prefix for a secret path — mirrors the logic in sealed-secrets.ts. - */ -function findGroupPrefix(secretPath: string): string | undefined { - const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) - if (teamMatch) return `teamConfig.${teamMatch[1]}` - const appsMatch = secretPath.match(/^apps\.([^.]+)/) - if (appsMatch) return `apps.${appsMatch[1]}` - const [firstSegment] = secretPath.split('.') - if (firstSegment && firstSegment !== 'kms' && firstSegment !== 'users') return firstSegment - return undefined -} - -function resolveSecretTargetPath(secretName: string, spec: Record): string | undefined { - // team-{name}-settings-secrets → teamConfig.{name} - const teamMatch = secretName.match(/^team-(.+)-settings-secrets$/) - if (teamMatch) return `teamConfig.${teamMatch[1]}` - - // {name}-secrets → apps.{name} or {name} - const nameMatch = secretName.match(/^(.+)-secrets$/) - if (!nameMatch) return undefined - - const [, name] = nameMatch - if (get(spec, `apps.${name}`) !== undefined) return `apps.${name}` - if (get(spec, name) !== undefined) return name - - return undefined -} - export function extractTeamDirectory(filePath: string): string { const match = filePath.match(/\/teams\/([^/]+)/) if (match === null) throw new Error(`Cannot extract team name from ${filePath} string`) diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index d1c6b7d9fb..d5f4aed1d0 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -314,19 +314,6 @@ describe('Installer', () => { // Should assume installed when verification can't be performed expect(state.isInstalled).toBe(true) }) - - test('should return true when git verification fails (gitea not ready)', async () => { - ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ - data: { status: 'completed' }, - }) - // getStoredGitRepoConfig throws (cluster issues) - ;(gitConfig.getStoredGitRepoConfig as jest.Mock).mockRejectedValue(new Error('connection refused')) - - const isInstalled = await installer.isInstalled() - - // Should assume installed when verification can't be performed - expect(isInstalled).toBe(true) - }) }) describe('setEnvAndCreateSecrets', () => { diff --git a/versions.yaml b/versions.yaml index ebf3ce350b..f3c2a59e11 100644 --- a/versions.yaml +++ b/versions.yaml @@ -1,6 +1,6 @@ -api: APL-523 -console: APL-523 -consoleLogin: APL-523 -tasks: APL-523 +api: main +console: main +consoleLogin: main +tasks: main tools: main aplCharts: main From 6ce26a1c06c0fa7172e9be43b33ce14966478556 Mon Sep 17 00:00:00 2001 From: Jehoszafat Zimnowoda <17126497+j-zimnowoda@users.noreply.github.com> Date: Mon, 4 May 2026 15:08:38 +0200 Subject: [PATCH 65/66] chore: improve agents.md --- .github/copilot-instructions.md | 157 +++----------------------------- AGENTS.md | 2 +- 2 files changed, 16 insertions(+), 143 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 46d8a14e4a..bc1cc635b8 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -4,145 +4,18 @@ APL Core (App Platform for Linode) is a Kubernetes platform that integrates 30+ cloud-native applications (Istio, Argo CD, Keycloak, Tekton, Harbor, etc.) into a cohesive, multi-tenant PaaS. The codebase is a hybrid of TypeScript (CLI/operators), Helm charts, Helmfile manifests, and Go templates. -**Core Architecture:** User configuration (`env/` directory) → Helmfile bases → Helmfile releases → Helm charts → Kubernetes manifests - -## Critical Development Patterns - -### Values Flow (3-Stage Merge) - -Values are loaded in a strict 3-stage pipeline (see [ADR-2021-10-18](../adr/2021-10-18-defaults-and-derived.md)): - -1. **Defaults** (`helmfile.d/snippets/defaults.yaml`) - Static defaults, will eventually come from schema -2. **User Input** (`$ENV_DIR/env/**/*.yaml`) - User-provided configuration (NEVER write defaults/derived values here) -3. **Derived Values** (`helmfile.d/snippets/derived.gotmpl`) - Computed from defaults + user input - -**Critical Rule:** User input directory (`$ENV_DIR`) contains ONLY user-supplied values. Defaults and derived values must never be written back to `$ENV_DIR`. - -### Helmfile Release Patterns - -All Helmfile specs in `helmfile.d/` execute alphabetically. Use reusable anchors from `helmfile.d/snippets/templates.gotmpl`: - -- `*default` - Standard chart deployment. Values merged: `charts/{name}/values.yaml` → `values/{name}/{name}.gotmpl` → `.Values.apps.{name}._rawValues` -- `*raw` - Deploy additional K8s manifests (operators + CRs) from `values/{name}/{name}-raw.gotmpl` -- `*rawCR` - Deploy custom resources using the `raw-cr` chart -- `*jobs` - Deploy jobs to the `maintenance` namespace using `values/jobs/{name}.gotmpl` - -### Schema-Driven Validation - -All user-configurable parameters MUST be defined in `values-schema.yaml` (JSON Schema). Run `npm run validate-values` to validate. The schema serves as both validation and documentation. - -## CLI Commands & Workflow - -### Essential Commands - -```bash -# Bootstrap a new values repo (creates $ENV_DIR with defaults) -otomi bootstrap - -# Validate user configuration against schema -otomi validate-values - -# Validate rendered Kubernetes manifests -otomi validate-templates [-l name=myapp] - -# Render values for inspection -otomi values - -# Render chart values for a specific app -otomi x helmfile -l name=myapp write-values - -# Deploy all charts (or use -l name=myapp for selective deploy) -otomi apply [-l name=myapp] - -# Generate diff before applying -otomi diff [-l name=myapp] - -# Deploy to cluster (initial setup) -otomi install -``` - -### Development Setup - -```bash -# Install dependencies (helmfile, helm, kubectl, etc.) -npm run install-deps - -# Run CLI locally (bypass Docker) -export IN_DOCKER=false -export ENV_DIR=$PWD/tests/fixtures -export NODE_ENV=test - -# Compile TypeScript -npm run compile - -# Run tests -npm test -``` - -## Integrating a New Core App - -1. **Add Helm chart** to `charts/{myapp}/` (or vendor from upstream) -2. **Create values template** at `values/{myapp}/{myapp}.gotmpl` -3. **Define Helmfile release** in appropriate `helmfile.d/helmfile-*.yaml` file: - ```yaml - releases: - - name: myapp - installed: {{ .Values.apps.myapp.enabled }} - namespace: my-namespace - <<: *default # or *raw, *rawCR, *jobs - ``` -4. **Add schema** for user-configurable properties in `values-schema.yaml` under `.definitions.apps.properties.myapp` -5. **Configure defaults** in `helmfile.d/snippets/defaults.yaml` under `apps.myapp` -6. **Add namespace** (if needed) to `core.yaml` at `k8s.namespaces` -7. **Configure ingress** (if needed) in `core.yaml` at `adminApps` or `teamApps` - -## Docker-Based Execution - -The `binzx/otomi` script wraps all commands in Docker by default: - -- Uses `linode/apl-core:${otomi_version}` image -- Mounts `$ENV_DIR` as `/home/app/env/` -- Set `IN_DOCKER=false` to run locally (useful for cloud provider auth plugins) - -## Testing Strategy - -- Unit tests: `npm test` (Jest, located in `src/**/*.test.ts`) -- Integration tests: Use fixtures in `tests/fixtures/` with `NODE_ENV=test` -- Template validation: `otomi validate-templates` (validates all rendered manifests against K8s schemas) -- Policy tests: `npm run test:opa` (Rego policy testing) - -## Key Files & Directories - -| Path | Purpose | -| ---------------------- | -------------------------------------------- | -| `src/cmd/*.ts` | CLI command implementations | -| `helmfile.d/` | Helmfile specs (execute alphabetically) | -| `helmfile.d/snippets/` | Reusable templates, defaults, derived values | -| `charts/` | Helm charts (vendored and custom) | -| `values/` | Value templates for each chart | -| `values-schema.yaml` | JSON Schema for user configuration | -| `core.yaml` | Namespaces, ingress, team apps config | -| `binzx/otomi` | Bash wrapper for Docker-based execution | -| `adr/` | Architectural Decision Records | - -## Common Gotchas - -- **Helmfile labels:** Use `-l name=myapp` to select specific releases (not `-l app=myapp`) -- **Raw values override:** Use `apps.{name}._rawValues` to override chart values not in schema (use sparingly) -- **YAML anchors:** Search for `&anchorname` to find anchor definitions when you see `<<: *anchorname` -- **Keycloak integration:** Use `_derived.oidcBaseUrl`, `apps.keycloak.idp.clientID/clientSecret` for SSO -- **Untrusted CA:** Check `_derived.untrustedCA` to conditionally disable cert verification - -## Debugging Tips - -- Check deployment state: `otomi status` -- View traces on errors: Collected automatically in `otomi apply` failures -- Inspect Helmfile output: `otomi x helmfile -l name=myapp template` -- Local development: Use `$PWD/tests/fixtures` as `$ENV_DIR` -- Enable verbose logging: Add `-v` flag to any command - -## References - -- Full development guide: [docs/development.md](../docs/development.md) -- Architectural decisions: [adr/index.md](../adr/index.md) -- Public docs: https://techdocs.akamai.com/app-platform/docs/welcome +## Knowledge Base + +Use AGENTS.md files as your primary reference for understanding the codebase structure, conventions, and critical patterns. Each AGENTS.md file provides a comprehensive overview of its respective directory. + +| Path | Focus | +| ---------------------------------------------------------------- | --------------------------------------------------- | +| [`AGENTS.md`](AGENTS.md) | High level design | +| [`src/AGENTS.md`](src/AGENTS.md) | TypeScript source structure, conventions, dev setup | +| [`src/cmd/AGENTS.md`](src/cmd/AGENTS.md) | CLI command inventory, patterns | +| [`src/common/AGENTS.md`](src/common/AGENTS.md) | Shared utility modules, dependency graph | +| [`src/operator/AGENTS.md`](src/operator/AGENTS.md) | GitOps operator architecture, execution flow | +| [`helmfile.d/AGENTS.md`](helmfile.d/AGENTS.md) | Helmfile release phases, execution order | +| [`helmfile.d/snippets/AGENTS.md`](helmfile.d/snippets/AGENTS.md) | Critical templates, defaults, derived values | +| [`charts/AGENTS.md`](charts/AGENTS.md) | Custom vs vendored chart inventory | +| [`charts/team-ns/AGENTS.md`](charts/team-ns/AGENTS.md) | Team namespace chart (most complex) | diff --git a/AGENTS.md b/AGENTS.md index ab1d353100..1395bd96fd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # APL Core — AI Agent Index -> **Akamai App Platform (APL)** is a Kubernetes PaaS that integrates 30+ cloud-native applications into a cohesive, multi-tenant platform. +> **App Platform (APL)** is a Kubernetes PaaS that integrates 30+ cloud-native applications into a cohesive, multi-tenant platform. > This index provides the context AI agents need to perform software development tasks efficiently. **Stack:** TypeScript CLI + Kubernetes Operator + Helmfile/Helm + Go Templates From 67ad9be3fe968c233835665a334ebb90205b5e86 Mon Sep 17 00:00:00 2001 From: Jehoszafat Zimnowoda <17126497+j-zimnowoda@users.noreply.github.com> Date: Mon, 4 May 2026 15:17:02 +0200 Subject: [PATCH 66/66] chore: improve agents.md --- AGENTS.md | 26 +++++++++++++------------- charts/team-ns/AGENTS.md | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1395bd96fd..4a1b2bed01 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -200,7 +200,7 @@ Current schema version: **60** (see `versions.specVersion` in defaults.yaml) ## 3. Agent Definitions -### Agent: Network Policy Definor +### Agent: Network Policy Expert **Scope:** Create, modify, or troubleshoot Kubernetes NetworkPolicies for platform applications and teams. @@ -286,15 +286,15 @@ Current schema version: **60** (see `versions.specVersion` in defaults.yaml) --- -### Agent: Ingress Configurator +### Agent: Ingress Expert -**Scope:** Configure ingress routes, gateways, HTTPRoutes, OAuth2 authentication, and domain management. +**Scope:** Configure gateways, HTTPRoutes, OAuth2 authentication, and domain management. **Key Files:** | File | Purpose | |------|---------| -| `core.yaml` → `adminApps` | Admin app ingress definitions | -| `core.yaml` → `teamApps` | Team app ingress definitions | +| `core.yaml` → `adminApps` | Deprecated | +| `core.yaml` → `teamApps` | Deprecated | | `helmfile.d/snippets/routes.gotmpl` | HTTPRoute template (parentRefs, auth rules) | | `helmfile.d/snippets/authpolicy-oauth2-ext.gotmpl` | OAuth2 external auth policy template | | `helmfile.d/snippets/authpolicy-jwt.gotmpl` | JWT authentication policy template | @@ -302,16 +302,16 @@ Current schema version: **60** (see `versions.specVersion` in defaults.yaml) | `helmfile.d/snippets/derived.gotmpl` | Computed domain names, gateway names, TLS | | `helmfile.d/snippets/domains.gotmpl` | Domain configuration helpers | | `charts/team-ns/templates/routes.yaml` | Team service route rendering | -| `charts/team-ns/templates/ingress.yaml` | Team ingress resources | +| `charts/team-ns/templates/ingress.yaml` | Deprecated | | `charts/kubernetes-gateways/` | Gateway API gateway definitions | -| `charts/istio-gateway/` | Istio-specific gateway chart | -| `charts/ingress-nginx/` | NGINX ingress controller chart | -| `values/ingress-nginx/` | NGINX ingress values | -| `values/istio-gateway/` | Istio gateway values | +| `charts/istio-gateway/` | Deprecated | +| `charts/ingress-nginx/` | Deprecated | +| `values/ingress-nginx/` | Deprecated| +| `values/istio-gateway/` | Deprecated | | `values/kubernetes-gateways/` | Gateway API values | | `values-schema.yaml` → `definitions.service` | Service/ingress schema | | `values-schema.yaml` → `definitions.ingressClassParameters` | Ingress class config | -| `values-schema.yaml` → `properties.ingress` | Platform ingress schema | +| `values-schema.yaml` → `properties.ingress` | Platform ingress class schema | **Concepts:** @@ -374,7 +374,7 @@ apps: | Definition | Purpose | |------------|---------| | `resources` | CPU/memory requests+limits | -| `rawValues` | Escape-hatch for unschema'd chart values | +| `rawValues` | define collection of raw Kubernetes manifests | | `image` / `imageSimple` | Container image config | | `idName` | Lowercase DNS-safe name pattern | | `domain` | Domain pattern | @@ -614,7 +614,7 @@ changes: --- -### Agent: Monitoring Stack Configurator +### Agent: Monitoring Stack Expert **Scope:** Configure Prometheus, Grafana, Alertmanager, Loki, and OpenTelemetry. diff --git a/charts/team-ns/AGENTS.md b/charts/team-ns/AGENTS.md index 9252a96cf9..e470ded6df 100644 --- a/charts/team-ns/AGENTS.md +++ b/charts/team-ns/AGENTS.md @@ -9,7 +9,7 @@ networking, security policies, and CI/CD (ArgoCD + Tekton) integration. - `_helpers.tpl` — Label logic, domain math, Docker/volume config generation. - `rbac.yaml` — Massive policy (SAs: team, kubectl, tekton; RoleBindings). - `routes.yaml` — Gateway API `HTTPRoute` for team services. - - `ingress.yaml` — Legacy/Standard Ingress resources. + - `ingress.yaml` — Legacy/Standard Ingress resources. (Deprecated) - `limitrange.yaml` — Container resource defaults. - `quota.yaml` — Team resource constraints (skipped for `team-admin`). - `argocd/` — Team Apps/Projects for GitOps lifecycle.