From db921d70b61f0499c4e01fc76804dc40da513109 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 4 Feb 2026 16:13:09 +0100 Subject: [PATCH 01/20] test: sealed secrets --- src/cmd/bootstrap.test.ts | 11 +- src/cmd/bootstrap.ts | 13 +- src/common/sealed-secrets.test.ts | 413 ++++++++++++++++++++++++++++++ src/common/sealed-secrets.ts | 375 +++++++++++++++++++++++++++ 4 files changed, 802 insertions(+), 10 deletions(-) create mode 100644 src/common/sealed-secrets.test.ts create mode 100644 src/common/sealed-secrets.ts diff --git a/src/cmd/bootstrap.test.ts b/src/cmd/bootstrap.test.ts index 4d61225a86..f1f469abd2 100644 --- a/src/cmd/bootstrap.test.ts +++ b/src/cmd/bootstrap.test.ts @@ -12,6 +12,8 @@ import { processValues, } from './bootstrap' +jest.mock('@linode/kubeseal-encrypt') + const { terminal } = stubs jest.mock('src/common/envalid', () => ({ @@ -37,6 +39,7 @@ describe('Bootstrapping values', () => { }), }), bootstrapSops: jest.fn(), + bootstrapSealedSecrets: jest.fn(), copyBasicFiles: jest.fn(), copyFile: jest.fn(), createCustomCA: jest.fn(), @@ -59,14 +62,14 @@ describe('Bootstrapping values', () => { } }) it('should call relevant sub routines', async () => { - deps.processValues.mockReturnValue(values) + deps.processValues.mockReturnValue({ originalInput: values, allSecrets: {} }) deps.hfValues.mockReturnValue(values) await bootstrap(deps) expect(deps.copyBasicFiles).toHaveBeenCalled() - expect(deps.bootstrapSops).toHaveBeenCalled() + expect(deps.bootstrapSealedSecrets).toHaveBeenCalled() }) it('should copy only skeleton files to env dir if it is empty or nonexisting', async () => { - deps.processValues.mockReturnValue(undefined) + deps.processValues.mockReturnValue({ originalInput: undefined, allSecrets: {} }) await bootstrap(deps) expect(deps.hfValues).toHaveBeenCalledTimes(0) }) @@ -385,7 +388,7 @@ describe('Bootstrapping values', () => { { id: 'user2', initialPassword: 'generated-password' }, ], }) - expect(res).toEqual({ + expect(res.originalInput).toEqual({ cluster: { name: 'bla', provider: 'dida' }, users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], }) diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index 9c30aea8f2..d2aecf9cfa 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -14,6 +14,7 @@ import { env, isCli } from 'src/common/envalid' import { hfValues } from 'src/common/hf' import { createK8sSecret, getDeploymentState, getK8sSecret, secretId } from 'src/common/k8s' import { getKmsSettings } from 'src/common/repo' +import { bootstrapSealedSecrets } from 'src/common/sealed-secrets' import { ensureTeamGitOpsDirectories, getFilename, gucci, isCore, loadYaml, rootDir } from 'src/common/utils' import { generateSecrets, writeValues } from 'src/common/values' import { BasicArguments, setParsedArgs } from 'src/common/yargs' @@ -291,7 +292,7 @@ export const processValues = async ( addInitialPasswords, addPlatformAdmin, }, -): Promise> => { +): Promise<{ originalInput: Record; allSecrets: Record }> => { const d = deps.terminal(`cmd:${cmdName}:processValues`) const { VALUES_INPUT } = env d.log(`Loading app values from ${VALUES_INPUT}`) @@ -324,7 +325,7 @@ export const processValues = async ( // and do some context dependent post processing: // to support potential failing chart install we store secrets on cluster if (!(env.isDev && env.DISABLE_SYNC)) await deps.createK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, 'otomi', allSecrets) - return originalInput + return { originalInput, allSecrets } } // create file structure based on file entry @@ -419,6 +420,7 @@ export const bootstrap = async ( hfValues, writeValues, bootstrapSops, + bootstrapSealedSecrets, migrate, encrypt, decrypt, @@ -434,10 +436,10 @@ export const bootstrap = async ( } await deps.copyBasicFiles() await deps.migrate() - const originalValues = await deps.processValues() + const { originalInput, allSecrets } = await deps.processValues() await deps.handleFileEntry() - await deps.bootstrapSops() - await ensureTeamGitOpsDirectories(ENV_DIR, originalValues) + await deps.bootstrapSealedSecrets(allSecrets, ENV_DIR) + await ensureTeamGitOpsDirectories(ENV_DIR, originalInput) d.log(`Done bootstrapping values`) } @@ -456,7 +458,6 @@ export const module = { handler: async (argv: BasicArguments): Promise => { setParsedArgs(argv) await prepareEnvironment({ skipAllPreChecks: true }) - await decrypt() await bootstrap() await bootstrapGit() }, diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts new file mode 100644 index 0000000000..29d937ed4a --- /dev/null +++ b/src/common/sealed-secrets.test.ts @@ -0,0 +1,413 @@ +import { pki } from 'node-forge' +import stubs from 'src/test-stubs' +import { + APP_NAMESPACE_MAP, + bootstrapSealedSecrets, + buildSecretToNamespaceMap, + createSealedSecretManifest, + createSealedSecretsKeySecret, + generateSealedSecretsKeyPair, + getPemFromCertificate, + writeSealedSecretManifests, +} from './sealed-secrets' + +const { terminal } = stubs + +jest.mock('@linode/kubeseal-encrypt', () => ({ + encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), +})) + +jest.mock('zx', () => ({ + $: jest.fn().mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ stderr: '', exitCode: 0 }), + }), + }), +})) + +jest.mock('src/common/envalid', () => ({ + env: {}, +})) + +describe('sealed-secrets', () => { + describe('generateSealedSecretsKeyPair', () => { + it('should generate a valid key pair with certificate and private key', () => { + const mockCert = { + publicKey: {}, + serialNumber: '', + validity: { notBefore: new Date(), notAfter: new Date() }, + sign: jest.fn(), + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + } + const mockKeys = { + publicKey: { n: {}, e: {} }, + privateKey: { d: {}, p: {}, q: {} }, + } + const deps = { + terminal, + pki: { + rsa: { generateKeyPair: jest.fn().mockReturnValue(mockKeys) }, + createCertificate: jest.fn().mockReturnValue(mockCert), + certificateToPem: jest.fn().mockReturnValue('-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n'), + privateKeyToPem: jest + .fn() + .mockReturnValue('-----BEGIN RSA PRIVATE KEY-----\nfake\n-----END RSA PRIVATE KEY-----\n'), + } as unknown as typeof pki, + } + + const result = generateSealedSecretsKeyPair(deps) + + expect(deps.pki.rsa.generateKeyPair).toHaveBeenCalledWith(4096) + expect(deps.pki.createCertificate).toHaveBeenCalled() + expect(mockCert.sign).toHaveBeenCalled() + expect(result.certificate).toContain('BEGIN CERTIFICATE') + expect(result.privateKey).toContain('BEGIN RSA PRIVATE KEY') + }) + + it('should set 10-year validity', () => { + const mockCert = { + publicKey: {}, + serialNumber: '', + validity: { notBefore: new Date(), notAfter: new Date() }, + sign: jest.fn(), + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + } + const mockKeys = { + publicKey: {}, + privateKey: {}, + } + const deps = { + terminal, + pki: { + rsa: { generateKeyPair: jest.fn().mockReturnValue(mockKeys) }, + createCertificate: jest.fn().mockReturnValue(mockCert), + certificateToPem: jest.fn().mockReturnValue('cert'), + privateKeyToPem: jest.fn().mockReturnValue('key'), + } as unknown as typeof pki, + } + + generateSealedSecretsKeyPair(deps) + + const notBefore = mockCert.validity.notBefore.getFullYear() + const notAfter = mockCert.validity.notAfter.getFullYear() + expect(notAfter - notBefore).toBe(10) + }) + }) + + describe('getPemFromCertificate', () => { + it('should extract SPKI public key from a certificate', () => { + // Generate a real key pair and certificate for this test + const keys = pki.rsa.generateKeyPair(2048) + const cert = pki.createCertificate() + cert.publicKey = keys.publicKey + cert.serialNumber = '01' + cert.validity.notBefore = new Date() + cert.validity.notAfter = new Date() + cert.validity.notAfter.setFullYear(cert.validity.notBefore.getFullYear() + 1) + const attrs = [{ name: 'commonName', value: 'test' }] + cert.setSubject(attrs) + cert.setIssuer(attrs) + cert.sign(keys.privateKey) + const certPem = pki.certificateToPem(cert) + + const result = getPemFromCertificate(certPem) + + expect(result).toContain('BEGIN PUBLIC KEY') + expect(result).toContain('END PUBLIC KEY') + }) + }) + + describe('createSealedSecretsKeySecret', () => { + it('should call kubectl to create namespace and TLS secret', async () => { + const mockResult = { stderr: '', exitCode: 0 } + const mockQuiet = jest.fn().mockResolvedValue(mockResult) + const mockNothrow = jest.fn().mockReturnValue({ quiet: mockQuiet }) + const mock$ = jest.fn().mockReturnValue({ nothrow: mockNothrow }) + const deps = { + $: mock$ as any, + terminal, + writeFile: jest.fn(), + mkdir: jest.fn(), + } + + await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) + + // Should have been called 3 times: namespace creation, secret creation, labeling + expect(mock$).toHaveBeenCalledTimes(3) + expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.crt', 'cert-pem') + expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.key', 'key-pem') + }) + }) + + describe('buildSecretToNamespaceMap', () => { + it('should group secrets by namespace and secret name', async () => { + const secrets = { + apps: { + harbor: { adminPassword: 'harbor-pass', secretKey: 'harbor-secret' }, + gitea: { adminPassword: 'gitea-pass' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue([ + 'apps.harbor.adminPassword', + 'apps.harbor.secretKey', + 'apps.gitea.adminPassword', + ]), + } + + const result = await buildSecretToNamespaceMap(secrets, [], deps) + + expect(result).toHaveLength(2) + + const harborMapping = result.find((m) => m.namespace === 'harbor') + expect(harborMapping).toBeDefined() + expect(harborMapping!.secretName).toBe('harbor-secrets') + expect(harborMapping!.data).toHaveProperty('apps_harbor_adminPassword', 'harbor-pass') + expect(harborMapping!.data).toHaveProperty('apps_harbor_secretKey', 'harbor-secret') + + const giteaMapping = result.find((m) => m.namespace === 'gitea') + expect(giteaMapping).toBeDefined() + expect(giteaMapping!.secretName).toBe('gitea-secrets') + expect(giteaMapping!.data).toHaveProperty('apps_gitea_adminPassword', 'gitea-pass') + }) + + it('should skip kms.sops paths', async () => { + const secrets = { + kms: { sops: { provider: 'age', age: { publicKey: 'pk', privateKey: 'sk' } } }, + apps: { harbor: { adminPassword: 'pass' } }, + } + const deps = { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue(['kms.sops.provider', 'kms.sops.age.publicKey', 'apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], deps) + + expect(result).toHaveLength(1) + expect(result[0].namespace).toBe('harbor') + }) + + it('should skip users path', async () => { + const secrets = { + users: [{ email: 'admin@example.com' }], + apps: { gitea: { adminPassword: 'pass' } }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['users', 'apps.gitea.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], deps) + + expect(result).toHaveLength(1) + expect(result[0].namespace).toBe('gitea') + }) + + it('should handle teamConfig dynamic paths', async () => { + const secrets = { + teamConfig: { + 'team-alpha': { someSecret: 'value' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['teamConfig.team-alpha.someSecret']), + } + + const result = await buildSecretToNamespaceMap(secrets, ['team-alpha'], deps) + + expect(result).toHaveLength(1) + expect(result[0].namespace).toBe('team-team-alpha') + expect(result[0].secretName).toBe('team-settings-secrets') + }) + + it('should filter out mappings with no data', async () => { + const secrets = {} + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], deps) + + expect(result).toHaveLength(0) + }) + }) + + describe('createSealedSecretManifest', () => { + it('should produce correct SealedSecret structure', async () => { + const mapping = { + namespace: 'harbor', + secretName: 'harbor-secrets', + data: { adminPassword: 'my-password', secretKey: 'my-secret' }, + } + const deps = { + encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), + } + + const result = await createSealedSecretManifest('mock-pem', mapping, deps) + + expect(result.apiVersion).toBe('bitnami.com/v1alpha1') + expect(result.kind).toBe('SealedSecret') + expect(result.metadata.name).toBe('harbor-secrets') + expect(result.metadata.namespace).toBe('harbor') + expect(result.metadata.annotations['sealedsecrets.bitnami.com/namespace-wide']).toBe('true') + expect(result.spec.encryptedData.adminPassword).toBe('encrypted-value') + expect(result.spec.encryptedData.secretKey).toBe('encrypted-value') + expect(result.spec.template.type).toBe('Opaque') + expect(result.spec.template.metadata.name).toBe('harbor-secrets') + expect(result.spec.template.metadata.namespace).toBe('harbor') + }) + + it('should call encryptSecretItem for each data key', async () => { + const mapping = { + namespace: 'gitea', + secretName: 'gitea-secrets', + data: { key1: 'val1', key2: 'val2', key3: 'val3' }, + } + const deps = { + encryptSecretItem: jest.fn().mockResolvedValue('enc'), + } + + await createSealedSecretManifest('pem', mapping, deps) + + expect(deps.encryptSecretItem).toHaveBeenCalledTimes(3) + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'gitea', 'val1') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'gitea', 'val2') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'gitea', 'val3') + }) + }) + + describe('writeSealedSecretManifests', () => { + it('should write each manifest to the correct directory', async () => { + const manifests = [ + { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: 'harbor-secrets', + namespace: 'harbor', + }, + spec: { + encryptedData: { key: 'enc' }, + template: { + immutable: false, + metadata: { name: 'harbor-secrets', namespace: 'harbor' }, + type: 'Opaque', + }, + }, + }, + ] + const deps = { + mkdir: jest.fn(), + writeFile: jest.fn(), + objectToYaml: jest.fn().mockReturnValue('yaml-content'), + terminal, + } + + await writeSealedSecretManifests(manifests, '/test', deps) + + expect(deps.mkdir).toHaveBeenCalledWith('/test/env/sealedsecrets/harbor', { recursive: true }) + expect(deps.writeFile).toHaveBeenCalledWith('/test/env/sealedsecrets/harbor/harbor-secrets.yaml', 'yaml-content') + }) + }) + + describe('bootstrapSealedSecrets', () => { + it('should orchestrate all steps in sequence', async () => { + const secrets = { + apps: { harbor: { adminPassword: 'pass' } }, + } + const mockMapping = { + namespace: 'harbor', + secretName: 'harbor-secrets', + data: { adminPassword: 'pass' }, + } + const mockManifest = { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: 'harbor-secrets', + namespace: 'harbor', + }, + spec: { + encryptedData: { adminPassword: 'encrypted' }, + template: { + immutable: false, + metadata: { name: 'harbor-secrets', namespace: 'harbor' }, + type: 'Opaque', + }, + }, + } + + const deps = { + terminal, + generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ + certificate: 'cert-pem', + privateKey: 'key-pem', + }), + getPemFromCertificate: jest.fn().mockReturnValue('spki-pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), + createSealedSecretManifest: jest.fn().mockResolvedValue(mockManifest), + writeSealedSecretManifests: jest.fn(), + encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), + } + + await bootstrapSealedSecrets(secrets, '/test', deps) + + expect(deps.generateSealedSecretsKeyPair).toHaveBeenCalled() + expect(deps.createSealedSecretsKeySecret).toHaveBeenCalledWith('cert-pem', 'key-pem') + expect(deps.getPemFromCertificate).toHaveBeenCalledWith('cert-pem') + expect(deps.buildSecretToNamespaceMap).toHaveBeenCalledWith(secrets, []) + expect(deps.createSealedSecretManifest).toHaveBeenCalledWith('spki-pem', mockMapping, { + encryptSecretItem: deps.encryptSecretItem, + }) + expect(deps.writeSealedSecretManifests).toHaveBeenCalledWith([mockManifest], '/test') + }) + + it('should extract team names from secrets', async () => { + const secrets = { + teamConfig: { + alpha: { secret: 'val' }, + beta: { secret: 'val' }, + }, + } + + const deps = { + terminal, + generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ + certificate: 'cert', + privateKey: 'key', + }), + getPemFromCertificate: jest.fn().mockReturnValue('pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), + createSealedSecretManifest: jest.fn(), + writeSealedSecretManifests: jest.fn(), + encryptSecretItem: jest.fn(), + } + + await bootstrapSealedSecrets(secrets, '/test', deps) + + expect(deps.buildSecretToNamespaceMap).toHaveBeenCalledWith(secrets, ['alpha', 'beta']) + }) + }) + + describe('APP_NAMESPACE_MAP', () => { + it('should have expected mappings', () => { + expect(APP_NAMESPACE_MAP['apps.harbor']).toBe('harbor') + expect(APP_NAMESPACE_MAP['apps.gitea']).toBe('gitea') + expect(APP_NAMESPACE_MAP['apps.oauth2-proxy']).toBe('istio-system') + expect(APP_NAMESPACE_MAP['apps.loki']).toBe('monitoring') + expect(APP_NAMESPACE_MAP['otomi']).toBe('otomi') + expect(APP_NAMESPACE_MAP['dns']).toBe('external-dns') + expect(APP_NAMESPACE_MAP['cluster']).toBe('cert-manager') + }) + }) +}) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts new file mode 100644 index 0000000000..c58f357e3f --- /dev/null +++ b/src/common/sealed-secrets.ts @@ -0,0 +1,375 @@ +import { encryptSecretItem } from '@linode/kubeseal-encrypt' +import { X509Certificate } from 'crypto' +import { mkdir, writeFile } from 'fs/promises' +import { get } from 'lodash' +import { pki } from 'node-forge' +import { terminal } from 'src/common/debug' +import { flattenObject, getSchemaSecretsPaths } from 'src/common/utils' +import { objectToYaml } from 'src/common/values' +import { $ } from 'zx' + +const cmdName = 'sealed-secrets' + +export interface SecretMapping { + namespace: string + secretName: string + data: Record +} + +export interface SealedSecretManifest { + apiVersion: string + kind: string + metadata: { + annotations: Record + name: string + namespace: string + } + spec: { + encryptedData: Record + template: { + immutable: boolean + metadata: { name: string; namespace: string } + type: string + } + } +} + +/** + * Mapping from secret path prefix to target Kubernetes namespace. + * Dynamic entries like `teamConfig.{teamId}` are handled separately. + */ +export const APP_NAMESPACE_MAP: Record = { + 'apps.harbor': 'harbor', + 'apps.gitea': 'gitea', + 'apps.keycloak': 'keycloak', + 'apps.grafana': 'grafana', + 'apps.loki': 'monitoring', + 'apps.oauth2-proxy': 'istio-system', + 'apps.oauth2-proxy-redis': 'istio-system', + 'apps.prometheus': 'monitoring', + 'apps.otomi-api': 'otomi', + 'apps.cert-manager': 'cert-manager', + 'apps.kubeflow-pipelines': 'kfp', + otomi: 'otomi', + oidc: 'otomi', + smtp: 'otomi', + dns: 'external-dns', + obj: 'otomi', + license: 'otomi', + users: 'keycloak', + alerts: 'monitoring', + cluster: 'cert-manager', +} + +/** + * Generate an RSA 4096-bit key pair and self-signed X.509 certificate for Sealed Secrets. + * Follows the pattern from createCustomCA() in bootstrap.ts. + */ +export const generateSealedSecretsKeyPair = (deps = { terminal, pki }): { certificate: string; privateKey: string } => { + const d = deps.terminal(`common:${cmdName}:generateSealedSecretsKeyPair`) + d.info('Generating sealed-secrets RSA key pair') + + const keys = deps.pki.rsa.generateKeyPair(4096) + const cert = deps.pki.createCertificate() + cert.publicKey = keys.publicKey + cert.serialNumber = '01' + cert.validity.notBefore = new Date() + cert.validity.notAfter = new Date() + cert.validity.notAfter.setFullYear(cert.validity.notBefore.getFullYear() + 10) + + const attrs = [ + { name: 'countryName', value: 'NL' }, + { shortName: 'ST', value: 'Utrecht' }, + { name: 'localityName', value: 'Utrecht' }, + { name: 'organizationName', value: 'APL' }, + { shortName: 'OU', value: 'SealedSecrets' }, + ] + cert.setSubject(attrs) + cert.setIssuer(attrs) + cert.setExtensions([ + { + name: 'basicConstraints', + cA: true, + }, + { + name: 'keyUsage', + keyCertSign: true, + digitalSignature: true, + keyEncipherment: true, + }, + ]) + cert.sign(keys.privateKey) + + const certificate = deps.pki.certificateToPem(cert).replaceAll('\r\n', '\n') + const privateKey = deps.pki.privateKeyToPem(keys.privateKey).replaceAll('\r\n', '\n') + + d.info('Generated sealed-secrets key pair') + return { certificate, privateKey } +} + +/** + * Extract SPKI PEM public key from a PEM-encoded X.509 certificate. + * Uses Node.js crypto.X509Certificate (same approach as getSealedSecretsPEM() in k8s.ts). + */ +export const getPemFromCertificate = (certificate: string): string => { + const x509 = new X509Certificate(certificate) + const exported = x509.publicKey.export({ format: 'pem', type: 'spki' }) + return typeof exported === 'string' ? exported : exported.toString('utf-8') +} + +/** + * Create the sealed-secrets namespace and TLS secret in Kubernetes. + * The controller will pick up this pre-created key on startup. + */ +export const createSealedSecretsKeySecret = async ( + certificate: string, + privateKey: string, + deps = { $, terminal, writeFile, mkdir }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:createSealedSecretsKeySecret`) + d.info('Creating sealed-secrets namespace and TLS secret') + + // Create namespace + await deps.$`kubectl create namespace sealed-secrets --dry-run=client -o yaml | kubectl apply -f -`.nothrow().quiet() + + // Write temp files for kubectl create secret tls + const tmpDir = '/tmp/sealed-secrets-bootstrap' + await deps.mkdir(tmpDir, { recursive: true }) + const certPath = `${tmpDir}/tls.crt` + const keyPath = `${tmpDir}/tls.key` + await deps.writeFile(certPath, certificate) + await deps.writeFile(keyPath, privateKey) + + // Create the TLS secret + const result = + await deps.$`kubectl create secret tls sealed-secrets-key -n sealed-secrets --cert=${certPath} --key=${keyPath} --dry-run=client -o yaml | kubectl apply -f -` + .nothrow() + .quiet() + if (result.stderr) d.error(result.stderr) + + // Label the secret so the controller picks it up + const labelResult = + await deps.$`kubectl label secret sealed-secrets-key -n sealed-secrets sealedsecrets.bitnami.com/sealed-secrets-key=active --overwrite` + .nothrow() + .quiet() + if (labelResult.stderr) d.error(labelResult.stderr) + + d.info('Created sealed-secrets TLS secret with key label') +} + +/** + * Resolve the namespace for a given secret path using APP_NAMESPACE_MAP. + * Handles dynamic teamConfig paths. + */ +const resolveNamespace = (secretPath: string): string | undefined => { + // Check for teamConfig dynamic paths + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) { + return `team-${teamMatch[1]}` + } + + // Find the longest matching prefix in APP_NAMESPACE_MAP + const sortedKeys = Object.keys(APP_NAMESPACE_MAP).sort((a, b) => b.length - a.length) + for (const prefix of sortedKeys) { + if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { + return APP_NAMESPACE_MAP[prefix] + } + } + + return undefined +} + +/** + * Derive a K8s secret name from the secret path prefix. + */ +const deriveSecretName = (secretPath: string): string => { + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) { + return 'team-settings-secrets' + } + + // Map specific path prefixes to secret names + const nameMap: Record = { + 'apps.harbor': 'harbor-secrets', + 'apps.gitea': 'gitea-secrets', + 'apps.keycloak': 'keycloak-secrets', + 'apps.grafana': 'grafana-secrets', + 'apps.loki': 'loki-secrets', + 'apps.oauth2-proxy': 'oauth2-proxy-secrets', + 'apps.oauth2-proxy-redis': 'oauth2-proxy-redis-secrets', + 'apps.prometheus': 'prometheus-secrets', + 'apps.otomi-api': 'otomi-api-secrets', + 'apps.cert-manager': 'cert-manager-secrets', + 'apps.kubeflow-pipelines': 'kubeflow-pipelines-secrets', + otomi: 'otomi-platform-secrets', + oidc: 'oidc-secrets', + smtp: 'smtp-secrets', + dns: 'dns-secrets', + obj: 'obj-storage-secrets', + license: 'license-secrets', + users: 'users-secrets', + alerts: 'alerts-secrets', + cluster: 'cluster-secrets', + } + + const sortedKeys = Object.keys(nameMap).sort((a, b) => b.length - a.length) + for (const prefix of sortedKeys) { + if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { + return nameMap[prefix] + } + } + + // Fallback: derive from first two path segments + const parts = secretPath.split('.') + return `${parts.slice(0, 2).join('-')}-secrets` +} + +/** + * Build a mapping from secrets to their target namespaces and K8s secret names. + * Groups secret paths by namespace and secret name. + */ +export const buildSecretToNamespaceMap = async ( + secrets: Record, + teams: string[], + deps = { getSchemaSecretsPaths }, +): Promise => { + const secretPaths = await deps.getSchemaSecretsPaths(teams) + const flat = flattenObject(secrets) + + // Group by namespace + secretName + const groupMap = new Map() + + for (const secretPath of secretPaths) { + // Skip SOPS-related paths + if (secretPath.startsWith('kms.sops')) continue + // Skip 'users' path — not a simple key-value secret + if (secretPath === 'users') continue + + const namespace = resolveNamespace(secretPath) + if (!namespace) continue + + const secretName = deriveSecretName(secretPath) + const groupKey = `${namespace}/${secretName}` + + if (!groupMap.has(groupKey)) { + groupMap.set(groupKey, { namespace, secretName, data: {} }) + } + + const mapping = groupMap.get(groupKey)! + + // Find all flat keys that match this secret path + for (const [flatKey, value] of Object.entries(flat)) { + if (flatKey === secretPath || flatKey.startsWith(`${secretPath}.`)) { + // Use the leaf key name as the data key + const dataKey = flatKey.replace(/\./g, '_') + if (value !== undefined && value !== null && value !== '') { + mapping.data[dataKey] = String(value) + } + } + } + } + + // Filter out empty mappings + return Array.from(groupMap.values()).filter((m) => Object.keys(m.data).length > 0) +} + +/** + * Create a SealedSecret manifest by encrypting each data value. + * Follows the pattern from createCatalogSealedSecret() in migrate.ts. + */ +export const createSealedSecretManifest = async ( + pem: string, + mapping: SecretMapping, + deps = { encryptSecretItem }, +): Promise => { + const encryptedData: Record = {} + for (const [key, value] of Object.entries(mapping.data)) { + encryptedData[key] = await deps.encryptSecretItem(pem, mapping.namespace, value) + } + + return { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: mapping.secretName, + namespace: mapping.namespace, + }, + spec: { + encryptedData, + template: { + immutable: false, + metadata: { name: mapping.secretName, namespace: mapping.namespace }, + type: 'Opaque', + }, + }, + } +} + +/** + * Write SealedSecret manifests to the env/sealedsecrets directory. + */ +export const writeSealedSecretManifests = async ( + manifests: SealedSecretManifest[], + envDir: string, + deps = { mkdir, writeFile, objectToYaml, terminal }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:writeSealedSecretManifests`) + + for (const manifest of manifests) { + const dir = `${envDir}/env/sealedsecrets/${manifest.metadata.namespace}` + await deps.mkdir(dir, { recursive: true }) + const filePath = `${dir}/${manifest.metadata.name}.yaml` + d.info(`Writing sealed secret to ${filePath}`) + await deps.writeFile(filePath, deps.objectToYaml(manifest)) + } +} + +/** + * Orchestrator: bootstrap sealed secrets for the platform. + * Replaces bootstrapSops(). + */ +export const bootstrapSealedSecrets = async ( + secrets: Record, + envDir: string, + deps = { + terminal, + generateSealedSecretsKeyPair, + getPemFromCertificate, + createSealedSecretsKeySecret, + buildSecretToNamespaceMap, + createSealedSecretManifest, + writeSealedSecretManifests, + encryptSecretItem, + }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:bootstrapSealedSecrets`) + d.info('Bootstrapping sealed secrets') + + // 1. Generate RSA key pair + self-signed X.509 certificate + const { certificate, privateKey } = deps.generateSealedSecretsKeyPair() + + // 2 & 3. Create namespace and store key pair as K8s TLS secret + await deps.createSealedSecretsKeySecret(certificate, privateKey) + + // 4. Extract SPKI PEM public key from certificate + const pem = deps.getPemFromCertificate(certificate) + + // 5. Build secret-to-namespace mapping + const teams = Object.keys(get(secrets, 'teamConfig', {}) as Record) + const mappings = await deps.buildSecretToNamespaceMap(secrets, teams) + + // 6. Create SealedSecret manifests + const manifests: SealedSecretManifest[] = [] + for (const mapping of mappings) { + const manifest = await deps.createSealedSecretManifest(pem, mapping, { + encryptSecretItem: deps.encryptSecretItem, + }) + manifests.push(manifest) + } + + // 7. Write SealedSecret manifests to disk + await deps.writeSealedSecretManifests(manifests, envDir) + + d.info(`Bootstrapped ${manifests.length} sealed secret manifests`) +} From c9578aa92aced3a659df5cca936480ba5dfe4d64 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 4 Feb 2026 16:20:03 +0100 Subject: [PATCH 02/20] test: sealed secrets --- src/cmd/bootstrap.ts | 2 +- src/common/sealed-secrets.test.ts | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index d2aecf9cfa..f71e6e7b9d 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -2,7 +2,7 @@ import { randomUUID } from 'crypto' import { existsSync } from 'fs' import { copyFile, cp, mkdir, readFile, writeFile } from 'fs/promises' import { generate as generatePassword } from 'generate-password' -import { cloneDeep, get, isEmpty, merge, set } from 'lodash' +import { cloneDeep, get, merge, set } from 'lodash' import { pki } from 'node-forge' import path from 'path' import { bootstrapGit } from 'src/common/bootstrap' diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 29d937ed4a..c937d20d6f 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -154,11 +154,7 @@ describe('sealed-secrets', () => { const deps = { getSchemaSecretsPaths: jest .fn() - .mockResolvedValue([ - 'apps.harbor.adminPassword', - 'apps.harbor.secretKey', - 'apps.gitea.adminPassword', - ]), + .mockResolvedValue(['apps.harbor.adminPassword', 'apps.harbor.secretKey', 'apps.gitea.adminPassword']), } const result = await buildSecretToNamespaceMap(secrets, [], deps) From 24618a06f85e36f0e6e078216d7d7e8d8d4f654a Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 4 Feb 2026 17:01:35 +0100 Subject: [PATCH 03/20] test: sealed secrets --- src/common/sealed-secrets.test.ts | 4 ++-- src/common/sealed-secrets.ts | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index c937d20d6f..6d6fa8880d 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -308,8 +308,8 @@ describe('sealed-secrets', () => { await writeSealedSecretManifests(manifests, '/test', deps) - expect(deps.mkdir).toHaveBeenCalledWith('/test/env/sealedsecrets/harbor', { recursive: true }) - expect(deps.writeFile).toHaveBeenCalledWith('/test/env/sealedsecrets/harbor/harbor-secrets.yaml', 'yaml-content') + expect(deps.mkdir).toHaveBeenCalledWith('/test/env/manifests/ns/harbor', { recursive: true }) + expect(deps.writeFile).toHaveBeenCalledWith('/test/env/manifests/ns/harbor/harbor-secrets.yaml', 'yaml-content') }) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index c58f357e3f..c305be8380 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -307,7 +307,7 @@ export const createSealedSecretManifest = async ( } /** - * Write SealedSecret manifests to the env/sealedsecrets directory. + * Write SealedSecret manifests to the env/manifests/ns directory. */ export const writeSealedSecretManifests = async ( manifests: SealedSecretManifest[], @@ -317,7 +317,8 @@ export const writeSealedSecretManifests = async ( const d = deps.terminal(`common:${cmdName}:writeSealedSecretManifests`) for (const manifest of manifests) { - const dir = `${envDir}/env/sealedsecrets/${manifest.metadata.namespace}` + // /env/manifests/ns/argocd/ + const dir = `${envDir}/env/manifests/ns/${manifest.metadata.namespace}` await deps.mkdir(dir, { recursive: true }) const filePath = `${dir}/${manifest.metadata.name}.yaml` d.info(`Writing sealed secret to ${filePath}`) From d34988ffe63b2d99fd9d9f0d96aec3f0d119625c Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 4 Feb 2026 21:29:18 +0100 Subject: [PATCH 04/20] test: env.old.gotmpl --- .values/.gitattributes | 1 - .values/.gitignore | 1 + chart/apl/templates/sops-secrets.yaml | 35 --------------------------- helmfile.d/snippets/env.old.gotmpl | 21 ++++++---------- helmfile.d/snippets/sops-env.gotmpl | 24 ------------------ src/cmd/bootstrap.ts | 5 ---- tpl/.sops.yaml.gotmpl | 3 --- values/otomi-api/otomi-api.gotmpl | 3 --- 8 files changed, 8 insertions(+), 85 deletions(-) delete mode 100644 .values/.gitattributes delete mode 100644 chart/apl/templates/sops-secrets.yaml delete mode 100644 tpl/.sops.yaml.gotmpl diff --git a/.values/.gitattributes b/.values/.gitattributes deleted file mode 100644 index 4f684e3799..0000000000 --- a/.values/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -secrets.*.yaml diff=sopsdiffer diff --git a/.values/.gitignore b/.values/.gitignore index c1e96df987..169ac1d5dc 100644 --- a/.values/.gitignore +++ b/.values/.gitignore @@ -14,3 +14,4 @@ core.yaml env/status.yaml env/bootstrap.yaml values-repo.yaml +secrets.*.yaml diff --git a/chart/apl/templates/sops-secrets.yaml b/chart/apl/templates/sops-secrets.yaml deleted file mode 100644 index 7544e0ec6c..0000000000 --- a/chart/apl/templates/sops-secrets.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{- $kms := .Values.kms | default dict }} -{{- if hasKey $kms "sops" }} -{{- $v := $kms.sops }} -apiVersion: v1 -kind: Secret -metadata: - name: apl-sops-secrets - namespace: apl-operator -type: Opaque -data: -{{- with $v.azure }} - AZURE_CLIENT_ID: {{ .clientId | b64enc }} - AZURE_CLIENT_SECRET: {{ .clientSecret | b64enc }} -{{- with .tenantId }} - AZURE_TENANT_ID: {{ . | b64enc }}{{ end }} -{{- with .environment }} - AZURE_ENVIRONMENT: {{ . | b64enc }}{{ end }} -{{- end }} -{{- with $v.aws }} - AWS_ACCESS_KEY_ID: {{ .accessKey | b64enc }} - AWS_SECRET_ACCESS_KEY: {{ .secretKey | b64enc }} -{{- with .region }} - AWS_REGION: {{ . | b64enc }}{{ end }} -{{- end }} -{{- with $v.age }} - SOPS_AGE_KEY: {{ .privateKey | b64enc }} -{{- end }} -{{- with $v.google }} - GCLOUD_SERVICE_KEY: {{ .accountJson | b64enc }} -{{- with .project }} - GOOGLE_PROJECT: {{ . | b64enc }}{{ end }} -{{- with .region }} - GOOGLE_REGION: {{ . | b64enc }}{{ end }} -{{- end }} -{{- end }} diff --git a/helmfile.d/snippets/env.old.gotmpl b/helmfile.d/snippets/env.old.gotmpl index c0c5fbdda7..e1d73a6140 100644 --- a/helmfile.d/snippets/env.old.gotmpl +++ b/helmfile.d/snippets/env.old.gotmpl @@ -7,12 +7,10 @@ {{- /* We relocated teamConfig.teams to teamConfig, so one time we might have to fall back to our previous location */}} {{- /* TODO:[teamConfig] deprecate somewhere in da future */}} {{- if hasKey $t.teamConfig "teams" }}{{ $teams = keys $t.teamConfig.teams }}{{ end }} -{{- $hasSops := eq (exec "bash" (list "-c" "( test -f $ENV_DIR/.sops.yaml && echo 'true' ) || echo 'false'") | trim) "true" }} {{- $apps := (exec "bash" (list "-c" "find $ENV_DIR/env/apps -name '*.yaml' -not -name 'secrets.*.yaml'")) | splitList "\n" }} {{- $databases := (exec "bash" (list "-c" "find $ENV_DIR/env/databases -name '*.yaml' || true ")) | splitList "\n" }} {{- $appsSecret := (exec "bash" (list "-c" "find $ENV_DIR/env/apps -name 'secrets.*.yaml'")) | splitList "\n" }} -{{- $ext := ($hasSops | ternary ".dec" "") }} -{{- $teamFileExists := eq (exec "bash" (list "-c" (printf "( test -f $ENV_DIR/env/secrets.teams.yaml%s && echo 'true' ) || echo 'false'" $ext)) | trim) "true" }} +{{- $teamFileExists := eq (exec "bash" (list "-c" "( test -f $ENV_DIR/env/secrets.teams.yaml && echo 'true' ) || echo 'false'") | trim) "true" }} helmDefaults: atomic: true historyMax: 3 @@ -34,8 +32,7 @@ environments: - {{ $ENV_DIR }}/env/status.yaml {{- end }} {{- if eq (exec "bash" (list "-c" "( test -f $ENV_DIR/env/secrets.license.yaml && echo 'true' ) || echo 'false'") | trim) "true" }} - - {{ $ENV_DIR }}/env/secrets.license.yaml{{ $ext }} -{{- end }} + - {{ $ENV_DIR }}/env/secrets.license.yaml{{- end }} {{- range $app := $apps }}{{ if ne $app "" }} - {{ $app }} @@ -50,17 +47,13 @@ environments: {{- end }} {{- end }} {{- end }} -{{- if eq (exec "bash" (list "-c" (printf "( test -f $ENV_DIR/env/secrets.users.yaml%s && echo 'true' ) || echo 'false'" (default "" $ext))) | trim) "true" }} - - {{ $ENV_DIR }}/env/secrets.users.yaml{{ $ext }} -{{- end }} +{{- if eq (exec "bash" (list "-c" "( test -f $ENV_DIR/env/secrets.users.yaml && echo 'true' ) || echo 'false'") | trim) "true" }} + - {{ $ENV_DIR }}/env/secrets.users.yaml{{- end }} {{- if eq (exec "bash" (list "-c" "( test -f $ENV_DIR/env/secrets.settings.yaml && echo 'true' ) || echo 'false'") | trim) "true" }} - - {{ $ENV_DIR }}/env/secrets.settings.yaml{{ $ext }} -{{- end }} + - {{ $ENV_DIR }}/env/secrets.settings.yaml{{- end }} {{- if $teamFileExists }} {{- if eq (exec "bash" (list "-c" "( test -f $ENV_DIR/env/secrets.teams.yaml && echo 'true' ) || echo 'false'") | trim) "true" }} - - {{ $ENV_DIR }}/env/secrets.teams.yaml{{ $ext }} - {{- end }} + - {{ $ENV_DIR }}/env/secrets.teams.yaml {{- end }} {{- end }} {{- range $app := $appsSecret }}{{ if ne $app "" }}{{ $file := $app | replace (print $ENV_DIR "/env/apps/") "" }} - - {{ $ENV_DIR }}/env/apps/{{ $file }}{{ $ext }} -{{- end }}{{ end }} + - {{ $ENV_DIR }}/env/apps/{{ $file }}{{- end }}{{ end }} diff --git a/helmfile.d/snippets/sops-env.gotmpl b/helmfile.d/snippets/sops-env.gotmpl index aa9e2ff866..e69de29bb2 100644 --- a/helmfile.d/snippets/sops-env.gotmpl +++ b/helmfile.d/snippets/sops-env.gotmpl @@ -1,24 +0,0 @@ -{{- with . | get "azure" nil }} -AZURE_CLIENT_ID: {{ .clientId }} -AZURE_CLIENT_SECRET: {{ .clientSecret }} -{{- with . | get "tenantId" nil }} -AZURE_TENANT_ID: {{ . }}{{ end }} -{{- with . | get "environment" nil }} -AZURE_ENVIRONMENT: {{ . }}{{ end }} -{{- end }} -{{- with . | get "aws" nil }} -AWS_ACCESS_KEY_ID: {{ .accessKey }} -AWS_SECRET_ACCESS_KEY: {{ .secretKey }} -{{- with . | get "region" nil }} -AWS_REGION: {{ . }}{{ end }} -{{- end }} -{{- with . | get "age" nil }} -SOPS_AGE_KEY: {{ .privateKey }} -{{- end }} -{{- with . | get "google" nil }} -GCLOUD_SERVICE_KEY: '{{ .accountJson | replace "\n" "" }}' -{{- with . | get "project" nil }} -GOOGLE_PROJECT: {{ . }}{{ end }} -{{- with . | get "region" nil }} -GOOGLE_REGION: {{ . }}{{ end }} -{{- end }} diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index f71e6e7b9d..ee60cecbde 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -85,11 +85,6 @@ export const bootstrapSops = async ( await deps.writeFile(targetPath, output) d.log(`Ready generating sops files. The configuration is written to: ${targetPath}`) - d.info('Copying sops related files') - // add sops related files - const file = '.gitattributes' - await deps.copyFile(`${rootDir}/.values/${file}`, `${envDir}/${file}`) - // prepare some credential files the first time and crypt some if (!exists) { if (isCli || env.OTOMI_DEV) { diff --git a/tpl/.sops.yaml.gotmpl b/tpl/.sops.yaml.gotmpl deleted file mode 100644 index cf6fc9a6b2..0000000000 --- a/tpl/.sops.yaml.gotmpl +++ /dev/null @@ -1,3 +0,0 @@ -creation_rules: - - path_regex: ^(.*/)?secrets.*.yaml(\.dec)?$ - {{ .provider }}: {{ .keys }} diff --git a/values/otomi-api/otomi-api.gotmpl b/values/otomi-api/otomi-api.gotmpl index e8681d450a..e3155e12bb 100644 --- a/values/otomi-api/otomi-api.gotmpl +++ b/values/otomi-api/otomi-api.gotmpl @@ -3,11 +3,9 @@ {{- $o := $v.apps | get "otomi-api" }} {{- $g := $v.apps.gitea }} {{- $cm := $v.apps | get "cert-manager" }} -{{- $sops := $v | get "kms.sops" dict }} {{- $giteaValuesUrl := "http://gitea-http.gitea.svc.cluster.local:3000/otomi/values" }} {{- $giteaValuesPublilcUrl := printf "https://gitea.%s/otomi/values" $v.cluster.domainSuffix }} {{- $defaultPlatformAdminEmail := printf "platform-admin@%s" $v.cluster.domainSuffix }} -{{- $sopsEnv := tpl (readFile "../../helmfile.d/snippets/sops-env.gotmpl") $sops }} {{- $version := $v.versions | get "api" }} {{- $isSemver := regexMatch "^[0-9.]+" $version }} {{- $coreVersion := $v.otomi.version }} @@ -41,7 +39,6 @@ secrets: GIT_USER: otomi-admin GIT_EMAIL: not@us.ed GIT_PASSWORD: {{ $g.adminPassword | quote}} - {{- $sopsEnv | nindent 2 }} env: DEFAULT_PLATFORM_ADMIN_EMAIL: {{ $defaultPlatformAdminEmail }} From a50af0a564b915e7611717443a7a049e42246a64 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Feb 2026 09:09:57 +0100 Subject: [PATCH 05/20] test: sealed secrets --- chart/apl/templates/deployment.yaml | 4 - helmfile.d/helmfile-03.databases.yaml.gotmpl | 7 - src/cmd/bootstrap.ts | 2 +- src/common/sealed-secrets.test.ts | 147 ++++++++++++--- src/common/sealed-secrets.ts | 167 +++++++++++++++--- .../gitea-db-secret-raw.gotmpl | 14 +- values/gitea/gitea-raw.gotmpl | 7 - 7 files changed, 262 insertions(+), 86 deletions(-) diff --git a/chart/apl/templates/deployment.yaml b/chart/apl/templates/deployment.yaml index 2b34ecc718..34186124a4 100644 --- a/chart/apl/templates/deployment.yaml +++ b/chart/apl/templates/deployment.yaml @@ -75,13 +75,9 @@ spec: value: {{ .Values.operator.pollIntervalMs | default "30000" | quote }} - name: RECONCILE_INTERVAL_MS value: {{ .Values.operator.reconcileIntervalMs | default "300000" | quote }} - {{- if hasKey $kms "sops" }} envFrom: - - secretRef: - name: apl-sops-secrets - secretRef: name: gitea-credentials - {{- end }} volumeMounts: - name: otomi-values mountPath: /home/app/stack/env diff --git a/helmfile.d/helmfile-03.databases.yaml.gotmpl b/helmfile.d/helmfile-03.databases.yaml.gotmpl index 3c718d9283..9f136b297f 100644 --- a/helmfile.d/helmfile-03.databases.yaml.gotmpl +++ b/helmfile.d/helmfile-03.databases.yaml.gotmpl @@ -14,13 +14,6 @@ bases: {{- $k := $a.keycloak }} releases: - - name: gitea-db-secret-artifacts - installed: true - namespace: gitea - labels: - pkg: gitea - app: core - <<: *raw - name: gitea-otomi-db installed: true namespace: gitea diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index ee60cecbde..4da62b6ad2 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -433,7 +433,7 @@ export const bootstrap = async ( await deps.migrate() const { originalInput, allSecrets } = await deps.processValues() await deps.handleFileEntry() - await deps.bootstrapSealedSecrets(allSecrets, ENV_DIR) + await deps.bootstrapSealedSecrets(allSecrets, ENV_DIR, originalInput) await ensureTeamGitOpsDirectories(ENV_DIR, originalInput) d.log(`Done bootstrapping values`) } diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 6d6fa8880d..79fe4453a9 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -2,6 +2,7 @@ import { pki } from 'node-forge' import stubs from 'src/test-stubs' import { APP_NAMESPACE_MAP, + APP_SECRET_OVERRIDES, bootstrapSealedSecrets, buildSecretToNamespaceMap, createSealedSecretManifest, @@ -144,33 +145,23 @@ describe('sealed-secrets', () => { }) describe('buildSecretToNamespaceMap', () => { - it('should group secrets by namespace and secret name', async () => { + it('should group secrets by namespace and secret name with leaf key naming', async () => { const secrets = { apps: { harbor: { adminPassword: 'harbor-pass', secretKey: 'harbor-secret' }, - gitea: { adminPassword: 'gitea-pass' }, }, } const deps = { - getSchemaSecretsPaths: jest - .fn() - .mockResolvedValue(['apps.harbor.adminPassword', 'apps.harbor.secretKey', 'apps.gitea.adminPassword']), + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.adminPassword', 'apps.harbor.secretKey']), } - const result = await buildSecretToNamespaceMap(secrets, [], deps) - - expect(result).toHaveLength(2) + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) const harborMapping = result.find((m) => m.namespace === 'harbor') expect(harborMapping).toBeDefined() expect(harborMapping!.secretName).toBe('harbor-secrets') - expect(harborMapping!.data).toHaveProperty('apps_harbor_adminPassword', 'harbor-pass') - expect(harborMapping!.data).toHaveProperty('apps_harbor_secretKey', 'harbor-secret') - - const giteaMapping = result.find((m) => m.namespace === 'gitea') - expect(giteaMapping).toBeDefined() - expect(giteaMapping!.secretName).toBe('gitea-secrets') - expect(giteaMapping!.data).toHaveProperty('apps_gitea_adminPassword', 'gitea-pass') + expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') + expect(harborMapping!.data).toHaveProperty('secretKey', 'harbor-secret') }) it('should skip kms.sops paths', async () => { @@ -184,7 +175,7 @@ describe('sealed-secrets', () => { .mockResolvedValue(['kms.sops.provider', 'kms.sops.age.publicKey', 'apps.harbor.adminPassword']), } - const result = await buildSecretToNamespaceMap(secrets, [], deps) + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) expect(result).toHaveLength(1) expect(result[0].namespace).toBe('harbor') @@ -193,16 +184,16 @@ describe('sealed-secrets', () => { it('should skip users path', async () => { const secrets = { users: [{ email: 'admin@example.com' }], - apps: { gitea: { adminPassword: 'pass' } }, + apps: { harbor: { adminPassword: 'pass' } }, } const deps = { - getSchemaSecretsPaths: jest.fn().mockResolvedValue(['users', 'apps.gitea.adminPassword']), + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['users', 'apps.harbor.adminPassword']), } - const result = await buildSecretToNamespaceMap(secrets, [], deps) + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) expect(result).toHaveLength(1) - expect(result[0].namespace).toBe('gitea') + expect(result[0].namespace).toBe('harbor') }) it('should handle teamConfig dynamic paths', async () => { @@ -215,7 +206,7 @@ describe('sealed-secrets', () => { getSchemaSecretsPaths: jest.fn().mockResolvedValue(['teamConfig.team-alpha.someSecret']), } - const result = await buildSecretToNamespaceMap(secrets, ['team-alpha'], deps) + const result = await buildSecretToNamespaceMap(secrets, ['team-alpha'], undefined, deps) expect(result).toHaveLength(1) expect(result[0].namespace).toBe('team-team-alpha') @@ -228,10 +219,95 @@ describe('sealed-secrets', () => { getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.adminPassword']), } - const result = await buildSecretToNamespaceMap(secrets, [], deps) + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) expect(result).toHaveLength(0) }) + + it('should use leaf key naming for nested secret paths', async () => { + const secrets = { + apps: { + harbor: { core: { secret: 'core-secret-val' } }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.core.secret']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].data).toHaveProperty('core_secret', 'core-secret-val') + }) + + it('should skip overridden prefixes and generate override secrets instead', async () => { + const secrets = { + apps: { + gitea: { adminPassword: 'gitea-pass', postgresqlPassword: 'pg-pass' }, + harbor: { adminPassword: 'harbor-pass' }, + }, + } + const allValues = { + apps: { + gitea: { adminUsername: 'admin', adminPassword: 'gitea-pass', postgresqlPassword: 'pg-pass' }, + harbor: { adminPassword: 'harbor-pass' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue([ + 'apps.gitea.adminPassword', + 'apps.gitea.postgresqlPassword', + 'apps.harbor.adminPassword', + ]), + } + + const result = await buildSecretToNamespaceMap(secrets, [], allValues, deps) + + // Harbor should use default convention + const harborMapping = result.find((m) => m.secretName === 'harbor-secrets') + expect(harborMapping).toBeDefined() + expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') + + // Gitea should NOT have a gitea-secrets mapping + const giteaDefaultMapping = result.find((m) => m.secretName === 'gitea-secrets') + expect(giteaDefaultMapping).toBeUndefined() + + // Gitea should have override-based secrets + const giteaAdminMapping = result.find((m) => m.secretName === 'gitea-admin-secret') + expect(giteaAdminMapping).toBeDefined() + expect(giteaAdminMapping!.namespace).toBe('gitea') + expect(giteaAdminMapping!.data).toEqual({ username: 'admin', password: 'gitea-pass' }) + + const giteaDbMapping = result.find((m) => m.secretName === 'gitea-db-secret') + expect(giteaDbMapping).toBeDefined() + expect(giteaDbMapping!.namespace).toBe('gitea') + expect(giteaDbMapping!.data).toEqual({ username: 'gitea', password: 'pg-pass' }) + }) + + it('should include static values in override secrets', async () => { + const secrets = { + apps: { + gitea: { postgresqlPassword: 'pg-pass' }, + }, + } + const allValues = { + apps: { + gitea: { postgresqlPassword: 'pg-pass' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.gitea.postgresqlPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], allValues, deps) + + const giteaDbMapping = result.find((m) => m.secretName === 'gitea-db-secret') + expect(giteaDbMapping).toBeDefined() + expect(giteaDbMapping!.data.username).toBe('gitea') + expect(giteaDbMapping!.data.password).toBe('pg-pass') + }) }) describe('createSealedSecretManifest', () => { @@ -355,12 +431,12 @@ describe('sealed-secrets', () => { encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), } - await bootstrapSealedSecrets(secrets, '/test', deps) + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) expect(deps.generateSealedSecretsKeyPair).toHaveBeenCalled() expect(deps.createSealedSecretsKeySecret).toHaveBeenCalledWith('cert-pem', 'key-pem') expect(deps.getPemFromCertificate).toHaveBeenCalledWith('cert-pem') - expect(deps.buildSecretToNamespaceMap).toHaveBeenCalledWith(secrets, []) + expect(deps.buildSecretToNamespaceMap).toHaveBeenCalledWith(secrets, [], undefined) expect(deps.createSealedSecretManifest).toHaveBeenCalledWith('spki-pem', mockMapping, { encryptSecretItem: deps.encryptSecretItem, }) @@ -389,9 +465,9 @@ describe('sealed-secrets', () => { encryptSecretItem: jest.fn(), } - await bootstrapSealedSecrets(secrets, '/test', deps) + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) - expect(deps.buildSecretToNamespaceMap).toHaveBeenCalledWith(secrets, ['alpha', 'beta']) + expect(deps.buildSecretToNamespaceMap).toHaveBeenCalledWith(secrets, ['alpha', 'beta'], undefined) }) }) @@ -406,4 +482,23 @@ describe('sealed-secrets', () => { expect(APP_NAMESPACE_MAP['cluster']).toBe('cert-manager') }) }) + + describe('APP_SECRET_OVERRIDES', () => { + it('should have gitea overrides configured', () => { + expect(APP_SECRET_OVERRIDES['apps.gitea']).toBeDefined() + expect(APP_SECRET_OVERRIDES['apps.gitea']).toHaveLength(2) + + const adminSecret = APP_SECRET_OVERRIDES['apps.gitea'].find((o) => o.secretName === 'gitea-admin-secret') + expect(adminSecret).toBeDefined() + expect(adminSecret!.namespace).toBe('gitea') + expect(adminSecret!.data.username).toEqual({ valuePath: 'apps.gitea.adminUsername' }) + expect(adminSecret!.data.password).toEqual({ valuePath: 'apps.gitea.adminPassword' }) + + const dbSecret = APP_SECRET_OVERRIDES['apps.gitea'].find((o) => o.secretName === 'gitea-db-secret') + expect(dbSecret).toBeDefined() + expect(dbSecret!.namespace).toBe('gitea') + expect(dbSecret!.data.username).toEqual({ static: 'gitea' }) + expect(dbSecret!.data.password).toEqual({ valuePath: 'apps.gitea.postgresqlPassword' }) + }) + }) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index c305be8380..3ed223fa94 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -61,6 +61,39 @@ export const APP_NAMESPACE_MAP: Record = { cluster: 'cert-manager', } +/** + * Per-app secret override configuration. + * When an app requires specific K8s Secret names and key layouts + * (e.g. gitea expects `gitea-admin-secret` with `username`/`password` keys), + * define overrides here instead of using the default `{app}-secrets` convention. + */ +interface SecretOverrideEntry { + secretName: string + namespace: string + data: Record +} + +export const APP_SECRET_OVERRIDES: Record = { + 'apps.gitea': [ + { + secretName: 'gitea-admin-secret', + namespace: 'gitea', + data: { + username: { valuePath: 'apps.gitea.adminUsername' }, + password: { valuePath: 'apps.gitea.adminPassword' }, + }, + }, + { + secretName: 'gitea-db-secret', + namespace: 'gitea', + data: { + username: { static: 'gitea' }, + password: { valuePath: 'apps.gitea.postgresqlPassword' }, + }, + }, + ], +} + /** * Generate an RSA 4096-bit key pair and self-signed X.509 certificate for Sealed Secrets. * Follows the pattern from createCustomCA() in bootstrap.ts. @@ -179,6 +212,55 @@ const resolveNamespace = (secretPath: string): string | undefined => { return undefined } +// Map specific path prefixes to secret names +const SECRET_NAME_MAP: Record = { + 'apps.harbor': 'harbor-secrets', + 'apps.gitea': 'gitea-secrets', + 'apps.keycloak': 'keycloak-secrets', + 'apps.grafana': 'grafana-secrets', + 'apps.loki': 'loki-secrets', + 'apps.oauth2-proxy': 'oauth2-proxy-secrets', + 'apps.oauth2-proxy-redis': 'oauth2-proxy-redis-secrets', + 'apps.prometheus': 'prometheus-secrets', + 'apps.otomi-api': 'otomi-api-secrets', + 'apps.cert-manager': 'cert-manager-secrets', + 'apps.kubeflow-pipelines': 'kubeflow-pipelines-secrets', + otomi: 'otomi-platform-secrets', + oidc: 'oidc-secrets', + smtp: 'smtp-secrets', + dns: 'dns-secrets', + obj: 'obj-storage-secrets', + license: 'license-secrets', + users: 'users-secrets', + alerts: 'alerts-secrets', + cluster: 'cluster-secrets', +} + +/** + * Find the group prefix for a secret path. + * Returns the prefix that maps to the secret name (e.g., 'apps.harbor' for 'apps.harbor.adminPassword'). + */ +const findGroupPrefix = (secretPath: string): string | undefined => { + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) { + return `teamConfig.${teamMatch[1]}` + } + + const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) + for (const prefix of sortedKeys) { + if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { + return prefix + } + } + + // Fallback: use first two path segments + const parts = secretPath.split('.') + if (parts.length >= 2) { + return parts.slice(0, 2).join('.') + } + return undefined +} + /** * Derive a K8s secret name from the secret path prefix. */ @@ -188,34 +270,10 @@ const deriveSecretName = (secretPath: string): string => { return 'team-settings-secrets' } - // Map specific path prefixes to secret names - const nameMap: Record = { - 'apps.harbor': 'harbor-secrets', - 'apps.gitea': 'gitea-secrets', - 'apps.keycloak': 'keycloak-secrets', - 'apps.grafana': 'grafana-secrets', - 'apps.loki': 'loki-secrets', - 'apps.oauth2-proxy': 'oauth2-proxy-secrets', - 'apps.oauth2-proxy-redis': 'oauth2-proxy-redis-secrets', - 'apps.prometheus': 'prometheus-secrets', - 'apps.otomi-api': 'otomi-api-secrets', - 'apps.cert-manager': 'cert-manager-secrets', - 'apps.kubeflow-pipelines': 'kubeflow-pipelines-secrets', - otomi: 'otomi-platform-secrets', - oidc: 'oidc-secrets', - smtp: 'smtp-secrets', - dns: 'dns-secrets', - obj: 'obj-storage-secrets', - license: 'license-secrets', - users: 'users-secrets', - alerts: 'alerts-secrets', - cluster: 'cluster-secrets', - } - - const sortedKeys = Object.keys(nameMap).sort((a, b) => b.length - a.length) + const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) for (const prefix of sortedKeys) { if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { - return nameMap[prefix] + return SECRET_NAME_MAP[prefix] } } @@ -231,19 +289,26 @@ const deriveSecretName = (secretPath: string): string => { export const buildSecretToNamespaceMap = async ( secrets: Record, teams: string[], + allValues?: Record, deps = { getSchemaSecretsPaths }, ): Promise => { const secretPaths = await deps.getSchemaSecretsPaths(teams) const flat = flattenObject(secrets) + const allFlat = allValues ? flattenObject(allValues) : flat // Group by namespace + secretName const groupMap = new Map() + // Determine which secret path prefixes have overrides + const overriddenPrefixes = Object.keys(APP_SECRET_OVERRIDES) + for (const secretPath of secretPaths) { // Skip SOPS-related paths if (secretPath.startsWith('kms.sops')) continue // Skip 'users' path — not a simple key-value secret if (secretPath === 'users') continue + // Skip overridden prefixes — they are handled separately below + if (overriddenPrefixes.some((p) => secretPath === p || secretPath.startsWith(`${p}.`))) continue const namespace = resolveNamespace(secretPath) if (!namespace) continue @@ -257,11 +322,18 @@ export const buildSecretToNamespaceMap = async ( const mapping = groupMap.get(groupKey)! + // Find the group prefix (e.g., 'apps.harbor' for 'apps.harbor.adminPassword') + const groupPrefix = findGroupPrefix(secretPath) + // Find all flat keys that match this secret path for (const [flatKey, value] of Object.entries(flat)) { if (flatKey === secretPath || flatKey.startsWith(`${secretPath}.`)) { - // Use the leaf key name as the data key - const dataKey = flatKey.replace(/\./g, '_') + // Use leaf key: strip the group prefix to get relative path + const relativePath = + groupPrefix && (flatKey === groupPrefix || flatKey.startsWith(`${groupPrefix}.`)) + ? flatKey.slice(groupPrefix.length + 1) + : flatKey + const dataKey = relativePath.replace(/\./g, '_') if (value !== undefined && value !== null && value !== '') { mapping.data[dataKey] = String(value) } @@ -269,6 +341,42 @@ export const buildSecretToNamespaceMap = async ( } } + // Process APP_SECRET_OVERRIDES + for (const [, overrides] of Object.entries(APP_SECRET_OVERRIDES)) { + for (const override of overrides) { + const data: Record = {} + let hasValuePathData = false + + // First pass: collect valuePath data + for (const [key, source] of Object.entries(override.data)) { + if (!('static' in source)) { + const value = allFlat[source.valuePath] + if (value !== undefined && value !== null && value !== '') { + data[key] = String(value) + hasValuePathData = true + } + } + } + + // Only add static values if we have at least one valuePath value + if (hasValuePathData) { + for (const [key, source] of Object.entries(override.data)) { + if ('static' in source) { + data[key] = source.static + } + } + } + + if (Object.keys(data).length > 0) { + groupMap.set(`${override.namespace}/${override.secretName}`, { + namespace: override.namespace, + secretName: override.secretName, + data, + }) + } + } + } + // Filter out empty mappings return Array.from(groupMap.values()).filter((m) => Object.keys(m.data).length > 0) } @@ -333,6 +441,7 @@ export const writeSealedSecretManifests = async ( export const bootstrapSealedSecrets = async ( secrets: Record, envDir: string, + allValues?: Record, deps = { terminal, generateSealedSecretsKeyPair, @@ -358,7 +467,7 @@ export const bootstrapSealedSecrets = async ( // 5. Build secret-to-namespace mapping const teams = Object.keys(get(secrets, 'teamConfig', {}) as Record) - const mappings = await deps.buildSecretToNamespaceMap(secrets, teams) + const mappings = await deps.buildSecretToNamespaceMap(secrets, teams, allValues) // 6. Create SealedSecret manifests const manifests: SealedSecretManifest[] = [] diff --git a/values/gitea-db-secret/gitea-db-secret-raw.gotmpl b/values/gitea-db-secret/gitea-db-secret-raw.gotmpl index 792c355e5f..c9acfc0dd8 100644 --- a/values/gitea-db-secret/gitea-db-secret-raw.gotmpl +++ b/values/gitea-db-secret/gitea-db-secret-raw.gotmpl @@ -1,12 +1,2 @@ -{{- $v := .Values }} -{{- $g := $v.apps.gitea }} - -resources: -- apiVersion: v1 - kind: Secret - type: kubernetes.io/basic-auth - metadata: - name: gitea-db-secret - data: - username: "{{ "gitea" | b64enc }}" - password: "{{ $g.postgresqlPassword | b64enc }}" +# This file is intentionally empty. +# gitea-db-secret is now managed via SealedSecrets (see APP_SECRET_OVERRIDES in sealed-secrets.ts) diff --git a/values/gitea/gitea-raw.gotmpl b/values/gitea/gitea-raw.gotmpl index 0b766b4a9c..6a0bd3ef9a 100644 --- a/values/gitea/gitea-raw.gotmpl +++ b/values/gitea/gitea-raw.gotmpl @@ -21,13 +21,6 @@ resources: data: ca-certificates.crt: {{ .Values._derived.caCert | b64enc }} {{- end }} -- apiVersion: v1 - kind: Secret - metadata: - name: gitea-admin-secret - data: - username: "{{ $g.adminUsername | b64enc }}" - password: "{{ $g.adminPassword | b64enc }}" # DB / app backup resources {{- if eq $obj.type "linode" }} - apiVersion: v1 From fab27278dda577eaaeb38086cb74df08add01c6c Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Feb 2026 10:02:34 +0100 Subject: [PATCH 06/20] test: sealed secrets --- src/cmd/install.ts | 21 ++++++++ src/common/sealed-secrets.ts | 94 +++++++++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 2de0cbd1c5..bd7638cdb0 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -5,6 +5,7 @@ import { logLevelString, terminal } from 'src/common/debug' import { env } from 'src/common/envalid' import { deployEssential, hf, HF_DEFAULT_SYNC_ARGS } from 'src/common/hf' import { applyServerSide, getDeploymentState, getHelmReleases, setDeploymentState, waitForCRD } from 'src/common/k8s' +import { applySealedSecretManifestsFromDir } from 'src/common/sealed-secrets' import { getFilename, rootDir } from 'src/common/utils' import { getImageTagFromValues, getPackageVersion, writeValuesToFile } from 'src/common/values' import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs' @@ -82,6 +83,26 @@ export const installAll = async () => { { streams: { stdout: d.stream.log, stderr: d.stream.error } }, ) + // Deploy sealed-secrets controller first (needs to be ready before applying SealedSecrets) + d.info('Deploying sealed-secrets controller') + await hf( + { + fileOpts: 'helmfile.d/helmfile-01.init.yaml.gotmpl', + labelOpts: ['name=sealed-secrets'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + + // Wait for SealedSecret CRD to be established + d.info('Waiting for SealedSecret CRD to be ready') + await retryInstallStep(waitForCRD, 'sealedsecrets.bitnami.com') + + // Apply SealedSecret manifests from disk (generated during bootstrap) + d.info('Applying SealedSecret manifests') + await applySealedSecretManifestsFromDir(env.ENV_DIR) + d.info('Deploying charts containing label app=core') await hf( { diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 3ed223fa94..6af278f837 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -1,8 +1,10 @@ import { encryptSecretItem } from '@linode/kubeseal-encrypt' import { X509Certificate } from 'crypto' -import { mkdir, writeFile } from 'fs/promises' +import { existsSync } from 'fs' +import { mkdir, readdir, readFile, writeFile } from 'fs/promises' import { get } from 'lodash' import { pki } from 'node-forge' +import { join } from 'path' import { terminal } from 'src/common/debug' import { flattenObject, getSchemaSecretsPaths } from 'src/common/utils' import { objectToYaml } from 'src/common/values' @@ -434,6 +436,94 @@ export const writeSealedSecretManifests = async ( } } +/** + * Apply SealedSecret manifests to the Kubernetes cluster. + * Creates namespaces if needed and applies the SealedSecret resources. + */ +export const applySealedSecretManifests = async ( + manifests: SealedSecretManifest[], + deps = { $, terminal, objectToYaml }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:applySealedSecretManifests`) + + // Group manifests by namespace + const byNamespace = new Map() + for (const manifest of manifests) { + const ns = manifest.metadata.namespace + if (!byNamespace.has(ns)) { + byNamespace.set(ns, []) + } + byNamespace.get(ns)!.push(manifest) + } + + // Ensure namespaces exist and apply manifests + for (const [namespace, nsManifests] of byNamespace) { + d.info(`Ensuring namespace ${namespace} exists`) + await deps.$`kubectl create namespace ${namespace} --dry-run=client -o yaml | kubectl apply -f -`.nothrow().quiet() + + for (const manifest of nsManifests) { + d.info(`Applying SealedSecret ${manifest.metadata.name} to namespace ${namespace}`) + const yaml = deps.objectToYaml(manifest) + const result = await deps.$`echo ${yaml} | kubectl apply -f -`.nothrow().quiet() + if (result.exitCode !== 0) { + d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${result.stderr}`) + } + } + } + + d.info(`Applied ${manifests.length} SealedSecret manifests to cluster`) +} + +/** + * Read and apply all SealedSecret manifests from the env/manifests/ns directory. + * This should be called during install, after the sealed-secrets controller is deployed. + */ +export const applySealedSecretManifestsFromDir = async ( + envDir: string, + deps = { $, terminal, readdir, readFile, existsSync }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) + const manifestsDir = join(envDir, 'env/manifests/ns') + + if (!deps.existsSync(manifestsDir)) { + d.info(`No SealedSecret manifests directory found at ${manifestsDir}`) + return + } + + d.info(`Applying SealedSecret manifests from ${manifestsDir}`) + + // Read all namespace directories + const namespaces = await deps.readdir(manifestsDir, { withFileTypes: true }) + let appliedCount = 0 + + for (const nsEntry of namespaces) { + if (!nsEntry.isDirectory()) continue + const namespace = nsEntry.name + const nsDir = join(manifestsDir, namespace) + + // Ensure namespace exists + d.info(`Ensuring namespace ${namespace} exists`) + await deps.$`kubectl create namespace ${namespace} --dry-run=client -o yaml | kubectl apply -f -`.nothrow().quiet() + + // Read all YAML files in the namespace directory + const files = await deps.readdir(nsDir) + for (const file of files) { + if (!file.endsWith('.yaml') && !file.endsWith('.yml')) continue + const filePath = join(nsDir, file) + d.info(`Applying SealedSecret from ${filePath}`) + + const result = await deps.$`kubectl apply -f ${filePath}`.nothrow().quiet() + if (result.exitCode !== 0) { + d.error(`Failed to apply SealedSecret from ${filePath}: ${result.stderr}`) + } else { + appliedCount += 1 + } + } + } + + d.info(`Applied ${appliedCount} SealedSecret manifests from directory`) +} + /** * Orchestrator: bootstrap sealed secrets for the platform. * Replaces bootstrapSops(). @@ -479,6 +569,8 @@ export const bootstrapSealedSecrets = async ( } // 7. Write SealedSecret manifests to disk + // Note: These manifests are applied later during install, after the sealed-secrets + // controller is deployed and the SealedSecret CRD is available. await deps.writeSealedSecretManifests(manifests, envDir) d.info(`Bootstrapped ${manifests.length} sealed secret manifests`) From 5e7e6b35470e5801fc0c562bbca62ba13ea3189a Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Feb 2026 10:37:18 +0100 Subject: [PATCH 07/20] test: sealed secrets --- src/common/sealed-secrets.test.ts | 26 +++++++++++++++++++++++++- src/common/sealed-secrets.ts | 19 +++++++++++++++---- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 79fe4453a9..907b7f6682 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -308,6 +308,30 @@ describe('sealed-secrets', () => { expect(giteaDbMapping!.data.username).toBe('gitea') expect(giteaDbMapping!.data.password).toBe('pg-pass') }) + + it('should use default values when valuePath not found but actual value exists', async () => { + // When adminPassword exists but adminUsername doesn't, username should use default + const secrets = { + apps: { + gitea: { adminPassword: 'gitea-pass' }, + }, + } + const allValues = { + apps: { + gitea: { adminPassword: 'gitea-pass' }, // no adminUsername provided + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.gitea.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], allValues, deps) + + const giteaAdminMapping = result.find((m) => m.secretName === 'gitea-admin-secret') + expect(giteaAdminMapping).toBeDefined() + expect(giteaAdminMapping!.data.username).toBe('otomi-admin') // default value + expect(giteaAdminMapping!.data.password).toBe('gitea-pass') + }) }) describe('createSealedSecretManifest', () => { @@ -491,7 +515,7 @@ describe('sealed-secrets', () => { const adminSecret = APP_SECRET_OVERRIDES['apps.gitea'].find((o) => o.secretName === 'gitea-admin-secret') expect(adminSecret).toBeDefined() expect(adminSecret!.namespace).toBe('gitea') - expect(adminSecret!.data.username).toEqual({ valuePath: 'apps.gitea.adminUsername' }) + expect(adminSecret!.data.username).toEqual({ valuePath: 'apps.gitea.adminUsername', default: 'otomi-admin' }) expect(adminSecret!.data.password).toEqual({ valuePath: 'apps.gitea.adminPassword' }) const dbSecret = APP_SECRET_OVERRIDES['apps.gitea'].find((o) => o.secretName === 'gitea-db-secret') diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 6af278f837..3ee89b3c22 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -72,7 +72,7 @@ export const APP_NAMESPACE_MAP: Record = { interface SecretOverrideEntry { secretName: string namespace: string - data: Record + data: Record } export const APP_SECRET_OVERRIDES: Record = { @@ -81,7 +81,7 @@ export const APP_SECRET_OVERRIDES: Record = { secretName: 'gitea-admin-secret', namespace: 'gitea', data: { - username: { valuePath: 'apps.gitea.adminUsername' }, + username: { valuePath: 'apps.gitea.adminUsername', default: 'otomi-admin' }, password: { valuePath: 'apps.gitea.adminPassword' }, }, }, @@ -349,19 +349,30 @@ export const buildSecretToNamespaceMap = async ( const data: Record = {} let hasValuePathData = false - // First pass: collect valuePath data + // First pass: collect valuePath data and track if any actual values were found + const pendingDefaults: Array<{ key: string; defaultValue: string }> = [] for (const [key, source] of Object.entries(override.data)) { if (!('static' in source)) { const value = allFlat[source.valuePath] if (value !== undefined && value !== null && value !== '') { data[key] = String(value) hasValuePathData = true + } else if (source.default !== undefined) { + // Queue default value - will be added only if we have at least one actual value + pendingDefaults.push({ key, defaultValue: source.default }) } } } - // Only add static values if we have at least one valuePath value + // Only add defaults and static values if we have at least one actual valuePath value if (hasValuePathData) { + // Add pending defaults + for (const { key, defaultValue } of pendingDefaults) { + if (!(key in data)) { + data[key] = defaultValue + } + } + // Add static values for (const [key, source] of Object.entries(override.data)) { if ('static' in source) { data[key] = source.static From ef02d3ac4f11c7a8f21f1d7b9a69935b5a69efa9 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Feb 2026 11:00:37 +0100 Subject: [PATCH 08/20] test: sealed secrets --- src/common/sealed-secrets.test.ts | 87 +++++++++++++++++++++++++++---- src/common/sealed-secrets.ts | 72 ++++++++++++++++++++----- 2 files changed, 136 insertions(+), 23 deletions(-) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 907b7f6682..28ce051e70 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -123,11 +123,25 @@ describe('sealed-secrets', () => { }) describe('createSealedSecretsKeySecret', () => { - it('should call kubectl to create namespace and TLS secret', async () => { - const mockResult = { stderr: '', exitCode: 0 } - const mockQuiet = jest.fn().mockResolvedValue(mockResult) + it('should create secret if it does not exist', async () => { + const mockQuiet = jest.fn().mockResolvedValue({ stderr: '', exitCode: 0 }) const mockNothrow = jest.fn().mockReturnValue({ quiet: mockQuiet }) - const mock$ = jest.fn().mockReturnValue({ nothrow: mockNothrow }) + // First call (namespace): success, Second call (check exists): not found (exitCode 1) + // Third call (create): success, Fourth call (label): success + const mock$ = jest + .fn() + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // namespace + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 1 }) }), + }) // check exists - NOT FOUND + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // create + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // label const deps = { $: mock$ as any, terminal, @@ -137,11 +151,33 @@ describe('sealed-secrets', () => { await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) - // Should have been called 3 times: namespace creation, secret creation, labeling - expect(mock$).toHaveBeenCalledTimes(3) + expect(mock$).toHaveBeenCalledTimes(4) expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.crt', 'cert-pem') expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.key', 'key-pem') }) + + it('should skip creation if secret already exists', async () => { + const mock$ = jest + .fn() + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // namespace + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // check exists - FOUND + const deps = { + $: mock$ as any, + terminal, + writeFile: jest.fn(), + mkdir: jest.fn(), + } + + await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) + + // Should only call namespace and check exists, not create or label + expect(mock$).toHaveBeenCalledTimes(2) + expect(deps.writeFile).not.toHaveBeenCalled() + }) }) describe('buildSecretToNamespaceMap', () => { @@ -414,7 +450,7 @@ describe('sealed-secrets', () => { }) describe('bootstrapSealedSecrets', () => { - it('should orchestrate all steps in sequence', async () => { + it('should generate new key pair when no existing cert found', async () => { const secrets = { apps: { harbor: { adminPassword: 'pass' } }, } @@ -443,6 +479,7 @@ describe('sealed-secrets', () => { const deps = { terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), // No existing cert generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ certificate: 'cert-pem', privateKey: 'key-pem', @@ -457,16 +494,43 @@ describe('sealed-secrets', () => { await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + expect(deps.getExistingSealedSecretsCert).toHaveBeenCalled() expect(deps.generateSealedSecretsKeyPair).toHaveBeenCalled() expect(deps.createSealedSecretsKeySecret).toHaveBeenCalledWith('cert-pem', 'key-pem') expect(deps.getPemFromCertificate).toHaveBeenCalledWith('cert-pem') - expect(deps.buildSecretToNamespaceMap).toHaveBeenCalledWith(secrets, [], undefined) - expect(deps.createSealedSecretManifest).toHaveBeenCalledWith('spki-pem', mockMapping, { - encryptSecretItem: deps.encryptSecretItem, - }) expect(deps.writeSealedSecretManifests).toHaveBeenCalledWith([mockManifest], '/test') }) + it('should use existing cert when found', async () => { + const secrets = { + apps: { harbor: { adminPassword: 'pass' } }, + } + const mockMapping = { + namespace: 'harbor', + secretName: 'harbor-secrets', + data: { adminPassword: 'pass' }, + } + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue('existing-cert-pem'), // Existing cert found + generateSealedSecretsKeyPair: jest.fn(), + getPemFromCertificate: jest.fn().mockReturnValue('existing-spki-pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), + createSealedSecretManifest: jest.fn().mockResolvedValue({}), + writeSealedSecretManifests: jest.fn(), + encryptSecretItem: jest.fn(), + } + + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + + expect(deps.getExistingSealedSecretsCert).toHaveBeenCalled() + expect(deps.generateSealedSecretsKeyPair).not.toHaveBeenCalled() // Should NOT generate new key + expect(deps.createSealedSecretsKeySecret).not.toHaveBeenCalled() // Should NOT create secret + expect(deps.getPemFromCertificate).toHaveBeenCalledWith('existing-cert-pem') + }) + it('should extract team names from secrets', async () => { const secrets = { teamConfig: { @@ -477,6 +541,7 @@ describe('sealed-secrets', () => { const deps = { terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ certificate: 'cert', privateKey: 'key', diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 3ee89b3c22..40d9b381cf 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -152,9 +152,38 @@ export const getPemFromCertificate = (certificate: string): string => { return typeof exported === 'string' ? exported : exported.toString('utf-8') } +/** + * Get the existing sealed-secrets certificate from the cluster if it exists. + * Returns the certificate PEM string or undefined if not found. + */ +export const getExistingSealedSecretsCert = async (deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:getExistingSealedSecretsCert`) + + const result = + await deps.$`kubectl get secret sealed-secrets-key -n sealed-secrets -o jsonpath='{.data.tls\\.crt}' 2>/dev/null` + .nothrow() + .quiet() + + if (result.exitCode !== 0 || !result.stdout || result.stdout === '') { + d.info('No existing sealed-secrets-key found') + return undefined + } + + try { + const certBase64 = result.stdout.replace(/'/g, '') + const cert = Buffer.from(certBase64, 'base64').toString('utf-8') + d.info('Found existing sealed-secrets-key certificate') + return cert + } catch { + d.warn('Failed to decode existing certificate') + return undefined + } +} + /** * Create the sealed-secrets namespace and TLS secret in Kubernetes. * The controller will pick up this pre-created key on startup. + * IMPORTANT: This only creates the secret if it doesn't already exist. */ export const createSealedSecretsKeySecret = async ( certificate: string, @@ -162,11 +191,19 @@ export const createSealedSecretsKeySecret = async ( deps = { $, terminal, writeFile, mkdir }, ): Promise => { const d = deps.terminal(`common:${cmdName}:createSealedSecretsKeySecret`) - d.info('Creating sealed-secrets namespace and TLS secret') // Create namespace await deps.$`kubectl create namespace sealed-secrets --dry-run=client -o yaml | kubectl apply -f -`.nothrow().quiet() + // Check if secret already exists + const existingSecret = await deps.$`kubectl get secret sealed-secrets-key -n sealed-secrets`.nothrow().quiet() + if (existingSecret.exitCode === 0) { + d.info('sealed-secrets-key already exists, skipping creation') + return + } + + d.info('Creating sealed-secrets TLS secret') + // Write temp files for kubectl create secret tls const tmpDir = '/tmp/sealed-secrets-bootstrap' await deps.mkdir(tmpDir, { recursive: true }) @@ -175,12 +212,15 @@ export const createSealedSecretsKeySecret = async ( await deps.writeFile(certPath, certificate) await deps.writeFile(keyPath, privateKey) - // Create the TLS secret + // Create the TLS secret (only if it doesn't exist) const result = - await deps.$`kubectl create secret tls sealed-secrets-key -n sealed-secrets --cert=${certPath} --key=${keyPath} --dry-run=client -o yaml | kubectl apply -f -` + await deps.$`kubectl create secret tls sealed-secrets-key -n sealed-secrets --cert=${certPath} --key=${keyPath}` .nothrow() .quiet() - if (result.stderr) d.error(result.stderr) + if (result.exitCode !== 0) { + d.error(`Failed to create sealed-secrets-key: ${result.stderr}`) + return + } // Label the secret so the controller picks it up const labelResult = @@ -548,6 +588,7 @@ export const bootstrapSealedSecrets = async ( generateSealedSecretsKeyPair, getPemFromCertificate, createSealedSecretsKeySecret, + getExistingSealedSecretsCert, buildSecretToNamespaceMap, createSealedSecretManifest, writeSealedSecretManifests, @@ -557,14 +598,21 @@ export const bootstrapSealedSecrets = async ( const d = deps.terminal(`common:${cmdName}:bootstrapSealedSecrets`) d.info('Bootstrapping sealed secrets') - // 1. Generate RSA key pair + self-signed X.509 certificate - const { certificate, privateKey } = deps.generateSealedSecretsKeyPair() - - // 2 & 3. Create namespace and store key pair as K8s TLS secret - await deps.createSealedSecretsKeySecret(certificate, privateKey) - - // 4. Extract SPKI PEM public key from certificate - const pem = deps.getPemFromCertificate(certificate) + // 1. Check if there's an existing sealed-secrets key in the cluster + const existingCert = await deps.getExistingSealedSecretsCert() + + let pem: string + if (existingCert) { + // Use existing certificate for encryption + d.info('Using existing sealed-secrets certificate') + pem = deps.getPemFromCertificate(existingCert) + } else { + // Generate new key pair and create the secret + d.info('Generating new sealed-secrets key pair') + const { certificate, privateKey } = deps.generateSealedSecretsKeyPair() + await deps.createSealedSecretsKeySecret(certificate, privateKey) + pem = deps.getPemFromCertificate(certificate) + } // 5. Build secret-to-namespace mapping const teams = Object.keys(get(secrets, 'teamConfig', {}) as Record) From 3146d3a0003a20e12d96db1d2e1e45de3d06051f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Feb 2026 14:21:34 +0100 Subject: [PATCH 09/20] test: sealed secrets --- src/common/values.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/values.ts b/src/common/values.ts index 5e2a0d7fc5..7a597f5eea 100644 --- a/src/common/values.ts +++ b/src/common/values.ts @@ -168,7 +168,7 @@ export const writeValues = async (inValues: Record, overwrite = fal d.debug('secrets: ', JSON.stringify(secrets, null, 2)) // from the plain values const plainValues = omit(values, cleanSecretPaths) as any - await saveValues(env.ENV_DIR, plainValues, secrets) + await saveValues(env.ENV_DIR, inValues, {}) d.info('All values were written to ENV_DIR') } From a657b0eec24a6f5be26d9330097dcecc8e00ba17 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Feb 2026 15:03:56 +0100 Subject: [PATCH 10/20] test: sealed secrets --- src/cmd/install.test.ts | 5 +++++ src/cmd/install.ts | 8 +++++++- src/common/sealed-secrets.ts | 26 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/cmd/install.test.ts b/src/cmd/install.test.ts index 2645cb2c9b..c97f0dbc2d 100644 --- a/src/cmd/install.test.ts +++ b/src/cmd/install.test.ts @@ -39,6 +39,11 @@ jest.mock('zx', () => ({ cd: jest.fn(), })) +jest.mock('src/common/sealed-secrets', () => ({ + applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue(undefined), + restartSealedSecretsController: jest.fn().mockResolvedValue(undefined), +})) + jest.mock('./commit', () => ({ commit: jest.fn(), deletePendingHelmReleases: jest.fn(), diff --git a/src/cmd/install.ts b/src/cmd/install.ts index bd7638cdb0..a9bafe1c8c 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -5,7 +5,7 @@ import { logLevelString, terminal } from 'src/common/debug' import { env } from 'src/common/envalid' import { deployEssential, hf, HF_DEFAULT_SYNC_ARGS } from 'src/common/hf' import { applyServerSide, getDeploymentState, getHelmReleases, setDeploymentState, waitForCRD } from 'src/common/k8s' -import { applySealedSecretManifestsFromDir } from 'src/common/sealed-secrets' +import { applySealedSecretManifestsFromDir, restartSealedSecretsController } from 'src/common/sealed-secrets' import { getFilename, rootDir } from 'src/common/utils' import { getImageTagFromValues, getPackageVersion, writeValuesToFile } from 'src/common/values' import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs' @@ -103,6 +103,12 @@ export const installAll = async () => { d.info('Applying SealedSecret manifests') await applySealedSecretManifestsFromDir(env.ENV_DIR) + // Restart the sealed-secrets controller to ensure it uses the correct key + // This is needed because the controller may have generated its own key before + // the bootstrap-created sealed-secrets-key secret was available + d.info('Restarting sealed-secrets controller') + await restartSealedSecretsController() + d.info('Deploying charts containing label app=core') await hf( { diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 40d9b381cf..f60e34a303 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -575,6 +575,32 @@ export const applySealedSecretManifestsFromDir = async ( d.info(`Applied ${appliedCount} SealedSecret manifests from directory`) } +/** + * Restart the sealed-secrets controller to ensure it uses the correct key. + * This is needed because if the controller starts before the sealed-secrets-key secret exists, + * it will generate its own key. Restarting forces it to pick up the existing key. + */ +export const restartSealedSecretsController = async (deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:restartSealedSecretsController`) + d.info('Restarting sealed-secrets controller to ensure correct key is used') + + const result = await deps.$`kubectl rollout restart deployment/sealed-secrets -n sealed-secrets`.nothrow().quiet() + if (result.exitCode !== 0) { + d.warn(`Failed to restart sealed-secrets controller: ${result.stderr}`) + return + } + + d.info('Waiting for sealed-secrets controller rollout') + const waitResult = await deps.$`kubectl rollout status deployment/sealed-secrets -n sealed-secrets --timeout=120s` + .nothrow() + .quiet() + if (waitResult.exitCode !== 0) { + d.warn(`Rollout status check failed: ${waitResult.stderr}`) + } else { + d.info('Sealed-secrets controller restarted successfully') + } +} + /** * Orchestrator: bootstrap sealed secrets for the platform. * Replaces bootstrapSops(). From 11aba23e236a5fa7b1a70f04fd5d945f133a13b1 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Feb 2026 19:17:33 +0100 Subject: [PATCH 11/20] test: sealed secrets & fix tls issue --- .../ingress-nginx/templates/clusterrole.yaml | 1 + values/ingress-nginx/ingress-nginx-raw.gotmpl | 34 ++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/charts/ingress-nginx/templates/clusterrole.yaml b/charts/ingress-nginx/templates/clusterrole.yaml index 51bc5002cc..15b2ac4b55 100644 --- a/charts/ingress-nginx/templates/clusterrole.yaml +++ b/charts/ingress-nginx/templates/clusterrole.yaml @@ -27,6 +27,7 @@ rules: - namespaces {{- end}} verbs: + - get - list - watch - apiGroups: diff --git a/values/ingress-nginx/ingress-nginx-raw.gotmpl b/values/ingress-nginx/ingress-nginx-raw.gotmpl index 28e89c9963..366b4d6389 100644 --- a/values/ingress-nginx/ingress-nginx-raw.gotmpl +++ b/values/ingress-nginx/ingress-nginx-raw.gotmpl @@ -7,10 +7,42 @@ resources: labels: app.kubernetes.io/component: controller name: {{ $ingress.className }} - {{- if eq $ingress.className $v.ingress.platformClass.className }} + {{- if eq $ingress.className $v.ingress.platformClass.className }} annotations: ingressclass.kubernetes.io/is-default-class: "true" {{- end }} spec: controller: "k8s.io/{{ $ingress.className }}" +{{- end }} +# ClusterRole to allow ingress controller to read TLS secrets from all namespaces +{{- range $ingress := $v.ingress.classes }} +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: ingress-nginx-{{ $ingress.className }}-secrets-reader + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx-{{ $ingress.className }} + rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: ingress-nginx-{{ $ingress.className }}-secrets-reader + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx-{{ $ingress.className }} + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-{{ $ingress.className }}-secrets-reader + subjects: + - kind: ServiceAccount + name: ingress-nginx-{{ $ingress.className }} + namespace: ingress {{- end }} \ No newline at end of file From 4c1e369513cafe8db0a53073c64325ac8ed36e61 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 5 Feb 2026 21:16:21 +0100 Subject: [PATCH 12/20] test: sealed secrets --- src/common/sealed-secrets.ts | 38 +++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index f60e34a303..c26232937c 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -12,6 +12,32 @@ import { $ } from 'zx' const cmdName = 'sealed-secrets' +/** + * Ensure a namespace exists. If it doesn't exist, create it with proper labels. + * This avoids overwriting labels on existing namespaces that were created by k8s-raw.gotmpl. + */ +export const ensureNamespaceExists = async (namespace: string, deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:ensureNamespaceExists`) + + // Check if namespace already exists + const existingNs = await deps.$`kubectl get namespace ${namespace}`.nothrow().quiet() + if (existingNs.exitCode === 0) { + d.debug(`Namespace ${namespace} already exists`) + return + } + + // Create namespace with proper label + d.info(`Creating namespace ${namespace}`) + const nsYaml = `apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} + labels: + name: ${namespace}` + + await deps.$`echo ${nsYaml} | kubectl apply -f -`.nothrow().quiet() +} + export interface SecretMapping { namespace: string secretName: string @@ -192,8 +218,8 @@ export const createSealedSecretsKeySecret = async ( ): Promise => { const d = deps.terminal(`common:${cmdName}:createSealedSecretsKeySecret`) - // Create namespace - await deps.$`kubectl create namespace sealed-secrets --dry-run=client -o yaml | kubectl apply -f -`.nothrow().quiet() + // Create namespace if it doesn't exist + await ensureNamespaceExists('sealed-secrets', { $: deps.$, terminal: deps.terminal }) // Check if secret already exists const existingSecret = await deps.$`kubectl get secret sealed-secrets-key -n sealed-secrets`.nothrow().quiet() @@ -509,8 +535,7 @@ export const applySealedSecretManifests = async ( // Ensure namespaces exist and apply manifests for (const [namespace, nsManifests] of byNamespace) { - d.info(`Ensuring namespace ${namespace} exists`) - await deps.$`kubectl create namespace ${namespace} --dry-run=client -o yaml | kubectl apply -f -`.nothrow().quiet() + await ensureNamespaceExists(namespace, { $: deps.$, terminal: deps.terminal }) for (const manifest of nsManifests) { d.info(`Applying SealedSecret ${manifest.metadata.name} to namespace ${namespace}`) @@ -552,9 +577,8 @@ export const applySealedSecretManifestsFromDir = async ( const namespace = nsEntry.name const nsDir = join(manifestsDir, namespace) - // Ensure namespace exists - d.info(`Ensuring namespace ${namespace} exists`) - await deps.$`kubectl create namespace ${namespace} --dry-run=client -o yaml | kubectl apply -f -`.nothrow().quiet() + // Ensure namespace exists with proper labels + await ensureNamespaceExists(namespace, { $: deps.$, terminal: deps.terminal }) // Read all YAML files in the namespace directory const files = await deps.readdir(nsDir) From f435dfd3547601f761e3e513ef0114ad132a624d Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Feb 2026 09:31:07 +0100 Subject: [PATCH 13/20] revert: unnecessary changes --- .values/.gitattributes | 1 + .values/.gitignore | 1 - helmfile.d/snippets/env.old.gotmpl | 21 ++++++++++++++------- 3 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 .values/.gitattributes diff --git a/.values/.gitattributes b/.values/.gitattributes new file mode 100644 index 0000000000..4f684e3799 --- /dev/null +++ b/.values/.gitattributes @@ -0,0 +1 @@ +secrets.*.yaml diff=sopsdiffer diff --git a/.values/.gitignore b/.values/.gitignore index 169ac1d5dc..c1e96df987 100644 --- a/.values/.gitignore +++ b/.values/.gitignore @@ -14,4 +14,3 @@ core.yaml env/status.yaml env/bootstrap.yaml values-repo.yaml -secrets.*.yaml diff --git a/helmfile.d/snippets/env.old.gotmpl b/helmfile.d/snippets/env.old.gotmpl index e1d73a6140..c0c5fbdda7 100644 --- a/helmfile.d/snippets/env.old.gotmpl +++ b/helmfile.d/snippets/env.old.gotmpl @@ -7,10 +7,12 @@ {{- /* We relocated teamConfig.teams to teamConfig, so one time we might have to fall back to our previous location */}} {{- /* TODO:[teamConfig] deprecate somewhere in da future */}} {{- if hasKey $t.teamConfig "teams" }}{{ $teams = keys $t.teamConfig.teams }}{{ end }} +{{- $hasSops := eq (exec "bash" (list "-c" "( test -f $ENV_DIR/.sops.yaml && echo 'true' ) || echo 'false'") | trim) "true" }} {{- $apps := (exec "bash" (list "-c" "find $ENV_DIR/env/apps -name '*.yaml' -not -name 'secrets.*.yaml'")) | splitList "\n" }} {{- $databases := (exec "bash" (list "-c" "find $ENV_DIR/env/databases -name '*.yaml' || true ")) | splitList "\n" }} {{- $appsSecret := (exec "bash" (list "-c" "find $ENV_DIR/env/apps -name 'secrets.*.yaml'")) | splitList "\n" }} -{{- $teamFileExists := eq (exec "bash" (list "-c" "( test -f $ENV_DIR/env/secrets.teams.yaml && echo 'true' ) || echo 'false'") | trim) "true" }} +{{- $ext := ($hasSops | ternary ".dec" "") }} +{{- $teamFileExists := eq (exec "bash" (list "-c" (printf "( test -f $ENV_DIR/env/secrets.teams.yaml%s && echo 'true' ) || echo 'false'" $ext)) | trim) "true" }} helmDefaults: atomic: true historyMax: 3 @@ -32,7 +34,8 @@ environments: - {{ $ENV_DIR }}/env/status.yaml {{- end }} {{- if eq (exec "bash" (list "-c" "( test -f $ENV_DIR/env/secrets.license.yaml && echo 'true' ) || echo 'false'") | trim) "true" }} - - {{ $ENV_DIR }}/env/secrets.license.yaml{{- end }} + - {{ $ENV_DIR }}/env/secrets.license.yaml{{ $ext }} +{{- end }} {{- range $app := $apps }}{{ if ne $app "" }} - {{ $app }} @@ -47,13 +50,17 @@ environments: {{- end }} {{- end }} {{- end }} -{{- if eq (exec "bash" (list "-c" "( test -f $ENV_DIR/env/secrets.users.yaml && echo 'true' ) || echo 'false'") | trim) "true" }} - - {{ $ENV_DIR }}/env/secrets.users.yaml{{- end }} +{{- if eq (exec "bash" (list "-c" (printf "( test -f $ENV_DIR/env/secrets.users.yaml%s && echo 'true' ) || echo 'false'" (default "" $ext))) | trim) "true" }} + - {{ $ENV_DIR }}/env/secrets.users.yaml{{ $ext }} +{{- end }} {{- if eq (exec "bash" (list "-c" "( test -f $ENV_DIR/env/secrets.settings.yaml && echo 'true' ) || echo 'false'") | trim) "true" }} - - {{ $ENV_DIR }}/env/secrets.settings.yaml{{- end }} + - {{ $ENV_DIR }}/env/secrets.settings.yaml{{ $ext }} +{{- end }} {{- if $teamFileExists }} {{- if eq (exec "bash" (list "-c" "( test -f $ENV_DIR/env/secrets.teams.yaml && echo 'true' ) || echo 'false'") | trim) "true" }} - - {{ $ENV_DIR }}/env/secrets.teams.yaml {{- end }} + - {{ $ENV_DIR }}/env/secrets.teams.yaml{{ $ext }} + {{- end }} {{- end }} {{- range $app := $appsSecret }}{{ if ne $app "" }}{{ $file := $app | replace (print $ENV_DIR "/env/apps/") "" }} - - {{ $ENV_DIR }}/env/apps/{{ $file }}{{- end }}{{ end }} + - {{ $ENV_DIR }}/env/apps/{{ $file }}{{ $ext }} +{{- end }}{{ end }} From 03c775664ca3a2e2a553870f54378e914c66e9fa Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Feb 2026 10:06:04 +0100 Subject: [PATCH 14/20] fix: sealed secrets --- src/common/sealed-secrets.test.ts | 32 +++++++++++++++++++++++++++++++ src/common/sealed-secrets.ts | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 28ce051e70..d13dc93ab3 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -345,6 +345,38 @@ describe('sealed-secrets', () => { expect(giteaDbMapping!.data.password).toBe('pg-pass') }) + it('should create gitea secrets on first install when generated passwords are only in secrets', async () => { + // On first install, allValues (originalInput) does NOT contain generated passwords, + // but secrets (allSecrets) does. The override lookup must check secrets first. + const secrets = { + apps: { + gitea: { adminPassword: 'generated-admin-pass', postgresqlPassword: 'generated-pg-pass' }, + }, + } + const allValues = { + apps: { + gitea: {}, // No generated passwords in originalInput on first install + }, + } + const deps = { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue(['apps.gitea.adminPassword', 'apps.gitea.postgresqlPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], allValues, deps) + + const giteaAdminMapping = result.find((m) => m.secretName === 'gitea-admin-secret') + expect(giteaAdminMapping).toBeDefined() + expect(giteaAdminMapping!.namespace).toBe('gitea') + expect(giteaAdminMapping!.data).toEqual({ username: 'otomi-admin', password: 'generated-admin-pass' }) + + const giteaDbMapping = result.find((m) => m.secretName === 'gitea-db-secret') + expect(giteaDbMapping).toBeDefined() + expect(giteaDbMapping!.namespace).toBe('gitea') + expect(giteaDbMapping!.data).toEqual({ username: 'gitea', password: 'generated-pg-pass' }) + }) + it('should use default values when valuePath not found but actual value exists', async () => { // When adminPassword exists but adminUsername doesn't, username should use default const secrets = { diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index c26232937c..0d69357c0f 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -419,7 +419,7 @@ export const buildSecretToNamespaceMap = async ( const pendingDefaults: Array<{ key: string; defaultValue: string }> = [] for (const [key, source] of Object.entries(override.data)) { if (!('static' in source)) { - const value = allFlat[source.valuePath] + const value = flat[source.valuePath] ?? allFlat[source.valuePath] if (value !== undefined && value !== null && value !== '') { data[key] = String(value) hasValuePathData = true From 90d88cfc4f330f5842c724c62b1b519239493039 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Feb 2026 13:43:03 +0100 Subject: [PATCH 15/20] test: sealed secrets --- package-lock.json | 37 +++++++++++++++++--- src/common/repo.test.ts | 72 +++++++++++++++++++++++++++++++++++++++ src/common/repo.ts | 64 +++++++++++++++++++++++++++++++--- src/common/values.test.ts | 46 ++++++++++++++++++++++++- src/common/values.ts | 61 ++++++++++++++++++++------------- 5 files changed, 246 insertions(+), 34 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0e397f1a02..741c56efb4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -200,6 +200,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -2465,6 +2466,7 @@ "integrity": "sha512-RsUFrSB0oQHEBnR8yarKIReUPwSu2ROpbjhdVKi4T/nQhMaS+TnIQPBwkMtb2r8A1KS2Hijw4D/4bV/XHoFQWw==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=20" } @@ -2546,7 +2548,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-css/-/dict-css-4.0.19.tgz", "integrity": "sha512-VYHtPnZt/Zd/ATbW3rtexWpBnHUohUrQOHff/2JBhsVgxOrksAxJnLAO43Q1ayLJBJUUwNVo+RU0sx0aaysZfg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-dart": { "version": "2.3.2", @@ -2686,14 +2689,16 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-html/-/dict-html-4.0.14.tgz", "integrity": "sha512-2bf7n+kS92g+cMKV0wr9o/Oq9n8JzU7CcrB96gIh2GHgnF+0xDOqO2W/1KeFAqOfqosoOVE48t+4dnEMkkoJ2Q==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-html-symbol-entities": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@cspell/dict-html-symbol-entities/-/dict-html-symbol-entities-4.0.5.tgz", "integrity": "sha512-429alTD4cE0FIwpMucvSN35Ld87HCyuM8mF731KU5Rm4Je2SG6hmVx7nkBsLyrmH3sQukTcr1GaiZsiEg8svPA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-java": { "version": "5.0.12", @@ -2891,7 +2896,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-typescript/-/dict-typescript-3.2.3.tgz", "integrity": "sha512-zXh1wYsNljQZfWWdSPYwQhpwiuW0KPW1dSd8idjMRvSD0aSvWWHoWlrMsmZeRl4qM4QCEAjua8+cjflm41cQBg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-vue": { "version": "3.0.5", @@ -4994,6 +5000,7 @@ "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -6583,7 +6590,8 @@ "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/json5": { "version": "0.0.29", @@ -6649,6 +6657,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -6848,6 +6857,7 @@ "integrity": "sha512-BtE0k6cjwjLZoZixN0t5AKP0kSzlGu7FctRXYuPAm//aaiZhmfq1JwdYpYr1brzEspYyFeF+8XF5j2VK6oalrA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.54.0", "@typescript-eslint/types": "8.54.0", @@ -7388,6 +7398,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -8269,6 +8280,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -10666,6 +10678,7 @@ "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", @@ -12034,6 +12047,7 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -12094,6 +12108,7 @@ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", "dev": true, "license": "MIT", + "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -12228,6 +12243,7 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -15729,6 +15745,7 @@ "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "30.2.0", "@jest/types": "30.2.0", @@ -17496,6 +17513,7 @@ "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", "license": "MIT", + "peer": true, "engines": { "node": ">= 10.16.0" } @@ -18539,6 +18557,7 @@ "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", "dev": true, "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -21619,6 +21638,7 @@ "dev": true, "inBundle": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -22674,6 +22694,7 @@ "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -23675,6 +23696,7 @@ "integrity": "sha512-WRgl5GcypwramYX4HV+eQGzUbD7UUbljVmS+5G1uMwX/wLgYuJAxGeerXJDMO2xshng4+FXqCgyB5QfClV6WjA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@semantic-release/commit-analyzer": "^13.0.1", "@semantic-release/error": "^4.0.0", @@ -25607,6 +25629,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -25810,6 +25833,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -26107,6 +26131,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -26286,6 +26311,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "napi-postinstall": "^0.2.4" }, @@ -26787,6 +26813,7 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz", "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=10.0.0" }, diff --git a/src/common/repo.test.ts b/src/common/repo.test.ts index 92d057b098..b19813bc24 100644 --- a/src/common/repo.test.ts +++ b/src/common/repo.test.ts @@ -11,6 +11,7 @@ import { hasCorrespondingDecryptedFile, renderManifest, renderManifestForSecrets, + resolveSecretPlaceholders, saveResourceGroupToFiles, sortTeamConfigArraysByName, sortUserArraysByName, @@ -898,3 +899,74 @@ describe('AplCatalog', () => { }) }) }) + +describe('resolveSecretPlaceholders', () => { + it('should resolve sealed placeholders from K8s secrets', async () => { + const values = { + apps: { gitea: { adminPassword: 'sealed:gitea/gitea-admin-secret/password', adminUsername: 'admin' } }, + } + const deps = { + getK8sSecret: jest.fn().mockResolvedValue({ password: 'real-pass', username: 'otomi-admin' }), + } + + const result = await resolveSecretPlaceholders(values, deps) + + expect(result.apps.gitea.adminPassword).toBe('real-pass') + expect(result.apps.gitea.adminUsername).toBe('admin') + expect(deps.getK8sSecret).toHaveBeenCalledWith('gitea-admin-secret', 'gitea') + }) + + it('should return values unchanged when no placeholders exist', async () => { + const values = { apps: { gitea: { adminPassword: 'plain-value' } } } + const deps = { getK8sSecret: jest.fn() } + + const result = await resolveSecretPlaceholders(values, deps) + expect(result.apps.gitea.adminPassword).toBe('plain-value') + expect(deps.getK8sSecret).not.toHaveBeenCalled() + }) + + it('should return values unchanged when K8s not available', async () => { + const values = { apps: { gitea: { adminPassword: 'sealed:gitea/gitea-admin-secret/password' } } } + const deps = { getK8sSecret: jest.fn().mockRejectedValue(new Error('no cluster')) } + + const result = await resolveSecretPlaceholders(values, deps) + expect(result.apps.gitea.adminPassword).toBe('sealed:gitea/gitea-admin-secret/password') + }) + + it('should cache K8s secret reads', async () => { + const values = { + apps: { + gitea: { + adminPassword: 'sealed:gitea/gitea-admin-secret/password', + adminUsername: 'sealed:gitea/gitea-admin-secret/username', + }, + }, + } + const deps = { + getK8sSecret: jest.fn().mockResolvedValue({ password: 'pass', username: 'admin' }), + } + + await resolveSecretPlaceholders(values, deps) + expect(deps.getK8sSecret).toHaveBeenCalledTimes(1) + }) + + it('should warn on invalid placeholder format', async () => { + const values = { apps: { gitea: { adminPassword: 'sealed:invalid-format' } } } + const deps = { getK8sSecret: jest.fn() } + + const result = await resolveSecretPlaceholders(values, deps) + expect(result.apps.gitea.adminPassword).toBe('sealed:invalid-format') + expect(deps.getK8sSecret).not.toHaveBeenCalled() + }) + + it('should not mutate original values', async () => { + const values = { apps: { gitea: { adminPassword: 'sealed:gitea/gitea-admin-secret/password' } } } + const deps = { + getK8sSecret: jest.fn().mockResolvedValue({ password: 'real-pass' }), + } + + const result = await resolveSecretPlaceholders(values, deps) + expect(result.apps.gitea.adminPassword).toBe('real-pass') + expect(values.apps.gitea.adminPassword).toBe('sealed:gitea/gitea-admin-secret/password') + }) +}) diff --git a/src/common/repo.ts b/src/common/repo.ts index f37993ca7b..2bb7779054 100644 --- a/src/common/repo.ts +++ b/src/common/repo.ts @@ -4,7 +4,9 @@ import { globSync } from 'glob' import jsonpath from 'jsonpath' import { cloneDeep, get, merge, omit, set } from 'lodash' import path from 'path' -import { getDirNames, loadYaml } from './utils' +import { terminal } from './debug' +import { getK8sSecret } from './k8s' +import { flattenObject, getDirNames, loadYaml } from './utils' import { objectToYaml, writeValuesToFile } from './values' export async function getTeamNames(envDir: string): Promise> { @@ -566,14 +568,68 @@ export function sortTeamConfigArraysByName(spec: Record): Record, + deps = { getK8sSecret }, +): Promise> { + const d = terminal('common:repo:resolveSecretPlaceholders') + const flat = flattenObject(values) + + const placeholders = Object.entries(flat).filter( + ([, value]) => typeof value === 'string' && (value as string).startsWith(SECRET_PLACEHOLDER_PREFIX), + ) + + if (placeholders.length === 0) return values + + const secretCache = new Map | undefined>() + const result = cloneDeep(values) + + for (const [valuePath, placeholder] of placeholders) { + const ref = (placeholder as string).replace(SECRET_PLACEHOLDER_PREFIX, '') + const parts = ref.split('/') + if (parts.length !== 3) { + d.warn(`Invalid secret placeholder format: ${placeholder}`) + continue + } + const [namespace, secretName, key] = parts + const cacheKey = `${namespace}/${secretName}` + + if (!secretCache.has(cacheKey)) { + try { + const secret = await deps.getK8sSecret(secretName, namespace) + secretCache.set(cacheKey, secret) + } catch { + d.warn(`Could not read K8s secret ${cacheKey}`) + secretCache.set(cacheKey, undefined) + } + } + + const secretData = secretCache.get(cacheKey) + if (secretData?.[key] !== undefined) { + set(result, valuePath, secretData[key]) + d.debug(`Resolved ${valuePath} from ${cacheKey}/${key}`) + } else { + d.warn(`Could not resolve placeholder: ${placeholder}`) + } + } + + return result +} + export async function setValuesFile( envDir: string, - deps = { pathExists: existsSync, loadValues, writeFile }, + deps = { pathExists: existsSync, loadValues, writeFile, resolveSecretPlaceholders }, ): Promise { const valuesPath = path.join(envDir, 'values-repo.yaml') - // if (await deps.pathExists(valuesPath)) return valuesPath const allValues = await deps.loadValues(envDir) - await deps.writeFile(valuesPath, objectToYaml(allValues)) + const resolved = await deps.resolveSecretPlaceholders(allValues) + await deps.writeFile(valuesPath, objectToYaml(resolved)) return valuesPath } diff --git a/src/common/values.test.ts b/src/common/values.test.ts index 06f76fa4e6..bb101f9979 100644 --- a/src/common/values.test.ts +++ b/src/common/values.test.ts @@ -1,5 +1,5 @@ import { cloneDeep, merge, set } from 'lodash' -import { generateSecrets } from 'src/common/values' +import { generateSecrets, replaceSecretsWithPlaceholders } from 'src/common/values' import stubs from 'src/test-stubs' const { terminal } = stubs @@ -73,3 +73,47 @@ describe('generateSecrets', () => { expect(res.nested.twoStage).toBe('exists') }) }) + +describe('replaceSecretsWithPlaceholders', () => { + it('should replace gitea secrets with sealed secret references', () => { + const input = { + apps: { + gitea: { adminPassword: 'real-pass', postgresqlPassword: 'db-pass', adminUsername: 'admin' }, + harbor: { adminPassword: 'harbor-pass' }, + }, + } + const result = replaceSecretsWithPlaceholders(input) + + expect(result.apps.gitea.adminPassword).toBe('sealed:gitea/gitea-admin-secret/password') + expect(result.apps.gitea.postgresqlPassword).toBe('sealed:gitea/gitea-db-secret/password') + expect(result.apps.gitea.adminUsername).toBe('sealed:gitea/gitea-admin-secret/username') + expect(result.apps.harbor.adminPassword).toBe('harbor-pass') + // Original should not be modified + expect(input.apps.gitea.adminPassword).toBe('real-pass') + }) + + it('should not replace already-placeholder values', () => { + const input = { + apps: { gitea: { adminPassword: 'sealed:gitea/gitea-admin-secret/password' } }, + } + const result = replaceSecretsWithPlaceholders(input) + expect(result.apps.gitea.adminPassword).toBe('sealed:gitea/gitea-admin-secret/password') + }) + + it('should not replace non-string values', () => { + const input = { + apps: { gitea: { adminPassword: 123 as any, postgresqlPassword: 'db-pass' } }, + } + const result = replaceSecretsWithPlaceholders(input) + expect(result.apps.gitea.adminPassword).toBe(123) + expect(result.apps.gitea.postgresqlPassword).toBe('sealed:gitea/gitea-db-secret/password') + }) + + it('should handle values without matching paths', () => { + const input = { + apps: { harbor: { adminPassword: 'harbor-pass' } }, + } + const result = replaceSecretsWithPlaceholders(input) + expect(result.apps.harbor.adminPassword).toBe('harbor-pass') + }) +}) diff --git a/src/common/values.ts b/src/common/values.ts index 7a597f5eea..b40626ab50 100644 --- a/src/common/values.ts +++ b/src/common/values.ts @@ -1,6 +1,6 @@ import { existsSync } from 'fs' import { mkdir, unlink, writeFile } from 'fs/promises' -import { cloneDeep, get, isEmpty, isEqual, merge, mergeWith, omit, pick, set } from 'lodash' +import { cloneDeep, get, isEmpty, isEqual, merge, mergeWith, pick, set } from 'lodash' import path from 'path' import { supportedK8sVersions } from 'src/supportedK8sVersions.json' import { stringify } from 'yaml' @@ -9,18 +9,9 @@ import { decrypt, encrypt } from './crypt' import { terminal } from './debug' import { env } from './envalid' import { hfValues } from './hf' -import { - extract, - flattenObject, - getSchemaSecretsPaths, - getValuesSchema, - gucci, - loadYaml, - pkg, - removeBlankAttributes, -} from './utils' - import { saveValues } from './repo' +import { APP_SECRET_OVERRIDES } from './sealed-secrets' +import { extract, flattenObject, getValuesSchema, gucci, loadYaml, pkg, removeBlankAttributes } from './utils' import { HelmArguments } from './yargs' export const objectToYaml = (obj: Record, indent = 4, lineWidth = 200): string => { @@ -158,21 +149,43 @@ export const writeValuesToFile = async ( export const writeValues = async (inValues: Record, overwrite = false): Promise => { const d = terminal('common:values:writeValues') d.debug('Writing values: ', inValues) - hasSops = existsSync(`${env.ENV_DIR}/.sops.yaml`) - const values = inValues - const teams = Object.keys(get(inValues, 'teamConfig', {})) - const cleanSecretPaths = await getSchemaSecretsPaths(teams) - d.debug('cleanSecretPaths: ', cleanSecretPaths) - // separate out the secrets - const secrets = removeBlankAttributes(pick(values, cleanSecretPaths)) - d.debug('secrets: ', JSON.stringify(secrets, null, 2)) - // from the plain values - const plainValues = omit(values, cleanSecretPaths) as any - await saveValues(env.ENV_DIR, inValues, {}) - + const valuesToWrite = replaceSecretsWithPlaceholders(inValues) + await saveValues(env.ENV_DIR, valuesToWrite, {}) d.info('All values were written to ENV_DIR') } +/** + * Builds a mapping from valuePath → sealed:// + * using APP_SECRET_OVERRIDES configuration. + */ +function buildSecretPlaceholderMap(): Map { + const map = new Map() + for (const [, overrides] of Object.entries(APP_SECRET_OVERRIDES)) { + for (const override of overrides) { + for (const [key, source] of Object.entries(override.data)) { + if (!('static' in source) && source.valuePath) { + map.set(source.valuePath, `sealed:${override.namespace}/${override.secretName}/${key}`) + } + } + } + } + return map +} + +export const replaceSecretsWithPlaceholders = (values: Record): Record => { + const result = cloneDeep(values) + const placeholderMap = buildSecretPlaceholderMap() + + for (const [valuePath, placeholder] of placeholderMap) { + const value = get(result, valuePath) + if (value !== undefined && typeof value === 'string' && !value.startsWith('sealed:')) { + set(result, valuePath, placeholder) + } + } + + return result +} + export const deriveSecrets = async (values: Record = {}): Promise> => { // Some secrets needs to be drived from the generated secrets From aa3f05b3d320161e40f0849ee25356d7a0fc7a3c Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Feb 2026 14:46:11 +0100 Subject: [PATCH 16/20] test: sealed secrets --- src/cmd/commit.ts | 4 ++-- src/common/bootstrap.ts | 2 +- src/common/values.ts | 25 +++++++++++++++++++++++-- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 43db20e0a2..d91ec7eb52 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -57,7 +57,7 @@ const commitAndPush = async (values: Record, branch: string, initia const d = terminal(`cmd:${cmdName}:commitAndPush`) d.info('Committing values') const message = initialInstall ? 'otomi commit' : 'updated values [ci skip]' - const { password } = getRepo(values) + const { password } = await getRepo(values) cd(env.ENV_DIR) try { try { @@ -132,7 +132,7 @@ export const commit = async (initialInstall: boolean, overrideArgs?: HelmArgumen await validateValues(overrideArgs) d.info('Preparing values') const values = (await hfValues()) as Record - const { branch, remote, username, email } = getRepo(values) + const { branch, remote, username, email } = await getRepo(values) if (initialInstall) { // we call this here again, as we might not have completed (happens upon first install): await bootstrapGit(values) diff --git a/src/common/bootstrap.ts b/src/common/bootstrap.ts index 2773d3ea3f..d485cbd4c3 100644 --- a/src/common/bootstrap.ts +++ b/src/common/bootstrap.ts @@ -22,7 +22,7 @@ export const bootstrapGit = async (inValues?: Record): Promise) - const { remote, branch, email, username, password } = getRepo(values) + const { remote, branch, email, username, password } = await getRepo(values) cd(env.ENV_DIR) if (existsSync(`${env.ENV_DIR}/.git`)) { d.info(`Git repo was already bootstrapped, setting identity just in case`) diff --git a/src/common/values.ts b/src/common/values.ts index b40626ab50..efe0b4e3f0 100644 --- a/src/common/values.ts +++ b/src/common/values.ts @@ -9,6 +9,7 @@ import { decrypt, encrypt } from './crypt' import { terminal } from './debug' import { env } from './envalid' import { hfValues } from './hf' +import { getK8sSecret } from './k8s' import { saveValues } from './repo' import { APP_SECRET_OVERRIDES } from './sealed-secrets' import { extract, flattenObject, getValuesSchema, gucci, loadYaml, pkg, removeBlankAttributes } from './utils' @@ -54,7 +55,27 @@ export interface Repo { branch: string } -export const getRepo = (values: Record): Repo => { +/** + * Resolves a single sealed:namespace/secretName/key placeholder to its actual value + * by reading the corresponding K8s Secret. Returns the original value if not a placeholder + * or if resolution fails. + */ +const resolveSinglePlaceholder = async (value: string, deps = { getK8sSecret }): Promise => { + if (!value || !value.startsWith('sealed:')) return value + const ref = value.replace('sealed:', '') + const parts = ref.split('/') + if (parts.length !== 3) return value + const [namespace, secretName, key] = parts + try { + const secret = await deps.getK8sSecret(secretName, namespace) + if (secret?.[key] !== undefined) return String(secret[key]) + } catch { + // K8s not available, return placeholder as-is + } + return value +} + +export const getRepo = async (values: Record, deps = { getK8sSecret }): Promise => { const giteaEnabled = values?.apps?.gitea?.enabled ?? true const byor = !!values?.apps?.['otomi-api']?.git if (!giteaEnabled && !byor) { @@ -74,7 +95,7 @@ export const getRepo = (values: Record): Repo => { branch = otomiApiGit?.branch ?? branch } else { username = 'otomi-admin' - password = values?.apps?.gitea?.adminPassword + password = await resolveSinglePlaceholder(String(values?.apps?.gitea?.adminPassword ?? ''), deps) email = `pipeline@cluster.local` const gitUrl = env.GIT_URL const gitPort = env.GIT_PORT From 31cf3d8ff313937b89e652184ee2bd06bcad274e Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Feb 2026 15:23:29 +0100 Subject: [PATCH 17/20] test: sealed secrets --- src/cmd/migrate.ts | 8 ++++++-- src/common/values.ts | 2 +- src/operator/installer.ts | 7 +++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 898d6c235c..5fa0abb4e0 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -14,7 +14,7 @@ import { env } from 'src/common/envalid' import { hf, HF_DEFAULT_SYNC_ARGS, hfValues } from 'src/common/hf' import { getFileMap, getTeamNames, saveResourceGroupToFiles, saveValues } from 'src/common/repo' import { getFilename, getSchemaSecretsPaths, gucci, loadYaml, rootDir } from 'src/common/utils' -import { objectToYaml, writeValues, writeValuesToFile } from 'src/common/values' +import { objectToYaml, resolveSinglePlaceholder, writeValues, writeValuesToFile } from 'src/common/values' import { BasicArguments, getParsedArgs, setParsedArgs } from 'src/common/yargs' import { v4 as uuidv4 } from 'uuid' import { parse } from 'yaml' @@ -712,7 +712,11 @@ const setDefaultAplCatalog = async (values: Record): Promise let secretCreated = false if (useGiteaCatalog) { try { - await createCatalogSealedSecret(d, gitea as { adminUsername: string; adminPassword: string }) + const resolvedGitea = { + adminUsername: await resolveSinglePlaceholder(String(gitea!.adminUsername)), + adminPassword: await resolveSinglePlaceholder(String(gitea!.adminPassword)), + } + await createCatalogSealedSecret(d, resolvedGitea) secretCreated = true } catch (error) { d.error('Failed to create catalog sealed secret, continuing without it:', error) diff --git a/src/common/values.ts b/src/common/values.ts index efe0b4e3f0..7b7e142a58 100644 --- a/src/common/values.ts +++ b/src/common/values.ts @@ -60,7 +60,7 @@ export interface Repo { * by reading the corresponding K8s Secret. Returns the original value if not a placeholder * or if resolution fails. */ -const resolveSinglePlaceholder = async (value: string, deps = { getK8sSecret }): Promise => { +export const resolveSinglePlaceholder = async (value: string, deps = { getK8sSecret }): Promise => { if (!value || !value.startsWith('sealed:')) return value const ref = value.replace('sealed:', '') const parts = ref.split('/') diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 63c2eb6cdd..f4e57a2c60 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -2,6 +2,7 @@ import * as process from 'node:process' import { terminal } from '../common/debug' import { hfValues } from '../common/hf' import { createUpdateConfigMap, createUpdateGenericSecret, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' +import { resolveSinglePlaceholder } from '../common/values' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' @@ -115,8 +116,10 @@ export class Installer { this.d.debug('Extracting credentials from installation values') const values = (await hfValues()) as Record - const gitUsername: string = values.apps.gitea?.adminUsername || 'otomi-admin' - const gitPassword: string = values.apps.gitea?.adminPassword + const gitUsername: string = await resolveSinglePlaceholder( + String(values.apps.gitea?.adminUsername || 'otomi-admin'), + ) + const gitPassword: string = await resolveSinglePlaceholder(String(values.apps.gitea?.adminPassword ?? '')) if (!gitUsername || !gitPassword) { throw new Error('Git credentials not found in values') From a98cf716d64ccc614e5de2d944996f3e2987ef9c Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Feb 2026 18:06:17 +0100 Subject: [PATCH 18/20] test: sealed secrets --- src/cmd/install.test.ts | 18 ++++++++++ src/cmd/install.ts | 73 +++++++++++++++++++++++++++++++++++++-- src/operator/installer.ts | 3 +- 3 files changed, 91 insertions(+), 3 deletions(-) diff --git a/src/cmd/install.test.ts b/src/cmd/install.test.ts index c97f0dbc2d..cd7888e98c 100644 --- a/src/cmd/install.test.ts +++ b/src/cmd/install.test.ts @@ -17,6 +17,7 @@ jest.mock('src/common/k8s', () => ({ applyServerSide: jest.fn(), restartOtomiApiDeployment: jest.fn(), waitForCRD: jest.fn(), + getK8sSecret: jest.fn().mockResolvedValue({ password: 'test', username: 'test' }), k8s: { app: jest.fn(), }, @@ -42,6 +43,23 @@ jest.mock('zx', () => ({ jest.mock('src/common/sealed-secrets', () => ({ applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue(undefined), restartSealedSecretsController: jest.fn().mockResolvedValue(undefined), + APP_SECRET_OVERRIDES: { + 'apps.gitea': [ + { + secretName: 'gitea-admin-secret', + namespace: 'gitea', + data: { + username: { valuePath: 'apps.gitea.adminUsername', default: 'otomi-admin' }, + password: { valuePath: 'apps.gitea.adminPassword' }, + }, + }, + { + secretName: 'gitea-db-secret', + namespace: 'gitea', + data: { username: { static: 'gitea' }, password: { valuePath: 'apps.gitea.postgresqlPassword' } }, + }, + ], + }, })) jest.mock('./commit', () => ({ diff --git a/src/cmd/install.ts b/src/cmd/install.ts index a9bafe1c8c..147a44e21c 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -4,8 +4,19 @@ import { cleanupHandler, prepareEnvironment } from 'src/common/cli' import { logLevelString, terminal } from 'src/common/debug' import { env } from 'src/common/envalid' import { deployEssential, hf, HF_DEFAULT_SYNC_ARGS } from 'src/common/hf' -import { applyServerSide, getDeploymentState, getHelmReleases, setDeploymentState, waitForCRD } from 'src/common/k8s' -import { applySealedSecretManifestsFromDir, restartSealedSecretsController } from 'src/common/sealed-secrets' +import { + applyServerSide, + getDeploymentState, + getHelmReleases, + getK8sSecret, + setDeploymentState, + waitForCRD, +} from 'src/common/k8s' +import { + APP_SECRET_OVERRIDES, + applySealedSecretManifestsFromDir, + restartSealedSecretsController, +} from 'src/common/sealed-secrets' import { getFilename, rootDir } from 'src/common/utils' import { getImageTagFromValues, getPackageVersion, writeValuesToFile } from 'src/common/values' import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs' @@ -45,6 +56,58 @@ const retryInstallStep = async ( ) } +/** + * Wait for SealedSecrets controller to decrypt SealedSecret resources into K8s Secrets. + * Polls all secrets defined in APP_SECRET_OVERRIDES until they exist in the cluster. + */ +const waitForSealedSecrets = async ( + timeoutMs = 120000, + intervalMs = 3000, + deps = { getK8sSecret, terminal }, +): Promise => { + const d = deps.terminal(`cmd:${cmdName}:waitForSealedSecrets`) + + // Build list of secrets to wait for from APP_SECRET_OVERRIDES + const secretsToWait: Array<{ namespace: string; secretName: string }> = [] + for (const overrides of Object.values(APP_SECRET_OVERRIDES)) { + for (const override of overrides) { + secretsToWait.push({ namespace: override.namespace, secretName: override.secretName }) + } + } + + if (secretsToWait.length === 0) { + d.info('No sealed secrets to wait for') + return + } + + d.info(`Waiting for ${secretsToWait.length} sealed secrets to be decrypted`) + const start = Date.now() + + while (Date.now() - start < timeoutMs) { + const pending: string[] = [] + for (const { namespace, secretName } of secretsToWait) { + try { + const secret = await deps.getK8sSecret(secretName, namespace) + if (!secret) { + pending.push(`${namespace}/${secretName}`) + } + } catch { + pending.push(`${namespace}/${secretName}`) + } + } + + if (pending.length === 0) { + d.info('All sealed secrets have been decrypted') + return + } + + d.info(`Still waiting for sealed secrets: ${pending.join(', ')}`) + await new Promise((resolve) => setTimeout(resolve, intervalMs)) + } + + throw new Error(`Timed out waiting for sealed secrets to be decrypted after ${timeoutMs}ms`) +} + export const installAll = async () => { const d = terminal(`cmd:${cmdName}:installAll`) const prevState = await getDeploymentState() @@ -109,6 +172,12 @@ export const installAll = async () => { d.info('Restarting sealed-secrets controller') await restartSealedSecretsController() + // Wait for SealedSecrets controller to decrypt all SealedSecret resources into K8s Secrets. + // This is critical: subsequent steps (hfValues, commit, getRepo) resolve sealed: placeholders + // by reading these K8s Secrets. Without this wait, placeholder resolution fails silently. + d.info('Waiting for sealed secrets to be decrypted into K8s Secrets') + await waitForSealedSecrets() + d.info('Deploying charts containing label app=core') await hf( { diff --git a/src/operator/installer.ts b/src/operator/installer.ts index f4e57a2c60..24e94638c2 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -86,7 +86,8 @@ export class Installer { status, attempt: attempt.toString(), timestamp: new Date().toISOString(), - ...(error && { error }), + // Always include error field to prevent stale values from StrategicMergePatch + error: error ?? '', } await createUpdateConfigMap(k8s.core(), 'apl-installation-status', 'apl-operator', data) From 6b1c101fab8b3f0a75d171048ba37141c69f58fb Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Feb 2026 21:21:41 +0100 Subject: [PATCH 19/20] test: sealed secrets --- src/cmd/commit.ts | 5 +-- src/cmd/install.test.ts | 19 +++++++--- src/cmd/install.ts | 9 +++++ src/operator/installer.test.ts | 65 +++++++++++++++++++++++++++++++++- src/operator/installer.ts | 32 ++++++++++++++++- 5 files changed, 121 insertions(+), 9 deletions(-) diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index d91ec7eb52..9113329451 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -78,8 +78,9 @@ const commitAndPush = async (values: Record, branch: string, initia } await $`git commit -m ${message} --no-verify`.quiet() } catch (e) { - d.log('commitAndPush error ', e?.message?.replace(password, '****')) - return + const errorMsg = `commitAndPush error: ${e?.message?.replace(password, '****')}` + d.error(errorMsg) + throw new Error(errorMsg) } if (values._derived?.untrustedCA) process.env.GIT_SSL_NO_VERIFY = '1' await retry( diff --git a/src/cmd/install.test.ts b/src/cmd/install.test.ts index cd7888e98c..fe6979ddbc 100644 --- a/src/cmd/install.test.ts +++ b/src/cmd/install.test.ts @@ -35,10 +35,20 @@ jest.mock('src/common/hf', () => ({ HF_DEFAULT_SYNC_ARGS: ['apply', '--sync-args', '--include-needs'], })) -jest.mock('zx', () => ({ - $: jest.fn(), - cd: jest.fn(), -})) +jest.mock('zx', () => { + const mockResult = { exitCode: 0, stdout: '', stderr: '' } + const createMockProcessPromise = () => { + const promise = Promise.resolve(mockResult) + const chainable: any = promise + chainable.nothrow = jest.fn().mockReturnValue(chainable) + chainable.quiet = jest.fn().mockReturnValue(chainable) + return chainable + } + return { + $: jest.fn().mockImplementation(() => createMockProcessPromise()), + cd: jest.fn(), + } +}) jest.mock('src/common/sealed-secrets', () => ({ applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue(undefined), @@ -129,7 +139,6 @@ describe('Install command', () => { stderr: '', }) mockDeps.deployEssential.mockResolvedValue(true) - mockDeps.$.mockResolvedValue(undefined) }) describe('module configuration', () => { diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 147a44e21c..aec3865010 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -193,6 +193,15 @@ export const installAll = async () => { if (!(env.isDev && env.DISABLE_SYNC)) { await commit(true) + + // Verify the git push actually succeeded by checking the remote branch exists + d.info('Verifying git push succeeded') + const verifyResult = await $`git -C ${env.ENV_DIR} ls-remote --exit-code --heads origin main`.nothrow().quiet() + if (verifyResult.exitCode !== 0) { + throw new Error('Git push verification failed: remote branch main does not exist after commit') + } + d.info('Git push verified successfully') + const initialData = await initialSetupData() await retryInstallStep(createCredentialsSecret, initialData.secretName, initialData.username, initialData.password) await retryInstallStep(createWelcomeConfigMap, initialData.secretName, initialData.domainSuffix) diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index 59bdd81a08..32a400ba9c 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -3,6 +3,31 @@ import * as k8s from '../common/k8s' import { AplOperations } from './apl-operations' import { Installer } from './installer' +const mockZx = jest.fn().mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }), + }), +}) + +jest.mock('zx', () => ({ + $: (...args: any[]) => mockZx(...args), +})) + +jest.mock('../common/envalid', () => ({ + env: { + GIT_PROTOCOL: 'http', + GIT_URL: 'gitea-http.gitea.svc.cluster.local', + GIT_PORT: '3000', + }, +})) + +jest.mock('./validators', () => ({ + operatorEnv: { + GIT_ORG: 'otomi', + GIT_REPO: 'values', + }, +})) + jest.mock('../common/debug', () => ({ terminal: jest.fn().mockImplementation(() => ({ info: jest.fn(), @@ -228,10 +253,18 @@ describe('Installer', () => { }) describe('isInstalled', () => { - test('should return completed status when ConfigMap exists', async () => { + test('should return completed status when ConfigMap exists and git repo has main branch', async () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ data: { status: 'completed' }, }) + // getK8sSecret returns credentials for git verification + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ GIT_USERNAME: 'admin', GIT_PASSWORD: 'pass' }) + // git ls-remote succeeds (main branch exists) + mockZx.mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 0 }), + }), + }) const isInstalled = await installer.isInstalled() @@ -240,6 +273,23 @@ describe('Installer', () => { expect(mockAplOps.install).not.toHaveBeenCalled() }) + test('should return false when status is completed but git repo has no main branch', async () => { + ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ + data: { status: 'completed' }, + }) + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ GIT_USERNAME: 'admin', GIT_PASSWORD: 'pass' }) + // git ls-remote fails (main branch does not exist) + mockZx.mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 2 }), + }), + }) + + const isInstalled = await installer.isInstalled() + + expect(isInstalled).toBe(false) + }) + test('should return true when ConfigMap does not exist', async () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue(null) @@ -272,6 +322,19 @@ describe('Installer', () => { expect(k8s.getK8sConfigMap).toHaveBeenCalledWith('apl-operator', 'apl-installation-status', mockCoreApi) expect(isInstalled).toBe(false) }) + + test('should return true when git verification fails (gitea not ready)', async () => { + ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ + data: { status: 'completed' }, + }) + // getK8sSecret throws (cluster issues) + ;(k8s.getK8sSecret as jest.Mock).mockRejectedValue(new Error('connection refused')) + + const isInstalled = await installer.isInstalled() + + // Should assume installed when verification can't be performed + expect(isInstalled).toBe(true) + }) }) describe('setEnvAndCreateSecrets', () => { diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 24e94638c2..0cedb4a3e7 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -1,10 +1,13 @@ import * as process from 'node:process' +import { $ } from 'zx' import { terminal } from '../common/debug' +import { env } from '../common/envalid' import { hfValues } from '../common/hf' import { createUpdateConfigMap, createUpdateGenericSecret, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' import { resolveSinglePlaceholder } from '../common/values' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' +import { operatorEnv } from './validators' export interface GitCredentials { username: string @@ -27,7 +30,34 @@ export class Installer { await this.updateInstallationStatus('completed', -1) return true } - return installStatus === 'completed' + if (installStatus === 'completed') { + // Verify the git repo actually has content - the previous install may have + // marked status as completed but the pod was killed before the git push finished + const gitRepoHasContent = await this.verifyGitRepoHasMainBranch() + if (!gitRepoHasContent) { + this.d.warn('Installation marked as completed but git repo has no main branch - will re-install') + return false + } + return true + } + return false + } + + private async verifyGitRepoHasMainBranch(): Promise { + try { + // Get credentials from K8s secret (created by Helm at deploy time) + const creds = await getK8sSecret('gitea-credentials', 'apl-operator') + const username = creds?.GIT_USERNAME ?? 'otomi-admin' + const password = creds?.GIT_PASSWORD ?? '' + const repoUrl = `${env.GIT_PROTOCOL}://${username}:${password}@${env.GIT_URL}:${env.GIT_PORT}/${operatorEnv.GIT_ORG}/${operatorEnv.GIT_REPO}.git` + const result = await $`git ls-remote --exit-code --heads ${repoUrl} main`.nothrow().quiet() + return result.exitCode === 0 + } catch { + // If we can't check (e.g. gitea not ready yet), assume it's fine + // The operator will detect the issue later during git polling + this.d.warn('Could not verify git repo - gitea may not be ready yet') + return true + } } public async initialize() { From 7495fd32f09396943a44624ca236cf40c213da41 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 9 Feb 2026 23:59:27 +0100 Subject: [PATCH 20/20] test: sealed secrets --- src/cmd/commit.ts | 6 ++++++ src/common/values.ts | 10 ++++++---- src/operator/installer.ts | 4 ++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 9113329451..d9dfcb1a77 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -137,6 +137,12 @@ export const commit = async (initialInstall: boolean, overrideArgs?: HelmArgumen if (initialInstall) { // we call this here again, as we might not have completed (happens upon first install): await bootstrapGit(values) + // Always update the remote URL after bootstrap - the initial bootstrapGit() (called during + // the bootstrap phase before install) may have set the URL with unresolved placeholder + // passwords because K8s secrets didn't exist yet. Now that secrets are decrypted, + // we need to update the URL with the real credentials. + cd(env.ENV_DIR) + await $`git remote set-url origin ${remote}`.nothrow().quiet() } else { cd(env.ENV_DIR) await setIdentity(username, email) diff --git a/src/common/values.ts b/src/common/values.ts index 7b7e142a58..d9bde66e83 100644 --- a/src/common/values.ts +++ b/src/common/values.ts @@ -60,8 +60,9 @@ export interface Repo { * by reading the corresponding K8s Secret. Returns the original value if not a placeholder * or if resolution fails. */ -export const resolveSinglePlaceholder = async (value: string, deps = { getK8sSecret }): Promise => { +export const resolveSinglePlaceholder = async (value: string, deps = { getK8sSecret, terminal }): Promise => { if (!value || !value.startsWith('sealed:')) return value + const d = deps.terminal('common:values:resolveSinglePlaceholder') const ref = value.replace('sealed:', '') const parts = ref.split('/') if (parts.length !== 3) return value @@ -69,13 +70,14 @@ export const resolveSinglePlaceholder = async (value: string, deps = { getK8sSec try { const secret = await deps.getK8sSecret(secretName, namespace) if (secret?.[key] !== undefined) return String(secret[key]) - } catch { - // K8s not available, return placeholder as-is + d.warn(`Could not resolve placeholder ${value}: key '${key}' not found in secret ${namespace}/${secretName}`) + } catch (e) { + d.warn(`Could not resolve placeholder ${value}: K8s secret ${namespace}/${secretName} not available`) } return value } -export const getRepo = async (values: Record, deps = { getK8sSecret }): Promise => { +export const getRepo = async (values: Record, deps = { getK8sSecret, terminal }): Promise => { const giteaEnabled = values?.apps?.gitea?.enabled ?? true const byor = !!values?.apps?.['otomi-api']?.git if (!giteaEnabled && !byor) { diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 0cedb4a3e7..d784545b6b 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -156,6 +156,10 @@ export class Installer { throw new Error('Git credentials not found in values') } + if (gitPassword.startsWith('sealed:') || gitUsername.startsWith('sealed:')) { + throw new Error('Git credentials contain unresolved sealed secret placeholders - K8s secrets may not be ready') + } + await createUpdateGenericSecret(k8s.core(), 'gitea-credentials', 'apl-operator', { GIT_USERNAME: gitUsername, GIT_PASSWORD: gitPassword,