diff --git a/chart/apl/templates/deployment.yaml b/chart/apl/templates/deployment.yaml index 2b34ecc718..34186124a4 100644 --- a/chart/apl/templates/deployment.yaml +++ b/chart/apl/templates/deployment.yaml @@ -75,13 +75,9 @@ spec: value: {{ .Values.operator.pollIntervalMs | default "30000" | quote }} - name: RECONCILE_INTERVAL_MS value: {{ .Values.operator.reconcileIntervalMs | default "300000" | quote }} - {{- if hasKey $kms "sops" }} envFrom: - - secretRef: - name: apl-sops-secrets - secretRef: name: gitea-credentials - {{- end }} volumeMounts: - name: otomi-values mountPath: /home/app/stack/env diff --git a/chart/apl/templates/sops-secrets.yaml b/chart/apl/templates/sops-secrets.yaml deleted file mode 100644 index 7544e0ec6c..0000000000 --- a/chart/apl/templates/sops-secrets.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{- $kms := .Values.kms | default dict }} -{{- if hasKey $kms "sops" }} -{{- $v := $kms.sops }} -apiVersion: v1 -kind: Secret -metadata: - name: apl-sops-secrets - namespace: apl-operator -type: Opaque -data: -{{- with $v.azure }} - AZURE_CLIENT_ID: {{ .clientId | b64enc }} - AZURE_CLIENT_SECRET: {{ .clientSecret | b64enc }} -{{- with .tenantId }} - AZURE_TENANT_ID: {{ . | b64enc }}{{ end }} -{{- with .environment }} - AZURE_ENVIRONMENT: {{ . | b64enc }}{{ end }} -{{- end }} -{{- with $v.aws }} - AWS_ACCESS_KEY_ID: {{ .accessKey | b64enc }} - AWS_SECRET_ACCESS_KEY: {{ .secretKey | b64enc }} -{{- with .region }} - AWS_REGION: {{ . | b64enc }}{{ end }} -{{- end }} -{{- with $v.age }} - SOPS_AGE_KEY: {{ .privateKey | b64enc }} -{{- end }} -{{- with $v.google }} - GCLOUD_SERVICE_KEY: {{ .accountJson | b64enc }} -{{- with .project }} - GOOGLE_PROJECT: {{ . | b64enc }}{{ end }} -{{- with .region }} - GOOGLE_REGION: {{ . | b64enc }}{{ end }} -{{- end }} -{{- end }} diff --git a/charts/ingress-nginx/templates/clusterrole.yaml b/charts/ingress-nginx/templates/clusterrole.yaml index 51bc5002cc..15b2ac4b55 100644 --- a/charts/ingress-nginx/templates/clusterrole.yaml +++ b/charts/ingress-nginx/templates/clusterrole.yaml @@ -27,6 +27,7 @@ rules: - namespaces {{- end}} verbs: + - get - list - watch - apiGroups: diff --git a/helmfile.d/helmfile-03.databases.yaml.gotmpl b/helmfile.d/helmfile-03.databases.yaml.gotmpl index 3c718d9283..9f136b297f 100644 --- a/helmfile.d/helmfile-03.databases.yaml.gotmpl +++ b/helmfile.d/helmfile-03.databases.yaml.gotmpl @@ -14,13 +14,6 @@ bases: {{- $k := $a.keycloak }} releases: - - name: gitea-db-secret-artifacts - installed: true - namespace: gitea - labels: - pkg: gitea - app: core - <<: *raw - name: gitea-otomi-db installed: true namespace: gitea diff --git a/helmfile.d/snippets/sops-env.gotmpl b/helmfile.d/snippets/sops-env.gotmpl index aa9e2ff866..e69de29bb2 100644 --- a/helmfile.d/snippets/sops-env.gotmpl +++ b/helmfile.d/snippets/sops-env.gotmpl @@ -1,24 +0,0 @@ -{{- with . | get "azure" nil }} -AZURE_CLIENT_ID: {{ .clientId }} -AZURE_CLIENT_SECRET: {{ .clientSecret }} -{{- with . | get "tenantId" nil }} -AZURE_TENANT_ID: {{ . }}{{ end }} -{{- with . | get "environment" nil }} -AZURE_ENVIRONMENT: {{ . }}{{ end }} -{{- end }} -{{- with . | get "aws" nil }} -AWS_ACCESS_KEY_ID: {{ .accessKey }} -AWS_SECRET_ACCESS_KEY: {{ .secretKey }} -{{- with . | get "region" nil }} -AWS_REGION: {{ . }}{{ end }} -{{- end }} -{{- with . | get "age" nil }} -SOPS_AGE_KEY: {{ .privateKey }} -{{- end }} -{{- with . | get "google" nil }} -GCLOUD_SERVICE_KEY: '{{ .accountJson | replace "\n" "" }}' -{{- with . | get "project" nil }} -GOOGLE_PROJECT: {{ . }}{{ end }} -{{- with . | get "region" nil }} -GOOGLE_REGION: {{ . }}{{ end }} -{{- end }} diff --git a/src/cmd/bootstrap.test.ts b/src/cmd/bootstrap.test.ts index 4d61225a86..f1f469abd2 100644 --- a/src/cmd/bootstrap.test.ts +++ b/src/cmd/bootstrap.test.ts @@ -12,6 +12,8 @@ import { processValues, } from './bootstrap' +jest.mock('@linode/kubeseal-encrypt') + const { terminal } = stubs jest.mock('src/common/envalid', () => ({ @@ -37,6 +39,7 @@ describe('Bootstrapping values', () => { }), }), bootstrapSops: jest.fn(), + bootstrapSealedSecrets: jest.fn(), copyBasicFiles: jest.fn(), copyFile: jest.fn(), createCustomCA: jest.fn(), @@ -59,14 +62,14 @@ describe('Bootstrapping values', () => { } }) it('should call relevant sub routines', async () => { - deps.processValues.mockReturnValue(values) + deps.processValues.mockReturnValue({ originalInput: values, allSecrets: {} }) deps.hfValues.mockReturnValue(values) await bootstrap(deps) expect(deps.copyBasicFiles).toHaveBeenCalled() - expect(deps.bootstrapSops).toHaveBeenCalled() + expect(deps.bootstrapSealedSecrets).toHaveBeenCalled() }) it('should copy only skeleton files to env dir if it is empty or nonexisting', async () => { - deps.processValues.mockReturnValue(undefined) + deps.processValues.mockReturnValue({ originalInput: undefined, allSecrets: {} }) await bootstrap(deps) expect(deps.hfValues).toHaveBeenCalledTimes(0) }) @@ -385,7 +388,7 @@ describe('Bootstrapping values', () => { { id: 'user2', initialPassword: 'generated-password' }, ], }) - expect(res).toEqual({ + expect(res.originalInput).toEqual({ cluster: { name: 'bla', provider: 'dida' }, users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }], }) diff --git a/src/cmd/bootstrap.ts b/src/cmd/bootstrap.ts index 9c30aea8f2..4da62b6ad2 100644 --- a/src/cmd/bootstrap.ts +++ b/src/cmd/bootstrap.ts @@ -2,7 +2,7 @@ import { randomUUID } from 'crypto' import { existsSync } from 'fs' import { copyFile, cp, mkdir, readFile, writeFile } from 'fs/promises' import { generate as generatePassword } from 'generate-password' -import { cloneDeep, get, isEmpty, merge, set } from 'lodash' +import { cloneDeep, get, merge, set } from 'lodash' import { pki } from 'node-forge' import path from 'path' import { bootstrapGit } from 'src/common/bootstrap' @@ -14,6 +14,7 @@ import { env, isCli } from 'src/common/envalid' import { hfValues } from 'src/common/hf' import { createK8sSecret, getDeploymentState, getK8sSecret, secretId } from 'src/common/k8s' import { getKmsSettings } from 'src/common/repo' +import { bootstrapSealedSecrets } from 'src/common/sealed-secrets' import { ensureTeamGitOpsDirectories, getFilename, gucci, isCore, loadYaml, rootDir } from 'src/common/utils' import { generateSecrets, writeValues } from 'src/common/values' import { BasicArguments, setParsedArgs } from 'src/common/yargs' @@ -84,11 +85,6 @@ export const bootstrapSops = async ( await deps.writeFile(targetPath, output) d.log(`Ready generating sops files. The configuration is written to: ${targetPath}`) - d.info('Copying sops related files') - // add sops related files - const file = '.gitattributes' - await deps.copyFile(`${rootDir}/.values/${file}`, `${envDir}/${file}`) - // prepare some credential files the first time and crypt some if (!exists) { if (isCli || env.OTOMI_DEV) { @@ -291,7 +287,7 @@ export const processValues = async ( addInitialPasswords, addPlatformAdmin, }, -): Promise> => { +): Promise<{ originalInput: Record; allSecrets: Record }> => { const d = deps.terminal(`cmd:${cmdName}:processValues`) const { VALUES_INPUT } = env d.log(`Loading app values from ${VALUES_INPUT}`) @@ -324,7 +320,7 @@ export const processValues = async ( // and do some context dependent post processing: // to support potential failing chart install we store secrets on cluster if (!(env.isDev && env.DISABLE_SYNC)) await deps.createK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, 'otomi', allSecrets) - return originalInput + return { originalInput, allSecrets } } // create file structure based on file entry @@ -419,6 +415,7 @@ export const bootstrap = async ( hfValues, writeValues, bootstrapSops, + bootstrapSealedSecrets, migrate, encrypt, decrypt, @@ -434,10 +431,10 @@ export const bootstrap = async ( } await deps.copyBasicFiles() await deps.migrate() - const originalValues = await deps.processValues() + const { originalInput, allSecrets } = await deps.processValues() await deps.handleFileEntry() - await deps.bootstrapSops() - await ensureTeamGitOpsDirectories(ENV_DIR, originalValues) + await deps.bootstrapSealedSecrets(allSecrets, ENV_DIR, originalInput) + await ensureTeamGitOpsDirectories(ENV_DIR, originalInput) d.log(`Done bootstrapping values`) } @@ -456,7 +453,6 @@ export const module = { handler: async (argv: BasicArguments): Promise => { setParsedArgs(argv) await prepareEnvironment({ skipAllPreChecks: true }) - await decrypt() await bootstrap() await bootstrapGit() }, diff --git a/src/cmd/commit.ts b/src/cmd/commit.ts index 4376499db2..677c398117 100644 --- a/src/cmd/commit.ts +++ b/src/cmd/commit.ts @@ -58,7 +58,7 @@ const commitAndPush = async (values: Record, branch: string, initia const d = terminal(`cmd:${cmdName}:commitAndPush`) d.info('Committing values') const message = initialInstall ? 'otomi commit' : 'updated values [ci skip]' - const { password } = getRepo(values) + const { password } = await getRepo(values) cd(env.ENV_DIR) try { try { @@ -79,8 +79,9 @@ const commitAndPush = async (values: Record, branch: string, initia } await $`git commit -m ${message} --no-verify`.quiet() } catch (e) { - d.log('commitAndPush error ', e?.message?.replace(password, '****')) - return + const errorMsg = `commitAndPush error: ${e?.message?.replace(password, '****')}` + d.error(errorMsg) + throw new Error(errorMsg) } if (values._derived?.untrustedCA) process.env.GIT_SSL_NO_VERIFY = '1' await retry( @@ -133,10 +134,16 @@ export const commit = async (initialInstall: boolean, overrideArgs?: HelmArgumen await validateValues(overrideArgs) d.info('Preparing values') const values = (await hfValues()) as Record - const { branch, remote, username, email } = getRepo(values) + const { branch, remote, username, email } = await getRepo(values) if (initialInstall) { // we call this here again, as we might not have completed (happens upon first install): await bootstrapGit(values) + // Always update the remote URL after bootstrap - the initial bootstrapGit() (called during + // the bootstrap phase before install) may have set the URL with unresolved placeholder + // passwords because K8s secrets didn't exist yet. Now that secrets are decrypted, + // we need to update the URL with the real credentials. + cd(env.ENV_DIR) + await $`git remote set-url origin ${remote}`.nothrow().quiet() } else { cd(env.ENV_DIR) await setIdentity(username, email) diff --git a/src/cmd/install.test.ts b/src/cmd/install.test.ts index 2645cb2c9b..fe6979ddbc 100644 --- a/src/cmd/install.test.ts +++ b/src/cmd/install.test.ts @@ -17,6 +17,7 @@ jest.mock('src/common/k8s', () => ({ applyServerSide: jest.fn(), restartOtomiApiDeployment: jest.fn(), waitForCRD: jest.fn(), + getK8sSecret: jest.fn().mockResolvedValue({ password: 'test', username: 'test' }), k8s: { app: jest.fn(), }, @@ -34,9 +35,41 @@ jest.mock('src/common/hf', () => ({ HF_DEFAULT_SYNC_ARGS: ['apply', '--sync-args', '--include-needs'], })) -jest.mock('zx', () => ({ - $: jest.fn(), - cd: jest.fn(), +jest.mock('zx', () => { + const mockResult = { exitCode: 0, stdout: '', stderr: '' } + const createMockProcessPromise = () => { + const promise = Promise.resolve(mockResult) + const chainable: any = promise + chainable.nothrow = jest.fn().mockReturnValue(chainable) + chainable.quiet = jest.fn().mockReturnValue(chainable) + return chainable + } + return { + $: jest.fn().mockImplementation(() => createMockProcessPromise()), + cd: jest.fn(), + } +}) + +jest.mock('src/common/sealed-secrets', () => ({ + applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue(undefined), + restartSealedSecretsController: jest.fn().mockResolvedValue(undefined), + APP_SECRET_OVERRIDES: { + 'apps.gitea': [ + { + secretName: 'gitea-admin-secret', + namespace: 'gitea', + data: { + username: { valuePath: 'apps.gitea.adminUsername', default: 'otomi-admin' }, + password: { valuePath: 'apps.gitea.adminPassword' }, + }, + }, + { + secretName: 'gitea-db-secret', + namespace: 'gitea', + data: { username: { static: 'gitea' }, password: { valuePath: 'apps.gitea.postgresqlPassword' } }, + }, + ], + }, })) jest.mock('./commit', () => ({ @@ -106,7 +139,6 @@ describe('Install command', () => { stderr: '', }) mockDeps.deployEssential.mockResolvedValue(true) - mockDeps.$.mockResolvedValue(undefined) }) describe('module configuration', () => { diff --git a/src/cmd/install.ts b/src/cmd/install.ts index 2de0cbd1c5..aec3865010 100644 --- a/src/cmd/install.ts +++ b/src/cmd/install.ts @@ -4,7 +4,19 @@ import { cleanupHandler, prepareEnvironment } from 'src/common/cli' import { logLevelString, terminal } from 'src/common/debug' import { env } from 'src/common/envalid' import { deployEssential, hf, HF_DEFAULT_SYNC_ARGS } from 'src/common/hf' -import { applyServerSide, getDeploymentState, getHelmReleases, setDeploymentState, waitForCRD } from 'src/common/k8s' +import { + applyServerSide, + getDeploymentState, + getHelmReleases, + getK8sSecret, + setDeploymentState, + waitForCRD, +} from 'src/common/k8s' +import { + APP_SECRET_OVERRIDES, + applySealedSecretManifestsFromDir, + restartSealedSecretsController, +} from 'src/common/sealed-secrets' import { getFilename, rootDir } from 'src/common/utils' import { getImageTagFromValues, getPackageVersion, writeValuesToFile } from 'src/common/values' import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs' @@ -44,6 +56,58 @@ const retryInstallStep = async ( ) } +/** + * Wait for SealedSecrets controller to decrypt SealedSecret resources into K8s Secrets. + * Polls all secrets defined in APP_SECRET_OVERRIDES until they exist in the cluster. + */ +const waitForSealedSecrets = async ( + timeoutMs = 120000, + intervalMs = 3000, + deps = { getK8sSecret, terminal }, +): Promise => { + const d = deps.terminal(`cmd:${cmdName}:waitForSealedSecrets`) + + // Build list of secrets to wait for from APP_SECRET_OVERRIDES + const secretsToWait: Array<{ namespace: string; secretName: string }> = [] + for (const overrides of Object.values(APP_SECRET_OVERRIDES)) { + for (const override of overrides) { + secretsToWait.push({ namespace: override.namespace, secretName: override.secretName }) + } + } + + if (secretsToWait.length === 0) { + d.info('No sealed secrets to wait for') + return + } + + d.info(`Waiting for ${secretsToWait.length} sealed secrets to be decrypted`) + const start = Date.now() + + while (Date.now() - start < timeoutMs) { + const pending: string[] = [] + for (const { namespace, secretName } of secretsToWait) { + try { + const secret = await deps.getK8sSecret(secretName, namespace) + if (!secret) { + pending.push(`${namespace}/${secretName}`) + } + } catch { + pending.push(`${namespace}/${secretName}`) + } + } + + if (pending.length === 0) { + d.info('All sealed secrets have been decrypted') + return + } + + d.info(`Still waiting for sealed secrets: ${pending.join(', ')}`) + await new Promise((resolve) => setTimeout(resolve, intervalMs)) + } + + throw new Error(`Timed out waiting for sealed secrets to be decrypted after ${timeoutMs}ms`) +} + export const installAll = async () => { const d = terminal(`cmd:${cmdName}:installAll`) const prevState = await getDeploymentState() @@ -82,6 +146,38 @@ export const installAll = async () => { { streams: { stdout: d.stream.log, stderr: d.stream.error } }, ) + // Deploy sealed-secrets controller first (needs to be ready before applying SealedSecrets) + d.info('Deploying sealed-secrets controller') + await hf( + { + fileOpts: 'helmfile.d/helmfile-01.init.yaml.gotmpl', + labelOpts: ['name=sealed-secrets'], + logLevel: logLevelString(), + args: hfArgs, + }, + { streams: { stdout: d.stream.log, stderr: d.stream.error } }, + ) + + // Wait for SealedSecret CRD to be established + d.info('Waiting for SealedSecret CRD to be ready') + await retryInstallStep(waitForCRD, 'sealedsecrets.bitnami.com') + + // Apply SealedSecret manifests from disk (generated during bootstrap) + d.info('Applying SealedSecret manifests') + await applySealedSecretManifestsFromDir(env.ENV_DIR) + + // Restart the sealed-secrets controller to ensure it uses the correct key + // This is needed because the controller may have generated its own key before + // the bootstrap-created sealed-secrets-key secret was available + d.info('Restarting sealed-secrets controller') + await restartSealedSecretsController() + + // Wait for SealedSecrets controller to decrypt all SealedSecret resources into K8s Secrets. + // This is critical: subsequent steps (hfValues, commit, getRepo) resolve sealed: placeholders + // by reading these K8s Secrets. Without this wait, placeholder resolution fails silently. + d.info('Waiting for sealed secrets to be decrypted into K8s Secrets') + await waitForSealedSecrets() + d.info('Deploying charts containing label app=core') await hf( { @@ -97,6 +193,15 @@ export const installAll = async () => { if (!(env.isDev && env.DISABLE_SYNC)) { await commit(true) + + // Verify the git push actually succeeded by checking the remote branch exists + d.info('Verifying git push succeeded') + const verifyResult = await $`git -C ${env.ENV_DIR} ls-remote --exit-code --heads origin main`.nothrow().quiet() + if (verifyResult.exitCode !== 0) { + throw new Error('Git push verification failed: remote branch main does not exist after commit') + } + d.info('Git push verified successfully') + const initialData = await initialSetupData() await retryInstallStep(createCredentialsSecret, initialData.secretName, initialData.username, initialData.password) await retryInstallStep(createWelcomeConfigMap, initialData.secretName, initialData.domainSuffix) diff --git a/src/cmd/migrate.ts b/src/cmd/migrate.ts index 898d6c235c..5fa0abb4e0 100644 --- a/src/cmd/migrate.ts +++ b/src/cmd/migrate.ts @@ -14,7 +14,7 @@ import { env } from 'src/common/envalid' import { hf, HF_DEFAULT_SYNC_ARGS, hfValues } from 'src/common/hf' import { getFileMap, getTeamNames, saveResourceGroupToFiles, saveValues } from 'src/common/repo' import { getFilename, getSchemaSecretsPaths, gucci, loadYaml, rootDir } from 'src/common/utils' -import { objectToYaml, writeValues, writeValuesToFile } from 'src/common/values' +import { objectToYaml, resolveSinglePlaceholder, writeValues, writeValuesToFile } from 'src/common/values' import { BasicArguments, getParsedArgs, setParsedArgs } from 'src/common/yargs' import { v4 as uuidv4 } from 'uuid' import { parse } from 'yaml' @@ -712,7 +712,11 @@ const setDefaultAplCatalog = async (values: Record): Promise let secretCreated = false if (useGiteaCatalog) { try { - await createCatalogSealedSecret(d, gitea as { adminUsername: string; adminPassword: string }) + const resolvedGitea = { + adminUsername: await resolveSinglePlaceholder(String(gitea!.adminUsername)), + adminPassword: await resolveSinglePlaceholder(String(gitea!.adminPassword)), + } + await createCatalogSealedSecret(d, resolvedGitea) secretCreated = true } catch (error) { d.error('Failed to create catalog sealed secret, continuing without it:', error) diff --git a/src/common/bootstrap.ts b/src/common/bootstrap.ts index 2773d3ea3f..d485cbd4c3 100644 --- a/src/common/bootstrap.ts +++ b/src/common/bootstrap.ts @@ -22,7 +22,7 @@ export const bootstrapGit = async (inValues?: Record): Promise) - const { remote, branch, email, username, password } = getRepo(values) + const { remote, branch, email, username, password } = await getRepo(values) cd(env.ENV_DIR) if (existsSync(`${env.ENV_DIR}/.git`)) { d.info(`Git repo was already bootstrapped, setting identity just in case`) diff --git a/src/common/repo.test.ts b/src/common/repo.test.ts index 92d057b098..b19813bc24 100644 --- a/src/common/repo.test.ts +++ b/src/common/repo.test.ts @@ -11,6 +11,7 @@ import { hasCorrespondingDecryptedFile, renderManifest, renderManifestForSecrets, + resolveSecretPlaceholders, saveResourceGroupToFiles, sortTeamConfigArraysByName, sortUserArraysByName, @@ -898,3 +899,74 @@ describe('AplCatalog', () => { }) }) }) + +describe('resolveSecretPlaceholders', () => { + it('should resolve sealed placeholders from K8s secrets', async () => { + const values = { + apps: { gitea: { adminPassword: 'sealed:gitea/gitea-admin-secret/password', adminUsername: 'admin' } }, + } + const deps = { + getK8sSecret: jest.fn().mockResolvedValue({ password: 'real-pass', username: 'otomi-admin' }), + } + + const result = await resolveSecretPlaceholders(values, deps) + + expect(result.apps.gitea.adminPassword).toBe('real-pass') + expect(result.apps.gitea.adminUsername).toBe('admin') + expect(deps.getK8sSecret).toHaveBeenCalledWith('gitea-admin-secret', 'gitea') + }) + + it('should return values unchanged when no placeholders exist', async () => { + const values = { apps: { gitea: { adminPassword: 'plain-value' } } } + const deps = { getK8sSecret: jest.fn() } + + const result = await resolveSecretPlaceholders(values, deps) + expect(result.apps.gitea.adminPassword).toBe('plain-value') + expect(deps.getK8sSecret).not.toHaveBeenCalled() + }) + + it('should return values unchanged when K8s not available', async () => { + const values = { apps: { gitea: { adminPassword: 'sealed:gitea/gitea-admin-secret/password' } } } + const deps = { getK8sSecret: jest.fn().mockRejectedValue(new Error('no cluster')) } + + const result = await resolveSecretPlaceholders(values, deps) + expect(result.apps.gitea.adminPassword).toBe('sealed:gitea/gitea-admin-secret/password') + }) + + it('should cache K8s secret reads', async () => { + const values = { + apps: { + gitea: { + adminPassword: 'sealed:gitea/gitea-admin-secret/password', + adminUsername: 'sealed:gitea/gitea-admin-secret/username', + }, + }, + } + const deps = { + getK8sSecret: jest.fn().mockResolvedValue({ password: 'pass', username: 'admin' }), + } + + await resolveSecretPlaceholders(values, deps) + expect(deps.getK8sSecret).toHaveBeenCalledTimes(1) + }) + + it('should warn on invalid placeholder format', async () => { + const values = { apps: { gitea: { adminPassword: 'sealed:invalid-format' } } } + const deps = { getK8sSecret: jest.fn() } + + const result = await resolveSecretPlaceholders(values, deps) + expect(result.apps.gitea.adminPassword).toBe('sealed:invalid-format') + expect(deps.getK8sSecret).not.toHaveBeenCalled() + }) + + it('should not mutate original values', async () => { + const values = { apps: { gitea: { adminPassword: 'sealed:gitea/gitea-admin-secret/password' } } } + const deps = { + getK8sSecret: jest.fn().mockResolvedValue({ password: 'real-pass' }), + } + + const result = await resolveSecretPlaceholders(values, deps) + expect(result.apps.gitea.adminPassword).toBe('real-pass') + expect(values.apps.gitea.adminPassword).toBe('sealed:gitea/gitea-admin-secret/password') + }) +}) diff --git a/src/common/repo.ts b/src/common/repo.ts index f37993ca7b..2bb7779054 100644 --- a/src/common/repo.ts +++ b/src/common/repo.ts @@ -4,7 +4,9 @@ import { globSync } from 'glob' import jsonpath from 'jsonpath' import { cloneDeep, get, merge, omit, set } from 'lodash' import path from 'path' -import { getDirNames, loadYaml } from './utils' +import { terminal } from './debug' +import { getK8sSecret } from './k8s' +import { flattenObject, getDirNames, loadYaml } from './utils' import { objectToYaml, writeValuesToFile } from './values' export async function getTeamNames(envDir: string): Promise> { @@ -566,14 +568,68 @@ export function sortTeamConfigArraysByName(spec: Record): Record, + deps = { getK8sSecret }, +): Promise> { + const d = terminal('common:repo:resolveSecretPlaceholders') + const flat = flattenObject(values) + + const placeholders = Object.entries(flat).filter( + ([, value]) => typeof value === 'string' && (value as string).startsWith(SECRET_PLACEHOLDER_PREFIX), + ) + + if (placeholders.length === 0) return values + + const secretCache = new Map | undefined>() + const result = cloneDeep(values) + + for (const [valuePath, placeholder] of placeholders) { + const ref = (placeholder as string).replace(SECRET_PLACEHOLDER_PREFIX, '') + const parts = ref.split('/') + if (parts.length !== 3) { + d.warn(`Invalid secret placeholder format: ${placeholder}`) + continue + } + const [namespace, secretName, key] = parts + const cacheKey = `${namespace}/${secretName}` + + if (!secretCache.has(cacheKey)) { + try { + const secret = await deps.getK8sSecret(secretName, namespace) + secretCache.set(cacheKey, secret) + } catch { + d.warn(`Could not read K8s secret ${cacheKey}`) + secretCache.set(cacheKey, undefined) + } + } + + const secretData = secretCache.get(cacheKey) + if (secretData?.[key] !== undefined) { + set(result, valuePath, secretData[key]) + d.debug(`Resolved ${valuePath} from ${cacheKey}/${key}`) + } else { + d.warn(`Could not resolve placeholder: ${placeholder}`) + } + } + + return result +} + export async function setValuesFile( envDir: string, - deps = { pathExists: existsSync, loadValues, writeFile }, + deps = { pathExists: existsSync, loadValues, writeFile, resolveSecretPlaceholders }, ): Promise { const valuesPath = path.join(envDir, 'values-repo.yaml') - // if (await deps.pathExists(valuesPath)) return valuesPath const allValues = await deps.loadValues(envDir) - await deps.writeFile(valuesPath, objectToYaml(allValues)) + const resolved = await deps.resolveSecretPlaceholders(allValues) + await deps.writeFile(valuesPath, objectToYaml(resolved)) return valuesPath } diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts new file mode 100644 index 0000000000..d13dc93ab3 --- /dev/null +++ b/src/common/sealed-secrets.test.ts @@ -0,0 +1,625 @@ +import { pki } from 'node-forge' +import stubs from 'src/test-stubs' +import { + APP_NAMESPACE_MAP, + APP_SECRET_OVERRIDES, + bootstrapSealedSecrets, + buildSecretToNamespaceMap, + createSealedSecretManifest, + createSealedSecretsKeySecret, + generateSealedSecretsKeyPair, + getPemFromCertificate, + writeSealedSecretManifests, +} from './sealed-secrets' + +const { terminal } = stubs + +jest.mock('@linode/kubeseal-encrypt', () => ({ + encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), +})) + +jest.mock('zx', () => ({ + $: jest.fn().mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ stderr: '', exitCode: 0 }), + }), + }), +})) + +jest.mock('src/common/envalid', () => ({ + env: {}, +})) + +describe('sealed-secrets', () => { + describe('generateSealedSecretsKeyPair', () => { + it('should generate a valid key pair with certificate and private key', () => { + const mockCert = { + publicKey: {}, + serialNumber: '', + validity: { notBefore: new Date(), notAfter: new Date() }, + sign: jest.fn(), + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + } + const mockKeys = { + publicKey: { n: {}, e: {} }, + privateKey: { d: {}, p: {}, q: {} }, + } + const deps = { + terminal, + pki: { + rsa: { generateKeyPair: jest.fn().mockReturnValue(mockKeys) }, + createCertificate: jest.fn().mockReturnValue(mockCert), + certificateToPem: jest.fn().mockReturnValue('-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n'), + privateKeyToPem: jest + .fn() + .mockReturnValue('-----BEGIN RSA PRIVATE KEY-----\nfake\n-----END RSA PRIVATE KEY-----\n'), + } as unknown as typeof pki, + } + + const result = generateSealedSecretsKeyPair(deps) + + expect(deps.pki.rsa.generateKeyPair).toHaveBeenCalledWith(4096) + expect(deps.pki.createCertificate).toHaveBeenCalled() + expect(mockCert.sign).toHaveBeenCalled() + expect(result.certificate).toContain('BEGIN CERTIFICATE') + expect(result.privateKey).toContain('BEGIN RSA PRIVATE KEY') + }) + + it('should set 10-year validity', () => { + const mockCert = { + publicKey: {}, + serialNumber: '', + validity: { notBefore: new Date(), notAfter: new Date() }, + sign: jest.fn(), + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + } + const mockKeys = { + publicKey: {}, + privateKey: {}, + } + const deps = { + terminal, + pki: { + rsa: { generateKeyPair: jest.fn().mockReturnValue(mockKeys) }, + createCertificate: jest.fn().mockReturnValue(mockCert), + certificateToPem: jest.fn().mockReturnValue('cert'), + privateKeyToPem: jest.fn().mockReturnValue('key'), + } as unknown as typeof pki, + } + + generateSealedSecretsKeyPair(deps) + + const notBefore = mockCert.validity.notBefore.getFullYear() + const notAfter = mockCert.validity.notAfter.getFullYear() + expect(notAfter - notBefore).toBe(10) + }) + }) + + describe('getPemFromCertificate', () => { + it('should extract SPKI public key from a certificate', () => { + // Generate a real key pair and certificate for this test + const keys = pki.rsa.generateKeyPair(2048) + const cert = pki.createCertificate() + cert.publicKey = keys.publicKey + cert.serialNumber = '01' + cert.validity.notBefore = new Date() + cert.validity.notAfter = new Date() + cert.validity.notAfter.setFullYear(cert.validity.notBefore.getFullYear() + 1) + const attrs = [{ name: 'commonName', value: 'test' }] + cert.setSubject(attrs) + cert.setIssuer(attrs) + cert.sign(keys.privateKey) + const certPem = pki.certificateToPem(cert) + + const result = getPemFromCertificate(certPem) + + expect(result).toContain('BEGIN PUBLIC KEY') + expect(result).toContain('END PUBLIC KEY') + }) + }) + + describe('createSealedSecretsKeySecret', () => { + it('should create secret if it does not exist', async () => { + const mockQuiet = jest.fn().mockResolvedValue({ stderr: '', exitCode: 0 }) + const mockNothrow = jest.fn().mockReturnValue({ quiet: mockQuiet }) + // First call (namespace): success, Second call (check exists): not found (exitCode 1) + // Third call (create): success, Fourth call (label): success + const mock$ = jest + .fn() + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // namespace + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 1 }) }), + }) // check exists - NOT FOUND + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // create + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // label + const deps = { + $: mock$ as any, + terminal, + writeFile: jest.fn(), + mkdir: jest.fn(), + } + + await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) + + expect(mock$).toHaveBeenCalledTimes(4) + expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.crt', 'cert-pem') + expect(deps.writeFile).toHaveBeenCalledWith('/tmp/sealed-secrets-bootstrap/tls.key', 'key-pem') + }) + + it('should skip creation if secret already exists', async () => { + const mock$ = jest + .fn() + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // namespace + .mockReturnValueOnce({ + nothrow: jest.fn().mockReturnValue({ quiet: jest.fn().mockResolvedValue({ exitCode: 0 }) }), + }) // check exists - FOUND + const deps = { + $: mock$ as any, + terminal, + writeFile: jest.fn(), + mkdir: jest.fn(), + } + + await createSealedSecretsKeySecret('cert-pem', 'key-pem', deps) + + // Should only call namespace and check exists, not create or label + expect(mock$).toHaveBeenCalledTimes(2) + expect(deps.writeFile).not.toHaveBeenCalled() + }) + }) + + describe('buildSecretToNamespaceMap', () => { + it('should group secrets by namespace and secret name with leaf key naming', async () => { + const secrets = { + apps: { + harbor: { adminPassword: 'harbor-pass', secretKey: 'harbor-secret' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.adminPassword', 'apps.harbor.secretKey']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + const harborMapping = result.find((m) => m.namespace === 'harbor') + expect(harborMapping).toBeDefined() + expect(harborMapping!.secretName).toBe('harbor-secrets') + expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') + expect(harborMapping!.data).toHaveProperty('secretKey', 'harbor-secret') + }) + + it('should skip kms.sops paths', async () => { + const secrets = { + kms: { sops: { provider: 'age', age: { publicKey: 'pk', privateKey: 'sk' } } }, + apps: { harbor: { adminPassword: 'pass' } }, + } + const deps = { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue(['kms.sops.provider', 'kms.sops.age.publicKey', 'apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].namespace).toBe('harbor') + }) + + it('should skip users path', async () => { + const secrets = { + users: [{ email: 'admin@example.com' }], + apps: { harbor: { adminPassword: 'pass' } }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['users', 'apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].namespace).toBe('harbor') + }) + + it('should handle teamConfig dynamic paths', async () => { + const secrets = { + teamConfig: { + 'team-alpha': { someSecret: 'value' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['teamConfig.team-alpha.someSecret']), + } + + const result = await buildSecretToNamespaceMap(secrets, ['team-alpha'], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].namespace).toBe('team-team-alpha') + expect(result[0].secretName).toBe('team-settings-secrets') + }) + + it('should filter out mappings with no data', async () => { + const secrets = {} + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(0) + }) + + it('should use leaf key naming for nested secret paths', async () => { + const secrets = { + apps: { + harbor: { core: { secret: 'core-secret-val' } }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.harbor.core.secret']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], undefined, deps) + + expect(result).toHaveLength(1) + expect(result[0].data).toHaveProperty('core_secret', 'core-secret-val') + }) + + it('should skip overridden prefixes and generate override secrets instead', async () => { + const secrets = { + apps: { + gitea: { adminPassword: 'gitea-pass', postgresqlPassword: 'pg-pass' }, + harbor: { adminPassword: 'harbor-pass' }, + }, + } + const allValues = { + apps: { + gitea: { adminUsername: 'admin', adminPassword: 'gitea-pass', postgresqlPassword: 'pg-pass' }, + harbor: { adminPassword: 'harbor-pass' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue([ + 'apps.gitea.adminPassword', + 'apps.gitea.postgresqlPassword', + 'apps.harbor.adminPassword', + ]), + } + + const result = await buildSecretToNamespaceMap(secrets, [], allValues, deps) + + // Harbor should use default convention + const harborMapping = result.find((m) => m.secretName === 'harbor-secrets') + expect(harborMapping).toBeDefined() + expect(harborMapping!.data).toHaveProperty('adminPassword', 'harbor-pass') + + // Gitea should NOT have a gitea-secrets mapping + const giteaDefaultMapping = result.find((m) => m.secretName === 'gitea-secrets') + expect(giteaDefaultMapping).toBeUndefined() + + // Gitea should have override-based secrets + const giteaAdminMapping = result.find((m) => m.secretName === 'gitea-admin-secret') + expect(giteaAdminMapping).toBeDefined() + expect(giteaAdminMapping!.namespace).toBe('gitea') + expect(giteaAdminMapping!.data).toEqual({ username: 'admin', password: 'gitea-pass' }) + + const giteaDbMapping = result.find((m) => m.secretName === 'gitea-db-secret') + expect(giteaDbMapping).toBeDefined() + expect(giteaDbMapping!.namespace).toBe('gitea') + expect(giteaDbMapping!.data).toEqual({ username: 'gitea', password: 'pg-pass' }) + }) + + it('should include static values in override secrets', async () => { + const secrets = { + apps: { + gitea: { postgresqlPassword: 'pg-pass' }, + }, + } + const allValues = { + apps: { + gitea: { postgresqlPassword: 'pg-pass' }, + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.gitea.postgresqlPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], allValues, deps) + + const giteaDbMapping = result.find((m) => m.secretName === 'gitea-db-secret') + expect(giteaDbMapping).toBeDefined() + expect(giteaDbMapping!.data.username).toBe('gitea') + expect(giteaDbMapping!.data.password).toBe('pg-pass') + }) + + it('should create gitea secrets on first install when generated passwords are only in secrets', async () => { + // On first install, allValues (originalInput) does NOT contain generated passwords, + // but secrets (allSecrets) does. The override lookup must check secrets first. + const secrets = { + apps: { + gitea: { adminPassword: 'generated-admin-pass', postgresqlPassword: 'generated-pg-pass' }, + }, + } + const allValues = { + apps: { + gitea: {}, // No generated passwords in originalInput on first install + }, + } + const deps = { + getSchemaSecretsPaths: jest + .fn() + .mockResolvedValue(['apps.gitea.adminPassword', 'apps.gitea.postgresqlPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], allValues, deps) + + const giteaAdminMapping = result.find((m) => m.secretName === 'gitea-admin-secret') + expect(giteaAdminMapping).toBeDefined() + expect(giteaAdminMapping!.namespace).toBe('gitea') + expect(giteaAdminMapping!.data).toEqual({ username: 'otomi-admin', password: 'generated-admin-pass' }) + + const giteaDbMapping = result.find((m) => m.secretName === 'gitea-db-secret') + expect(giteaDbMapping).toBeDefined() + expect(giteaDbMapping!.namespace).toBe('gitea') + expect(giteaDbMapping!.data).toEqual({ username: 'gitea', password: 'generated-pg-pass' }) + }) + + it('should use default values when valuePath not found but actual value exists', async () => { + // When adminPassword exists but adminUsername doesn't, username should use default + const secrets = { + apps: { + gitea: { adminPassword: 'gitea-pass' }, + }, + } + const allValues = { + apps: { + gitea: { adminPassword: 'gitea-pass' }, // no adminUsername provided + }, + } + const deps = { + getSchemaSecretsPaths: jest.fn().mockResolvedValue(['apps.gitea.adminPassword']), + } + + const result = await buildSecretToNamespaceMap(secrets, [], allValues, deps) + + const giteaAdminMapping = result.find((m) => m.secretName === 'gitea-admin-secret') + expect(giteaAdminMapping).toBeDefined() + expect(giteaAdminMapping!.data.username).toBe('otomi-admin') // default value + expect(giteaAdminMapping!.data.password).toBe('gitea-pass') + }) + }) + + describe('createSealedSecretManifest', () => { + it('should produce correct SealedSecret structure', async () => { + const mapping = { + namespace: 'harbor', + secretName: 'harbor-secrets', + data: { adminPassword: 'my-password', secretKey: 'my-secret' }, + } + const deps = { + encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), + } + + const result = await createSealedSecretManifest('mock-pem', mapping, deps) + + expect(result.apiVersion).toBe('bitnami.com/v1alpha1') + expect(result.kind).toBe('SealedSecret') + expect(result.metadata.name).toBe('harbor-secrets') + expect(result.metadata.namespace).toBe('harbor') + expect(result.metadata.annotations['sealedsecrets.bitnami.com/namespace-wide']).toBe('true') + expect(result.spec.encryptedData.adminPassword).toBe('encrypted-value') + expect(result.spec.encryptedData.secretKey).toBe('encrypted-value') + expect(result.spec.template.type).toBe('Opaque') + expect(result.spec.template.metadata.name).toBe('harbor-secrets') + expect(result.spec.template.metadata.namespace).toBe('harbor') + }) + + it('should call encryptSecretItem for each data key', async () => { + const mapping = { + namespace: 'gitea', + secretName: 'gitea-secrets', + data: { key1: 'val1', key2: 'val2', key3: 'val3' }, + } + const deps = { + encryptSecretItem: jest.fn().mockResolvedValue('enc'), + } + + await createSealedSecretManifest('pem', mapping, deps) + + expect(deps.encryptSecretItem).toHaveBeenCalledTimes(3) + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'gitea', 'val1') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'gitea', 'val2') + expect(deps.encryptSecretItem).toHaveBeenCalledWith('pem', 'gitea', 'val3') + }) + }) + + describe('writeSealedSecretManifests', () => { + it('should write each manifest to the correct directory', async () => { + const manifests = [ + { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: 'harbor-secrets', + namespace: 'harbor', + }, + spec: { + encryptedData: { key: 'enc' }, + template: { + immutable: false, + metadata: { name: 'harbor-secrets', namespace: 'harbor' }, + type: 'Opaque', + }, + }, + }, + ] + const deps = { + mkdir: jest.fn(), + writeFile: jest.fn(), + objectToYaml: jest.fn().mockReturnValue('yaml-content'), + terminal, + } + + await writeSealedSecretManifests(manifests, '/test', deps) + + expect(deps.mkdir).toHaveBeenCalledWith('/test/env/manifests/ns/harbor', { recursive: true }) + expect(deps.writeFile).toHaveBeenCalledWith('/test/env/manifests/ns/harbor/harbor-secrets.yaml', 'yaml-content') + }) + }) + + describe('bootstrapSealedSecrets', () => { + it('should generate new key pair when no existing cert found', async () => { + const secrets = { + apps: { harbor: { adminPassword: 'pass' } }, + } + const mockMapping = { + namespace: 'harbor', + secretName: 'harbor-secrets', + data: { adminPassword: 'pass' }, + } + const mockManifest = { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: 'harbor-secrets', + namespace: 'harbor', + }, + spec: { + encryptedData: { adminPassword: 'encrypted' }, + template: { + immutable: false, + metadata: { name: 'harbor-secrets', namespace: 'harbor' }, + type: 'Opaque', + }, + }, + } + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), // No existing cert + generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ + certificate: 'cert-pem', + privateKey: 'key-pem', + }), + getPemFromCertificate: jest.fn().mockReturnValue('spki-pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), + createSealedSecretManifest: jest.fn().mockResolvedValue(mockManifest), + writeSealedSecretManifests: jest.fn(), + encryptSecretItem: jest.fn().mockResolvedValue('encrypted'), + } + + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + + expect(deps.getExistingSealedSecretsCert).toHaveBeenCalled() + expect(deps.generateSealedSecretsKeyPair).toHaveBeenCalled() + expect(deps.createSealedSecretsKeySecret).toHaveBeenCalledWith('cert-pem', 'key-pem') + expect(deps.getPemFromCertificate).toHaveBeenCalledWith('cert-pem') + expect(deps.writeSealedSecretManifests).toHaveBeenCalledWith([mockManifest], '/test') + }) + + it('should use existing cert when found', async () => { + const secrets = { + apps: { harbor: { adminPassword: 'pass' } }, + } + const mockMapping = { + namespace: 'harbor', + secretName: 'harbor-secrets', + data: { adminPassword: 'pass' }, + } + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue('existing-cert-pem'), // Existing cert found + generateSealedSecretsKeyPair: jest.fn(), + getPemFromCertificate: jest.fn().mockReturnValue('existing-spki-pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), + createSealedSecretManifest: jest.fn().mockResolvedValue({}), + writeSealedSecretManifests: jest.fn(), + encryptSecretItem: jest.fn(), + } + + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + + expect(deps.getExistingSealedSecretsCert).toHaveBeenCalled() + expect(deps.generateSealedSecretsKeyPair).not.toHaveBeenCalled() // Should NOT generate new key + expect(deps.createSealedSecretsKeySecret).not.toHaveBeenCalled() // Should NOT create secret + expect(deps.getPemFromCertificate).toHaveBeenCalledWith('existing-cert-pem') + }) + + it('should extract team names from secrets', async () => { + const secrets = { + teamConfig: { + alpha: { secret: 'val' }, + beta: { secret: 'val' }, + }, + } + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), + generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ + certificate: 'cert', + privateKey: 'key', + }), + getPemFromCertificate: jest.fn().mockReturnValue('pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), + createSealedSecretManifest: jest.fn(), + writeSealedSecretManifests: jest.fn(), + encryptSecretItem: jest.fn(), + } + + await bootstrapSealedSecrets(secrets, '/test', undefined, deps) + + expect(deps.buildSecretToNamespaceMap).toHaveBeenCalledWith(secrets, ['alpha', 'beta'], undefined) + }) + }) + + describe('APP_NAMESPACE_MAP', () => { + it('should have expected mappings', () => { + expect(APP_NAMESPACE_MAP['apps.harbor']).toBe('harbor') + expect(APP_NAMESPACE_MAP['apps.gitea']).toBe('gitea') + expect(APP_NAMESPACE_MAP['apps.oauth2-proxy']).toBe('istio-system') + expect(APP_NAMESPACE_MAP['apps.loki']).toBe('monitoring') + expect(APP_NAMESPACE_MAP['otomi']).toBe('otomi') + expect(APP_NAMESPACE_MAP['dns']).toBe('external-dns') + expect(APP_NAMESPACE_MAP['cluster']).toBe('cert-manager') + }) + }) + + describe('APP_SECRET_OVERRIDES', () => { + it('should have gitea overrides configured', () => { + expect(APP_SECRET_OVERRIDES['apps.gitea']).toBeDefined() + expect(APP_SECRET_OVERRIDES['apps.gitea']).toHaveLength(2) + + const adminSecret = APP_SECRET_OVERRIDES['apps.gitea'].find((o) => o.secretName === 'gitea-admin-secret') + expect(adminSecret).toBeDefined() + expect(adminSecret!.namespace).toBe('gitea') + expect(adminSecret!.data.username).toEqual({ valuePath: 'apps.gitea.adminUsername', default: 'otomi-admin' }) + expect(adminSecret!.data.password).toEqual({ valuePath: 'apps.gitea.adminPassword' }) + + const dbSecret = APP_SECRET_OVERRIDES['apps.gitea'].find((o) => o.secretName === 'gitea-db-secret') + expect(dbSecret).toBeDefined() + expect(dbSecret!.namespace).toBe('gitea') + expect(dbSecret!.data.username).toEqual({ static: 'gitea' }) + expect(dbSecret!.data.password).toEqual({ valuePath: 'apps.gitea.postgresqlPassword' }) + }) + }) +}) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts new file mode 100644 index 0000000000..0d69357c0f --- /dev/null +++ b/src/common/sealed-secrets.ts @@ -0,0 +1,686 @@ +import { encryptSecretItem } from '@linode/kubeseal-encrypt' +import { X509Certificate } from 'crypto' +import { existsSync } from 'fs' +import { mkdir, readdir, readFile, writeFile } from 'fs/promises' +import { get } from 'lodash' +import { pki } from 'node-forge' +import { join } from 'path' +import { terminal } from 'src/common/debug' +import { flattenObject, getSchemaSecretsPaths } from 'src/common/utils' +import { objectToYaml } from 'src/common/values' +import { $ } from 'zx' + +const cmdName = 'sealed-secrets' + +/** + * Ensure a namespace exists. If it doesn't exist, create it with proper labels. + * This avoids overwriting labels on existing namespaces that were created by k8s-raw.gotmpl. + */ +export const ensureNamespaceExists = async (namespace: string, deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:ensureNamespaceExists`) + + // Check if namespace already exists + const existingNs = await deps.$`kubectl get namespace ${namespace}`.nothrow().quiet() + if (existingNs.exitCode === 0) { + d.debug(`Namespace ${namespace} already exists`) + return + } + + // Create namespace with proper label + d.info(`Creating namespace ${namespace}`) + const nsYaml = `apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} + labels: + name: ${namespace}` + + await deps.$`echo ${nsYaml} | kubectl apply -f -`.nothrow().quiet() +} + +export interface SecretMapping { + namespace: string + secretName: string + data: Record +} + +export interface SealedSecretManifest { + apiVersion: string + kind: string + metadata: { + annotations: Record + name: string + namespace: string + } + spec: { + encryptedData: Record + template: { + immutable: boolean + metadata: { name: string; namespace: string } + type: string + } + } +} + +/** + * Mapping from secret path prefix to target Kubernetes namespace. + * Dynamic entries like `teamConfig.{teamId}` are handled separately. + */ +export const APP_NAMESPACE_MAP: Record = { + 'apps.harbor': 'harbor', + 'apps.gitea': 'gitea', + 'apps.keycloak': 'keycloak', + 'apps.grafana': 'grafana', + 'apps.loki': 'monitoring', + 'apps.oauth2-proxy': 'istio-system', + 'apps.oauth2-proxy-redis': 'istio-system', + 'apps.prometheus': 'monitoring', + 'apps.otomi-api': 'otomi', + 'apps.cert-manager': 'cert-manager', + 'apps.kubeflow-pipelines': 'kfp', + otomi: 'otomi', + oidc: 'otomi', + smtp: 'otomi', + dns: 'external-dns', + obj: 'otomi', + license: 'otomi', + users: 'keycloak', + alerts: 'monitoring', + cluster: 'cert-manager', +} + +/** + * Per-app secret override configuration. + * When an app requires specific K8s Secret names and key layouts + * (e.g. gitea expects `gitea-admin-secret` with `username`/`password` keys), + * define overrides here instead of using the default `{app}-secrets` convention. + */ +interface SecretOverrideEntry { + secretName: string + namespace: string + data: Record +} + +export const APP_SECRET_OVERRIDES: Record = { + 'apps.gitea': [ + { + secretName: 'gitea-admin-secret', + namespace: 'gitea', + data: { + username: { valuePath: 'apps.gitea.adminUsername', default: 'otomi-admin' }, + password: { valuePath: 'apps.gitea.adminPassword' }, + }, + }, + { + secretName: 'gitea-db-secret', + namespace: 'gitea', + data: { + username: { static: 'gitea' }, + password: { valuePath: 'apps.gitea.postgresqlPassword' }, + }, + }, + ], +} + +/** + * Generate an RSA 4096-bit key pair and self-signed X.509 certificate for Sealed Secrets. + * Follows the pattern from createCustomCA() in bootstrap.ts. + */ +export const generateSealedSecretsKeyPair = (deps = { terminal, pki }): { certificate: string; privateKey: string } => { + const d = deps.terminal(`common:${cmdName}:generateSealedSecretsKeyPair`) + d.info('Generating sealed-secrets RSA key pair') + + const keys = deps.pki.rsa.generateKeyPair(4096) + const cert = deps.pki.createCertificate() + cert.publicKey = keys.publicKey + cert.serialNumber = '01' + cert.validity.notBefore = new Date() + cert.validity.notAfter = new Date() + cert.validity.notAfter.setFullYear(cert.validity.notBefore.getFullYear() + 10) + + const attrs = [ + { name: 'countryName', value: 'NL' }, + { shortName: 'ST', value: 'Utrecht' }, + { name: 'localityName', value: 'Utrecht' }, + { name: 'organizationName', value: 'APL' }, + { shortName: 'OU', value: 'SealedSecrets' }, + ] + cert.setSubject(attrs) + cert.setIssuer(attrs) + cert.setExtensions([ + { + name: 'basicConstraints', + cA: true, + }, + { + name: 'keyUsage', + keyCertSign: true, + digitalSignature: true, + keyEncipherment: true, + }, + ]) + cert.sign(keys.privateKey) + + const certificate = deps.pki.certificateToPem(cert).replaceAll('\r\n', '\n') + const privateKey = deps.pki.privateKeyToPem(keys.privateKey).replaceAll('\r\n', '\n') + + d.info('Generated sealed-secrets key pair') + return { certificate, privateKey } +} + +/** + * Extract SPKI PEM public key from a PEM-encoded X.509 certificate. + * Uses Node.js crypto.X509Certificate (same approach as getSealedSecretsPEM() in k8s.ts). + */ +export const getPemFromCertificate = (certificate: string): string => { + const x509 = new X509Certificate(certificate) + const exported = x509.publicKey.export({ format: 'pem', type: 'spki' }) + return typeof exported === 'string' ? exported : exported.toString('utf-8') +} + +/** + * Get the existing sealed-secrets certificate from the cluster if it exists. + * Returns the certificate PEM string or undefined if not found. + */ +export const getExistingSealedSecretsCert = async (deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:getExistingSealedSecretsCert`) + + const result = + await deps.$`kubectl get secret sealed-secrets-key -n sealed-secrets -o jsonpath='{.data.tls\\.crt}' 2>/dev/null` + .nothrow() + .quiet() + + if (result.exitCode !== 0 || !result.stdout || result.stdout === '') { + d.info('No existing sealed-secrets-key found') + return undefined + } + + try { + const certBase64 = result.stdout.replace(/'/g, '') + const cert = Buffer.from(certBase64, 'base64').toString('utf-8') + d.info('Found existing sealed-secrets-key certificate') + return cert + } catch { + d.warn('Failed to decode existing certificate') + return undefined + } +} + +/** + * Create the sealed-secrets namespace and TLS secret in Kubernetes. + * The controller will pick up this pre-created key on startup. + * IMPORTANT: This only creates the secret if it doesn't already exist. + */ +export const createSealedSecretsKeySecret = async ( + certificate: string, + privateKey: string, + deps = { $, terminal, writeFile, mkdir }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:createSealedSecretsKeySecret`) + + // Create namespace if it doesn't exist + await ensureNamespaceExists('sealed-secrets', { $: deps.$, terminal: deps.terminal }) + + // Check if secret already exists + const existingSecret = await deps.$`kubectl get secret sealed-secrets-key -n sealed-secrets`.nothrow().quiet() + if (existingSecret.exitCode === 0) { + d.info('sealed-secrets-key already exists, skipping creation') + return + } + + d.info('Creating sealed-secrets TLS secret') + + // Write temp files for kubectl create secret tls + const tmpDir = '/tmp/sealed-secrets-bootstrap' + await deps.mkdir(tmpDir, { recursive: true }) + const certPath = `${tmpDir}/tls.crt` + const keyPath = `${tmpDir}/tls.key` + await deps.writeFile(certPath, certificate) + await deps.writeFile(keyPath, privateKey) + + // Create the TLS secret (only if it doesn't exist) + const result = + await deps.$`kubectl create secret tls sealed-secrets-key -n sealed-secrets --cert=${certPath} --key=${keyPath}` + .nothrow() + .quiet() + if (result.exitCode !== 0) { + d.error(`Failed to create sealed-secrets-key: ${result.stderr}`) + return + } + + // Label the secret so the controller picks it up + const labelResult = + await deps.$`kubectl label secret sealed-secrets-key -n sealed-secrets sealedsecrets.bitnami.com/sealed-secrets-key=active --overwrite` + .nothrow() + .quiet() + if (labelResult.stderr) d.error(labelResult.stderr) + + d.info('Created sealed-secrets TLS secret with key label') +} + +/** + * Resolve the namespace for a given secret path using APP_NAMESPACE_MAP. + * Handles dynamic teamConfig paths. + */ +const resolveNamespace = (secretPath: string): string | undefined => { + // Check for teamConfig dynamic paths + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) { + return `team-${teamMatch[1]}` + } + + // Find the longest matching prefix in APP_NAMESPACE_MAP + const sortedKeys = Object.keys(APP_NAMESPACE_MAP).sort((a, b) => b.length - a.length) + for (const prefix of sortedKeys) { + if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { + return APP_NAMESPACE_MAP[prefix] + } + } + + return undefined +} + +// Map specific path prefixes to secret names +const SECRET_NAME_MAP: Record = { + 'apps.harbor': 'harbor-secrets', + 'apps.gitea': 'gitea-secrets', + 'apps.keycloak': 'keycloak-secrets', + 'apps.grafana': 'grafana-secrets', + 'apps.loki': 'loki-secrets', + 'apps.oauth2-proxy': 'oauth2-proxy-secrets', + 'apps.oauth2-proxy-redis': 'oauth2-proxy-redis-secrets', + 'apps.prometheus': 'prometheus-secrets', + 'apps.otomi-api': 'otomi-api-secrets', + 'apps.cert-manager': 'cert-manager-secrets', + 'apps.kubeflow-pipelines': 'kubeflow-pipelines-secrets', + otomi: 'otomi-platform-secrets', + oidc: 'oidc-secrets', + smtp: 'smtp-secrets', + dns: 'dns-secrets', + obj: 'obj-storage-secrets', + license: 'license-secrets', + users: 'users-secrets', + alerts: 'alerts-secrets', + cluster: 'cluster-secrets', +} + +/** + * Find the group prefix for a secret path. + * Returns the prefix that maps to the secret name (e.g., 'apps.harbor' for 'apps.harbor.adminPassword'). + */ +const findGroupPrefix = (secretPath: string): string | undefined => { + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) { + return `teamConfig.${teamMatch[1]}` + } + + const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) + for (const prefix of sortedKeys) { + if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { + return prefix + } + } + + // Fallback: use first two path segments + const parts = secretPath.split('.') + if (parts.length >= 2) { + return parts.slice(0, 2).join('.') + } + return undefined +} + +/** + * Derive a K8s secret name from the secret path prefix. + */ +const deriveSecretName = (secretPath: string): string => { + const teamMatch = secretPath.match(/^teamConfig\.([^.]+)/) + if (teamMatch) { + return 'team-settings-secrets' + } + + const sortedKeys = Object.keys(SECRET_NAME_MAP).sort((a, b) => b.length - a.length) + for (const prefix of sortedKeys) { + if (secretPath === prefix || secretPath.startsWith(`${prefix}.`)) { + return SECRET_NAME_MAP[prefix] + } + } + + // Fallback: derive from first two path segments + const parts = secretPath.split('.') + return `${parts.slice(0, 2).join('-')}-secrets` +} + +/** + * Build a mapping from secrets to their target namespaces and K8s secret names. + * Groups secret paths by namespace and secret name. + */ +export const buildSecretToNamespaceMap = async ( + secrets: Record, + teams: string[], + allValues?: Record, + deps = { getSchemaSecretsPaths }, +): Promise => { + const secretPaths = await deps.getSchemaSecretsPaths(teams) + const flat = flattenObject(secrets) + const allFlat = allValues ? flattenObject(allValues) : flat + + // Group by namespace + secretName + const groupMap = new Map() + + // Determine which secret path prefixes have overrides + const overriddenPrefixes = Object.keys(APP_SECRET_OVERRIDES) + + for (const secretPath of secretPaths) { + // Skip SOPS-related paths + if (secretPath.startsWith('kms.sops')) continue + // Skip 'users' path — not a simple key-value secret + if (secretPath === 'users') continue + // Skip overridden prefixes — they are handled separately below + if (overriddenPrefixes.some((p) => secretPath === p || secretPath.startsWith(`${p}.`))) continue + + const namespace = resolveNamespace(secretPath) + if (!namespace) continue + + const secretName = deriveSecretName(secretPath) + const groupKey = `${namespace}/${secretName}` + + if (!groupMap.has(groupKey)) { + groupMap.set(groupKey, { namespace, secretName, data: {} }) + } + + const mapping = groupMap.get(groupKey)! + + // Find the group prefix (e.g., 'apps.harbor' for 'apps.harbor.adminPassword') + const groupPrefix = findGroupPrefix(secretPath) + + // Find all flat keys that match this secret path + for (const [flatKey, value] of Object.entries(flat)) { + if (flatKey === secretPath || flatKey.startsWith(`${secretPath}.`)) { + // Use leaf key: strip the group prefix to get relative path + const relativePath = + groupPrefix && (flatKey === groupPrefix || flatKey.startsWith(`${groupPrefix}.`)) + ? flatKey.slice(groupPrefix.length + 1) + : flatKey + const dataKey = relativePath.replace(/\./g, '_') + if (value !== undefined && value !== null && value !== '') { + mapping.data[dataKey] = String(value) + } + } + } + } + + // Process APP_SECRET_OVERRIDES + for (const [, overrides] of Object.entries(APP_SECRET_OVERRIDES)) { + for (const override of overrides) { + const data: Record = {} + let hasValuePathData = false + + // First pass: collect valuePath data and track if any actual values were found + const pendingDefaults: Array<{ key: string; defaultValue: string }> = [] + for (const [key, source] of Object.entries(override.data)) { + if (!('static' in source)) { + const value = flat[source.valuePath] ?? allFlat[source.valuePath] + if (value !== undefined && value !== null && value !== '') { + data[key] = String(value) + hasValuePathData = true + } else if (source.default !== undefined) { + // Queue default value - will be added only if we have at least one actual value + pendingDefaults.push({ key, defaultValue: source.default }) + } + } + } + + // Only add defaults and static values if we have at least one actual valuePath value + if (hasValuePathData) { + // Add pending defaults + for (const { key, defaultValue } of pendingDefaults) { + if (!(key in data)) { + data[key] = defaultValue + } + } + // Add static values + for (const [key, source] of Object.entries(override.data)) { + if ('static' in source) { + data[key] = source.static + } + } + } + + if (Object.keys(data).length > 0) { + groupMap.set(`${override.namespace}/${override.secretName}`, { + namespace: override.namespace, + secretName: override.secretName, + data, + }) + } + } + } + + // Filter out empty mappings + return Array.from(groupMap.values()).filter((m) => Object.keys(m.data).length > 0) +} + +/** + * Create a SealedSecret manifest by encrypting each data value. + * Follows the pattern from createCatalogSealedSecret() in migrate.ts. + */ +export const createSealedSecretManifest = async ( + pem: string, + mapping: SecretMapping, + deps = { encryptSecretItem }, +): Promise => { + const encryptedData: Record = {} + for (const [key, value] of Object.entries(mapping.data)) { + encryptedData[key] = await deps.encryptSecretItem(pem, mapping.namespace, value) + } + + return { + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { + annotations: { 'sealedsecrets.bitnami.com/namespace-wide': 'true' }, + name: mapping.secretName, + namespace: mapping.namespace, + }, + spec: { + encryptedData, + template: { + immutable: false, + metadata: { name: mapping.secretName, namespace: mapping.namespace }, + type: 'Opaque', + }, + }, + } +} + +/** + * Write SealedSecret manifests to the env/manifests/ns directory. + */ +export const writeSealedSecretManifests = async ( + manifests: SealedSecretManifest[], + envDir: string, + deps = { mkdir, writeFile, objectToYaml, terminal }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:writeSealedSecretManifests`) + + for (const manifest of manifests) { + // /env/manifests/ns/argocd/ + const dir = `${envDir}/env/manifests/ns/${manifest.metadata.namespace}` + await deps.mkdir(dir, { recursive: true }) + const filePath = `${dir}/${manifest.metadata.name}.yaml` + d.info(`Writing sealed secret to ${filePath}`) + await deps.writeFile(filePath, deps.objectToYaml(manifest)) + } +} + +/** + * Apply SealedSecret manifests to the Kubernetes cluster. + * Creates namespaces if needed and applies the SealedSecret resources. + */ +export const applySealedSecretManifests = async ( + manifests: SealedSecretManifest[], + deps = { $, terminal, objectToYaml }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:applySealedSecretManifests`) + + // Group manifests by namespace + const byNamespace = new Map() + for (const manifest of manifests) { + const ns = manifest.metadata.namespace + if (!byNamespace.has(ns)) { + byNamespace.set(ns, []) + } + byNamespace.get(ns)!.push(manifest) + } + + // Ensure namespaces exist and apply manifests + for (const [namespace, nsManifests] of byNamespace) { + await ensureNamespaceExists(namespace, { $: deps.$, terminal: deps.terminal }) + + for (const manifest of nsManifests) { + d.info(`Applying SealedSecret ${manifest.metadata.name} to namespace ${namespace}`) + const yaml = deps.objectToYaml(manifest) + const result = await deps.$`echo ${yaml} | kubectl apply -f -`.nothrow().quiet() + if (result.exitCode !== 0) { + d.error(`Failed to apply SealedSecret ${manifest.metadata.name}: ${result.stderr}`) + } + } + } + + d.info(`Applied ${manifests.length} SealedSecret manifests to cluster`) +} + +/** + * Read and apply all SealedSecret manifests from the env/manifests/ns directory. + * This should be called during install, after the sealed-secrets controller is deployed. + */ +export const applySealedSecretManifestsFromDir = async ( + envDir: string, + deps = { $, terminal, readdir, readFile, existsSync }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:applySealedSecretManifestsFromDir`) + const manifestsDir = join(envDir, 'env/manifests/ns') + + if (!deps.existsSync(manifestsDir)) { + d.info(`No SealedSecret manifests directory found at ${manifestsDir}`) + return + } + + d.info(`Applying SealedSecret manifests from ${manifestsDir}`) + + // Read all namespace directories + const namespaces = await deps.readdir(manifestsDir, { withFileTypes: true }) + let appliedCount = 0 + + for (const nsEntry of namespaces) { + if (!nsEntry.isDirectory()) continue + const namespace = nsEntry.name + const nsDir = join(manifestsDir, namespace) + + // Ensure namespace exists with proper labels + await ensureNamespaceExists(namespace, { $: deps.$, terminal: deps.terminal }) + + // Read all YAML files in the namespace directory + const files = await deps.readdir(nsDir) + for (const file of files) { + if (!file.endsWith('.yaml') && !file.endsWith('.yml')) continue + const filePath = join(nsDir, file) + d.info(`Applying SealedSecret from ${filePath}`) + + const result = await deps.$`kubectl apply -f ${filePath}`.nothrow().quiet() + if (result.exitCode !== 0) { + d.error(`Failed to apply SealedSecret from ${filePath}: ${result.stderr}`) + } else { + appliedCount += 1 + } + } + } + + d.info(`Applied ${appliedCount} SealedSecret manifests from directory`) +} + +/** + * Restart the sealed-secrets controller to ensure it uses the correct key. + * This is needed because if the controller starts before the sealed-secrets-key secret exists, + * it will generate its own key. Restarting forces it to pick up the existing key. + */ +export const restartSealedSecretsController = async (deps = { $, terminal }): Promise => { + const d = deps.terminal(`common:${cmdName}:restartSealedSecretsController`) + d.info('Restarting sealed-secrets controller to ensure correct key is used') + + const result = await deps.$`kubectl rollout restart deployment/sealed-secrets -n sealed-secrets`.nothrow().quiet() + if (result.exitCode !== 0) { + d.warn(`Failed to restart sealed-secrets controller: ${result.stderr}`) + return + } + + d.info('Waiting for sealed-secrets controller rollout') + const waitResult = await deps.$`kubectl rollout status deployment/sealed-secrets -n sealed-secrets --timeout=120s` + .nothrow() + .quiet() + if (waitResult.exitCode !== 0) { + d.warn(`Rollout status check failed: ${waitResult.stderr}`) + } else { + d.info('Sealed-secrets controller restarted successfully') + } +} + +/** + * Orchestrator: bootstrap sealed secrets for the platform. + * Replaces bootstrapSops(). + */ +export const bootstrapSealedSecrets = async ( + secrets: Record, + envDir: string, + allValues?: Record, + deps = { + terminal, + generateSealedSecretsKeyPair, + getPemFromCertificate, + createSealedSecretsKeySecret, + getExistingSealedSecretsCert, + buildSecretToNamespaceMap, + createSealedSecretManifest, + writeSealedSecretManifests, + encryptSecretItem, + }, +): Promise => { + const d = deps.terminal(`common:${cmdName}:bootstrapSealedSecrets`) + d.info('Bootstrapping sealed secrets') + + // 1. Check if there's an existing sealed-secrets key in the cluster + const existingCert = await deps.getExistingSealedSecretsCert() + + let pem: string + if (existingCert) { + // Use existing certificate for encryption + d.info('Using existing sealed-secrets certificate') + pem = deps.getPemFromCertificate(existingCert) + } else { + // Generate new key pair and create the secret + d.info('Generating new sealed-secrets key pair') + const { certificate, privateKey } = deps.generateSealedSecretsKeyPair() + await deps.createSealedSecretsKeySecret(certificate, privateKey) + pem = deps.getPemFromCertificate(certificate) + } + + // 5. Build secret-to-namespace mapping + const teams = Object.keys(get(secrets, 'teamConfig', {}) as Record) + const mappings = await deps.buildSecretToNamespaceMap(secrets, teams, allValues) + + // 6. Create SealedSecret manifests + const manifests: SealedSecretManifest[] = [] + for (const mapping of mappings) { + const manifest = await deps.createSealedSecretManifest(pem, mapping, { + encryptSecretItem: deps.encryptSecretItem, + }) + manifests.push(manifest) + } + + // 7. Write SealedSecret manifests to disk + // Note: These manifests are applied later during install, after the sealed-secrets + // controller is deployed and the SealedSecret CRD is available. + await deps.writeSealedSecretManifests(manifests, envDir) + + d.info(`Bootstrapped ${manifests.length} sealed secret manifests`) +} diff --git a/src/common/values.test.ts b/src/common/values.test.ts index 06f76fa4e6..bb101f9979 100644 --- a/src/common/values.test.ts +++ b/src/common/values.test.ts @@ -1,5 +1,5 @@ import { cloneDeep, merge, set } from 'lodash' -import { generateSecrets } from 'src/common/values' +import { generateSecrets, replaceSecretsWithPlaceholders } from 'src/common/values' import stubs from 'src/test-stubs' const { terminal } = stubs @@ -73,3 +73,47 @@ describe('generateSecrets', () => { expect(res.nested.twoStage).toBe('exists') }) }) + +describe('replaceSecretsWithPlaceholders', () => { + it('should replace gitea secrets with sealed secret references', () => { + const input = { + apps: { + gitea: { adminPassword: 'real-pass', postgresqlPassword: 'db-pass', adminUsername: 'admin' }, + harbor: { adminPassword: 'harbor-pass' }, + }, + } + const result = replaceSecretsWithPlaceholders(input) + + expect(result.apps.gitea.adminPassword).toBe('sealed:gitea/gitea-admin-secret/password') + expect(result.apps.gitea.postgresqlPassword).toBe('sealed:gitea/gitea-db-secret/password') + expect(result.apps.gitea.adminUsername).toBe('sealed:gitea/gitea-admin-secret/username') + expect(result.apps.harbor.adminPassword).toBe('harbor-pass') + // Original should not be modified + expect(input.apps.gitea.adminPassword).toBe('real-pass') + }) + + it('should not replace already-placeholder values', () => { + const input = { + apps: { gitea: { adminPassword: 'sealed:gitea/gitea-admin-secret/password' } }, + } + const result = replaceSecretsWithPlaceholders(input) + expect(result.apps.gitea.adminPassword).toBe('sealed:gitea/gitea-admin-secret/password') + }) + + it('should not replace non-string values', () => { + const input = { + apps: { gitea: { adminPassword: 123 as any, postgresqlPassword: 'db-pass' } }, + } + const result = replaceSecretsWithPlaceholders(input) + expect(result.apps.gitea.adminPassword).toBe(123) + expect(result.apps.gitea.postgresqlPassword).toBe('sealed:gitea/gitea-db-secret/password') + }) + + it('should handle values without matching paths', () => { + const input = { + apps: { harbor: { adminPassword: 'harbor-pass' } }, + } + const result = replaceSecretsWithPlaceholders(input) + expect(result.apps.harbor.adminPassword).toBe('harbor-pass') + }) +}) diff --git a/src/common/values.ts b/src/common/values.ts index 5e2a0d7fc5..d9bde66e83 100644 --- a/src/common/values.ts +++ b/src/common/values.ts @@ -1,6 +1,6 @@ import { existsSync } from 'fs' import { mkdir, unlink, writeFile } from 'fs/promises' -import { cloneDeep, get, isEmpty, isEqual, merge, mergeWith, omit, pick, set } from 'lodash' +import { cloneDeep, get, isEmpty, isEqual, merge, mergeWith, pick, set } from 'lodash' import path from 'path' import { supportedK8sVersions } from 'src/supportedK8sVersions.json' import { stringify } from 'yaml' @@ -9,18 +9,10 @@ import { decrypt, encrypt } from './crypt' import { terminal } from './debug' import { env } from './envalid' import { hfValues } from './hf' -import { - extract, - flattenObject, - getSchemaSecretsPaths, - getValuesSchema, - gucci, - loadYaml, - pkg, - removeBlankAttributes, -} from './utils' - +import { getK8sSecret } from './k8s' import { saveValues } from './repo' +import { APP_SECRET_OVERRIDES } from './sealed-secrets' +import { extract, flattenObject, getValuesSchema, gucci, loadYaml, pkg, removeBlankAttributes } from './utils' import { HelmArguments } from './yargs' export const objectToYaml = (obj: Record, indent = 4, lineWidth = 200): string => { @@ -63,7 +55,29 @@ export interface Repo { branch: string } -export const getRepo = (values: Record): Repo => { +/** + * Resolves a single sealed:namespace/secretName/key placeholder to its actual value + * by reading the corresponding K8s Secret. Returns the original value if not a placeholder + * or if resolution fails. + */ +export const resolveSinglePlaceholder = async (value: string, deps = { getK8sSecret, terminal }): Promise => { + if (!value || !value.startsWith('sealed:')) return value + const d = deps.terminal('common:values:resolveSinglePlaceholder') + const ref = value.replace('sealed:', '') + const parts = ref.split('/') + if (parts.length !== 3) return value + const [namespace, secretName, key] = parts + try { + const secret = await deps.getK8sSecret(secretName, namespace) + if (secret?.[key] !== undefined) return String(secret[key]) + d.warn(`Could not resolve placeholder ${value}: key '${key}' not found in secret ${namespace}/${secretName}`) + } catch (e) { + d.warn(`Could not resolve placeholder ${value}: K8s secret ${namespace}/${secretName} not available`) + } + return value +} + +export const getRepo = async (values: Record, deps = { getK8sSecret, terminal }): Promise => { const giteaEnabled = values?.apps?.gitea?.enabled ?? true const byor = !!values?.apps?.['otomi-api']?.git if (!giteaEnabled && !byor) { @@ -83,7 +97,7 @@ export const getRepo = (values: Record): Repo => { branch = otomiApiGit?.branch ?? branch } else { username = 'otomi-admin' - password = values?.apps?.gitea?.adminPassword + password = await resolveSinglePlaceholder(String(values?.apps?.gitea?.adminPassword ?? ''), deps) email = `pipeline@cluster.local` const gitUrl = env.GIT_URL const gitPort = env.GIT_PORT @@ -158,21 +172,43 @@ export const writeValuesToFile = async ( export const writeValues = async (inValues: Record, overwrite = false): Promise => { const d = terminal('common:values:writeValues') d.debug('Writing values: ', inValues) - hasSops = existsSync(`${env.ENV_DIR}/.sops.yaml`) - const values = inValues - const teams = Object.keys(get(inValues, 'teamConfig', {})) - const cleanSecretPaths = await getSchemaSecretsPaths(teams) - d.debug('cleanSecretPaths: ', cleanSecretPaths) - // separate out the secrets - const secrets = removeBlankAttributes(pick(values, cleanSecretPaths)) - d.debug('secrets: ', JSON.stringify(secrets, null, 2)) - // from the plain values - const plainValues = omit(values, cleanSecretPaths) as any - await saveValues(env.ENV_DIR, plainValues, secrets) - + const valuesToWrite = replaceSecretsWithPlaceholders(inValues) + await saveValues(env.ENV_DIR, valuesToWrite, {}) d.info('All values were written to ENV_DIR') } +/** + * Builds a mapping from valuePath → sealed:// + * using APP_SECRET_OVERRIDES configuration. + */ +function buildSecretPlaceholderMap(): Map { + const map = new Map() + for (const [, overrides] of Object.entries(APP_SECRET_OVERRIDES)) { + for (const override of overrides) { + for (const [key, source] of Object.entries(override.data)) { + if (!('static' in source) && source.valuePath) { + map.set(source.valuePath, `sealed:${override.namespace}/${override.secretName}/${key}`) + } + } + } + } + return map +} + +export const replaceSecretsWithPlaceholders = (values: Record): Record => { + const result = cloneDeep(values) + const placeholderMap = buildSecretPlaceholderMap() + + for (const [valuePath, placeholder] of placeholderMap) { + const value = get(result, valuePath) + if (value !== undefined && typeof value === 'string' && !value.startsWith('sealed:')) { + set(result, valuePath, placeholder) + } + } + + return result +} + export const deriveSecrets = async (values: Record = {}): Promise> => { // Some secrets needs to be drived from the generated secrets diff --git a/src/operator/installer.test.ts b/src/operator/installer.test.ts index 59bdd81a08..32a400ba9c 100644 --- a/src/operator/installer.test.ts +++ b/src/operator/installer.test.ts @@ -3,6 +3,31 @@ import * as k8s from '../common/k8s' import { AplOperations } from './apl-operations' import { Installer } from './installer' +const mockZx = jest.fn().mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }), + }), +}) + +jest.mock('zx', () => ({ + $: (...args: any[]) => mockZx(...args), +})) + +jest.mock('../common/envalid', () => ({ + env: { + GIT_PROTOCOL: 'http', + GIT_URL: 'gitea-http.gitea.svc.cluster.local', + GIT_PORT: '3000', + }, +})) + +jest.mock('./validators', () => ({ + operatorEnv: { + GIT_ORG: 'otomi', + GIT_REPO: 'values', + }, +})) + jest.mock('../common/debug', () => ({ terminal: jest.fn().mockImplementation(() => ({ info: jest.fn(), @@ -228,10 +253,18 @@ describe('Installer', () => { }) describe('isInstalled', () => { - test('should return completed status when ConfigMap exists', async () => { + test('should return completed status when ConfigMap exists and git repo has main branch', async () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ data: { status: 'completed' }, }) + // getK8sSecret returns credentials for git verification + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ GIT_USERNAME: 'admin', GIT_PASSWORD: 'pass' }) + // git ls-remote succeeds (main branch exists) + mockZx.mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 0 }), + }), + }) const isInstalled = await installer.isInstalled() @@ -240,6 +273,23 @@ describe('Installer', () => { expect(mockAplOps.install).not.toHaveBeenCalled() }) + test('should return false when status is completed but git repo has no main branch', async () => { + ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ + data: { status: 'completed' }, + }) + ;(k8s.getK8sSecret as jest.Mock).mockResolvedValue({ GIT_USERNAME: 'admin', GIT_PASSWORD: 'pass' }) + // git ls-remote fails (main branch does not exist) + mockZx.mockReturnValue({ + nothrow: jest.fn().mockReturnValue({ + quiet: jest.fn().mockResolvedValue({ exitCode: 2 }), + }), + }) + + const isInstalled = await installer.isInstalled() + + expect(isInstalled).toBe(false) + }) + test('should return true when ConfigMap does not exist', async () => { ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue(null) @@ -272,6 +322,19 @@ describe('Installer', () => { expect(k8s.getK8sConfigMap).toHaveBeenCalledWith('apl-operator', 'apl-installation-status', mockCoreApi) expect(isInstalled).toBe(false) }) + + test('should return true when git verification fails (gitea not ready)', async () => { + ;(k8s.getK8sConfigMap as jest.Mock).mockResolvedValue({ + data: { status: 'completed' }, + }) + // getK8sSecret throws (cluster issues) + ;(k8s.getK8sSecret as jest.Mock).mockRejectedValue(new Error('connection refused')) + + const isInstalled = await installer.isInstalled() + + // Should assume installed when verification can't be performed + expect(isInstalled).toBe(true) + }) }) describe('setEnvAndCreateSecrets', () => { diff --git a/src/operator/installer.ts b/src/operator/installer.ts index 63c2eb6cdd..d784545b6b 100644 --- a/src/operator/installer.ts +++ b/src/operator/installer.ts @@ -1,9 +1,13 @@ import * as process from 'node:process' +import { $ } from 'zx' import { terminal } from '../common/debug' +import { env } from '../common/envalid' import { hfValues } from '../common/hf' import { createUpdateConfigMap, createUpdateGenericSecret, getK8sConfigMap, getK8sSecret, k8s } from '../common/k8s' +import { resolveSinglePlaceholder } from '../common/values' import { AplOperations } from './apl-operations' import { getErrorMessage } from './utils' +import { operatorEnv } from './validators' export interface GitCredentials { username: string @@ -26,7 +30,34 @@ export class Installer { await this.updateInstallationStatus('completed', -1) return true } - return installStatus === 'completed' + if (installStatus === 'completed') { + // Verify the git repo actually has content - the previous install may have + // marked status as completed but the pod was killed before the git push finished + const gitRepoHasContent = await this.verifyGitRepoHasMainBranch() + if (!gitRepoHasContent) { + this.d.warn('Installation marked as completed but git repo has no main branch - will re-install') + return false + } + return true + } + return false + } + + private async verifyGitRepoHasMainBranch(): Promise { + try { + // Get credentials from K8s secret (created by Helm at deploy time) + const creds = await getK8sSecret('gitea-credentials', 'apl-operator') + const username = creds?.GIT_USERNAME ?? 'otomi-admin' + const password = creds?.GIT_PASSWORD ?? '' + const repoUrl = `${env.GIT_PROTOCOL}://${username}:${password}@${env.GIT_URL}:${env.GIT_PORT}/${operatorEnv.GIT_ORG}/${operatorEnv.GIT_REPO}.git` + const result = await $`git ls-remote --exit-code --heads ${repoUrl} main`.nothrow().quiet() + return result.exitCode === 0 + } catch { + // If we can't check (e.g. gitea not ready yet), assume it's fine + // The operator will detect the issue later during git polling + this.d.warn('Could not verify git repo - gitea may not be ready yet') + return true + } } public async initialize() { @@ -85,7 +116,8 @@ export class Installer { status, attempt: attempt.toString(), timestamp: new Date().toISOString(), - ...(error && { error }), + // Always include error field to prevent stale values from StrategicMergePatch + error: error ?? '', } await createUpdateConfigMap(k8s.core(), 'apl-installation-status', 'apl-operator', data) @@ -115,13 +147,19 @@ export class Installer { this.d.debug('Extracting credentials from installation values') const values = (await hfValues()) as Record - const gitUsername: string = values.apps.gitea?.adminUsername || 'otomi-admin' - const gitPassword: string = values.apps.gitea?.adminPassword + const gitUsername: string = await resolveSinglePlaceholder( + String(values.apps.gitea?.adminUsername || 'otomi-admin'), + ) + const gitPassword: string = await resolveSinglePlaceholder(String(values.apps.gitea?.adminPassword ?? '')) if (!gitUsername || !gitPassword) { throw new Error('Git credentials not found in values') } + if (gitPassword.startsWith('sealed:') || gitUsername.startsWith('sealed:')) { + throw new Error('Git credentials contain unresolved sealed secret placeholders - K8s secrets may not be ready') + } + await createUpdateGenericSecret(k8s.core(), 'gitea-credentials', 'apl-operator', { GIT_USERNAME: gitUsername, GIT_PASSWORD: gitPassword, diff --git a/tpl/.sops.yaml.gotmpl b/tpl/.sops.yaml.gotmpl deleted file mode 100644 index cf6fc9a6b2..0000000000 --- a/tpl/.sops.yaml.gotmpl +++ /dev/null @@ -1,3 +0,0 @@ -creation_rules: - - path_regex: ^(.*/)?secrets.*.yaml(\.dec)?$ - {{ .provider }}: {{ .keys }} diff --git a/values/gitea-db-secret/gitea-db-secret-raw.gotmpl b/values/gitea-db-secret/gitea-db-secret-raw.gotmpl index 792c355e5f..c9acfc0dd8 100644 --- a/values/gitea-db-secret/gitea-db-secret-raw.gotmpl +++ b/values/gitea-db-secret/gitea-db-secret-raw.gotmpl @@ -1,12 +1,2 @@ -{{- $v := .Values }} -{{- $g := $v.apps.gitea }} - -resources: -- apiVersion: v1 - kind: Secret - type: kubernetes.io/basic-auth - metadata: - name: gitea-db-secret - data: - username: "{{ "gitea" | b64enc }}" - password: "{{ $g.postgresqlPassword | b64enc }}" +# This file is intentionally empty. +# gitea-db-secret is now managed via SealedSecrets (see APP_SECRET_OVERRIDES in sealed-secrets.ts) diff --git a/values/gitea/gitea-raw.gotmpl b/values/gitea/gitea-raw.gotmpl index 0b766b4a9c..6a0bd3ef9a 100644 --- a/values/gitea/gitea-raw.gotmpl +++ b/values/gitea/gitea-raw.gotmpl @@ -21,13 +21,6 @@ resources: data: ca-certificates.crt: {{ .Values._derived.caCert | b64enc }} {{- end }} -- apiVersion: v1 - kind: Secret - metadata: - name: gitea-admin-secret - data: - username: "{{ $g.adminUsername | b64enc }}" - password: "{{ $g.adminPassword | b64enc }}" # DB / app backup resources {{- if eq $obj.type "linode" }} - apiVersion: v1 diff --git a/values/ingress-nginx/ingress-nginx-raw.gotmpl b/values/ingress-nginx/ingress-nginx-raw.gotmpl index 28e89c9963..366b4d6389 100644 --- a/values/ingress-nginx/ingress-nginx-raw.gotmpl +++ b/values/ingress-nginx/ingress-nginx-raw.gotmpl @@ -7,10 +7,42 @@ resources: labels: app.kubernetes.io/component: controller name: {{ $ingress.className }} - {{- if eq $ingress.className $v.ingress.platformClass.className }} + {{- if eq $ingress.className $v.ingress.platformClass.className }} annotations: ingressclass.kubernetes.io/is-default-class: "true" {{- end }} spec: controller: "k8s.io/{{ $ingress.className }}" +{{- end }} +# ClusterRole to allow ingress controller to read TLS secrets from all namespaces +{{- range $ingress := $v.ingress.classes }} +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: ingress-nginx-{{ $ingress.className }}-secrets-reader + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx-{{ $ingress.className }} + rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: ingress-nginx-{{ $ingress.className }}-secrets-reader + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx-{{ $ingress.className }} + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-{{ $ingress.className }}-secrets-reader + subjects: + - kind: ServiceAccount + name: ingress-nginx-{{ $ingress.className }} + namespace: ingress {{- end }} \ No newline at end of file diff --git a/values/otomi-api/otomi-api.gotmpl b/values/otomi-api/otomi-api.gotmpl index e8681d450a..e3155e12bb 100644 --- a/values/otomi-api/otomi-api.gotmpl +++ b/values/otomi-api/otomi-api.gotmpl @@ -3,11 +3,9 @@ {{- $o := $v.apps | get "otomi-api" }} {{- $g := $v.apps.gitea }} {{- $cm := $v.apps | get "cert-manager" }} -{{- $sops := $v | get "kms.sops" dict }} {{- $giteaValuesUrl := "http://gitea-http.gitea.svc.cluster.local:3000/otomi/values" }} {{- $giteaValuesPublilcUrl := printf "https://gitea.%s/otomi/values" $v.cluster.domainSuffix }} {{- $defaultPlatformAdminEmail := printf "platform-admin@%s" $v.cluster.domainSuffix }} -{{- $sopsEnv := tpl (readFile "../../helmfile.d/snippets/sops-env.gotmpl") $sops }} {{- $version := $v.versions | get "api" }} {{- $isSemver := regexMatch "^[0-9.]+" $version }} {{- $coreVersion := $v.otomi.version }} @@ -41,7 +39,6 @@ secrets: GIT_USER: otomi-admin GIT_EMAIL: not@us.ed GIT_PASSWORD: {{ $g.adminPassword | quote}} - {{- $sopsEnv | nindent 2 }} env: DEFAULT_PLATFORM_ADMIN_EMAIL: {{ $defaultPlatformAdminEmail }}