From d1ae644c5eb5bcb2aed70a3f4cb219d9996480ed Mon Sep 17 00:00:00 2001 From: pwistbac Date: Wed, 24 Jun 2026 09:35:33 +0300 Subject: [PATCH 1/6] Add documentation for how to migrate from minio to seaweedfs (#763) --- docs/minio-to-seaweed-migration-guide.md | 212 +++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 docs/minio-to-seaweed-migration-guide.md diff --git a/docs/minio-to-seaweed-migration-guide.md b/docs/minio-to-seaweed-migration-guide.md new file mode 100644 index 00000000..be323056 --- /dev/null +++ b/docs/minio-to-seaweed-migration-guide.md @@ -0,0 +1,212 @@ +# Minio to SeaweedFS Migration Guide + +**Estimated Time:** 20-30 minutes +**Complexity:** Medium (3 manual steps) +**Risk Level:** Medium (Data loss possible if not done correctly) + +## Overview + +This guide helps cluster managers migrate from minio s3 to seaweedfs s3 by upgrading cluster-forge from v2.1.3 to v2.2.0. The migration involves only **2 manual steps** and takes approximately **20-30 minutes**. + +> **Important:** This migration will briefly affect access to s3 data. Plan accordingly and notify your team. + +## Prerequisites + +Before starting, verify your cluster meets these requirements: + +```bash + kubectl -n minio-tenant-default get pods + kubectl -n minio-operator get pods +``` + +We should have 1 or more healthy pods named minio-operator-**** and one or more healthy pods named default-minio-tenant-pool-****. + +**Required Access:** +- kubectl admin permissions +- ArgoCD web UI access +- Gitea web UI access +- Current cluster-forge version v2.1.3 + +## Migration Process + +### Step 1: Upgrade cluster-forge + +**Navigate to Gitea:** +1. Open your browser and go to `https://gitea.` +2. Login with your credentials +3. Navigate to the `cluster-org/cluster-values` repository +4. Open the file: `values.yaml` +5. Click the **"Edit"** button (pencil icon) + +**Make the following changes:** + +**Change 1: Update targetRevision** + +Find: +```yaml +targetRevision: v2.1.3 +``` + +Change to: +```yaml +targetRevision: v2.2.0 +``` + +**Change 2: Update enabledApps list** + +In the `enabledApps:` section: + +**Comment out** these apps (add `#` at the beginning): +```yaml +enabledApps: + # - minio-operator + # - minio-tenant + # - minio-tenant-config +``` + +**Add** these new apps: +```yaml +enabledApps: + - seaweedfs-crds + - seaweedfs-operator + - seaweedfs-config +``` + +**Complete example:** +```yaml +targetRevision: v2.2.0 + +enabledApps: + # Old gateway apps (commented out for migration) + # - minio-operator + # - minio-tenant + # - minio-tenant-config + + # New gateway apps + - seaweedfs-crds + - seaweedfs-operator + - seaweedfs-config + + # Other apps (unchanged) + - cert-manager + - cert-manager-config + - metallb + - metallb-config + # ... rest of your enabled apps +``` + +**Commit the changes:** +1. Scroll to the bottom of the edit page +2. Add commit message: `feat: migrate cluster-forge to v2.2.0` +3. Click **"Commit Changes"** + +> ** Important:** After committing, the cluster-forge ArgoCD application will show as "OutOfSync" or "Degraded". This is expected and will be synced after a while as long as autosync is active. Otherwise you can click "Sync" on the application page. + +> ** Important:** The new ArgoCD application seaweedfs-config will continue showing "OutOfSync" even after it reaches a healthy state. This is due to an unused component called seaweed-ingress being in a persistent "Progressing" state. The ingress can not be disabled in our current version of SeaweedFS-operator, but it will be disabled in an upcoming release of cluster-forge + +#### Verifying that seaweed is running +Before continuing, make sure all seaweedfs pods are running. The amount of pods might differ with your cluster size setting. + +```bash +kubectl -n seaweedfs-instance get pods + +NAME READY STATUS RESTARTS +seaweed-filer-0 1/1 Running xxx +seaweed-master-0 1/1 Running xxx +seaweed-master-1 1/1 Running xxx +seaweed-master-2 1/1 Running xxx +seaweed-volume-0 1/1 Running xxx +seaweed-volume-1 1/1 Running xxx +seaweed-volume-2 1/1 Running xxx +seaweedfs-admin-9b64677fd-j6b5h 1/1 Running xxx +``` + + +```bash +kubectl -n seaweedfs-operator get pods + +NAME READY STATUS RESTARTS AGE +seaweedfs-operator-***** 1/1 Running xxx xxx +``` + +Once the pods are running, you can open the seaweed admin interface to see the contents of the buckets (initially empty). + +**Navigate to Seaweed-Admin:** +1. Open your browser and go to `https://seaweed-admin.` +2. Find the value of the password called `admin-password` in the secret called `seaweedfs-admin-secret` in the namespace `seaweedfs-instance` +3. Login with the credentials username: `admin`, password: `` +4. Navigate to Buckets and confirm that the `default-bucket` bucket exists + +***Verify that AIWB API is using seaweed s3*** +1. Using kubectl or k9s, inspect the pod `aiwb-api-****`, in the namespace `aiwb`. +2. Check that the environment variable `MINIO_URL` has the value `http://filer-s3.seaweedfs-instance.svc.cluster.local:80` + +### Step 2: Run the Minio->Seaweed migration Job (5-10 minutes) + +Download the latest version of cluster-forge (unless you already have it). +```bash +git clone https://github.com/silogen/cluster-forge.git +cd cd cluster-forge/scripts/utils +kubectl apply -f mirror-minio-to-seaweedfs-job.yaml +``` + +**Expected Output:** +``` +🔧 MinIO to SeaweedFS S3 Mirror Job +================================================ +... +"Waiting for SeaweedFS S3 service to be ready..." +... +"Starting mirror operation..." +... +✓ Mirror operation completed successfully! +... +Cleanup complete. +``` +**Navigate to Seaweed-Admin:** +1. Go back to Seaweed-Admin as described above +2. Open Buckets and choose "default-bucket" +3. Verify that all the data has been mirrored + +### Step 5: Add seaweedfs storage to AIRM (optional) +***Navigate to AIRM UI*** +1. Open your browser and go to `https://airmui.` +2. Find the value of the password called `USER_PASSWORD` in the secret called `airm-user-credentials` in the namespace `airm` +3. Login with the credentials username: `devuser@`, password: `` +4. Navigate to Storage and click Add storage -> S3 Bucket + Use the following config: + - `Bucket URL`: `http://filer-s3.seaweedfs-instance.svc.cluster.local:80` + - `Storage secret` Your existing secret with the same minio credentials (or a new one) + - `Access Key Name`: `minio-access-key` + - `Secret Key Name`: `minio-secret-key` + +### Step 6: Test uploading/downloading datasets from AIWB (optional) +Make sure you can download your datasets from AIWB and that you can upload new datasets. + +### Step 4: Clean Up Old Minio Resources (3-5 minutes) (optional) +Once you have verified that all the data is mirrored to Seaweed, you can safely remove the minio applications. + +*** Important: Make sure you removed the minio ArgoCD applications in Step 1. If you didn't, just go and remove the minio applications in your values.yaml file. The argocd application needs to be removed to allow deleting the resources. *** + +#### Remove the old minio Tenant and PVC (deleting the data) + +```bash +kubectl -n minio-tenant-default delete tenant default-minio-tenant +> tenant.minio.min.io "default-minio-tenant" deleted from minio-tenant-default namespace +kubectl -n minio-tenant-default delete pvc -l v1.min.io/tenant=default-minio-tenant +> persistentvolumeclaim "data0-default-minio-tenant-pool-0-0" deleted from minio-tenant-default namespace +``` +#### Remove the minio operator, the CRDs, and the minio namespaces + +``` +kubectl -n minio-operator delete deployment/minio-operator +> deployment.apps "minio-operator" deleted from minio-operator namespace +kubectl delete namespace minio-tenant-default +> namespace "minio-tenant-default" deleted +kubectl delete namespace minio-operator +> namespace "minio-operator" deleted + +kubectl delete crds tenants.minio.min.io +> customresourcedefinition.apiextensions.k8s.io "tenants.minio.min.io" deleted +``` + From b628b22615be2144d4365d9bf46a7a7b45903bfa Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Wed, 24 Jun 2026 11:26:24 +0000 Subject: [PATCH 2/6] EAI-7043: fix UUID routing by not setting x-ai-eg-model when backend is present MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When x-ai-eg-backend is set (from body or as a direct header), x-ai-eg-model is intentionally skipped. This prevents model-name fallback rules in other HTTPRoutes from matching — Envoy Gateway maintains per-HTTPRoute ordering so an older route's model-name rule fires before a newer route's UUID rule when both have equal specificity. Without x-ai-eg-model, only the UUID rule and catch-all rules remain; the UUID rule (2 conditions) beats catch-all (0 conditions) by specificity regardless of creation order. --- .../envoy-extension-policy-model-header.yaml | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/sources/envoy-gateway-config/templates/envoy-extension-policy-model-header.yaml b/sources/envoy-gateway-config/templates/envoy-extension-policy-model-header.yaml index eb4400b9..fc792c15 100644 --- a/sources/envoy-gateway-config/templates/envoy-extension-policy-model-header.yaml +++ b/sources/envoy-gateway-config/templates/envoy-extension-policy-model-header.yaml @@ -1,15 +1,16 @@ {{- if .Values.aiGateway.enabled }} -# Lua filter that sets x-ai-eg-model from the request body before ext_authz runs, on the -# ai-gateway (where all AI traffic terminates and is authorized). +# Lua filter that extracts routing headers from the request body before ext_authz runs, +# on the ai-gateway (where all AI traffic terminates and is authorized). # -# The AI Gateway's ext_proc reads the body and sets x-ai-eg-model, but it is inserted via -# xdsTranslator post-hooks which run after filterOrder is applied. So ext_authz (cluster-auth) -# would run without x-ai-eg-model and fall through to a catch-all route with no auth -# annotations, allowing the request regardless of which model is targeted (EAI-6805). +# When "backend" (AIM UUID) is present in the body or already set as a request header, +# only x-ai-eg-backend is set and the filter returns early. x-ai-eg-model is intentionally +# NOT set in this case so that model-name fallback rules in other HTTPRoutes cannot match — +# the UUID rule wins by specificity over catch-all rules without any cross-route ordering +# dependency (EAI-7043). # -# This Lua filter runs early (ordered before ext_authz via the ai-gateway EnvoyProxy -# filterOrder), reads the model field from the JSON body, and sets x-ai-eg-model so -# cluster-auth can match the correct per-model route and enforce cluster-auth/allowed-group. +# When "backend" is absent, x-ai-eg-model is set from the body for standard +# OpenAI-compatible clients so cluster-auth can match the correct per-model route and +# enforce cluster-auth/allowed-group (EAI-6805). apiVersion: gateway.envoyproxy.io/v1alpha1 kind: EnvoyExtensionPolicy metadata: @@ -28,7 +29,17 @@ spec: if body == nil then return end - local model = tostring(body):match('"model"%s*:%s*"([^"]+)"') + local body_str = tostring(body) + local backend = body_str:match('"backend"%s*:%s*"([^"]+)"') + if backend ~= nil then + request_handle:headers():replace("x-ai-eg-backend", backend) + return + end + local existing_backend = request_handle:headers():get("x-ai-eg-backend") + if existing_backend ~= nil and existing_backend ~= "" then + return + end + local model = body_str:match('"model"%s*:%s*"([^"]+)"') if model ~= nil then request_handle:headers():replace("x-ai-eg-model", model) end From a31bc974bd26984bd10c23d63ef035f5fa65f426 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Wed, 24 Jun 2026 11:39:01 +0000 Subject: [PATCH 3/6] EAI-7043: preserve model attribution in access logs via x-ai-eg-model-log header --- .../templates/ai-gateway-proxy-config.yaml | 2 +- .../envoy-extension-policy-model-header.yaml | 23 ++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/sources/envoy-gateway-config/templates/ai-gateway-proxy-config.yaml b/sources/envoy-gateway-config/templates/ai-gateway-proxy-config.yaml index ec99311a..56b666d4 100644 --- a/sources/envoy-gateway-config/templates/ai-gateway-proxy-config.yaml +++ b/sources/envoy-gateway-config/templates/ai-gateway-proxy-config.yaml @@ -44,7 +44,7 @@ spec: authority: "%REQ(:AUTHORITY)%" backendHost: "%UPSTREAM_HOST%" backendCluster: "%UPSTREAM_CLUSTER%" - model: "%REQ(x-ai-eg-model)%" + model: "%REQ(x-ai-eg-model-log)%" llm_input_token: "%DYNAMIC_METADATA(io.envoy.ai_gateway:llm_input_token)%" llm_output_token: "%DYNAMIC_METADATA(io.envoy.ai_gateway:llm_output_token)%" llm_total_token: "%DYNAMIC_METADATA(io.envoy.ai_gateway:llm_total_token)%" diff --git a/sources/envoy-gateway-config/templates/envoy-extension-policy-model-header.yaml b/sources/envoy-gateway-config/templates/envoy-extension-policy-model-header.yaml index fc792c15..5c9b3ed8 100644 --- a/sources/envoy-gateway-config/templates/envoy-extension-policy-model-header.yaml +++ b/sources/envoy-gateway-config/templates/envoy-extension-policy-model-header.yaml @@ -1,14 +1,17 @@ {{- if .Values.aiGateway.enabled }} -# Lua filter that extracts routing headers from the request body before ext_authz runs, -# on the ai-gateway (where all AI traffic terminates and is authorized). +# Lua filter that extracts routing and logging headers from the request body before +# ext_authz runs, on the ai-gateway (where all AI traffic terminates and is authorized). +# +# x-ai-eg-model-log is always set from the body's "model" field (when present) so the +# access log can attribute requests to a model regardless of routing path. # # When "backend" (AIM UUID) is present in the body or already set as a request header, -# only x-ai-eg-backend is set and the filter returns early. x-ai-eg-model is intentionally -# NOT set in this case so that model-name fallback rules in other HTTPRoutes cannot match — -# the UUID rule wins by specificity over catch-all rules without any cross-route ordering -# dependency (EAI-7043). +# only x-ai-eg-backend is set for routing and the filter returns early. x-ai-eg-model is +# intentionally NOT set in this case so that model-name fallback rules in other HTTPRoutes +# cannot match — the UUID rule wins by specificity over catch-all rules without any +# cross-route ordering dependency (EAI-7043). # -# When "backend" is absent, x-ai-eg-model is set from the body for standard +# When "backend" is absent, x-ai-eg-model is also set from the body for standard # OpenAI-compatible clients so cluster-auth can match the correct per-model route and # enforce cluster-auth/allowed-group (EAI-6805). apiVersion: gateway.envoyproxy.io/v1alpha1 @@ -31,17 +34,21 @@ spec: end local body_str = tostring(body) local backend = body_str:match('"backend"%s*:%s*"([^"]+)"') + local model = body_str:match('"model"%s*:%s*"([^"]+)"') if backend ~= nil then request_handle:headers():replace("x-ai-eg-backend", backend) + if model ~= nil then + request_handle:headers():replace("x-ai-eg-model-log", model) + end return end local existing_backend = request_handle:headers():get("x-ai-eg-backend") if existing_backend ~= nil and existing_backend ~= "" then return end - local model = body_str:match('"model"%s*:%s*"([^"]+)"') if model ~= nil then request_handle:headers():replace("x-ai-eg-model", model) + request_handle:headers():replace("x-ai-eg-model-log", model) end end {{- end }} From c6f0f738ecb723985d1f26b0cb513c73aa8b7a2d Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Wed, 24 Jun 2026 13:01:46 +0000 Subject: [PATCH 4/6] EAI-7043: switch to 3-condition UUID rule, Lua sets both backend and model headers --- .../templates/ai-gateway-proxy-config.yaml | 2 +- .../envoy-extension-policy-model-header.yaml | 29 ++++--------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/sources/envoy-gateway-config/templates/ai-gateway-proxy-config.yaml b/sources/envoy-gateway-config/templates/ai-gateway-proxy-config.yaml index 56b666d4..ec99311a 100644 --- a/sources/envoy-gateway-config/templates/ai-gateway-proxy-config.yaml +++ b/sources/envoy-gateway-config/templates/ai-gateway-proxy-config.yaml @@ -44,7 +44,7 @@ spec: authority: "%REQ(:AUTHORITY)%" backendHost: "%UPSTREAM_HOST%" backendCluster: "%UPSTREAM_CLUSTER%" - model: "%REQ(x-ai-eg-model-log)%" + model: "%REQ(x-ai-eg-model)%" llm_input_token: "%DYNAMIC_METADATA(io.envoy.ai_gateway:llm_input_token)%" llm_output_token: "%DYNAMIC_METADATA(io.envoy.ai_gateway:llm_output_token)%" llm_total_token: "%DYNAMIC_METADATA(io.envoy.ai_gateway:llm_total_token)%" diff --git a/sources/envoy-gateway-config/templates/envoy-extension-policy-model-header.yaml b/sources/envoy-gateway-config/templates/envoy-extension-policy-model-header.yaml index 5c9b3ed8..02844207 100644 --- a/sources/envoy-gateway-config/templates/envoy-extension-policy-model-header.yaml +++ b/sources/envoy-gateway-config/templates/envoy-extension-policy-model-header.yaml @@ -1,19 +1,11 @@ {{- if .Values.aiGateway.enabled }} -# Lua filter that extracts routing and logging headers from the request body before -# ext_authz runs, on the ai-gateway (where all AI traffic terminates and is authorized). +# Lua filter that extracts routing headers from the request body before ext_authz runs, +# on the ai-gateway (where all AI traffic terminates and is authorized). # -# x-ai-eg-model-log is always set from the body's "model" field (when present) so the -# access log can attribute requests to a model regardless of routing path. -# -# When "backend" (AIM UUID) is present in the body or already set as a request header, -# only x-ai-eg-backend is set for routing and the filter returns early. x-ai-eg-model is -# intentionally NOT set in this case so that model-name fallback rules in other HTTPRoutes -# cannot match — the UUID rule wins by specificity over catch-all rules without any -# cross-route ordering dependency (EAI-7043). -# -# When "backend" is absent, x-ai-eg-model is also set from the body for standard -# OpenAI-compatible clients so cluster-auth can match the correct per-model route and -# enforce cluster-auth/allowed-group (EAI-6805). +# Both x-ai-eg-backend and x-ai-eg-model are set from the request body so that the +# 3-condition UUID rule (Host + x-ai-eg-backend + x-ai-eg-model) wins by specificity +# over any catch-all rules in other HTTPRoutes, regardless of HTTPRoute creation order. +# The model-name fallback rule has been removed — clients must supply a backend UUID. apiVersion: gateway.envoyproxy.io/v1alpha1 kind: EnvoyExtensionPolicy metadata: @@ -37,18 +29,9 @@ spec: local model = body_str:match('"model"%s*:%s*"([^"]+)"') if backend ~= nil then request_handle:headers():replace("x-ai-eg-backend", backend) - if model ~= nil then - request_handle:headers():replace("x-ai-eg-model-log", model) - end - return - end - local existing_backend = request_handle:headers():get("x-ai-eg-backend") - if existing_backend ~= nil and existing_backend ~= "" then - return end if model ~= nil then request_handle:headers():replace("x-ai-eg-model", model) - request_handle:headers():replace("x-ai-eg-model-log", model) end end {{- end }} From 810bda7e8808edae919bdb3e42dc0254abfb1e7f Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Wed, 24 Jun 2026 14:30:44 +0000 Subject: [PATCH 5/6] bump cluster-auth to 0.6.0-rc9 --- sources/cluster-auth/0.5.9/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/cluster-auth/0.5.9/values.yaml b/sources/cluster-auth/0.5.9/values.yaml index db9164c8..59b1115f 100644 --- a/sources/cluster-auth/0.5.9/values.yaml +++ b/sources/cluster-auth/0.5.9/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 image: repository: ghcr.io/silogen/cluster-auth pullPolicy: Always - tag: "0.6.0-rc8" + tag: "0.6.0-rc9" imagePullSecrets: [] nameOverride: "" From 80bea9c0e08fd34f417e5974e06fbaac018c2019 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Thu, 25 Jun 2026 04:29:34 +0000 Subject: [PATCH 6/6] bump cluster-auth to 0.6.0-rc10 --- sources/cluster-auth/0.5.9/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/cluster-auth/0.5.9/values.yaml b/sources/cluster-auth/0.5.9/values.yaml index 59b1115f..36d70e7a 100644 --- a/sources/cluster-auth/0.5.9/values.yaml +++ b/sources/cluster-auth/0.5.9/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 image: repository: ghcr.io/silogen/cluster-auth pullPolicy: Always - tag: "0.6.0-rc9" + tag: "0.6.0-rc10" imagePullSecrets: [] nameOverride: ""