feat(swift): multipart support

.github/workflows/build-image.yml

@@ -1,6 +1,5 @@
 name: build image
 
-
 on:
   push:
     branches:

.github/workflows/checks.yml

@@ -8,41 +8,38 @@ on:
     types: [opened, synchronize, reopened, ready_for_review]
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   DOCKER_CMD: docker
   UV_CACHE_DIR: /tmp/.uv-cache
 
 jobs:
   checks:
     runs-on: ubuntu-latest
-    env:
-      HERALD_SWIFTTEST_OS_USERNAME: ${{ secrets.OPENSTACK_USERNAME }}
-      HERALD_SWIFTTEST_OS_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD }}
-      HERALD_SWIFTTEST_OS_PROJECT_NAME: ${{ secrets.OPENSTACK_PROJECT }}
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
 
-      - name: Install Nix
-        uses: DeterminateSystems/nix-installer-action@v16
+      - uses: DeterminateSystems/nix-installer-action@v16
 
-      - name: Set up Nix cache
-        uses: DeterminateSystems/magic-nix-cache-action@v9
+      - uses: DeterminateSystems/flakehub-cache-action@v3
 
-      - name: Run pre-commit hooks via prek
+      - name: pre-commit hooks
         run: nix develop --command prek run --all-files
 
-      - name: Cache Deno
+      - name: deno cache
         uses: actions/cache@v4
         with:
           path: ~/.cache/deno
           key: ${{ runner.os }}-deno-${{ hashFiles('deno.lock') }}
           restore-keys: |
             ${{ runner.os }}-deno-
 
-      - name: Restore uv cache
+      - name: uv cache
         uses: actions/cache@v5
         with:
           path: /tmp/.uv-cache
@@ -51,43 +48,76 @@ jobs:
             uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }}
             uv-${{ runner.os }}
 
-      - name: Start services
-        run: nix develop --command deno run --allow-all x/compose-up.ts s3 db
+      - name: start container
+        run: nix develop --command deno run --allow-all x/compose-up.ts s3 swift db
 
-      - name: Wait for MinIO
+      - name: wait for services
         run: |
+          echo "Waiting for MinIO..."
           for i in {1..30}; do
-            if curl -f http://localhost:9000/minio/health/live; then
+            if curl -sf http://localhost:9000/minio/health/live; then
               echo "MinIO is ready"
+              break
+            fi
+            sleep 2
+          done || (echo "MinIO failed to start" && exit 1)
+
+          echo "Waiting for SAIO..."
+          for i in {1..60}; do
+            if curl -sf http://localhost:8080/healthcheck; then
+              echo "SAIO is ready"
               exit 0
             fi
-            echo "Waiting for MinIO..."
             sleep 2
           done
-          echo "MinIO failed to start"
+          echo "SAIO failed to start"
           exit 1
 
-      - name: Integration tests
+      - name: integration tests
         run: nix develop --command deno task test
 
-      - name: S3 Compatibility (MinIO)
-        run: nix develop --command deno run --allow-all x/s3-tests.ts --backend minio
+      - name: benchmarks
+        run: nix develop --command deno bench --allow-all benchmarks/
+
+      - name: s3-tests
+        run: |
+          # Run MinIO tests in background
+          nix develop --command deno run --allow-all x/s3-tests.ts --backend minio &
+          MINIO_PID=$!
+
+          # Run Swift tests in background against SAIO
+          nix develop --command deno run --allow-all x/s3-tests.ts --backend swift &
+          SWIFT_PID=$!
+
+          # Wait for both and capture exit codes
+          MINIO_EXIT=0
+          if ! wait $MINIO_PID; then
+            MINIO_EXIT=$?
+          fi
+
+          SWIFT_EXIT=0
+          if [ -n "$SWIFT_PID" ]; then
+            if ! wait $SWIFT_PID; then
+              SWIFT_EXIT=$?
+            fi
+          fi
 
-      - name: S3 Compatibility (Swift)
-        if: env.HERALD_SWIFTTEST_OS_USERNAME != ''
-        env:
-          HERALD_SWIFTTEST_OS_REGION_NAME: dc3-a
-          HERALD_SWIFTTEST_AUTH_URL: https://api.pub1.infomaniak.cloud/identity/v3
-        run: nix develop --command deno run --allow-all x/s3-tests.ts --backend swift
+          # Exit with error if either failed
+          if [ $MINIO_EXIT -ne 0 ] || [ $SWIFT_EXIT -ne 0 ]; then
+            echo "One or more compatibility tests failed (MinIO: $MINIO_EXIT, Swift: $SWIFT_EXIT)"
+            exit 1
+          fi
 
-      - name: Minimize uv cache
+      - name: prune uv cache
         run: nix develop --command uv cache prune --ci
 
-      - name: Dump logs on failure
+      - name: failure logs
         if: failure()
         run: |
-          echo "--- s3-tests/s3-tests.log ---"
+          echo "--- s3-tests/s3-tests.log (MinIO) ---"
           cat s3-tests/s3-tests.log || true
+          echo "--- s3-tests/s3-tests-swift.log (Swift) ---"
+          cat s3-tests/s3-tests-swift.log || true
           echo "--- s3-tests/herald-proxy.log ---"
           cat s3-tests/herald-proxy.log || true
           echo "--- s3-tests/herald-proxy-swift.log ---"

AGENTS.md

@@ -4,7 +4,7 @@
     all HTTP requests.
   - Prefer generators over effect piping.
   - Use methods on `Effect.Option` like `Option.isNone` instead of looking at
-    _tag.
+    `_tag`.
   - **NEVER** use standard `try/catch` or `try/finally` blocks around `yield*`
     in Effect generators. Use `Effect.addFinalizer`, `Effect.try`,
     `Effect.catchAll`, or `Effect.orElse`.
@@ -28,3 +28,6 @@
   agent.
 - Maintain strict type safety. Avoid "any" casts or requirement hacks.
 - Use the structured `Logger` layer for all diagnostic output.
+
+- Always fix deno lint and deno check issues before running tests, the type
+  system is there to help.

CONTRIBUTING.md

@@ -1,76 +1,84 @@
 # Contributing
 
+## Starting Services
+
+You can start the containers used for development using the provided scripts:
+
+```bash
+# Start MinIO and Redis
+deno run --allow-all x/compose-up.ts s3 db
+
+# Start Swift (SAIO)
+deno run --allow-all x/compose-up.ts swift
+```
+
+## Running Tests
+
+```bash
+# Run all tests
+deno task test
+
+# Run Swift integration tests specifically
+deno task test --filter "Swift/"
+```
+
+## Benchmarking
+
+```bash
+deno bench --allow-all benchmarks/
+```
+
 ## Repo Map
 
 - `src/Domain`: Core logic and data models. Contains Effect Schemas for global
-  configuration and logic for bucket matching.
+  configuration and logic for backend resolution/matching.
 
 - `src/Config`: Application configuration loading. Defines the HeraldConfig
   service layer.
 
 - `src/Services`: Shared service abstractions and implementations.
-
   - `src/Services/Backend.ts`: Generic storage backend interface with structured
     request/response types and domain-specific error types.
-
   - `src/Services/BackendResolver.ts`: Logic for dynamically providing the
     correct backend based on request context.
-
   - `src/Services/S3Xml.ts`: S3-compatible XML response formatting for errors,
     bucket listings, and object listings.
+  - `src/Services/BackendKeyValueStore.ts`: Abstraction for backend-specific
+    key-value storage.
 
-- `src/Backends/S3`: S3 protocol implementation.
-
-  - `src/Backends/S3/Backend.ts`: S3-specific implementation of the
-    BackendService using AWS SDK, handling MinIO metadata stripping and encoding
-    normalization.
-
-  - `src/Backends/S3/Client.ts`: Low-level AWS SDK S3 client management and
-    credential resolution.
-
-  - `src/Backends/S3/Signer.ts`: AWS Signature Version 4 implementation for
-    request signing.
+- `src/Backends`: Specific storage backend implementations.
+  - `src/Backends/S3`: S3 protocol implementation using AWS SDK.
+  - `src/Backends/Swift`: OpenStack Swift protocol implementation.
 
 - `src/Frontend`: HTTP ingress layer.
-
   - `src/Frontend/Api.ts`: HttpApi definition for the S3 compatibility layer.
-
   - `src/Frontend/Http.ts`: Main HTTP server setup and endpoint group
     registrations.
-
-  - `src/Frontend/Utils.ts`: Shared frontend helpers for backend resolution and
-    S3-compliant error mapping.
-
-  - `src/Frontend/Buckets/`: Handlers for bucket-level S3 operations (Create,
-    Delete, List, Head).
-
-  - `src/Frontend/Objects/`: Handlers for object-level S3 operations (Get, Put,
-    Delete, Head, List, Multi-Object Delete).
-
+  - `src/Frontend/Buckets/`: Handlers for bucket-level S3 operations.
+  - `src/Frontend/Objects/`: Handlers for object-level S3 operations, including
+    Multipart Upload (via `Post.ts`).
   - `src/Frontend/Health/`: Handlers for system health monitoring.
 
-- `tests/`: Test suite.
+- `src/Logging` & `src/Tracing.ts`: Diagnostic observability layers.
 
+- `tests/`: Test suite.
   - `tests/integration/`: End-to-end tests comparing Herald proxy behavior
     against a MinIO baseline using snapshots.
+  - `tests/config.test.ts`: Unit tests for configuration and backend resolution.
+  - `tests/utils.ts`: Shared test harness and snapshot normalization logic.
 
-  - `tests/config.test.ts`: Unit tests for configuration inheritance, glob
-    matching, and backend resolution.
-
-  - `tests/utils.ts`: Shared test harness, Effect-based assertions, and snapshot
-    normalization logic.
+- `benchmarks/`: Performance testing suite for evaluating proxy overhead and
+  streaming efficiency.
 
 - `x/`: CLI utilities and development scripts.
-
-  - `x/s3-tests.ts`: Orchestration script for running the ceph `s3-tests` suite
-    against the proxy.
-
-  - `x/snapdiff.ts`: Tool for comparing Herald proxy snapshots against baseline
+  - `x/dev.ts`: Main development entry point for running the proxy locally.
+  - `x/s3-tests.ts`: Orchestration for running the ceph `s3-tests` suite.
+  - `x/snapdiff.ts`: Tool for comparing proxy snapshots against baseline
     responses.
+  - `x/compose-up.ts` & `x/compose-down.ts`: Helpers for managing local Docker
+    dependencies.
 
-  - `x/utils.ts`: Shell scripting utilities powered by `dax`.
-
-- `tools/`: Infrastructure and development tools.
+- `chart/`: Helm charts for Kubernetes deployment.
 
-  - `tools/compose.yml`: Docker configuration for local development services
-    (MinIO, Redis).
+- `tools/`: Infrastructure and development tools (Docker Compose,
+  Containerfiles).

README.md

@@ -68,6 +68,78 @@ backends:
       # Glob pattern support within the map
       "test-*":
         region: us-east-1
+
+  # Example Swift backend
+  swift-storage:
+    protocol: swift
+    auth_url: http://keystone.example.com/v3
+    region: RegionOne
+    # Optional: override the Swift container name for all buckets in this backend
+    # container: my-fixed-container
+    credentials:
+      username: my-user
+      password: my-password
+      project_name: my-project
+      user_domain_name: Default
+      project_domain_name: Default
+    # Route all archive buckets to Swift
+    buckets: "archive-*"
+
+cors:
+  # Global CORS defaults
+  allowedOrigins: ["*"]
+  allowedMethods: ["GET", "PUT", "POST", "DELETE", "HEAD", "OPTIONS"]
+  allowedHeaders: ["*"]
+  exposedHeaders: ["*"]
+  maxAge: 3600
+  credentials: false
+```
+
+### CORS Configuration
+
+Herald supports fine-grained CORS control at three levels with the following
+precedence: **Bucket > Backend > Global**.
+
+- **Global**: Defined at the root of the config file under `cors`.
+- **Backend**: Defined within a backend block under `cors`. Overrides global
+  settings.
+- **Bucket**: Defined within a bucket definition under `cors`. Overrides both
+  backend and global settings.
+
+#### Default Behavior
+
+If no CORS configuration is provided at any level, **CORS is disabled** and
+Herald will not add any CORS-related headers to responses. Preflight `OPTIONS`
+requests will be passed through to the backend.
+
+If you enable CORS by providing configuration at any level, the following
+defaults are applied for any omitted fields:
+
+| Field            | Default Value                           | Description                                            |
+| ---------------- | --------------------------------------- | ------------------------------------------------------ |
+| `maxAge`         | `3600`                                  | Max age in seconds for preflight results               |
+| `allowedMethods` | `GET, PUT, POST, DELETE, HEAD, OPTIONS` | Allowed HTTP methods                                   |
+| `allowedHeaders` | (Mirrors request)                       | Defaults to mirroring `Access-Control-Request-Headers` |
+| `credentials`    | `false`                                 | Whether to allow credentials                           |
+| `allowedOrigins` | (None)                                  | Headers only added if `Origin` matches an entry        |
+
+Example with overrides:
+
+```yaml
+cors: # Global defaults
+  allowedOrigins: ["*"]
+  credentials: false
+
+backends:
+  prod:
+    protocol: s3
+    cors: # Backend-level override
+      allowedOrigins: ["https://app.example.com"]
+      credentials: true
+    buckets:
+      assets:
+        cors: # Bucket-level override
+          allowedOrigins: ["https://cdn.example.com"]
 ```
 
 ### Routing Logic
@@ -78,5 +150,5 @@ resolves the backend using the following priority:
 1. **Direct match**: Looks for `my-bucket` in all backends' `buckets` maps.
 2. **Glob match (map)**: Looks for glob patterns (like `test-*`) in all
    backends' `buckets` maps.
-3. **Glob match (string)**: If a backend has `buckets: "..."`, it checks if the
-   bucket name matches that pattern.
+3. **Glob match (string)**: If a backend has `buckets: "string-*"`, it checks if
+   the bucket name matches that pattern.

TODO.md

@@ -48,10 +48,11 @@ implementation.
 
 - [ ] **Multi-Object Delete**: Implementation of `POST /?delete`. _(Focus tests:
       `test_multi_object_delete`, `test_multi_object_delete_key_limit`)_
-- [ ] **Multipart Upload**: Support for `InitiateMultipartUpload`, `UploadPart`,
+- [x] **Multipart Upload**: Support for `InitiateMultipartUpload`, `UploadPart`,
       `CompleteMultipartUpload`, `AbortMultipartUpload`, and `ListParts`.
       _(Focus tests: `test_multipart_upload`, `test_multipart_upload_empty`,
       `test_abort_multipart_upload`)_
+  - [x] **Swift Multipart Upload**: Implement S3 multipart mapping to Swift SLO.
 - [ ] **GetObject Attributes**: Implementation of `GET /bucket/key?attributes`.
       _(Focus tests: `test_get_object_attributes`)_
 - [ ] **HeadObject Consistency**: Fix `404 Not Found` errors on existing objects