diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..775bcd1 --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +HERALD_SWIFTTEST_OS_AUTH=$TF_VAR_OS_PASSWORD +HERALD_SWIFTTEST_OS_PASSWORD=$TF_VAR_OS_PASSWORD +HERALD_SWIFTTEST_OS_PROJECT_NAME=$TF_VAR_OS_PROJECT_NAME +HERALD_SWIFTTEST_OS_USERNAME=$TF_VAR_OS_USERNAME +HEARLD_SWIFTTEST_AUTH_URL=https://api.pub1.infomaniak.cloud/identity/v3 +HEARLD_SWIFTTEST_OS_REGION_NAME=dc3-a diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 0000000..d38e054 --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,58 @@ +name: build image + +on: + push: + branches: + - main + paths: + - "src/**" + - "tools/**" + - ".github/workflows/build-image.yml" + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths: + - "src/**" + - "tools/**" + - ".github/workflows/build-image.yml" + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare .dockerignore + run: cp tools/Containerfile.containerignore .dockerignore + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./tools/Containerfile + push: true + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..8309150 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,123 @@ +name: checks + +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +env: + DOCKER_CMD: docker + UV_CACHE_DIR: /tmp/.uv-cache + +jobs: + checks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: DeterminateSystems/nix-installer-action@v16 + + - uses: DeterminateSystems/flakehub-cache-action@v3 + + - name: pre-commit hooks + run: nix develop --command prek run --all-files + + - name: deno cache + uses: actions/cache@v4 + with: + path: ~/.cache/deno + key: ${{ runner.os }}-deno-${{ hashFiles('deno.lock') }} + restore-keys: | + ${{ runner.os }}-deno- + + - name: uv cache + uses: actions/cache@v5 + with: + path: /tmp/.uv-cache + key: uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }} + restore-keys: | + uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }} + uv-${{ runner.os }} + + - name: start container + run: nix develop --command deno run --allow-all x/compose-up.ts s3 swift db + + - name: wait for services + run: | + echo "Waiting for MinIO..." + for i in {1..30}; do + if curl -sf http://localhost:9000/minio/health/live; then + echo "MinIO is ready" + break + fi + sleep 2 + done || (echo "MinIO failed to start" && exit 1) + + echo "Waiting for SAIO..." + for i in {1..60}; do + if curl -sf http://localhost:8080/healthcheck; then + echo "SAIO is ready" + exit 0 + fi + sleep 2 + done + echo "SAIO failed to start" + exit 1 + + - name: integration tests + run: nix develop --command deno task test + + - name: benchmarks + run: nix develop --command deno bench --allow-all benchmarks/ + + - name: s3-tests + if: false + run: | + set +e + + run_minio() { + echo "=== Running s3-tests (MinIO) ===" + nix develop --command deno run --allow-all x/s3-tests.ts --backend minio --no-abort + echo "--- s3-tests/s3-tests.log (MinIO) ---" + cat s3-tests/s3-tests.log || true + echo "--- s3-tests/herald-proxy.log (MinIO) ---" + cat s3-tests/herald-proxy.log || true + } + + run_swift() { + echo "=== Running s3-tests (Swift) ===" + nix develop --command deno run --allow-all x/s3-tests.ts --backend swift --no-abort + echo "--- s3-tests/s3-tests-swift.log (Swift) ---" + cat s3-tests/s3-tests-swift.log || true + echo "--- s3-tests/herald-proxy-swift.log (Swift) ---" + cat s3-tests/herald-proxy-swift.log || true + } + + run_minio & + pid_minio=$! + + run_swift & + pid_swift=$! + + wait $pid_minio + status_minio=$? + + wait $pid_swift + status_swift=$? + + # Fail the step if either failed + if [ $status_minio -ne 0 ] || [ $status_swift -ne 0 ]; then + # exit 1 + fi + + - name: prune uv cache + run: nix develop --command uv cache prune --ci diff --git a/.github/workflows/release-request.yml b/.github/workflows/release-request.yml deleted file mode 100644 index e11d30b..0000000 --- a/.github/workflows/release-request.yml +++ /dev/null @@ -1,159 +0,0 @@ -name: Prepare Release - -on: - workflow_dispatch: - push: - branches: - - main - -jobs: - check-version: - name: Check Commitizen Version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.version.outputs.version }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Configure Git - run: | - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - - - name: Get current version (without bumping or pushing) - id: version - uses: commitizen-tools/commitizen-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - push: false - dry_run: true - changelog: false - - prepare-release-pr: - name: Create Release Branch and PR - needs: check-version - if: ${{ needs.check-version.outputs.version != '' && github.ref == 'refs/heads/main' }} - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: main - - - name: Bump version using Commitizen - id: cz - uses: commitizen-tools/commitizen-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - git_name: ${{ github.actor }} - git_email: ${{ github.actor }}@users.noreply.github.com - push: false - changelog: true - dry_run: false - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v8 - with: - title: "Release ${{ steps.cz.outputs.version }}" - body: "Automated PR for version bump to ${{ steps.cz.outputs.version }}" - branch: "release-v${{ steps.cz.outputs.version }}" - delete-branch: true - - check-release: - runs-on: ubuntu-latest - # if: github.ref == 'refs/heads/main' && github.event_name == 'push' - outputs: - release: ${{ steps.check.outputs.release }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: main - - - name: Configure Git - run: | - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - - - name: Get current version - id: version - run: | - VERSION=$(yq '.commitizen.version' .cz.yaml) - echo "version=$VERSION" >> $GITHUB_OUTPUT - - - name: Check if GitHub release already exists - id: check - run: | - VERSION=${{ steps.version.outputs.version }} - echo "Detected version: $VERSION" - - RELEASE_EXISTS=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - https://api.github.com/repos/${{ github.repository }}/releases/tags/v$VERSION \ - | jq -r '.tag_name // empty') - - if [[ "$RELEASE_EXISTS" == "v$VERSION" ]]; then - echo "Release v$VERSION already exists." - echo "release=" >> $GITHUB_OUTPUT - else - echo "Release v$VERSION does not exist yet." - echo "release=$VERSION" >> $GITHUB_OUTPUT - fi - finalize-release: - name: Finalize Release - needs: check-release - if: ${{ needs.check-release.outputs.release != '' }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Tag and Push - run: | - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - git tag -a "v${{ needs.check-release.outputs.release }}" -m "Release v${{ needs.check-release.outputs.release }}" - git push origin "v${{ needs.check-release.outputs.release }}" - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - tag_name: "v${{ needs.check-release.outputs.release }}" - name: "Release v${{ needs.check-release.outputs.release }}" - body_path: "CHANGELOG.md" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - build-docker: - name: Build and Push Docker - needs: check-release - if: ${{ needs.check-release.outputs.release != '' }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and Push Docker - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ghcr.io/${{ github.repository_owner }}/herald:v${{ needs.check-release.outputs.release }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index f09fed5..0000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: test suite -run-name: test suite for ${{ github.event.pull_request.title || github.ref }} -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - types: - - opened - - reopened - - synchronize - - ready_for_review - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - DENO_V: 2.3.5 - GHJK_VERSION: "v0.3.2" - GHJK_ENV: "ci" - -jobs: - changes: - runs-on: ubuntu-latest - permissions: - pull-requests: read - steps: - - uses: actions/checkout@v6 - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - full: - - '.github/workflows/tests.yml' - - 'src/**' - - 'tests/**' - - 'examples/**' - outputs: - full: ${{ steps.filter.outputs.full }} - - pre-commit: - needs: changes - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - uses: denoland/setup-deno@v2 - with: - deno-version: ${{ env.DENO_V }} - - name: Install tofu - run: | - curl --proto '=https' --tlsv1.2 -fsSL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh - chmod +x install-opentofu.sh - ./install-opentofu.sh --install-method deb - rm -f install-opentofu.sh - - - shell: bash - run: | - python -m pip install --upgrade pip - pip install pre-commit - pre-commit install - deno --version - pre-commit run --all-files - - test-full: - needs: [changes] - if: ${{ needs.changes.outputs.full == 'true' && github.event.pull_request.draft == false }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: denoland/setup-deno@v2 - with: - deno-version: ${{ env.DENO_V }} - - name: Download Install Script - run: curl -fsSL "https://raw.github.com/metatypedev/ghjk/$GHJK_VERSION/install.sh" -o install.sh - - name: Execute Install Script - run: yes | bash install.sh - - run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - run: echo "BASH_ENV=$HOME/.local/share/ghjk/env.sh" >> "$GITHUB_ENV" - - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - name: Install tofu - run: | - curl --proto '=https' --tlsv1.2 -fsSL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh - chmod +x install-opentofu.sh - ./install-opentofu.sh --install-method deb - rm -f install-opentofu.sh - - uses: actions/setup-node@v6 - with: - node-version: 18 - - name: setup start-server-and-test - run: npm install -g start-server-and-test - - shell: bash - env: - AUTH_TYPE: "default" - LOG_LEVEL: "DEBUG" - ENV: "DEV" - S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} - S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} - OPENSTACK_USERNAME: ${{ secrets.OPENSTACK_USERNAME }} - OPENSTACK_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD }} - OPENSTACK_PROJECT: ${{ secrets.OPENSTACK_PROJECT }} - AWS_ACCESS_KEY_ID: ${{ secrets.OPENSTACK_USERNAME }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.OPENSTACK_PASSWORD }} - run: | - # run all tests - deno --version - ghjk x dev-compose s3 - sleep 20 - - deno install - - # ghjk x setup-auth - npx start-server-and-test 'deno serve -A --unstable-kv src/main.ts' http://0.0.0.0:8000/ 'deno test -A' diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d402a5f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "s3-tests"] + path = s3-tests + url = https://github.com/ceph/s3-tests diff --git a/.infisical.json b/.infisical.json new file mode 100644 index 0000000..a6af04a --- /dev/null +++ b/.infisical.json @@ -0,0 +1,5 @@ +{ + "workspaceId": "39bbe4e4-20c2-42fa-8a6c-1bcafcc74faf", + "defaultEnvironment": "", + "gitBranchToEnvironmentMapping": null +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 685422c..96b71d2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v6.0.0 hooks: - id: check-added-large-files exclude: tests/res @@ -34,16 +34,21 @@ repos: - id: deno-check name: Deno check language: system - entry: bash -c 'deno check src/ tests/ benchmarks/' + entry: bash -c 'deno check src/ tests/' pass_filenames: false types: - ts - repo: https://github.com/tofuutils/pre-commit-opentofu - rev: v1.0.3 + rev: v2.2.2 hooks: - id: tofu_fmt - - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.10.0.1 + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.36.0 hooks: - - id: shellcheck - args: ["--exclude=SC2154,SC2181"] + - id: check-dependabot + - id: check-github-workflows + # - repo: https://github.com/shellcheck-py/shellcheck-py + # rev: v0.10.0.1 + # hooks: + # - id: shellcheck + # args: ["--exclude=SC2154,SC2181"] diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..3ee405d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,35 @@ +- We're using the effects library https://effect.website/llms.txt + - Their HTTP implementation is described in ./HTTP_PLATFORM.md + - **ALWAYS** use `@effect/platform/HttpClient` instead of native `fetch` for + all HTTP requests. + - Prefer generators over effect piping. + - Use methods on `Effect.Option` like `Option.isNone` instead of looking at + `_tag`. + - **NEVER** use standard `try/catch` or `try/finally` blocks around `yield*` + in Effect generators. Use `Effect.addFinalizer`, `Effect.try`, + `Effect.catchAll`, or `Effect.orElse`. + - **ALWAYS** use the `Config` module from Effect for environment variable + access instead of `Deno.env.get`. +- **NEVER** assume default values using `??` or ternary operators for critical + configuration or external input (e.g., `bucket.region ?? "us-east-1"`, + `request.headers.host ?? "localhost"`). Always fail explicitly with a + descriptive error. +- Use `Effect.fail` or `Effect.die` instead of returning "unknown" or empty + strings when expected data is missing. +- When mapping external errors (like S3 SDK exceptions), be as specific as + possible. Avoid generic "Unknown" or "S3 error" messages. + +- Reference ./herald, ./s3proxy and ./s3-tests for S3 behavior and other S3 + proxy imps. +- Reference ./ghjk for Deno typescript conventions especially ./ghjk/tests/. +- Reference ./sample-http for how to do some things using the Effect library. + +- Prefer to preserve comments unless they are progress comments written by an + agent. +- Maintain strict type safety. Avoid "any" casts or requirement hacks. +- Use the structured `Logger` layer for all diagnostic output. + +- Always fix deno lint and deno check issues before running tests, the type + system is there to help. +- Never use `--no-check`. Treat the codebase like a Rust codebase. Live and die + by the type system. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..703051e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,84 @@ +# Contributing + +## Starting Services + +You can start the containers used for development using the provided scripts: + +```bash +# Start MinIO and Redis +deno run --allow-all x/compose-up.ts s3 db + +# Start Swift (SAIO) +deno run --allow-all x/compose-up.ts swift +``` + +## Running Tests + +```bash +# Run all tests +deno task test + +# Run Swift integration tests specifically +deno task test --filter "Swift/" +``` + +## Benchmarking + +```bash +deno bench --allow-all benchmarks/ +``` + +## Repo Map + +- `src/Domain`: Core logic and data models. Contains Effect Schemas for global + configuration and logic for backend resolution/matching. + +- `src/Config`: Application configuration loading. Defines the HeraldConfig + service layer. + +- `src/Services`: Shared service abstractions and implementations. + - `src/Services/Backend.ts`: Generic storage backend interface with structured + request/response types and domain-specific error types. + - `src/Services/BackendResolver.ts`: Logic for dynamically providing the + correct backend based on request context. + - `src/Services/S3Xml.ts`: S3-compatible XML response formatting for errors, + bucket listings, and object listings. + - `src/Services/BackendKeyValueStore.ts`: Abstraction for backend-specific + key-value storage. + +- `src/Backends`: Specific storage backend implementations. + - `src/Backends/S3`: S3 protocol implementation using AWS SDK. + - `src/Backends/Swift`: OpenStack Swift protocol implementation. + +- `src/Frontend`: HTTP ingress layer. + - `src/Frontend/Api.ts`: HttpApi definition for the S3 compatibility layer. + - `src/Frontend/Http.ts`: Main HTTP server setup and endpoint group + registrations. + - `src/Frontend/Buckets/`: Handlers for bucket-level S3 operations. + - `src/Frontend/Objects/`: Handlers for object-level S3 operations, including + Multipart Upload (via `Post.ts`). + - `src/Frontend/Health/`: Handlers for system health monitoring. + +- `src/Logging` & `src/Tracing.ts`: Diagnostic observability layers. + +- `tests/`: Test suite. + - `tests/integration/`: End-to-end tests comparing Herald proxy behavior + against a MinIO baseline using snapshots. + - `tests/config.test.ts`: Unit tests for configuration and backend resolution. + - `tests/utils.ts`: Shared test harness and snapshot normalization logic. + +- `benchmarks/`: Performance testing suite for evaluating proxy overhead and + streaming efficiency. + +- `x/`: CLI utilities and development scripts. + - `x/dev.ts`: Main development entry point for running the proxy locally. + - `x/s3-tests.ts`: Orchestration for running the ceph `s3-tests` suite. + - `x/snapdiff.ts`: Tool for comparing proxy snapshots against baseline + responses. + - `x/compose-up.ts` & `x/compose-down.ts`: Helpers for managing local Docker + dependencies. + +- `chart/`: Helm charts for Kubernetes deployment. + +- `tools/`: Infrastructure and development tools (Docker Compose, + Containerfiles). diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 4635e32..0000000 --- a/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# checkov:skip=CKV_DOCKER_2: Health check managed elsewhere -# checkov:skip=CKV_DOCKER_3: User settings managed elsewhere -FROM denoland/deno:alpine-2.3.5 - -WORKDIR /app - -COPY import_map.json deno.jsonc ./ - -COPY ./src ./src - -RUN deno cache ./src/main.ts -RUN ls -l src/main.ts - -ENTRYPOINT ["deno"] -CMD ["serve", "-A", "--unstable-kv", "src/main.ts"] diff --git a/README.md b/README.md index 12531c4..b44679e 100644 --- a/README.md +++ b/README.md @@ -18,307 +18,293 @@


-## Table of Contents - -- [ Overview](#Overview) -- [ Features](#Features) -- [ Project Structure](#Project-Structure) -- [ Getting Started](#Getting-Started) - - [ Prerequisites](#Prerequisites) - - [ Development](#Development) - - [ herald.yaml config file](#herald.yaml-config-file) - - [ Environment Variables](#Environment-Variables) - - [ Usage](#Usage) - - [ Testing](#Testing) -- [ Project Roadmap](#Project-Roadmap) -- [ Contributing](#Contributing) -- [ Acknowledgments](#Acknowledgments) - ---- - -## Overview - -Herald is an S3 proxy that allows communication to multiple storage services with different communication protocols using the S3 protocol. For instance, you can use herald to connect to an OpenStack swift storage service as you would to S3 storage services like AWS S3 and MinIO. While OpenStack has its own middleware to accept requests in S3 protocol, herald addresses some issues you will face using that S3 middleware. Currently, herald supports two types of backends(storage providers): S3 and OpenStack Swift. A comprehensive list of herald's features and capabilities are listed below. - -## Features - -- Multi Backend Compatibility: interacting with multiple storage backends using a single protocol. You can use the S3 protocol to communicate with storage services that don't necessarily support an S3 protocol natively. Herald supports S3 and OpenStack Swift Storage backends as of right now. -- Kubernetes Native Authentication: herald supports authentication using a service account provisioned by the Kubernetes server. You can register services in your cluster that can access herald and the resources managed by herald to allow a robust authentication pipeline easy to manage. -- Mirroring: a multi-backend commit is another handy feature in herald that allows you to mirror any operations you make to your storage services through herald. You can have a primary storage configured with replicas. The primary and replicas will be in sync and during times of unavailability, herald will use the replicas to fetch data. For write operations, after the operation has been completed, it will be mirrored to a replica storage. The mirroring is done using transactional tasks, stored in a message/task queue, which ensures the replicas are in sync. For read operations, the operation will be forwarded to replica storages if the primary is unavailable. The mirroring operations are done using web workers to avoid the impact on performance. -- IaC Support: Herald supports IaC tech stacks that use the S3 protocol to provision resources such as Terraform and OpenTofu. - -## Project Structure - -```sh -└── herald/ - ├── .github - │ ├── dependabot.yml - │ ├── pull_request_template.md - │ └── workflows - ├── Dockerfile - ├── LICENSE.md - ├── README.md - ├── benchmarks - │ ├── bench_saver.ts - │ ├── result.json - │ └── sdk - ├── deno.jsonc - ├── deno.lock - ├── docker-compose.yml - ├── examples - │ └── simple-bucket-test - ├── ghjk.ts - ├── herald-compose.yaml - ├── herald.yaml - ├── import_map.json - ├── src - │ ├── auth - │ ├── backends - │ ├── buckets - │ ├── config - │ ├── constants - │ ├── main.ts - │ ├── types - │ ├── utils - │ └── workers - ├── tests - │ ├── iac - │ ├── mirror - │ ├── s3 - │ ├── swift - │ └── utils - ├── tools - │ ├── compose - │ ├── deps.ts - │ └── s3-comparison - └── utils - ├── file.ts - └── s3.ts -```` - ---- - -## Getting Started - -### Prerequisites - -Before getting started with herald, ensure your runtime environment meets the following requirements: - -- **Programming Language:** TypeScript -- **Container Runtime:** Docker - -### Development - -Install herald using one of the following methods: - -**Build from source:** - -1. Clone the herald repository: - -```sh -❯ git clone https://github.com/expnt/herald +Herald is an S3 proxy that supports: + +- Protocol translation (S3 to S3, S3 to Swift). +- Backend routing based on bucket names. +- Flexible bucket mapping with glob support. + +## Quick start + +Run Herald in Docker with env-only config (no YAML). Point it at an +S3-compatible backend (e.g. [MinIO](https://min.io)) and use any S3 client +against Herald. + +```bash +# Start Herald (default backend: S3 at host's MinIO). Port 3000. +docker run -p 3000:3000 \ + -e HERALD_DEFAULT_PROTOCOL=s3 \ + -e HERALD_DEFAULT_ENDPOINT=http://host.docker.internal:9000 \ + -e HERALD_DEFAULT_REGION=us-east-1 \ + -e HERALD_DEFAULT_ACCESS_KEY_ID=minioadmin \ + -e HERALD_DEFAULT_SECRET_ACCESS_KEY=minioadmin \ + ghcr.io/expnt/herald:latest ``` -2. Navigate to the project directory: +Use the AWS CLI (or any S3 client) with Herald as the endpoint. The S3 API is +mounted at `/s3`; use path-style so bucket and key are in the path. -```sh -❯ cd herald +```bash +# List buckets via Herald +aws s3 ls --endpoint-url http://localhost:3000/s3 + +# List objects in a bucket +aws s3 ls --endpoint-url http://localhost:3000/s3 s3://my-bucket/ ``` -3. Install ghjk +**Images:** [ghcr.io/expnt/herald](https://ghcr.io/expnt/herald) **Helm chart:** +[chart/](chart/) for Kubernetes (chart may be outdated; update planned). -[ghjk](https://github.com/metatypedev/ghjk) is a developer environment management tool used to install dependencies required to run herald. +## Config -4. Install dependencies +Herald is configured via a YAML file (typically `herald.yaml`). The +configuration defines backends and how incoming requests are routed to them. -```sh -❯ ghjk p resolve -``` +```yaml +backends: + # Unique identifier for the backend + minio: + # Backend protocol: "s3" or "swift" + protocol: s3 + + # Base URL of the backend service + endpoint: http://127.0.0.1:9000 -5. Run services needed for herald. + # Default region for this backend + region: us-east-1 -We just spin a minio s3 server and a swift object storage container in docker. + # Authentication credentials for the backend + credentials: + accessKeyId: minioadmin + secretAccessKey: minioadmin -```sh -❯ ghjk dev-compose up all + # Bucket routing rules. + # Can be: + # 1. "*" to match all buckets not claimed by other backends + # 2. A glob pattern like "logs-*" + # 3. A map of bucket definitions for granular control + # Optional: auth for this backend (bucket > backend > global) + auth: + accessKeysRefs: [admin] + + buckets: + # Simple bucket mapping (inherits backend settings) + my-bucket: {} + + # Mapping with overrides; bucket-level auth overrides backend/global + external-data: + auth: + accessKeysRefs: [readonly] + # Map proxy bucket "external-data" to backend bucket "data-v1" + bucket_name: data-v1 + # Override endpoint for this specific bucket + endpoint: http://special-endpoint:9000 + # Override region + region: us-west-2 + + # Glob pattern support within the map + "test-*": + region: us-east-1 + + # Example Swift backend + swift-storage: + protocol: swift + auth_url: http://keystone.example.com/v3 + region: RegionOne + # Optional: override the Swift container name for all buckets in this backend + # container: my-fixed-container + credentials: + username: my-user + password: my-password + project_name: my-project + user_domain_name: Default + project_domain_name: Default + # Route all archive buckets to Swift + buckets: "archive-*" + +# Optional: require S3 SigV4 auth for incoming requests (see Auth section) +auth: + accessKeysRefs: [admin, readonly] + +cors: + # Global CORS defaults + allowedOrigins: ["*"] + allowedMethods: ["GET", "PUT", "POST", "DELETE", "HEAD", "OPTIONS"] + allowedHeaders: ["*"] + exposedHeaders: ["*"] + maxAge: 3600 + credentials: false ``` -6. Configure herald.yaml +### Auth (incoming request verification) -Configuration for the cloud services that herald connects with are defined here. Other configs such as the port it runs on, temporary dir for tests is also defined here. The object storage for a task store is also defined here. A serialized task store is saved in the specified storage service where durable tasks for mirroring tasks are stored. Service account names are also configured for jwk based authentication. This is a sample configuration file. +Herald can verify incoming S3 requests using AWS Signature Version 4 (SigV4). +When auth is configured, only requests signed with one of the configured access +keys are accepted. Credentials are never stored in the config file; you +reference them by name (_refs_) and supply the actual keys via environment +variables. -```yaml -port: 8000 -temp_dir: "./tmp" -backends: - minio_s3: - protocol: s3 - openstack_swift: - protocol: swift - exoscale_s3: - protocol: s3 +#### Precedence -task_store_backend: - endpoint: "http://localhost:9000" - region: local - forcePathStyle: true - bucket: task-store - credentials: - accessKeyId: minio - secretAccessKey: password - -service_accounts: - - name: "system:serviceaccount:dev-s3-herald:default" - buckets: - - s3-test - - s3-mirror-test - - swift-mirror-test - - iac-s3 - - name: "system:serviceaccount:stg-datacycle:datacycle-app-backend-sa" - buckets: - - s3-test - - s3-mirror-test - - iac-s3 - -buckets: - s3-test: - backend: minio_s3 - config: - endpoint: "http://localhost:9000" - region: local - forcePathStyle: true - bucket: s3-test - credentials: - accessKeyId: minio - secretAccessKey: password - -replicas: - - name: replica-0 - backend: minio_s3 - config: - endpoint: "http://localhost:9090" - region: local - forcePathStyle: true - bucket: s3-test - credentials: - accessKeyId: minio - secretAccessKey: password -``` +Auth is resolved at three levels with the same precedence as CORS: **Bucket > +Backend > Global**. The most specific definition wins (e.g. a bucket’s `auth` +overrides its backend’s `auth`). +#### Config shape -7. Run herald +At each level you set `auth.accessKeysRefs`: a list of ref names (strings). Each +ref maps to a pair of env vars: -```sh -❯ deno run src/main.ts -``` +- `HERALD_AUTH__ACCESS_KEY_ID` — access key id +- `HERALD_AUTH__SECRET_KEY` — secret key -### herald.yaml config file - -This configuration YAML file is used to set up and manage the Herald service, which interacts with various storage backends and service accounts. Below is a detailed description of the key sections in the file: - -- **port**: Specifies the port number (8000) on which herald will run. -- **temp_dir**: Defines the temporary directory (`./tmp`) used by the service. -- **backends**: Lists the supported storage backends, including `minio_s3`, `openstack_swift`, and `exoscale_s3`, each with its respective protocol. -- **task_store_backend**: Configures the backend for storing tasks, including the endpoint, region, path style, bucket name, and credentials (access key ID and secret access key). -- **service_accounts**: Defines the service accounts with access to specific buckets. Each service account has a name and a list of accessible buckets. -- **buckets**: Specifies the configuration for individual buckets, including the backend type, endpoint, region, path style, bucket name, and credentials. -- **replicas**: Configures replicas for redundancy and load balancing. Each replica has a name, backend type, and configuration similar to the buckets section. - -This configuration file allows for flexible and secure management of storage resources and service accounts, ensuring that the Herald service can interact with multiple storage backends and maintain high availability through replicas. - -### Environment Variables -| **Name** | **Default** | **Description** | -|----------------------------|------------------------------|-------------------------------| -| **debug** | — | Boolean string that enables or disables debug mode. | -| **log_level** | (optional) | Logging level; possible values: `NOTSET`, `DEBUG`, `INFO`, `WARN`, `ERROR`, `CRITICAL`. | -| **env** | `DEV` | Environment in which the application runs (`DEV` or `PROD`). | -| **k8s_api** | `https://kubernetes.default.svc` | URL for the Kubernetes API. | -| **cert_path** | `/var/run/secrets/kubernetes.io/serviceaccount/ca.crt` | File path to the Kubernetes service account CA certificate. | -| **config_file_path** | herald.yaml | Path to the Herald configuration file. | -| **service_account_token_path** | `/var/run/secrets/kubernetes.io/serviceaccount/token` | File path to the Kubernetes service account token. | -| **version** | `0.1` | Application version. | -| **sentry_dsn** | (optional) | DSN for Sentry, used for error tracking. | -| **sentry_sample_rate** | `1` | Sampling rate for Sentry events (numeric, 0 to 1). | -| **sentry_traces_sample_rate** | `1` | Sampling rate for Sentry traces (numeric, 0 to 1). | - -### Run using docker - -Pull the image first - -```sh -❯ docker pull ghcr.io/expnt/herald:latest -``` +`` is the ref name in UPPERCASE (e.g. ref `admin` → +`HERALD_AUTH_ADMIN_ACCESS_KEY_ID`). Only refs that have both env vars set are +used; missing refs are skipped. -Run herald using the following command: -**Using `docker`**   [](https://www.docker.com/) +Example: global `auth.accessKeysRefs: [admin, readonly]` with +`HERALD_AUTH_ADMIN_ACCESS_KEY_ID`, `HERALD_AUTH_ADMIN_SECRET_KEY` and +`HERALD_AUTH_READONLY_ACCESS_KEY_ID`, `HERALD_AUTH_READONLY_SECRET_KEY` set in +the environment allows requests signed with either key. You can override at +backend or bucket level (e.g. a backend that only accepts `admin`, or a bucket +that only accepts `readonly`). -```sh -❯ docker run -it expnt/herald:latest -``` +#### When auth is not configured -### Testing +If no `auth` is defined at any level for a request, Herald does not perform +SigV4 verification and the request is not gated by these credentials. -To run full tests, +### CORS Configuration -```sh -❯ deno test -A tests -``` +Herald supports fine-grained CORS control at three levels with the following +precedence: **Bucket > Backend > Global**. ---- - -## Project Roadmap - -- [x] **`Task 1`**: Mirroring -- [ ] **`Task 2`**: Event Notification. -- [ ] **`Task 3`**: Advanced Cache Policy - ---- - -## Contributing - -- **💬 [Join the Discussions](https://github.com/expnt/herald/discussions)**: Share your insights, provide feedback, or ask questions. -- **🐛 [Report Issues](https://github.com/expnt/herald/issues)**: Submit bugs found or log feature requests for the `herald` project. -- **💡 [Submit Pull Requests](https://github.com/expnt/herald/blob/main/CONTRIBUTING.md)**: Review open PRs, and submit your own PRs. - -
-Contributing Guidelines - -1. **Fork the Repository**: Start by forking the project repository to your github account. -2. **Clone Locally**: Clone the forked repository to your local machine using a git client. - ```sh - git clone https://github.com/expnt/herald - ``` -3. **Create a New Branch**: Always work on a new branch, giving it a descriptive name. - ```sh - git checkout -b new-feature-x - ``` -4. **Make Your Changes**: Develop and test your changes locally. -5. **Commit Your Changes**: Commit with a clear message describing your updates. - ```sh - git commit -m 'Implemented new feature x.' - ``` -6. **Push to github**: Push the changes to your forked repository. - ```sh - git push origin new-feature-x - ``` -7. **Submit a Pull Request**: Create a PR against the original project repository. Clearly describe the changes and their motivations. -8. **Review**: Once your PR is reviewed and approved, it will be merged into the main branch. Congratulations on your contribution! -
- -
-Contributor Graph -
-

- - - -

-
+- **Global**: Defined at the root of the config file under `cors`. +- **Backend**: Defined within a backend block under `cors`. Overrides global + settings. +- **Bucket**: Defined within a bucket definition under `cors`. Overrides both + backend and global settings. + +#### Default Behavior ---- +If no CORS configuration is provided at any level, **CORS is disabled** and +Herald will not add any CORS-related headers to responses. Preflight `OPTIONS` +requests will be passed through to the backend. -## Acknowledgments +If you enable CORS by providing configuration at any level, the following +defaults are applied for any omitted fields: -- List any resources, contributors, inspiration, etc. here. +| Field | Default Value | Description | +| ---------------- | --------------------------------------- | ------------------------------------------------------ | +| `maxAge` | `3600` | Max age in seconds for preflight results | +| `allowedMethods` | `GET, PUT, POST, DELETE, HEAD, OPTIONS` | Allowed HTTP methods | +| `allowedHeaders` | (Mirrors request) | Defaults to mirroring `Access-Control-Request-Headers` | +| `credentials` | `false` | Whether to allow credentials | +| `allowedOrigins` | (None) | Headers only added if `Origin` matches an entry | + +Example with overrides: + +```yaml +cors: # Global defaults + allowedOrigins: ["*"] + credentials: false + +backends: + prod: + protocol: s3 + cors: # Backend-level override + allowedOrigins: ["https://app.example.com"] + credentials: true + buckets: + assets: + cors: # Bucket-level override + allowedOrigins: ["https://cdn.example.com"] +``` ---- +### Routing Logic + +When a request comes in for a bucket (e.g., `GET /my-bucket/file.txt`), Herald +resolves the backend using the following priority: + +1. **Direct match**: Looks for `my-bucket` in all backends' `buckets` maps. +2. **Glob match (map)**: Looks for glob patterns (like `test-*`) in all + backends' `buckets` maps. +3. **Glob match (string)**: If a backend has `buckets: "string-*"`, it checks if + the bucket name matches that pattern. + +When several backends could match (e.g. two globs), the **first backend in +config order** wins. + +### Environment variable configuration + +Configuration can be supplied or overridden via environment variables; env is +merged with YAML at load time (env wins for the same path). All config-related +vars use the `HERALD_` prefix. Naming: `HERALD_` applies to the `default` +backend or global (for top-level keys like auth/CORS); `HERALD__` +applies to that backend. Keys are normalised (e.g. `AUTH_URL` → `auth_url`; +credential keys go under `credentials`). + +| Var | Purpose | Default | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | ------------- | +| `HERALD_CONFIG_PATH` | Path to YAML config file | `herald.yaml` | +| `HERALD_LOG_LEVEL` | Log level (e.g. `DEBUG`, `INFO`) | (none; INFO) | +| `PORT` | HTTP server port | `3000` | +| `HERALD_AUTH_ACCESS_KEYS_REFS` | Global auth: comma-separated ref names | — | +| `HERALD__AUTH_ACCESS_KEYS_REFS` | Backend auth: comma-separated ref names | — | +| `HERALD_AUTH__ACCESS_KEY_ID` | Access key for auth ref (SigV4) | — | +| `HERALD_AUTH__SECRET_KEY` | Secret key for auth ref (SigV4) | — | +| `HERALD_PROTOCOL`, `HERALD_ENDPOINT`, `HERALD_REGION`, `HERALD_BUCKETS` | Default backend (S3) | — | +| `HERALD__PROTOCOL`, `HERALD__ENDPOINT`, `HERALD__REGION`, `HERALD__BUCKETS` | Backend (S3) | — | +| `HERALD__ACCESS_KEY_ID`, `HERALD__SECRET_ACCESS_KEY` | Backend S3 credentials | — | +| `HERALD__AUTH_URL`, `HERALD__CONTAINER`, `HERALD__USERNAME`, `HERALD__PASSWORD`, `HERALD__PROJECT_NAME`, `HERALD__USER_DOMAIN_NAME`, `HERALD__PROJECT_DOMAIN_NAME` | Backend (Swift) | — | +| `HERALD_CORS_ALLOWED_ORIGINS`, `HERALD_CORS_ALLOWED_METHODS`, `HERALD_CORS_ALLOWED_HEADERS`, `HERALD_CORS_EXPOSED_HEADERS`, `HERALD_CORS_MAX_AGE`, `HERALD_CORS_CREDENTIALS` | Global CORS (lists comma-separated) | — | +| `HERALD__CORS_` | Backend CORS (same keys as above) | — | + +### Health and observability + +- **Health:** `GET /health` returns `{ "status": "ok" }`. Use it for + liveness/readiness. +- **Logging:** Set `HERALD_LOG_LEVEL` (e.g. `DEBUG`, `INFO`) to control log + verbosity. +- **Tracing:** Optional OpenTelemetry: set `OTEL_EXPORTER_OTLP_ENDPOINT` (and + `OTEL_SERVICE_NAME`, default `herald`) to export traces to an OTLP collector. + +## Deployment + +- **Docker:** Images are published at + [ghcr.io/expnt/herald](https://ghcr.io/expnt/herald). Use env vars (see table + above) or mount a `herald.yaml` and set `HERALD_CONFIG_PATH`. +- **Kubernetes:** A Helm chart is in [chart/](chart/). It may be outdated; + updates are planned. + +## Limitations + +Herald is an S3 proxy focused on routing, protocol translation, and core object +operations. The following are **not** currently supported (or are partial): + +- **Bucket subresources:** Bucket policies (`?policy`), lifecycle + (`?lifecycle`), versioning config (`?versioning`), tagging (`?tagging`), ACLs + (`?acl`), website (`?website`), public access block (`?publicAccessBlock`), + replication, logging, inventory, metrics, ownership controls. +- **Object subresources:** Object ACLs, tagging, legal hold, retention (Object + Lock), S3 Select. Copy Object (`x-amz-copy-source`) and Multi-Object Delete + (`POST ?delete`) are not implemented. +- **Object operations:** GetObjectAttributes (`?attributes`) is not implemented. + Checksum headers (`x-amz-checksum-*`) and conditional requests (`If-Match`, + etc.) are not fully supported. +- **List enhancements:** `encoding-type=url`, special delimiter handling, + ListObjectsV2 `FetchOwner`, unordered listing behavior may not match S3. +- **Auth & IAM:** No IAM policy evaluation, STS, or web identity federation. + Anonymous access for public buckets/objects is not implemented. Invalid or + missing SigV4 auth may not return 403/400 as expected. +- **Validation & protocol:** Bucket naming rules (length, format) are not + strictly enforced. HTTP 100 Continue (`Expect: 100-continue`) is not + supported. Some error codes and response fields may differ from S3. + +For the full list of missing functionality and focus tests (from the s3-tests +suite), see [TODO.md](TODO.md). + +## Prior art + +- https://github.com/gaul/s3proxy +- https://github.com/ceph/s3-tests diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..5be06f6 --- /dev/null +++ b/TODO.md @@ -0,0 +1,191 @@ +# Missing Functionality in Herald3 + +This list represents the S3 functionality that is currently missing in Herald3, +based on a comparison with the `s3-tests` suite and a review of the existing +implementation. + +## 1. Bucket Operations + +- [ ] **Bucket Policies**: Implementation of `GET/PUT/DELETE /?policy`. _(Focus + tests: `test_get_bucket_policy_status`, + `test_post_object_missing_policy_condition`)_ +- [ ] **CORS (Cross-Origin Resource Sharing)**: Implementation of + `GET/PUT/DELETE /?cors` and handling of `OPTIONS` preflight requests. + _(Focus tests: `test_set_cors`, `test_cors_origin_response`, + `test_cors_header_option`)_ +- [ ] **Lifecycle Management**: Implementation of `GET/PUT/DELETE /?lifecycle`. + _(Focus tests: `test_lifecycle_expiration`, `test_lifecycle_transition`)_ +- [ ] **Tagging**: Implementation of `GET/PUT/DELETE /?tagging` for buckets. + _(Focus tests: `test_bucket_tagging_create`, `test_bucket_tagging_get`)_ +- [ ] **Versioning Configuration**: Implementation of `GET/PUT /?versioning`. + (Basic `listVersions` is partially implemented). _(Focus tests: + `test_bucket_list_return_data_versioning`, + `test_versioning_concurrent_multi_object_delete`)_ +- [ ] **ACLs (Access Control Lists)**: Implementation of `GET/PUT /?acl` for + buckets. _(Focus tests: `test_bucket_acl_default`, + `test_put_bucket_acl_grant_group_read`, `test_bucket_header_acl_grants`)_ +- [ ] **Website Configuration**: Implementation of `GET/PUT/DELETE /?website`. + _(Focus tests: `test_website_configuration`, + `test_website_error_document`)_ +- [ ] **Public Access Block**: Implementation of + `GET/PUT/DELETE /?publicAccessBlock`. _(Focus tests: + `test_bucket_public_access_block`)_ +- [ ] **Bucket Listing Enhancements**: - [ ] **Encoding Type**: Support for + `?encoding-type=url` in `ListObjects` and `ListObjectsV2`. _(Focus tests: + `test_bucket_list_encoding_basic`, `test_bucket_listv2_encoding_basic`)_ - + [ ] **Special Characters in Delimiters**: Fix handling of percentage, + whitespace, and other special characters as delimiters. _(Focus tests: + `test_bucket_list_delimiter_percentage`, + `test_bucket_list_delimiter_whitespace`)_ - [ ] **V2 Fetch Owner**: + Support for `FetchOwner` parameter in `ListObjectsV2`. _(Focus tests: + `test_bucket_listv2_fetchowner_empty`)_ - [ ] **Unordered Listings**: + Ensure consistent behavior when listing objects in buckets with + non-standard ordering. _(Focus tests: `test_bucket_list_unordered`)_ +- [ ] **Replication Configuration**: Implementation of + `GET/PUT/DELETE /?replication`. +- [ ] **Notification Configuration (SNS)**: Implementation of + `GET/PUT /?notification`. +- [ ] **Logging Configuration**: Implementation of `GET/PUT /?logging`. _(Focus + tests: `test_bucket_logging_config`)_ +- [ ] **Inventory Configuration**: Implementation of + `GET/PUT/DELETE /?inventory`. +- [ ] **Metrics Configuration**: Implementation of `GET/PUT/DELETE /?metrics`. +- [ ] **Intelligent-Tiering Configuration**: Implementation of + `GET/PUT/DELETE /?intelligent-tiering`. +- [ ] **Ownership Controls**: Implementation of + `GET/PUT/DELETE /?ownershipControls`. + +## 2. Object Operations + +- [ ] **Multi-Object Delete**: Implementation of `POST /?delete`. _(Focus tests: + `test_multi_object_delete`, `test_multi_object_delete_key_limit`)_ +- [x] **Multipart Upload**: Support for `InitiateMultipartUpload`, `UploadPart`, + `CompleteMultipartUpload`, `AbortMultipartUpload`, and `ListParts`. + _(Focus tests: `test_multipart_upload`, `test_multipart_upload_empty`, + `test_abort_multipart_upload`)_ + - [x] **Swift Multipart Upload**: Implement S3 multipart mapping to Swift SLO. +- [ ] **GetObject Attributes**: Implementation of `GET /bucket/key?attributes`. + _(Focus tests: `test_get_object_attributes`)_ +- [ ] **HeadObject Consistency**: Fix `404 Not Found` errors on existing objects + during certain test sequences. _(Focus tests: + `test_object_head_zero_bytes`)_ +- [ ] **Unicode Metadata**: Fix support for non-ASCII characters in object + metadata. Currently failing across all backends. _(Focus tests: + `test_object_set_get_unicode_metadata`)_ +- [ ] **Copy Object**: Support for `PUT` with `x-amz-copy-source` header. + _(Focus tests: `test_object_copy`)_ +- [ ] **Tagging**: Implementation of `GET/PUT/DELETE /?tagging` for objects. + _(Focus tests: `test_object_tagging`)_ +- [ ] **ACLs (Access Control Lists)**: Implementation of `GET/PUT /?acl` for + objects. Currently failing due to missing XML parsing/formatting for + object-level ACLs. _(Focus tests: `test_object_acl_default`, + `test_object_acl_read`, `test_object_put_acl_mtime`)_ +- [ ] **Legal Hold & Retention**: Implementation of `GET/PUT /?legal-hold` and + `GET/PUT /?retention` (Object Lock). +- [ ] **Object Lock Configuration**: Implementation of `GET/PUT /?object-lock` + on objects. +- [ ] **S3 Select**: Implementation of `POST /?select&select-type=2`. +- [ ] **Checksums**: Support for `x-amz-checksum-sha1`, `x-amz-checksum-sha256`, + `x-amz-checksum-crc32`, and `x-amz-checksum-crc32c`. Currently failing + validation tests. _(Focus tests: `test_object_checksum_sha256`)_ + - [ ] **Fix S3 Buffering**: Refactor S3 `putObject` and `uploadPart` to stream + directly to the AWS SDK instead of collecting chunks into a + `Uint8Array`. + - [ ] **Fix Swift Validation Timing**: Move Swift checksum validation before + the final commit to avoid "zombie" objects (data persisted despite + failure). + - [ ] **Implement CRC64NVME**: Add the missing logic for CRC64NVME in the + `Checksum` service. + - [ ] **Validation on GET**: Implement "Check-on-Read" validation for `GET` + requests, supporting abrupt termination or trailers on mismatch. + - [ ] **Swift Header Cleanup**: Fix duplicate checksum headers in Swift + responses (remove `x-amz-meta-` versions of internal checksums). +- [ ] **Server-Side Encryption (SSE)**: Handling of + `x-amz-server-side-encryption`, + `x-amz-server-side-encryption-customer-algorithm`, etc. +- [ ] **Restore Object**: Support for `POST /?restore`. + +## 3. Authentication & IAM + +- [ ] **IAM Integration**: Full implementation of IAM policy evaluation for all + requests. +- [ ] **User Policies**: Support for user-specific IAM policies. +- [ ] **Security Token Service (STS)**: Implementation of `GetSessionToken`, + `AssumeRole`, etc. +- [ ] **Web Identity Federation**: Implementation of + `AssumeRoleWithWebIdentity`. +- [ ] **Anonymous Access**: Correctly handle anonymous requests for public + buckets/objects. _(Focus tests: `test_bucket_list_objects_anonymous`, + `test_post_object_anonymous_request`)_ + +## 4. Validation, Errors & Protocol + +- [ ] **HTTP 100 Continue**: Support for `Expect: 100-continue` (return 100 + before reading body). _(Focus tests: `test_100_continue`, + `test_100_continue_error_retry`)_ +- [ ] **SigV4 Request Validation**: Reject invalid or missing Authorization and + `x-amz-date` with 403/400. Many tests expect 403 for bad/missing auth. + _(Focus tests: `test_*_bad_authorization_*`, `test_*_bad_date_*_aws2`)_ +- [ ] **Content-Length Handling**: Require or correctly handle Content-Length + for PUT/POST; reject or accept requests with missing/invalid + Content-Length as per S3 behavior. _(Focus tests: + `test_object_create_bad_contentlength_none`, + `test_bucket_create_bad_contentlength_none`)_ +- [ ] **Special Key Names / Prefix**: Bucket create and list with special + characters in key names and prefix. _(Focus tests: + `test_bucket_create_special_key_names`, + `test_bucket_list_special_prefix`)_ +- [ ] **Bucket Naming Validation**: Implement strict S3 naming rules (no IP + addresses, no double dots, length 3-63, etc.). Currently many naming tests + fail. _(Focus tests: `test_bucket_create_naming_bad_ip`, + `test_bucket_create_naming_dns_dot_dot`, + `test_bucket_create_naming_bad_starts_nonalpha`)_ +- [ ] **Correct Error Codes**: Ensure accurate HTTP status codes for S3 errors. + - [ ] **409 Conflict**: Ensure `BucketAlreadyExists` and + `BucketAlreadyOwnedByYou` return 409. (Partially fixed for Swift create). + - [ ] **404 Not Found**: Ensure `NoSuchKey` and `NoSuchBucket` return 404 + with correct XML body. - [ ] **403 Forbidden**: Ensure `AccessDenied` + returns 403. +- [~] **Method POST Support (PostObject)**: S3 PostObject (POST at bucket root + with multipart/form-data, policy + signature) is implemented. Authenticated + form uploads return 204/200/201; invalid policy/signature return 403. (e.g. + `test_post_object_authenticated_request`). Fix harness logging and debug + first. _(Focus tests: `test_post_object_authenticated_request`; unit tests in + `tests/postobject.test.ts`)_ +- [ ] **Multipart Reliability**: Address `502 Bad Gateway` errors occurring + during `CreateMultipartUpload` and other multipart operations. _(Focus + tests: `test_multipart_upload`)_ +- [ ] **Conditional Requests**: Fix `If-Match`, `If-None-Match`, + `If-Modified-Since`, and `If-Unmodified-Since` behavior. Currently failing + to return `412 Precondition Failed` or `304 Not Modified` correctly. + _(Focus tests: `test_get_object_ifmatch_failed`, + `test_get_object_ifnonematch_good`, + `test_get_object_ifmodifiedsince_failed`)_ +- [ ] **Response Field Completeness**: Ensure expected XML/JSON fields like + `ChecksumSHA256`, `Rules`, `Errors`, and `x-amz-delete-marker` are present + in responses. +- [ ] **Metadata Handling**: Fix incorrect `BucketAlreadyOwnedByYou` errors + being returned on non-create operations (e.g., during `PutBucketPolicy`). + _(Focus tests: `test_bucket_list_return_data`)_ + +## 5. General Compatibility & Compliance + +- [ ] **Strict RFC 2616 Compliance**: Address tests tagged with + `fails_strict_rfc2616`. +- [ ] **S3Proxy Compatibility**: Address tests tagged with `fails_on_s3proxy` to + ensure broader compatibility. +- [ ] **Advanced Header Support**: Comprehensive support for headers like + `Cache-Control`, `Content-Disposition`, `Content-Encoding`, + `Content-Language`, and `Expires`. + +## 5. Non-Standard / Protocol Specific + +- [ ] **Append Object**: Implementation of `appendobject` (often found in + Ceph/RGW). + +## 6. Architectural & DevEx + +- [ ] **Configuration Hot-Reloading**: Implement a watcher for `herald.yaml` to + invalidate the `BackendResolver` cache on configuration changes. +- [ ] **Header Marshalling Abstraction**: Centralize S3 header parsing and + generation to reduce boilerplate in the Frontend handlers. diff --git a/benchmarks/buckets.bench.ts b/benchmarks/buckets.bench.ts new file mode 100644 index 0000000..09d1252 --- /dev/null +++ b/benchmarks/buckets.bench.ts @@ -0,0 +1,140 @@ +import { + CreateBucketCommand, + DeleteBucketCommand, + HeadBucketCommand, + ListBucketsCommand, +} from "@aws-sdk/client-s3"; +import { type BenchmarkCase, benchmarkHarness } from "./utils.ts"; +import type { GlobalConfig } from "../src/Domain/Config.ts"; +import { Effect } from "effect"; +import { HttpClientRequest } from "@effect/platform"; + +const benchConfig: GlobalConfig = { + backends: { + minio: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + buckets: "*", + }, + }, +}; + +const BUCKET_PREFIX = "bench-bucket-"; + +const cases: BenchmarkCase[] = [ + { + name: "create/new", + group: "buckets", + config: benchConfig, + fn: async (client, b) => { + const bucketName = `${BUCKET_PREFIX}${ + Math.random().toString(36).substring(7) + }`; + b.start(); + await client.send(new CreateBucketCommand({ Bucket: bucketName })); + b.end(); + // Cleanup after measurement + await client.send(new DeleteBucketCommand({ Bucket: bucketName })).catch( + () => {}, + ); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + const bucketName = `${BUCKET_PREFIX}${ + Math.random().toString(36).substring(7) + }`; + b.start(); + const request = HttpClientRequest.put(`${url}/${bucketName}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(request)); + b.end(); + // Cleanup + const delReq = HttpClientRequest.del(`${url}/${bucketName}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(delReq)).catch(() => {}); + }, + }, + { + name: "list/all", + group: "buckets", + config: benchConfig, + fn: async (client, b) => { + b.start(); + await client.send(new ListBucketsCommand({})); + b.end(); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.get(`${url}?format=json`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(request)); + b.end(); + }, + }, + { + name: "head/existing", + group: "buckets", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: "head-bucket" })) + .catch(() => {}); + }, + fn: async (client, b) => { + b.start(); + await client.send(new HeadBucketCommand({ Bucket: "head-bucket" })); + b.end(); + }, + teardown: async (client) => { + await client.send(new DeleteBucketCommand({ Bucket: "head-bucket" })) + .catch(() => {}); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.head(`${url}/head-bucket`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(request)); + b.end(); + }, + }, + { + name: "delete/existing", + group: "buckets", + config: benchConfig, + fn: async (client, b) => { + const bucketName = `delete-${Math.random().toString(36).substring(7)}`; + await client.send(new CreateBucketCommand({ Bucket: bucketName })); + b.start(); + await client.send(new DeleteBucketCommand({ Bucket: bucketName })); + b.end(); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + const bucketName = `delete-${Math.random().toString(36).substring(7)}`; + // Setup + const putReq = HttpClientRequest.put(`${url}/${bucketName}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(putReq)); + + b.start(); + const request = HttpClientRequest.del(`${url}/${bucketName}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(request)); + b.end(); + }, + }, +]; + +benchmarkHarness(cases); diff --git a/benchmarks/objects.bench.ts b/benchmarks/objects.bench.ts new file mode 100644 index 0000000..096e1f7 --- /dev/null +++ b/benchmarks/objects.bench.ts @@ -0,0 +1,551 @@ +import { + CompleteMultipartUploadCommand, + CreateBucketCommand, + CreateMultipartUploadCommand, + DeleteObjectCommand, + GetObjectCommand, + HeadObjectCommand, + PutObjectCommand, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import { type BenchmarkCase, benchmarkHarness } from "./utils.ts"; +import type { GlobalConfig } from "../src/Domain/Config.ts"; +import { Effect, Stream } from "effect"; +import { HttpClientRequest } from "@effect/platform"; + +const benchConfig: GlobalConfig = { + backends: { + minio: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + buckets: "*", + }, + }, +}; + +const BUCKET = "bench-bucket-objects"; +const DATA_1KB = new Uint8Array(1024).fill(97); +const DATA_1MB = new Uint8Array(1024 * 1024).fill(97); +const DATA_10MB = new Uint8Array(10 * 1024 * 1024).fill(97); + +const getLargeData = (sizeMb: number) => + new Uint8Array(sizeMb * 1024 * 1024).fill(97); + +const cases: BenchmarkCase[] = [ + // --- PutObject --- + { + name: "put/1kb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + b.start(); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "1kb.txt", + Body: DATA_1KB, + }), + ); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "1kb.txt" }), + ).catch(() => {}); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.put(`${url}/${BUCKET}/1kb.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + HttpClientRequest.bodyUint8Array(DATA_1KB), + ); + const response = await Effect.runPromise(client.execute(request)); + await response.text; // Ensure body is consumed + b.end(); + }, + }, + { + name: "put/1mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + b.start(); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "1mb.txt", + Body: DATA_1MB, + }), + ); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "1mb.txt" }), + ).catch(() => {}); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.put(`${url}/${BUCKET}/1mb.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + HttpClientRequest.bodyUint8Array(DATA_1MB), + ); + const response = await Effect.runPromise(client.execute(request)); + await response.text; + b.end(); + }, + }, + { + name: "put/10mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + b.start(); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "10mb.txt", + Body: DATA_10MB, + }), + ); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "10mb.txt" }), + ).catch(() => {}); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.put(`${url}/${BUCKET}/10mb.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + HttpClientRequest.bodyUint8Array(DATA_10MB), + ); + const response = await Effect.runPromise(client.execute(request)); + await response.text; + b.end(); + }, + }, + { + name: "put/100mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + const data = getLargeData(100); + b.start(); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "100mb.txt", + Body: data, + }), + ); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "100mb.txt" }), + ).catch(() => {}); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + const data = getLargeData(100); + b.start(); + const request = HttpClientRequest.put(`${url}/${BUCKET}/100mb.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + HttpClientRequest.bodyUint8Array(data), + ); + const response = await Effect.runPromise(client.execute(request)); + await response.text; + b.end(); + }, + }, + // --- HeadObject --- + { + name: "get/1kb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "get-1kb.txt", + Body: DATA_1KB, + }), + ); + }, + fn: async (client, b) => { + b.start(); + const res = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: "get-1kb.txt" }), + ); + await res.Body?.transformToByteArray(); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "get-1kb.txt" }), + ).catch(() => {}); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.get(`${url}/${BUCKET}/get-1kb.txt`) + .pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + const response = await Effect.runPromise(client.execute(request)); + await Effect.runPromise(Stream.runDrain(response.stream)); + b.end(); + }, + }, + { + name: "get/1mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "get-1mb.txt", + Body: DATA_1MB, + }), + ); + }, + fn: async (client, b) => { + b.start(); + const res = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: "get-1mb.txt" }), + ); + await res.Body?.transformToByteArray(); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "get-1mb.txt" }), + ).catch(() => {}); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.get(`${url}/${BUCKET}/get-1mb.txt`) + .pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + const response = await Effect.runPromise(client.execute(request)); + await Effect.runPromise(Stream.runDrain(response.stream)); + b.end(); + }, + }, + { + name: "get/10mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "get-10mb.txt", + Body: DATA_10MB, + }), + ); + }, + fn: async (client, b) => { + b.start(); + const res = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: "get-10mb.txt" }), + ); + await res.Body?.transformToByteArray(); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "get-10mb.txt" }), + ).catch(() => {}); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.get(`${url}/${BUCKET}/get-10mb.txt`) + .pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + const response = await Effect.runPromise(client.execute(request)); + await Effect.runPromise(Stream.runDrain(response.stream)); + b.end(); + }, + }, + { + name: "get/100mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "get-100mb.txt", + Body: getLargeData(100), + }), + ); + }, + fn: async (client, b) => { + b.start(); + const res = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: "get-100mb.txt" }), + ); + await res.Body?.transformToByteArray(); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "get-100mb.txt" }), + ).catch(() => {}); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.get(`${url}/${BUCKET}/get-100mb.txt`) + .pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + const response = await Effect.runPromise(client.execute(request)); + await Effect.runPromise(Stream.runDrain(response.stream)); + b.end(); + }, + }, + + // --- HeadObject --- + { + name: "head/existing", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "head.txt", + Body: DATA_1KB, + }), + ); + }, + fn: async (client, b) => { + b.start(); + await client.send( + new HeadObjectCommand({ Bucket: BUCKET, Key: "head.txt" }), + ); + b.end(); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.head(`${url}/${BUCKET}/head.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(request)); + b.end(); + }, + }, + + // --- DeleteObject --- + { + name: "delete/existing", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "delete.txt", + Body: DATA_1KB, + }), + ); + + b.start(); + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "delete.txt" }), + ); + b.end(); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + // Pre-upload for delete + const putReq = HttpClientRequest.put( + `${url}/${BUCKET}/delete-direct.txt`, + ).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + HttpClientRequest.bodyUint8Array(DATA_1KB), + ); + await Effect.runPromise(client.execute(putReq)); + + b.start(); + const request = HttpClientRequest.del( + `${url}/${BUCKET}/delete-direct.txt`, + ).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(request)); + b.end(); + }, + }, + + // --- Multipart Upload --- + { + name: "multipart/10mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + const key = "multipart-10mb.txt"; + const partSize = 5 * 1024 * 1024; + const body = new Uint8Array(partSize).fill(97); + + b.start(); + const { UploadId } = await client.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + const { ETag: etag1 } = await client.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: body, + }), + ); + const { ETag: etag2 } = await client.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 2, + Body: body, + }), + ); + await client.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { + Parts: [ + { ETag: etag1, PartNumber: 1 }, + { ETag: etag2, PartNumber: 2 }, + ], + }, + }), + ); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "multipart-10mb.txt" }), + ).catch(() => {}); + }, + }, + { + name: "multipart/100mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + const key = "multipart-100mb.txt"; + const partSize = 10 * 1024 * 1024; + const body = new Uint8Array(partSize).fill(97); + + b.start(); + const { UploadId } = await client.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + + const parts = await Promise.all( + Array.from({ length: 10 }, (_, i) => i + 1).map(async (i) => { + const { ETag } = await client.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: i, + Body: body, + }), + ); + return { ETag, PartNumber: i }; + }), + ); + + await client.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { + Parts: parts, + }, + }), + ); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "multipart-100mb.txt" }), + ).catch(() => {}); + }, + }, +]; + +benchmarkHarness(cases); diff --git a/benchmarks/utils.ts b/benchmarks/utils.ts new file mode 100644 index 0000000..33e343b --- /dev/null +++ b/benchmarks/utils.ts @@ -0,0 +1,382 @@ +import { S3Client } from "@aws-sdk/client-s3"; +import { Config, Effect, Layer, Logger, LogLevel, Option, Scope } from "effect"; +import { HttpHeraldLive } from "../src/Http.ts"; +import { HeraldConfig } from "../src/Config/Layer.ts"; +import { lookupBucket } from "../src/Domain/Config.ts"; +import { BackendResolver } from "../src/Services/BackendResolver.ts"; +import { S3ClientFactory } from "../src/Backends/S3/Client.ts"; +import { SwiftClient } from "../src/Backends/Swift/Client.ts"; +import { S3XmlLive } from "../src/Services/S3Xml.ts"; +import { Checksum } from "../src/Services/Checksum.ts"; +import { S3HeaderService } from "../src/Services/S3HeaderService.ts"; +import { HttpApiBuilder, HttpServer } from "@effect/platform"; +import { FetchHttpClient, HttpClient } from "@effect/platform"; +import type { GlobalConfig } from "../src/Domain/Config.ts"; + +export type BenchmarkCase = { + name: string; + config: GlobalConfig; + fn: (client: S3Client, b: Deno.BenchContext) => Promise; + // For direct comparisons that don't use S3 SDK + directSwiftFn?: ( + target: { url: string; token: string; container: string }, + client: HttpClient.HttpClient, + b: Deno.BenchContext, + ) => Promise; + setup?: (client: S3Client) => Promise; + teardown?: (client: S3Client) => Promise; + group?: string; + baseline?: boolean; + ignore?: boolean; + only?: boolean; +}; + +export const getSwiftConfig = () => + Effect.gen(function* () { + const authUrl = yield* Config.string("HERALD_SWIFTTEST_AUTH_URL").pipe( + Config.orElse(() => Config.string("OS_AUTH_URL")), + Config.withDefault("http://localhost:8080/auth/v1.0"), + Config.option, + ); + + const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_USERNAME")), + Config.orElse(() => Config.string("OS_USERNAME")), + Config.withDefault("test:tester"), + Config.option, + ); + const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PASSWORD")), + Config.orElse(() => Config.string("OS_PASSWORD")), + Config.withDefault("testing"), + Config.option, + ); + const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") + .pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PROJECT_NAME")), + Config.orElse(() => Config.string("OS_PROJECT_NAME")), + Config.option, + ); + const region = yield* Config.string("HERALD_SWIFTTEST_OS_REGION_NAME").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_REGION_NAME")), + Config.orElse(() => Config.string("OS_REGION_NAME")), + Config.withDefault("dc3-a"), + Config.option, + ); + + if ( + Option.isNone(username) || Option.isNone(password) || + Option.isNone(authUrl) + ) { + return Option.none(); + } + + const config: GlobalConfig = { + backends: { + swift: { + protocol: "swift", + auth_url: authUrl.value, + region: Option.getOrUndefined(region), + credentials: { + username: username.value, + password: password.value, + project_name: Option.getOrUndefined(projectName), + user_domain_name: "Default", + project_domain_name: "Default", + }, + buckets: "*", + }, + }, + }; + return Option.some(config); + }); + +export interface BenchHarness { + proxyUrl: string; + minioUrl: string; + directClient: S3Client; + proxyClient: S3Client; + // Raw swift target for direct comparisons + swiftTarget?: { url: string; token: string; container: string }; + httpClient?: HttpClient.HttpClient; +} + +export const makeBenchHarness = ( + config: GlobalConfig, +): Effect.Effect => + Effect.gen(function* () { + const benchCredentials = { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }; + + const HeraldConfigLive = Layer.succeed(HeraldConfig, { + raw: config, + lookupBucket: (name: string) => lookupBucket(config, name), + resolveAuth: () => Option.some([benchCredentials]), + resolveAuthForBackendId: () => Option.some([benchCredentials]), + }); + + const ApiWithRequirements = HttpHeraldLive.pipe( + Layer.provide(BackendResolver.Default), + Layer.provide(S3ClientFactory.Default), + Layer.provide(SwiftClient.Default), + Layer.provide(S3XmlLive), + Layer.provide(Checksum.Default), + Layer.provide(S3HeaderService.Default), + Layer.provide(HeraldConfigLive), + Layer.provide(FetchHttpClient.layer), + Layer.provide(Layer.succeed(FetchHttpClient.RequestInit, { + // @ts-ignore: duplex is required for streaming body in fetch + duplex: "half", + })), + Layer.provideMerge(HttpServer.layerContext), + Layer.provideMerge(Logger.minimumLogLevel(LogLevel.None)), + ); + + const webHandler = HttpApiBuilder.toWebHandler(ApiWithRequirements); + + const server = Deno.serve( + { port: 0, onListen: () => {} }, + async (req) => { + try { + return await webHandler.handler(req); + } catch (_e) { + return new Response("Internal Server Error", { status: 500 }); + } + }, + ); + + yield* Effect.addFinalizer(() => + Effect.tryPromise(() => server.shutdown()).pipe(Effect.orDie) + ); + yield* Effect.addFinalizer(() => + Effect.tryPromise(() => webHandler.dispose()).pipe(Effect.orDie) + ); + + const proxyUrl = `http://localhost:${server.addr.port}`; + const minioUrl = "http://localhost:9000"; + const credentials = { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }; + + const directClient = new S3Client({ + endpoint: minioUrl, + region: "us-east-1", + credentials, + forcePathStyle: true, + requestChecksumCalculation: "WHEN_REQUIRED", + responseChecksumValidation: "WHEN_REQUIRED", + }); + + const proxyClient = new S3Client({ + endpoint: proxyUrl, + region: "us-east-1", + credentials, + forcePathStyle: true, + requestChecksumCalculation: "WHEN_REQUIRED", + responseChecksumValidation: "WHEN_REQUIRED", + }); + + let swiftTarget: BenchHarness["swiftTarget"] = undefined; + let httpClient: HttpClient.HttpClient | undefined = undefined; + + // If swift is configured, get a token for direct benchmarks + const swiftBackendId = Object.keys(config.backends).find((k) => + config.backends[k].protocol === "swift" + ); + if (swiftBackendId) { + const swiftClient = yield* SwiftClient; + const authMeta = yield* swiftClient.getAuthMeta({ + backend_id: swiftBackendId, + }); + swiftTarget = { + url: authMeta.storageUrl, + token: authMeta.token, + container: "bench-bucket", // Fixed for bench + }; + httpClient = yield* HttpClient.HttpClient; + } + + return { + proxyUrl, + minioUrl, + directClient, + proxyClient, + swiftTarget, + httpClient, + }; + }).pipe( + // We need to provide the requirements for SwiftClient and HttpClient + Effect.provide(SwiftClient.Default), + Effect.provide(FetchHttpClient.layer), + Effect.provide(Layer.succeed(FetchHttpClient.RequestInit, { + // @ts-ignore: duplex is required for streaming body in fetch + duplex: "half", + })), + Effect.provide( + Layer.succeed(HeraldConfig, { + raw: config, + lookupBucket: (name: string) => lookupBucket(config, name), + resolveAuth: () => + Option.some([{ + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }]), + resolveAuthForBackendId: () => + Option.some([{ + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }]), + }), + ), + ); + +// Global state for harnesses to avoid iterative restarts +let minioHarness: BenchHarness | null = null; +let swiftHarness: BenchHarness | null = null; +let globalScope: Scope.Scope | null = null; + +// Check swift config once at the beginning +const swiftConfigOpt = await Effect.runPromise(getSwiftConfig()); + +async function ensureHarnesses(bc: BenchmarkCase) { + if (globalScope) return; + + globalScope = Effect.runSync(Scope.make()); + + minioHarness = await Effect.runPromise( + makeBenchHarness(bc.config).pipe( + Effect.provideService(Scope.Scope, globalScope), + ), + ); + + if (Option.isSome(swiftConfigOpt)) { + swiftHarness = await Effect.runPromise( + makeBenchHarness(swiftConfigOpt.value).pipe( + Effect.provideService(Scope.Scope, globalScope), + ), + ); + } +} + +export function benchmarkHarness(cases: BenchmarkCase[]) { + for (const bc of cases) { + const operationName = `${bc.group ? `${bc.group}/` : ""}${bc.name}`; + const s3Group = `${operationName} (S3)`; + const swiftGroup = `${operationName} (Swift)`; + + // 1. Baseline (Direct Minio) + Deno.bench({ + name: `Minio-Direct`, + group: s3Group, + ignore: bc.ignore, + only: bc.only, + fn: async (b) => { + await ensureHarnesses(bc); + const client = minioHarness!.directClient; + + try { + if (bc.setup) await bc.setup(client); + } catch (e) { + throw new Error(`Setup failed for ${operationName} (Baseline): ${e}`); + } + + await bc.fn(client, b); + + if (bc.teardown) { + await bc.teardown(client).catch(() => {}); + } + }, + }); + + // 2. Proxy (Herald + Minio) + Deno.bench({ + name: `Herald-Proxy`, + baseline: true, + group: s3Group, + ignore: bc.ignore, + only: bc.only, + fn: async (b) => { + await ensureHarnesses(bc); + const client = minioHarness!.proxyClient; + + try { + if (bc.setup) await bc.setup(client); + } catch (e) { + throw new Error(`Setup failed for ${operationName} (Proxy): ${e}`); + } + + await bc.fn(client, b); + + if (bc.teardown) { + await bc.teardown(client).catch(() => {}); + } + }, + }); + + // 3. Swift Proxy (Herald + Swift) + Deno.bench({ + name: `Swift-Proxy`, + group: swiftGroup, + baseline: true, + ignore: bc.ignore || Option.isNone(swiftConfigOpt), + only: bc.only, + fn: async (b) => { + await ensureHarnesses(bc); + if (!swiftHarness) return; + const client = swiftHarness.proxyClient; + + try { + if (bc.setup) await bc.setup(client); + } catch (e) { + throw new Error( + `Setup failed for ${operationName} (Swift-Proxy): ${e}`, + ); + } + + await bc.fn(client, b); + + if (bc.teardown) { + await bc.teardown(client).catch(() => {}); + } + }, + }); + + // 4. Swift Direct (Raw Swift API) + if (bc.directSwiftFn) { + Deno.bench({ + name: `Swift-Direct`, + group: swiftGroup, + ignore: bc.ignore || Option.isNone(swiftConfigOpt), + only: bc.only, + fn: async (b) => { + await ensureHarnesses(bc); + if ( + !swiftHarness || !swiftHarness.swiftTarget || + !swiftHarness.httpClient + ) return; + + try { + if (bc.setup) await bc.setup(swiftHarness.proxyClient); + } catch (e) { + throw new Error( + `Setup failed for ${operationName} (Swift-Direct): ${e}`, + ); + } + + await bc.directSwiftFn!( + swiftHarness.swiftTarget, + swiftHarness.httpClient, + b, + ); + + if (bc.teardown) { + await bc.teardown(swiftHarness.proxyClient).catch(() => {}); + } + }, + }); + } + } +} diff --git a/chart/Chart.yaml b/chart/Chart.yaml index f0c2a1e..51d9887 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 name: herald -description: A Helm chart for the herald application -version: 0.7.0 -appVersion: "1.0" +description: A Helm chart for Herald (S3 proxy with backend routing) +version: 0.11.0 +appVersion: "0.11.0" diff --git a/chart/README.md b/chart/README.md index e69de29..834b807 100644 --- a/chart/README.md +++ b/chart/README.md @@ -0,0 +1,39 @@ +# Herald Helm Chart + +Deploy [Herald](https://github.com/expnt/herald) (S3 proxy with backend routing) on Kubernetes. + +## Install + +```bash +# Install with default values (single replica, config from values) +helm install my-herald ./chart -n herald --create-namespace + +# Install with custom config file +helm install my-herald ./chart -n herald --create-namespace -f my-values.yaml +``` + +## Configuration + +| Value | Description | Default | +| ----- | ----------- | ------- | +| `config` | Herald [GlobalConfig](https://github.com/expnt/herald#config): `backends` (required), optional `cors`, `auth`. Rendered as `herald-config.yaml` in a ConfigMap. | `backends: {}` (you must set backends, e.g. S3 or openstack_swift) | +| `port` | App listen port (container port and health probes) | `3000` | +| `image.repository` | Container image | `ghcr.io/expnt/herald` | +| `image.tag` | Image tag | `v0.11.0` | +| `replicaCount` | Number of replicas | `1` | +| `service.port` | Service port | `80` | +| `ingress.enabled` | Create an Ingress | `true` | +| `extraEnv` | Additional env vars (e.g. `HERALD_LOG_LEVEL`, `HERALD__*` for backend creds) | `[]` | +| `extraEnvFrom` | Env from Secrets/ConfigMaps | `{}` | +| `resources` | Pod resource requests/limits | `{}` | + +Config schema: each backend has `protocol` (`s3` or `swift`), optional `endpoint`, `region`, `credentials`, and `buckets` (`"*"` or a map of bucket names to overrides). See the [main README](../README.md) for full config docs, env vars, auth, and CORS. + +## Endpoints + +- **Health:** `GET /health` returns `{ "status": "ok" }` (used for liveness/readiness). +- **S3 API:** Path prefix `/s3`. Use `https:///s3` as the S3 endpoint URL with path-style. + +## Service account + +The chart can create a Kubernetes ServiceAccount for the deployment (set `serviceAccount.create: true`). Herald itself does not use a custom “service account” concept; this is only for pod identity and RBAC. diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 08e8dea..36092bc 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -38,13 +38,17 @@ spec: {{- toYaml .Values.containerSecurityContext | nindent 12 }} ports: - name: http - containerPort: 8000 + containerPort: {{ .Values.port }} protocol: TCP envFrom: {{- with .Values.extraEnvFrom }} {{- toYaml . | nindent 12 }} {{- end }} env: + - name: HERALD_CONFIG_PATH + value: "/etc/herald/herald-config.yaml" + - name: PORT + value: {{ .Values.port | quote }} {{- with .Values.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} @@ -54,16 +58,19 @@ spec: {{- end }} livenessProbe: httpGet: - path: /health-check + path: /health port: http readinessProbe: httpGet: - path: /health-check + path: /health port: http resources: {{- toYaml .Values.resources | nindent 12 }} volumes: - {{- with .Values.volumes }} + - name: herald + configMap: + name: {{ include "herald.fullname" . }} + {{- with .Values.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/chart/templates/herald-config.yaml b/chart/templates/herald-config.yaml index 823b64f..751811e 100644 --- a/chart/templates/herald-config.yaml +++ b/chart/templates/herald-config.yaml @@ -1,10 +1,10 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ .Chart.Name }} + name: {{ include "herald.fullname" . }} namespace: {{ .Values.namespace }} + labels: + {{- include "herald.labels" . | nindent 4 }} data: - herald-config.yaml: - {{- with .Values.heraldConfig }} - {{- toYaml . | nindent 4 }} - {{- end }} + herald-config.yaml: | + {{- .Values.config | toYaml | nindent 4 }} diff --git a/chart/templates/serviceaccount.yaml b/chart/templates/serviceaccount.yaml index 25678c7..95e3a5f 100644 --- a/chart/templates/serviceaccount.yaml +++ b/chart/templates/serviceaccount.yaml @@ -4,7 +4,7 @@ kind: ServiceAccount metadata: name: {{ include "herald.serviceAccountName" . }} labels: - {{- include "generic.labels" . | nindent 4 }} + {{- include "herald.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/chart/values.yaml b/chart/values.yaml index e55ba44..2d338cf 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1,43 +1,27 @@ - name: herald namespace: herald replicaCount: 1 -heraldConfig: - port: 8000 - temp_dir: "./tmp" - task_store_backend: - endpoint: http://minio.herald:9000 - region: local - forcePathStyle: true - bucket: s3-test - credentials: - accessKeyId: "fromEnv:S3_ACCESS_KEY" - secretAccessKey: "fromEnv:S3_SECRET_KEY" - backends: - minio_s3: - protocol: s3 - openstack_swift: - protocol: swift - service_accounts: [] - default_bucket: "s3-test" - buckets: - s3-test: - backend: minio_s3 - config: - endpoint: http://minio.herald:9000 - region: local - forcePathStyle: true - bucket: s3-test - credentials: - accessKeyId: "fromEnv:S3_ACCESS_KEY" - secretAccessKey: "fromEnv:S3_SECRET_KEY" - replicas: [] +# Herald config (GlobalConfig). Rendered as herald-config.yaml in the ConfigMap. +# See repo README for full config docs. Backends each have protocol, endpoint?, region?, +# credentials?, buckets ("*" or map of bucket name to overrides). +# Override with your backends (e.g. S3/minio or openstack_swift). No default backend +# so Helm merge does not add a stray minio when you only pass e.g. openstack_swift. +config: + backends: {} + # Example S3 backend (uncomment and set credentials): + # minio: + # protocol: s3 + # endpoint: http://minio.herald:9000 + # region: us-east-1 + # credentials: { accessKeyId: "", secretAccessKey: "" } + # buckets: "*" + # Optional: cors, auth (accessKeysRefs) at root or per backend/bucket image: repository: ghcr.io/expnt/herald - tag: "v0.7.0" + tag: "v0.11.0" pullPolicy: IfNotPresent imagePullSecrets: [] @@ -56,22 +40,17 @@ deploymentAnnotations: {} podSecurityContext: {} securityContext: {} +containerSecurityContext: {} resources: {} -extraEnvFrom: {} -extraEnv: - - name: CONFIG_FILE_PATH - value: "/etc/herald/herald-config.yaml" - - name: AUTH_TYPE - value: "none" - - name: SENTRY_DSN - value: "" - - name: S3_ACCESS_KEY - value: "minio" - - name: S3_SECRET_KEY - value: "password" - -containerPort: 8000 +# envFrom: list of secretRef/configMapRef for env (e.g. backend credentials) +extraEnvFrom: [] +# Herald reads HERALD_CONFIG_PATH and PORT; these are set from the chart (see deployment). +# Add HERALD_* or other env here. Backend credentials can go in config or HERALD__*. +extraEnv: [] + +# App port (Herald default 3000). Used for containerPort and health probes. +port: 3000 service: type: ClusterIP @@ -87,19 +66,18 @@ ingress: - path: / pathType: ImplementationSpecific tls: [] - # - secretName: web-tls - # hosts: - # - chart-example.local +# - secretName: web-tls +# hosts: +# - chart-example.local volumeMounts: - name: herald mountPath: /etc/herald/ readOnly: true -volumes: - - name: herald - configMap: - name: herald +# Default volume (herald config) is defined in the deployment template with fullname. +# Add extra volumes here if needed. +extraVolumes: [] helmhookjob: enabled: false diff --git a/deno.jsonc b/deno.jsonc new file mode 100644 index 0000000..c18560e --- /dev/null +++ b/deno.jsonc @@ -0,0 +1,67 @@ +{ + "tasks": { + "dev": "deno run --allow-all --watch src/main.ts", + "test": "deno test --allow-all tests/", + "snapdiff": "deno run --allow-all x/snapdiff.ts" + }, + "imports": { + "@david/dax": "jsr:@david/dax@^0.44.2", + "@effect/platform": "npm:@effect/platform@^0.90.3", + "@effect/platform-node": "npm:@effect/platform-node@^0.96.0", + "@effect/opentelemetry": "npm:@effect/opentelemetry@^0.56.2", + "@effect/opentelemetry/NodeSdk": "npm:@effect/opentelemetry@^0.56.2/NodeSdk", + "@opentelemetry/exporter-trace-otlp-http": "npm:@opentelemetry/exporter-trace-otlp-http@^0.203.0", + "@opentelemetry/sdk-trace-base": "npm:@opentelemetry/sdk-trace-base@^2.0.1", + "@opentelemetry/sdk-trace-node": "npm:@opentelemetry/sdk-trace-node@^2.0.1", + "@smithy/node-http-handler": "npm:@smithy/node-http-handler@^4.4.8", + "@std/assert": "jsr:@std/assert@1", + "@std/yaml": "jsr:@std/yaml@^1.0.5", + "@std/path": "jsr:@std/path@^1.0.8", + "@std/fmt": "jsr:@std/fmt@^1.0.3", + "@std/testing": "jsr:@std/testing@^1.0.0", + "@smithy/signature-v4": "npm:@smithy/signature-v4@^4.2.0", + "@smithy/types": "npm:@smithy/types@^3.7.0", + "@aws-crypto/sha256": "npm:@aws-crypto/sha256-js@^5.2.0", + "@aws-sdk/client-s3": "npm:@aws-sdk/client-s3@^3.x", + "effect": "npm:effect@^3.17.7", + "xml2js": "npm:xml2js@0.6.2", + "node-http": "node:http", + "node-assert": "node:assert", + "node-crypto": "node:crypto", + "node-buffer": "node:buffer", + "node-stream": "node:stream", + "node-stream/web": "node:stream/web", + "jest-diff": "npm:jest-diff@^29.7.0", + "cliffy/ansi/": "https://deno.land/x/cliffy@v1.0.0-rc.3/ansi/" + }, + "compilerOptions": {}, + "fmt": { + "exclude": [ + "./chart/" + ] + }, + "lint": { + "exclude": [ + "x", + ".git", + "play.ts", + "vendor/**" + ], + "rules": { + "include": [ + "no-console", + "no-sync-fn-in-async-fn", + "no-external-import", + "no-inferrable-types", + "no-self-compare", + "no-throw-literal", + "verbatim-module-syntax", + "no-await-in-loop", + "ban-untagged-todo" + ], + "exclude": [ + // "no-explicit-any" + ] + } + } +} diff --git a/deno.lock b/deno.lock new file mode 100644 index 0000000..fcc418a --- /dev/null +++ b/deno.lock @@ -0,0 +1,1828 @@ +{ + "version": "5", + "specifiers": { + "jsr:@david/console-static-text@0.3": "0.3.0", + "jsr:@david/dax@~0.44.2": "0.44.2", + "jsr:@david/path@0.2": "0.2.0", + "jsr:@david/which@~0.4.1": "0.4.1", + "jsr:@std/assert@1": "1.0.16", + "jsr:@std/assert@^1.0.15": "1.0.16", + "jsr:@std/async@^1.0.15": "1.0.16", + "jsr:@std/bytes@^1.0.5": "1.0.6", + "jsr:@std/data-structures@^1.0.9": "1.0.9", + "jsr:@std/fmt@1": "1.0.8", + "jsr:@std/fmt@^1.0.3": "1.0.8", + "jsr:@std/fs@1": "1.0.21", + "jsr:@std/fs@^1.0.19": "1.0.21", + "jsr:@std/fs@^1.0.20": "1.0.21", + "jsr:@std/internal@^1.0.12": "1.0.12", + "jsr:@std/io@0.225": "0.225.2", + "jsr:@std/path@1": "1.1.4", + "jsr:@std/path@^1.0.8": "1.1.4", + "jsr:@std/path@^1.1.2": "1.1.4", + "jsr:@std/path@^1.1.4": "1.1.4", + "jsr:@std/testing@1": "1.0.16", + "jsr:@std/yaml@^1.0.5": "1.0.9", + "npm:@aws-crypto/sha256-js@^5.2.0": "5.2.0", + "npm:@aws-sdk/client-s3@*": "3.937.0", + "npm:@aws-sdk/client-s3@3": "3.937.0", + "npm:@effect/opentelemetry@~0.56.2": "0.56.6_@effect+platform@0.90.10__effect@3.19.14_@opentelemetry+sdk-trace-base@2.3.0__@opentelemetry+api@1.9.0_@opentelemetry+sdk-trace-node@2.3.0__@opentelemetry+api@1.9.0_@opentelemetry+semantic-conventions@1.38.0_effect@3.19.14", + "npm:@effect/platform-node@0.96": "0.96.1_@effect+cluster@0.48.16__@effect+platform@0.90.10___effect@3.19.14__@effect+rpc@0.69.5___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+sql@0.44.2___@effect+experimental@0.54.6____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+workflow@0.9.6___@effect+platform@0.90.10____effect@3.19.14___@effect+rpc@0.69.5____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___effect@3.19.14__effect@3.19.14_@effect+platform@0.90.10__effect@3.19.14_@effect+rpc@0.69.5__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_@effect+sql@0.44.2__@effect+experimental@0.54.6___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_effect@3.19.14", + "npm:@effect/platform@*": "0.90.10_effect@3.19.14", + "npm:@effect/platform@~0.90.3": "0.90.10_effect@3.19.14", + "npm:@opentelemetry/exporter-trace-otlp-http@0.203": "0.203.0_@opentelemetry+api@1.9.0", + "npm:@opentelemetry/sdk-trace-base@^2.0.1": "2.3.0_@opentelemetry+api@1.9.0", + "npm:@opentelemetry/sdk-trace-node@^2.0.1": "2.3.0_@opentelemetry+api@1.9.0", + "npm:@smithy/node-http-handler@^4.4.8": "4.4.8", + "npm:@smithy/signature-v4@^4.2.0": "4.2.4", + "npm:@smithy/types@^3.7.0": "3.7.2", + "npm:effect@*": "3.19.14", + "npm:effect@^3.17.7": "3.19.14", + "npm:jest-diff@*": "29.7.0", + "npm:jest-diff@^29.7.0": "29.7.0", + "npm:npm@*": "11.7.0", + "npm:xml2js@0.6.2": "0.6.2" + }, + "jsr": { + "@david/console-static-text@0.3.0": { + "integrity": "2dfb46ecee525755f7989f94ece30bba85bd8ffe3e8666abc1bf926e1ee0698d" + }, + "@david/dax@0.44.2": { + "integrity": "26f5985f66a4340d55fb05ca90a0063bb5f0d670a326e14cb33a974aafcbb8d9", + "dependencies": [ + "jsr:@david/console-static-text", + "jsr:@david/path", + "jsr:@david/which", + "jsr:@std/fmt@1", + "jsr:@std/fs@^1.0.20", + "jsr:@std/io", + "jsr:@std/path@1" + ] + }, + "@david/path@0.2.0": { + "integrity": "f2d7aa7f02ce5a55e27c09f9f1381794acb09d328f8d3c8a2e3ab3ffc294dccd", + "dependencies": [ + "jsr:@std/fs@1", + "jsr:@std/path@1" + ] + }, + "@david/which@0.4.1": { + "integrity": "896a682b111f92ab866cc70c5b4afab2f5899d2f9bde31ed00203b9c250f225e" + }, + "@std/assert@1.0.16": { + "integrity": "6a7272ed1eaa77defe76e5ff63ca705d9c495077e2d5fd0126d2b53fc5bd6532", + "dependencies": [ + "jsr:@std/internal" + ] + }, + "@std/async@1.0.16": { + "integrity": "6c9e43035313b67b5de43e2b3ee3eadb39a488a0a0a3143097f112e025d3ee9a" + }, + "@std/bytes@1.0.6": { + "integrity": "f6ac6adbd8ccd99314045f5703e23af0a68d7f7e58364b47d2c7f408aeb5820a" + }, + "@std/data-structures@1.0.9": { + "integrity": "033d6e17e64bf1f84a614e647c1b015fa2576ae3312305821e1a4cb20674bb4d" + }, + "@std/fmt@1.0.8": { + "integrity": "71e1fc498787e4434d213647a6e43e794af4fd393ef8f52062246e06f7e372b7" + }, + "@std/fs@1.0.21": { + "integrity": "d720fe1056d78d43065a4d6e0eeb2b19f34adb8a0bc7caf3a4dbf1d4178252cd", + "dependencies": [ + "jsr:@std/internal", + "jsr:@std/path@^1.1.4" + ] + }, + "@std/internal@1.0.12": { + "integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027" + }, + "@std/io@0.225.2": { + "integrity": "3c740cd4ee4c082e6cfc86458f47e2ab7cb353dc6234d5e9b1f91a2de5f4d6c7", + "dependencies": [ + "jsr:@std/bytes" + ] + }, + "@std/path@1.1.4": { + "integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5", + "dependencies": [ + "jsr:@std/internal" + ] + }, + "@std/testing@1.0.16": { + "integrity": "a917ffdeb5924c9be436dc78bc32e511760e14d3a96e49c607fc5ecca86d0092", + "dependencies": [ + "jsr:@std/assert@^1.0.15", + "jsr:@std/async", + "jsr:@std/data-structures", + "jsr:@std/fs@^1.0.19", + "jsr:@std/internal", + "jsr:@std/path@^1.1.2" + ] + }, + "@std/yaml@1.0.9": { + "integrity": "6bad3dc766dd85b4b37eabcba81b6aa4eac7a392792ae29abcfb0f90602d55bb" + } + }, + "npm": { + "@aws-crypto/crc32@5.2.0": { + "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", + "dependencies": [ + "@aws-crypto/util", + "@aws-sdk/types", + "tslib" + ] + }, + "@aws-crypto/crc32c@5.2.0": { + "integrity": "sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==", + "dependencies": [ + "@aws-crypto/util", + "@aws-sdk/types", + "tslib" + ] + }, + "@aws-crypto/sha1-browser@5.2.0": { + "integrity": "sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==", + "dependencies": [ + "@aws-crypto/supports-web-crypto", + "@aws-crypto/util", + "@aws-sdk/types", + "@aws-sdk/util-locate-window", + "@smithy/util-utf8@2.3.0", + "tslib" + ] + }, + "@aws-crypto/sha256-browser@5.2.0": { + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "dependencies": [ + "@aws-crypto/sha256-js", + "@aws-crypto/supports-web-crypto", + "@aws-crypto/util", + "@aws-sdk/types", + "@aws-sdk/util-locate-window", + "@smithy/util-utf8@2.3.0", + "tslib" + ] + }, + "@aws-crypto/sha256-js@5.2.0": { + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "dependencies": [ + "@aws-crypto/util", + "@aws-sdk/types", + "tslib" + ] + }, + "@aws-crypto/supports-web-crypto@5.2.0": { + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "dependencies": [ + "tslib" + ] + }, + "@aws-crypto/util@5.2.0": { + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": [ + "@aws-sdk/types", + "@smithy/util-utf8@2.3.0", + "tslib" + ] + }, + "@aws-sdk/client-s3@3.937.0": { + "integrity": "sha512-ioeNe6HSc7PxjsUQY7foSHmgesxM5KwAeUtPhIHgKx99nrM+7xYCfW4FMvHypUzz7ZOvqlCdH7CEAZ8ParBvVg==", + "dependencies": [ + "@aws-crypto/sha1-browser", + "@aws-crypto/sha256-browser", + "@aws-crypto/sha256-js", + "@aws-sdk/core", + "@aws-sdk/credential-provider-node", + "@aws-sdk/middleware-bucket-endpoint", + "@aws-sdk/middleware-expect-continue", + "@aws-sdk/middleware-flexible-checksums", + "@aws-sdk/middleware-host-header", + "@aws-sdk/middleware-location-constraint", + "@aws-sdk/middleware-logger", + "@aws-sdk/middleware-recursion-detection", + "@aws-sdk/middleware-sdk-s3", + "@aws-sdk/middleware-ssec", + "@aws-sdk/middleware-user-agent", + "@aws-sdk/region-config-resolver", + "@aws-sdk/signature-v4-multi-region", + "@aws-sdk/types", + "@aws-sdk/util-endpoints", + "@aws-sdk/util-user-agent-browser", + "@aws-sdk/util-user-agent-node", + "@smithy/config-resolver", + "@smithy/core", + "@smithy/eventstream-serde-browser", + "@smithy/eventstream-serde-config-resolver", + "@smithy/eventstream-serde-node", + "@smithy/fetch-http-handler", + "@smithy/hash-blob-browser", + "@smithy/hash-node", + "@smithy/hash-stream-node", + "@smithy/invalid-dependency", + "@smithy/md5-js", + "@smithy/middleware-content-length", + "@smithy/middleware-endpoint", + "@smithy/middleware-retry", + "@smithy/middleware-serde", + "@smithy/middleware-stack", + "@smithy/node-config-provider", + "@smithy/node-http-handler", + "@smithy/protocol-http@5.3.8", + "@smithy/smithy-client", + "@smithy/types@4.12.0", + "@smithy/url-parser", + "@smithy/util-base64", + "@smithy/util-body-length-browser", + "@smithy/util-body-length-node", + "@smithy/util-defaults-mode-browser", + "@smithy/util-defaults-mode-node", + "@smithy/util-endpoints", + "@smithy/util-middleware@4.2.8", + "@smithy/util-retry", + "@smithy/util-stream", + "@smithy/util-utf8@4.2.0", + "@smithy/util-waiter", + "tslib" + ] + }, + "@aws-sdk/client-sso@3.936.0": { + "integrity": "sha512-0G73S2cDqYwJVvqL08eakj79MZG2QRaB56Ul8/Ps9oQxllr7DMI1IQ/N3j3xjxgpq/U36pkoFZ8aK1n7Sbr3IQ==", + "dependencies": [ + "@aws-crypto/sha256-browser", + "@aws-crypto/sha256-js", + "@aws-sdk/core", + "@aws-sdk/middleware-host-header", + "@aws-sdk/middleware-logger", + "@aws-sdk/middleware-recursion-detection", + "@aws-sdk/middleware-user-agent", + "@aws-sdk/region-config-resolver", + "@aws-sdk/types", + "@aws-sdk/util-endpoints", + "@aws-sdk/util-user-agent-browser", + "@aws-sdk/util-user-agent-node", + "@smithy/config-resolver", + "@smithy/core", + "@smithy/fetch-http-handler", + "@smithy/hash-node", + "@smithy/invalid-dependency", + "@smithy/middleware-content-length", + "@smithy/middleware-endpoint", + "@smithy/middleware-retry", + "@smithy/middleware-serde", + "@smithy/middleware-stack", + "@smithy/node-config-provider", + "@smithy/node-http-handler", + "@smithy/protocol-http@5.3.8", + "@smithy/smithy-client", + "@smithy/types@4.12.0", + "@smithy/url-parser", + "@smithy/util-base64", + "@smithy/util-body-length-browser", + "@smithy/util-body-length-node", + "@smithy/util-defaults-mode-browser", + "@smithy/util-defaults-mode-node", + "@smithy/util-endpoints", + "@smithy/util-middleware@4.2.8", + "@smithy/util-retry", + "@smithy/util-utf8@4.2.0", + "tslib" + ] + }, + "@aws-sdk/core@3.936.0": { + "integrity": "sha512-eGJ2ySUMvgtOziHhDRDLCrj473RJoL4J1vPjVM3NrKC/fF3/LoHjkut8AAnKmrW6a2uTzNKubigw8dEnpmpERw==", + "dependencies": [ + "@aws-sdk/types", + "@aws-sdk/xml-builder", + "@smithy/core", + "@smithy/node-config-provider", + "@smithy/property-provider", + "@smithy/protocol-http@5.3.8", + "@smithy/signature-v4@5.3.5", + "@smithy/smithy-client", + "@smithy/types@4.12.0", + "@smithy/util-base64", + "@smithy/util-middleware@4.2.8", + "@smithy/util-utf8@4.2.0", + "tslib" + ] + }, + "@aws-sdk/credential-provider-env@3.936.0": { + "integrity": "sha512-dKajFuaugEA5i9gCKzOaVy9uTeZcApE+7Z5wdcZ6j40523fY1a56khDAUYkCfwqa7sHci4ccmxBkAo+fW1RChA==", + "dependencies": [ + "@aws-sdk/core", + "@aws-sdk/types", + "@smithy/property-provider", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/credential-provider-http@3.936.0": { + "integrity": "sha512-5FguODLXG1tWx/x8fBxH+GVrk7Hey2LbXV5h9SFzYCx/2h50URBm0+9hndg0Rd23+xzYe14F6SI9HA9c1sPnjg==", + "dependencies": [ + "@aws-sdk/core", + "@aws-sdk/types", + "@smithy/fetch-http-handler", + "@smithy/node-http-handler", + "@smithy/property-provider", + "@smithy/protocol-http@5.3.8", + "@smithy/smithy-client", + "@smithy/types@4.12.0", + "@smithy/util-stream", + "tslib" + ] + }, + "@aws-sdk/credential-provider-ini@3.936.0": { + "integrity": "sha512-TbUv56ERQQujoHcLMcfL0Q6bVZfYF83gu/TjHkVkdSlHPOIKaG/mhE2XZSQzXv1cud6LlgeBbfzVAxJ+HPpffg==", + "dependencies": [ + "@aws-sdk/core", + "@aws-sdk/credential-provider-env", + "@aws-sdk/credential-provider-http", + "@aws-sdk/credential-provider-login", + "@aws-sdk/credential-provider-process", + "@aws-sdk/credential-provider-sso", + "@aws-sdk/credential-provider-web-identity", + "@aws-sdk/nested-clients", + "@aws-sdk/types", + "@smithy/credential-provider-imds", + "@smithy/property-provider", + "@smithy/shared-ini-file-loader", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/credential-provider-login@3.936.0": { + "integrity": "sha512-8DVrdRqPyUU66gfV7VZNToh56ZuO5D6agWrkLQE/xbLJOm2RbeRgh6buz7CqV8ipRd6m+zCl9mM4F3osQLZn8Q==", + "dependencies": [ + "@aws-sdk/core", + "@aws-sdk/nested-clients", + "@aws-sdk/types", + "@smithy/property-provider", + "@smithy/protocol-http@5.3.8", + "@smithy/shared-ini-file-loader", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/credential-provider-node@3.936.0": { + "integrity": "sha512-rk/2PCtxX9xDsQW8p5Yjoca3StqmQcSfkmD7nQ61AqAHL1YgpSQWqHE+HjfGGiHDYKG7PvE33Ku2GyA7lEIJAw==", + "dependencies": [ + "@aws-sdk/credential-provider-env", + "@aws-sdk/credential-provider-http", + "@aws-sdk/credential-provider-ini", + "@aws-sdk/credential-provider-process", + "@aws-sdk/credential-provider-sso", + "@aws-sdk/credential-provider-web-identity", + "@aws-sdk/types", + "@smithy/credential-provider-imds", + "@smithy/property-provider", + "@smithy/shared-ini-file-loader", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/credential-provider-process@3.936.0": { + "integrity": "sha512-GpA4AcHb96KQK2PSPUyvChvrsEKiLhQ5NWjeef2IZ3Jc8JoosiedYqp6yhZR+S8cTysuvx56WyJIJc8y8OTrLA==", + "dependencies": [ + "@aws-sdk/core", + "@aws-sdk/types", + "@smithy/property-provider", + "@smithy/shared-ini-file-loader", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/credential-provider-sso@3.936.0": { + "integrity": "sha512-wHlEAJJvtnSyxTfNhN98JcU4taA1ED2JvuI2eePgawqBwS/Tzi0mhED1lvNIaWOkjfLd+nHALwszGrtJwEq4yQ==", + "dependencies": [ + "@aws-sdk/client-sso", + "@aws-sdk/core", + "@aws-sdk/token-providers", + "@aws-sdk/types", + "@smithy/property-provider", + "@smithy/shared-ini-file-loader", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/credential-provider-web-identity@3.936.0": { + "integrity": "sha512-v3qHAuoODkoRXsAF4RG+ZVO6q2P9yYBT4GMpMEfU9wXVNn7AIfwZgTwzSUfnjNiGva5BKleWVpRpJ9DeuLFbUg==", + "dependencies": [ + "@aws-sdk/core", + "@aws-sdk/nested-clients", + "@aws-sdk/types", + "@smithy/property-provider", + "@smithy/shared-ini-file-loader", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/middleware-bucket-endpoint@3.936.0": { + "integrity": "sha512-XLSVVfAorUxZh6dzF+HTOp4R1B5EQcdpGcPliWr0KUj2jukgjZEcqbBmjyMF/p9bmyQsONX80iURF1HLAlW0qg==", + "dependencies": [ + "@aws-sdk/types", + "@aws-sdk/util-arn-parser", + "@smithy/node-config-provider", + "@smithy/protocol-http@5.3.8", + "@smithy/types@4.12.0", + "@smithy/util-config-provider", + "tslib" + ] + }, + "@aws-sdk/middleware-expect-continue@3.936.0": { + "integrity": "sha512-Eb4ELAC23bEQLJmUMYnPWcjD3FZIsmz2svDiXEcxRkQU9r7NRID7pM7C5NPH94wOfiCk0b2Y8rVyFXW0lGQwbA==", + "dependencies": [ + "@aws-sdk/types", + "@smithy/protocol-http@5.3.8", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/middleware-flexible-checksums@3.936.0": { + "integrity": "sha512-l3GG6CrSQtMCM6fWY7foV3JQv0WJWT+3G6PSP3Ceb/KEE/5Lz5PrYFXTBf+bVoYL1b0bGjGajcgAXpstBmtHtQ==", + "dependencies": [ + "@aws-crypto/crc32", + "@aws-crypto/crc32c", + "@aws-crypto/util", + "@aws-sdk/core", + "@aws-sdk/types", + "@smithy/is-array-buffer@4.2.0", + "@smithy/node-config-provider", + "@smithy/protocol-http@5.3.8", + "@smithy/types@4.12.0", + "@smithy/util-middleware@4.2.8", + "@smithy/util-stream", + "@smithy/util-utf8@4.2.0", + "tslib" + ] + }, + "@aws-sdk/middleware-host-header@3.936.0": { + "integrity": "sha512-tAaObaAnsP1XnLGndfkGWFuzrJYuk9W0b/nLvol66t8FZExIAf/WdkT2NNAWOYxljVs++oHnyHBCxIlaHrzSiw==", + "dependencies": [ + "@aws-sdk/types", + "@smithy/protocol-http@5.3.8", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/middleware-location-constraint@3.936.0": { + "integrity": "sha512-SCMPenDtQMd9o5da9JzkHz838w3327iqXk3cbNnXWqnNRx6unyW8FL0DZ84gIY12kAyVHz5WEqlWuekc15ehfw==", + "dependencies": [ + "@aws-sdk/types", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/middleware-logger@3.936.0": { + "integrity": "sha512-aPSJ12d3a3Ea5nyEnLbijCaaYJT2QjQ9iW+zGh5QcZYXmOGWbKVyPSxmVOboZQG+c1M8t6d2O7tqrwzIq8L8qw==", + "dependencies": [ + "@aws-sdk/types", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/middleware-recursion-detection@3.936.0": { + "integrity": "sha512-l4aGbHpXM45YNgXggIux1HgsCVAvvBoqHPkqLnqMl9QVapfuSTjJHfDYDsx1Xxct6/m7qSMUzanBALhiaGO2fA==", + "dependencies": [ + "@aws-sdk/types", + "@aws/lambda-invoke-store", + "@smithy/protocol-http@5.3.8", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/middleware-sdk-s3@3.936.0": { + "integrity": "sha512-UQs/pVq4cOygsnKON0pOdSKIWkfgY0dzq4h+fR+xHi/Ng3XzxPJhWeAE6tDsKrcyQc1X8UdSbS70XkfGYr5hng==", + "dependencies": [ + "@aws-sdk/core", + "@aws-sdk/types", + "@aws-sdk/util-arn-parser", + "@smithy/core", + "@smithy/node-config-provider", + "@smithy/protocol-http@5.3.8", + "@smithy/signature-v4@5.3.5", + "@smithy/smithy-client", + "@smithy/types@4.12.0", + "@smithy/util-config-provider", + "@smithy/util-middleware@4.2.8", + "@smithy/util-stream", + "@smithy/util-utf8@4.2.0", + "tslib" + ] + }, + "@aws-sdk/middleware-ssec@3.936.0": { + "integrity": "sha512-/GLC9lZdVp05ozRik5KsuODR/N7j+W+2TbfdFL3iS+7un+gnP6hC8RDOZd6WhpZp7drXQ9guKiTAxkZQwzS8DA==", + "dependencies": [ + "@aws-sdk/types", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/middleware-user-agent@3.936.0": { + "integrity": "sha512-YB40IPa7K3iaYX0lSnV9easDOLPLh+fJyUDF3BH8doX4i1AOSsYn86L4lVldmOaSX+DwiaqKHpvk4wPBdcIPWw==", + "dependencies": [ + "@aws-sdk/core", + "@aws-sdk/types", + "@aws-sdk/util-endpoints", + "@smithy/core", + "@smithy/protocol-http@5.3.8", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/nested-clients@3.936.0": { + "integrity": "sha512-eyj2tz1XmDSLSZQ5xnB7cLTVKkSJnYAEoNDSUNhzWPxrBDYeJzIbatecOKceKCU8NBf8gWWZCK/CSY0mDxMO0A==", + "dependencies": [ + "@aws-crypto/sha256-browser", + "@aws-crypto/sha256-js", + "@aws-sdk/core", + "@aws-sdk/middleware-host-header", + "@aws-sdk/middleware-logger", + "@aws-sdk/middleware-recursion-detection", + "@aws-sdk/middleware-user-agent", + "@aws-sdk/region-config-resolver", + "@aws-sdk/types", + "@aws-sdk/util-endpoints", + "@aws-sdk/util-user-agent-browser", + "@aws-sdk/util-user-agent-node", + "@smithy/config-resolver", + "@smithy/core", + "@smithy/fetch-http-handler", + "@smithy/hash-node", + "@smithy/invalid-dependency", + "@smithy/middleware-content-length", + "@smithy/middleware-endpoint", + "@smithy/middleware-retry", + "@smithy/middleware-serde", + "@smithy/middleware-stack", + "@smithy/node-config-provider", + "@smithy/node-http-handler", + "@smithy/protocol-http@5.3.8", + "@smithy/smithy-client", + "@smithy/types@4.12.0", + "@smithy/url-parser", + "@smithy/util-base64", + "@smithy/util-body-length-browser", + "@smithy/util-body-length-node", + "@smithy/util-defaults-mode-browser", + "@smithy/util-defaults-mode-node", + "@smithy/util-endpoints", + "@smithy/util-middleware@4.2.8", + "@smithy/util-retry", + "@smithy/util-utf8@4.2.0", + "tslib" + ] + }, + "@aws-sdk/region-config-resolver@3.936.0": { + "integrity": "sha512-wOKhzzWsshXGduxO4pqSiNyL9oUtk4BEvjWm9aaq6Hmfdoydq6v6t0rAGHWPjFwy9z2haovGRi3C8IxdMB4muw==", + "dependencies": [ + "@aws-sdk/types", + "@smithy/config-resolver", + "@smithy/node-config-provider", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/signature-v4-multi-region@3.936.0": { + "integrity": "sha512-8qS0GFUqkmwO7JZ0P8tdluBmt1UTfYUah8qJXGzNh9n1Pcb0AIeT117cCSiCUtwk+gDbJvd4hhRIhJCNr5wgjg==", + "dependencies": [ + "@aws-sdk/middleware-sdk-s3", + "@aws-sdk/types", + "@smithy/protocol-http@5.3.8", + "@smithy/signature-v4@5.3.5", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/token-providers@3.936.0": { + "integrity": "sha512-vvw8+VXk0I+IsoxZw0mX9TMJawUJvEsg3EF7zcCSetwhNPAU8Xmlhv7E/sN/FgSmm7b7DsqKoW6rVtQiCs1PWQ==", + "dependencies": [ + "@aws-sdk/core", + "@aws-sdk/nested-clients", + "@aws-sdk/types", + "@smithy/property-provider", + "@smithy/shared-ini-file-loader", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/types@3.936.0": { + "integrity": "sha512-uz0/VlMd2pP5MepdrHizd+T+OKfyK4r3OA9JI+L/lPKg0YFQosdJNCKisr6o70E3dh8iMpFYxF1UN/4uZsyARg==", + "dependencies": [ + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/util-arn-parser@3.893.0": { + "integrity": "sha512-u8H4f2Zsi19DGnwj5FSZzDMhytYF/bCh37vAtBsn3cNDL3YG578X5oc+wSX54pM3tOxS+NY7tvOAo52SW7koUA==", + "dependencies": [ + "tslib" + ] + }, + "@aws-sdk/util-endpoints@3.936.0": { + "integrity": "sha512-0Zx3Ntdpu+z9Wlm7JKUBOzS9EunwKAb4KdGUQQxDqh5Lc3ta5uBoub+FgmVuzwnmBu9U1Os8UuwVTH0Lgu+P5w==", + "dependencies": [ + "@aws-sdk/types", + "@smithy/types@4.12.0", + "@smithy/url-parser", + "@smithy/util-endpoints", + "tslib" + ] + }, + "@aws-sdk/util-locate-window@3.893.0": { + "integrity": "sha512-T89pFfgat6c8nMmpI8eKjBcDcgJq36+m9oiXbcUzeU55MP9ZuGgBomGjGnHaEyF36jenW9gmg3NfZDm0AO2XPg==", + "dependencies": [ + "tslib" + ] + }, + "@aws-sdk/util-user-agent-browser@3.936.0": { + "integrity": "sha512-eZ/XF6NxMtu+iCma58GRNRxSq4lHo6zHQLOZRIeL/ghqYJirqHdenMOwrzPettj60KWlv827RVebP9oNVrwZbw==", + "dependencies": [ + "@aws-sdk/types", + "@smithy/types@4.12.0", + "bowser", + "tslib" + ] + }, + "@aws-sdk/util-user-agent-node@3.936.0": { + "integrity": "sha512-XOEc7PF9Op00pWV2AYCGDSu5iHgYjIO53Py2VUQTIvP7SRCaCsXmA33mjBvC2Ms6FhSyWNa4aK4naUGIz0hQcw==", + "dependencies": [ + "@aws-sdk/middleware-user-agent", + "@aws-sdk/types", + "@smithy/node-config-provider", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@aws-sdk/xml-builder@3.930.0": { + "integrity": "sha512-YIfkD17GocxdmlUVc3ia52QhcWuRIUJonbF8A2CYfcWNV3HzvAqpcPeC0bYUhkK+8e8YO1ARnLKZQE0TlwzorA==", + "dependencies": [ + "@smithy/types@4.12.0", + "fast-xml-parser", + "tslib" + ] + }, + "@aws/lambda-invoke-store@0.2.3": { + "integrity": "sha512-oLvsaPMTBejkkmHhjf09xTgk71mOqyr/409NKhRIL08If7AhVfUsJhVsx386uJaqNd42v9kWamQ9lFbkoC2dYw==" + }, + "@effect/cluster@0.48.16_@effect+platform@0.90.10__effect@3.19.14_@effect+rpc@0.69.5__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_@effect+sql@0.44.2__@effect+experimental@0.54.6___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_@effect+workflow@0.9.6__@effect+platform@0.90.10___effect@3.19.14__@effect+rpc@0.69.5___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__effect@3.19.14_effect@3.19.14": { + "integrity": "sha512-ZZkrSMVetOvlRDD8mPCX3IcVJtvUZBp6++lUKNGIT6LRIObRP4lVwtei85Z+4g49WpeLvJnSdH0zjPtGieFDHQ==", + "dependencies": [ + "@effect/platform", + "@effect/rpc", + "@effect/sql", + "@effect/workflow", + "effect" + ] + }, + "@effect/experimental@0.54.6_@effect+platform@0.90.10__effect@3.19.14_effect@3.19.14": { + "integrity": "sha512-UqHMvCQmrZT6kUVoUC0lqyno4Yad+j9hBGCdUjW84zkLwAq08tPqySiZUKRwY+Ae5B2Ab8rISYJH7nQvct9DMQ==", + "dependencies": [ + "@effect/platform", + "effect", + "uuid" + ] + }, + "@effect/opentelemetry@0.56.6_@effect+platform@0.90.10__effect@3.19.14_@opentelemetry+sdk-trace-base@2.3.0__@opentelemetry+api@1.9.0_@opentelemetry+sdk-trace-node@2.3.0__@opentelemetry+api@1.9.0_@opentelemetry+semantic-conventions@1.38.0_effect@3.19.14": { + "integrity": "sha512-cBi9frXujTIEGXChkl4VdQfvDe7QvzC18SM8wK0CKYSgH9ZL7v/F5f5/3fTSTfEdO9ZyBk73s5Jbbogab0Q01g==", + "dependencies": [ + "@effect/platform", + "@opentelemetry/sdk-trace-base@2.3.0_@opentelemetry+api@1.9.0", + "@opentelemetry/sdk-trace-node", + "@opentelemetry/semantic-conventions", + "effect" + ], + "optionalPeers": [ + "@opentelemetry/sdk-trace-base@2.3.0_@opentelemetry+api@1.9.0", + "@opentelemetry/sdk-trace-node" + ] + }, + "@effect/platform-node-shared@0.49.2_@effect+cluster@0.48.16__@effect+platform@0.90.10___effect@3.19.14__@effect+rpc@0.69.5___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+sql@0.44.2___@effect+experimental@0.54.6____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+workflow@0.9.6___@effect+platform@0.90.10____effect@3.19.14___@effect+rpc@0.69.5____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___effect@3.19.14__effect@3.19.14_@effect+platform@0.90.10__effect@3.19.14_@effect+rpc@0.69.5__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_@effect+sql@0.44.2__@effect+experimental@0.54.6___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_effect@3.19.14": { + "integrity": "sha512-uYlQi2swDV9hdHatr2Onov3G+VlEF+3+Qm9dvdOZiZNE1bVqvs/zs6LVT8Yrz/3Vq/4JPzGcN+acx0iiJo5ZVw==", + "dependencies": [ + "@effect/cluster", + "@effect/platform", + "@effect/rpc", + "@effect/sql", + "@parcel/watcher", + "effect", + "multipasta", + "ws" + ] + }, + "@effect/platform-node@0.96.1_@effect+cluster@0.48.16__@effect+platform@0.90.10___effect@3.19.14__@effect+rpc@0.69.5___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+sql@0.44.2___@effect+experimental@0.54.6____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+workflow@0.9.6___@effect+platform@0.90.10____effect@3.19.14___@effect+rpc@0.69.5____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___effect@3.19.14__effect@3.19.14_@effect+platform@0.90.10__effect@3.19.14_@effect+rpc@0.69.5__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_@effect+sql@0.44.2__@effect+experimental@0.54.6___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_effect@3.19.14": { + "integrity": "sha512-4nfB/XRJJ246MCdI7klTE/aVvA9txfI83RnymS7pNyoG4CXUKELi87JrkrWFTtOlewzt5UMWpmqsFmm2qHxx3A==", + "dependencies": [ + "@effect/cluster", + "@effect/platform", + "@effect/platform-node-shared", + "@effect/rpc", + "@effect/sql", + "effect", + "mime", + "undici", + "ws" + ] + }, + "@effect/platform@0.90.10_effect@3.19.14": { + "integrity": "sha512-QhDPgCaLfIMQKOCoCPQvRUS+Y34iYJ07jdZ/CBAvYFvg/iUBebsmFuHL63RCD/YZH9BuK/kqqLYAA3M0fmUEgg==", + "dependencies": [ + "effect", + "find-my-way-ts", + "msgpackr", + "multipasta" + ] + }, + "@effect/rpc@0.69.5_@effect+platform@0.90.10__effect@3.19.14_effect@3.19.14": { + "integrity": "sha512-LLCZP/aiaW4HeoIaoZuVZpJb/PFCwdJP21b3xP6l+1yoRVw8HlKYyfy/outRCF+BT4ndtY0/utFSeGWC21Qr7w==", + "dependencies": [ + "@effect/platform", + "effect" + ] + }, + "@effect/sql@0.44.2_@effect+experimental@0.54.6__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_@effect+platform@0.90.10__effect@3.19.14_effect@3.19.14": { + "integrity": "sha512-DEcvriHvj88zu7keruH9NcHQzam7yQzLNLJO6ucDXMCAwWzYZSJOsmkxBznRFv8ylFtccSclKH2fuj+wRKPjCQ==", + "dependencies": [ + "@effect/experimental", + "@effect/platform", + "effect", + "uuid" + ] + }, + "@effect/workflow@0.9.6_@effect+platform@0.90.10__effect@3.19.14_@effect+rpc@0.69.5__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_effect@3.19.14": { + "integrity": "sha512-uPBpSJ8NYwYA6VLZovfejwNik+2kAaoDtlPi+VTlxFMscWNYx+xlGiRg8CO/oa2pHCwkJYjOI27SGOlUawiz1w==", + "dependencies": [ + "@effect/platform", + "@effect/rpc", + "effect" + ] + }, + "@jest/schemas@29.6.3": { + "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "dependencies": [ + "@sinclair/typebox" + ] + }, + "@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3": { + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "os": ["darwin"], + "cpu": ["arm64"] + }, + "@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3": { + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "os": ["darwin"], + "cpu": ["x64"] + }, + "@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3": { + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3": { + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "os": ["linux"], + "cpu": ["arm"] + }, + "@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3": { + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "os": ["linux"], + "cpu": ["x64"] + }, + "@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3": { + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "os": ["win32"], + "cpu": ["x64"] + }, + "@opentelemetry/api-logs@0.203.0": { + "integrity": "sha512-9B9RU0H7Ya1Dx/Rkyc4stuBZSGVQF27WigitInx2QQoj6KUpEFYPKoWjdFTunJYxmXmh17HeBvbMa1EhGyPmqQ==", + "dependencies": [ + "@opentelemetry/api" + ] + }, + "@opentelemetry/api@1.9.0": { + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==" + }, + "@opentelemetry/context-async-hooks@2.3.0_@opentelemetry+api@1.9.0": { + "integrity": "sha512-hGcsT0qDP7Il1L+qT3JFpiGl1dCjF794Bb4yCRCYdr7XC0NwHtOF3ngF86Gk6TUnsakbyQsDQ0E/S4CU0F4d4g==", + "dependencies": [ + "@opentelemetry/api" + ] + }, + "@opentelemetry/core@2.0.1_@opentelemetry+api@1.9.0": { + "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/semantic-conventions" + ] + }, + "@opentelemetry/core@2.3.0_@opentelemetry+api@1.9.0": { + "integrity": "sha512-PcmxJQzs31cfD0R2dE91YGFcLxOSN4Bxz7gez5UwSUjCai8BwH/GI5HchfVshHkWdTkUs0qcaPJgVHKXUp7I3A==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/semantic-conventions" + ] + }, + "@opentelemetry/exporter-trace-otlp-http@0.203.0_@opentelemetry+api@1.9.0": { + "integrity": "sha512-ZDiaswNYo0yq/cy1bBLJFe691izEJ6IgNmkjm4C6kE9ub/OMQqDXORx2D2j8fzTBTxONyzusbaZlqtfmyqURPw==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/core@2.0.1_@opentelemetry+api@1.9.0", + "@opentelemetry/otlp-exporter-base", + "@opentelemetry/otlp-transformer", + "@opentelemetry/resources@2.0.1_@opentelemetry+api@1.9.0", + "@opentelemetry/sdk-trace-base@2.0.1_@opentelemetry+api@1.9.0" + ] + }, + "@opentelemetry/otlp-exporter-base@0.203.0_@opentelemetry+api@1.9.0": { + "integrity": "sha512-Wbxf7k+87KyvxFr5D7uOiSq/vHXWommvdnNE7vECO3tAhsA2GfOlpWINCMWUEPdHZ7tCXxw6Epp3vgx3jU7llQ==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/core@2.0.1_@opentelemetry+api@1.9.0", + "@opentelemetry/otlp-transformer" + ] + }, + "@opentelemetry/otlp-transformer@0.203.0_@opentelemetry+api@1.9.0": { + "integrity": "sha512-Y8I6GgoCna0qDQ2W6GCRtaF24SnvqvA8OfeTi7fqigD23u8Jpb4R5KFv/pRvrlGagcCLICMIyh9wiejp4TXu/A==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/api-logs", + "@opentelemetry/core@2.0.1_@opentelemetry+api@1.9.0", + "@opentelemetry/resources@2.0.1_@opentelemetry+api@1.9.0", + "@opentelemetry/sdk-logs", + "@opentelemetry/sdk-metrics", + "@opentelemetry/sdk-trace-base@2.0.1_@opentelemetry+api@1.9.0", + "protobufjs" + ] + }, + "@opentelemetry/resources@2.0.1_@opentelemetry+api@1.9.0": { + "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/core@2.0.1_@opentelemetry+api@1.9.0", + "@opentelemetry/semantic-conventions" + ] + }, + "@opentelemetry/resources@2.3.0_@opentelemetry+api@1.9.0": { + "integrity": "sha512-shlr2l5g+87J8wqYlsLyaUsgKVRO7RtX70Ckd5CtDOWtImZgaUDmf4Z2ozuSKQLM2wPDR0TE/3bPVBNJtRm/cQ==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/core@2.3.0_@opentelemetry+api@1.9.0", + "@opentelemetry/semantic-conventions" + ] + }, + "@opentelemetry/sdk-logs@0.203.0_@opentelemetry+api@1.9.0": { + "integrity": "sha512-vM2+rPq0Vi3nYA5akQD2f3QwossDnTDLvKbea6u/A2NZ3XDkPxMfo/PNrDoXhDUD/0pPo2CdH5ce/thn9K0kLw==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/api-logs", + "@opentelemetry/core@2.0.1_@opentelemetry+api@1.9.0", + "@opentelemetry/resources@2.0.1_@opentelemetry+api@1.9.0" + ] + }, + "@opentelemetry/sdk-metrics@2.0.1_@opentelemetry+api@1.9.0": { + "integrity": "sha512-wf8OaJoSnujMAHWR3g+/hGvNcsC16rf9s1So4JlMiFaFHiE4HpIA3oUh+uWZQ7CNuK8gVW/pQSkgoa5HkkOl0g==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/core@2.0.1_@opentelemetry+api@1.9.0", + "@opentelemetry/resources@2.0.1_@opentelemetry+api@1.9.0" + ] + }, + "@opentelemetry/sdk-trace-base@2.0.1_@opentelemetry+api@1.9.0": { + "integrity": "sha512-xYLlvk/xdScGx1aEqvxLwf6sXQLXCjk3/1SQT9X9AoN5rXRhkdvIFShuNNmtTEPRBqcsMbS4p/gJLNI2wXaDuQ==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/core@2.0.1_@opentelemetry+api@1.9.0", + "@opentelemetry/resources@2.0.1_@opentelemetry+api@1.9.0", + "@opentelemetry/semantic-conventions" + ] + }, + "@opentelemetry/sdk-trace-base@2.3.0_@opentelemetry+api@1.9.0": { + "integrity": "sha512-B0TQ2e9h0ETjpI+eGmCz8Ojb+lnYms0SE3jFwEKrN/PK4aSVHU28AAmnOoBmfub+I3jfgPwvDJgomBA5a7QehQ==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/core@2.3.0_@opentelemetry+api@1.9.0", + "@opentelemetry/resources@2.3.0_@opentelemetry+api@1.9.0", + "@opentelemetry/semantic-conventions" + ] + }, + "@opentelemetry/sdk-trace-node@2.3.0_@opentelemetry+api@1.9.0": { + "integrity": "sha512-oGsG3vIiC8zYjOWE4CgtS6d2gQhp4pT04AI9UL1wtJOxTSNVZiiIPgHnOp/qKJSwkD4YJHSohi6inSilPmGM2Q==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/context-async-hooks", + "@opentelemetry/core@2.3.0_@opentelemetry+api@1.9.0", + "@opentelemetry/sdk-trace-base@2.3.0_@opentelemetry+api@1.9.0" + ] + }, + "@opentelemetry/semantic-conventions@1.38.0": { + "integrity": "sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg==" + }, + "@parcel/watcher-android-arm64@2.5.1": { + "integrity": "sha512-KF8+j9nNbUN8vzOFDpRMsaKBHZ/mcjEjMToVMJOhTozkDonQFFrRcfdLWn6yWKCmJKmdVxSgHiYvTCef4/qcBA==", + "os": ["android"], + "cpu": ["arm64"] + }, + "@parcel/watcher-darwin-arm64@2.5.1": { + "integrity": "sha512-eAzPv5osDmZyBhou8PoF4i6RQXAfeKL9tjb3QzYuccXFMQU0ruIc/POh30ePnaOyD1UXdlKguHBmsTs53tVoPw==", + "os": ["darwin"], + "cpu": ["arm64"] + }, + "@parcel/watcher-darwin-x64@2.5.1": { + "integrity": "sha512-1ZXDthrnNmwv10A0/3AJNZ9JGlzrF82i3gNQcWOzd7nJ8aj+ILyW1MTxVk35Db0u91oD5Nlk9MBiujMlwmeXZg==", + "os": ["darwin"], + "cpu": ["x64"] + }, + "@parcel/watcher-freebsd-x64@2.5.1": { + "integrity": "sha512-SI4eljM7Flp9yPuKi8W0ird8TI/JK6CSxju3NojVI6BjHsTyK7zxA9urjVjEKJ5MBYC+bLmMcbAWlZ+rFkLpJQ==", + "os": ["freebsd"], + "cpu": ["x64"] + }, + "@parcel/watcher-linux-arm-glibc@2.5.1": { + "integrity": "sha512-RCdZlEyTs8geyBkkcnPWvtXLY44BCeZKmGYRtSgtwwnHR4dxfHRG3gR99XdMEdQ7KeiDdasJwwvNSF5jKtDwdA==", + "os": ["linux"], + "cpu": ["arm"] + }, + "@parcel/watcher-linux-arm-musl@2.5.1": { + "integrity": "sha512-6E+m/Mm1t1yhB8X412stiKFG3XykmgdIOqhjWj+VL8oHkKABfu/gjFj8DvLrYVHSBNC+/u5PeNrujiSQ1zwd1Q==", + "os": ["linux"], + "cpu": ["arm"] + }, + "@parcel/watcher-linux-arm64-glibc@2.5.1": { + "integrity": "sha512-LrGp+f02yU3BN9A+DGuY3v3bmnFUggAITBGriZHUREfNEzZh/GO06FF5u2kx8x+GBEUYfyTGamol4j3m9ANe8w==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "@parcel/watcher-linux-arm64-musl@2.5.1": { + "integrity": "sha512-cFOjABi92pMYRXS7AcQv9/M1YuKRw8SZniCDw0ssQb/noPkRzA+HBDkwmyOJYp5wXcsTrhxO0zq1U11cK9jsFg==", + "os": ["linux"], + "cpu": ["arm64"] + }, + "@parcel/watcher-linux-x64-glibc@2.5.1": { + "integrity": "sha512-GcESn8NZySmfwlTsIur+49yDqSny2IhPeZfXunQi48DMugKeZ7uy1FX83pO0X22sHntJ4Ub+9k34XQCX+oHt2A==", + "os": ["linux"], + "cpu": ["x64"] + }, + "@parcel/watcher-linux-x64-musl@2.5.1": { + "integrity": "sha512-n0E2EQbatQ3bXhcH2D1XIAANAcTZkQICBPVaxMeaCVBtOpBZpWJuf7LwyWPSBDITb7In8mqQgJ7gH8CILCURXg==", + "os": ["linux"], + "cpu": ["x64"] + }, + "@parcel/watcher-win32-arm64@2.5.1": { + "integrity": "sha512-RFzklRvmc3PkjKjry3hLF9wD7ppR4AKcWNzH7kXR7GUe0Igb3Nz8fyPwtZCSquGrhU5HhUNDr/mKBqj7tqA2Vw==", + "os": ["win32"], + "cpu": ["arm64"] + }, + "@parcel/watcher-win32-ia32@2.5.1": { + "integrity": "sha512-c2KkcVN+NJmuA7CGlaGD1qJh1cLfDnQsHjE89E60vUEMlqduHGCdCLJCID5geFVM0dOtA3ZiIO8BoEQmzQVfpQ==", + "os": ["win32"], + "cpu": ["ia32"] + }, + "@parcel/watcher-win32-x64@2.5.1": { + "integrity": "sha512-9lHBdJITeNR++EvSQVUcaZoWupyHfXe1jZvGZ06O/5MflPcuPLtEphScIBL+AiCWBO46tDSHzWyD0uDmmZqsgA==", + "os": ["win32"], + "cpu": ["x64"] + }, + "@parcel/watcher@2.5.1": { + "integrity": "sha512-dfUnCxiN9H4ap84DvD2ubjw+3vUNpstxa0TneY/Paat8a3R4uQZDLSvWjmznAY/DoahqTHl9V46HF/Zs3F29pg==", + "dependencies": [ + "detect-libc@1.0.3", + "is-glob", + "micromatch", + "node-addon-api" + ], + "optionalDependencies": [ + "@parcel/watcher-android-arm64", + "@parcel/watcher-darwin-arm64", + "@parcel/watcher-darwin-x64", + "@parcel/watcher-freebsd-x64", + "@parcel/watcher-linux-arm-glibc", + "@parcel/watcher-linux-arm-musl", + "@parcel/watcher-linux-arm64-glibc", + "@parcel/watcher-linux-arm64-musl", + "@parcel/watcher-linux-x64-glibc", + "@parcel/watcher-linux-x64-musl", + "@parcel/watcher-win32-arm64", + "@parcel/watcher-win32-ia32", + "@parcel/watcher-win32-x64" + ], + "scripts": true + }, + "@protobufjs/aspromise@1.1.2": { + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + }, + "@protobufjs/base64@1.1.2": { + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "@protobufjs/codegen@2.0.4": { + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "@protobufjs/eventemitter@1.1.0": { + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + }, + "@protobufjs/fetch@1.1.0": { + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dependencies": [ + "@protobufjs/aspromise", + "@protobufjs/inquire" + ] + }, + "@protobufjs/float@1.0.2": { + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + }, + "@protobufjs/inquire@1.1.0": { + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" + }, + "@protobufjs/path@1.1.2": { + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + }, + "@protobufjs/pool@1.1.0": { + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + }, + "@protobufjs/utf8@1.1.0": { + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + }, + "@sinclair/typebox@0.27.8": { + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==" + }, + "@smithy/abort-controller@4.2.8": { + "integrity": "sha512-peuVfkYHAmS5ybKxWcfraK7WBBP0J+rkfUcbHJJKQ4ir3UAUNQI+Y4Vt/PqSzGqgloJ5O1dk7+WzNL8wcCSXbw==", + "dependencies": [ + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/chunked-blob-reader-native@4.2.1": { + "integrity": "sha512-lX9Ay+6LisTfpLid2zZtIhSEjHMZoAR5hHCR4H7tBz/Zkfr5ea8RcQ7Tk4mi0P76p4cN+Btz16Ffno7YHpKXnQ==", + "dependencies": [ + "@smithy/util-base64", + "tslib" + ] + }, + "@smithy/chunked-blob-reader@5.2.0": { + "integrity": "sha512-WmU0TnhEAJLWvfSeMxBNe5xtbselEO8+4wG0NtZeL8oR21WgH1xiO37El+/Y+H/Ie4SCwBy3MxYWmOYaGgZueA==", + "dependencies": [ + "tslib" + ] + }, + "@smithy/config-resolver@4.4.3": { + "integrity": "sha512-ezHLe1tKLUxDJo2LHtDuEDyWXolw8WGOR92qb4bQdWq/zKenO5BvctZGrVJBK08zjezSk7bmbKFOXIVyChvDLw==", + "dependencies": [ + "@smithy/node-config-provider", + "@smithy/types@4.12.0", + "@smithy/util-config-provider", + "@smithy/util-endpoints", + "@smithy/util-middleware@4.2.8", + "tslib" + ] + }, + "@smithy/core@3.21.1": { + "integrity": "sha512-NUH8R4O6FkN8HKMojzbGg/5pNjsfTjlMmeFclyPfPaXXUrbr5TzhWgbf7t92wfrpCHRgpjyz7ffASIS3wX28aA==", + "dependencies": [ + "@smithy/middleware-serde", + "@smithy/protocol-http@5.3.8", + "@smithy/types@4.12.0", + "@smithy/util-base64", + "@smithy/util-body-length-browser", + "@smithy/util-middleware@4.2.8", + "@smithy/util-stream", + "@smithy/util-utf8@4.2.0", + "@smithy/uuid", + "tslib" + ] + }, + "@smithy/credential-provider-imds@4.2.5": { + "integrity": "sha512-BZwotjoZWn9+36nimwm/OLIcVe+KYRwzMjfhd4QT7QxPm9WY0HiOV8t/Wlh+HVUif0SBVV7ksq8//hPaBC/okQ==", + "dependencies": [ + "@smithy/node-config-provider", + "@smithy/property-provider", + "@smithy/types@4.12.0", + "@smithy/url-parser", + "tslib" + ] + }, + "@smithy/eventstream-codec@4.2.5": { + "integrity": "sha512-Ogt4Zi9hEbIP17oQMd68qYOHUzmH47UkK7q7Gl55iIm9oKt27MUGrC5JfpMroeHjdkOliOA4Qt3NQ1xMq/nrlA==", + "dependencies": [ + "@aws-crypto/crc32", + "@smithy/types@4.12.0", + "@smithy/util-hex-encoding@4.2.0", + "tslib" + ] + }, + "@smithy/eventstream-serde-browser@4.2.5": { + "integrity": "sha512-HohfmCQZjppVnKX2PnXlf47CW3j92Ki6T/vkAT2DhBR47e89pen3s4fIa7otGTtrVxmj7q+IhH0RnC5kpR8wtw==", + "dependencies": [ + "@smithy/eventstream-serde-universal", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/eventstream-serde-config-resolver@4.3.5": { + "integrity": "sha512-ibjQjM7wEXtECiT6my1xfiMH9IcEczMOS6xiCQXoUIYSj5b1CpBbJ3VYbdwDy8Vcg5JHN7eFpOCGk8nyZAltNQ==", + "dependencies": [ + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/eventstream-serde-node@4.2.5": { + "integrity": "sha512-+elOuaYx6F2H6x1/5BQP5ugv12nfJl66GhxON8+dWVUEDJ9jah/A0tayVdkLRP0AeSac0inYkDz5qBFKfVp2Gg==", + "dependencies": [ + "@smithy/eventstream-serde-universal", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/eventstream-serde-universal@4.2.5": { + "integrity": "sha512-G9WSqbST45bmIFaeNuP/EnC19Rhp54CcVdX9PDL1zyEB514WsDVXhlyihKlGXnRycmHNmVv88Bvvt4EYxWef/Q==", + "dependencies": [ + "@smithy/eventstream-codec", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/fetch-http-handler@5.3.9": { + "integrity": "sha512-I4UhmcTYXBrct03rwzQX1Y/iqQlzVQaPxWjCjula++5EmWq9YGBrx6bbGqluGc1f0XEfhSkiY4jhLgbsJUMKRA==", + "dependencies": [ + "@smithy/protocol-http@5.3.8", + "@smithy/querystring-builder", + "@smithy/types@4.12.0", + "@smithy/util-base64", + "tslib" + ] + }, + "@smithy/hash-blob-browser@4.2.6": { + "integrity": "sha512-8P//tA8DVPk+3XURk2rwcKgYwFvwGwmJH/wJqQiSKwXZtf/LiZK+hbUZmPj/9KzM+OVSwe4o85KTp5x9DUZTjw==", + "dependencies": [ + "@smithy/chunked-blob-reader", + "@smithy/chunked-blob-reader-native", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/hash-node@4.2.5": { + "integrity": "sha512-DpYX914YOfA3UDT9CN1BM787PcHfWRBB43fFGCYrZFUH0Jv+5t8yYl+Pd5PW4+QzoGEDvn5d5QIO4j2HyYZQSA==", + "dependencies": [ + "@smithy/types@4.12.0", + "@smithy/util-buffer-from@4.2.0", + "@smithy/util-utf8@4.2.0", + "tslib" + ] + }, + "@smithy/hash-stream-node@4.2.5": { + "integrity": "sha512-6+do24VnEyvWcGdHXomlpd0m8bfZePpUKBy7m311n+JuRwug8J4dCanJdTymx//8mi0nlkflZBvJe+dEO/O12Q==", + "dependencies": [ + "@smithy/types@4.12.0", + "@smithy/util-utf8@4.2.0", + "tslib" + ] + }, + "@smithy/invalid-dependency@4.2.5": { + "integrity": "sha512-2L2erASEro1WC5nV+plwIMxrTXpvpfzl4e+Nre6vBVRR2HKeGGcvpJyyL3/PpiSg+cJG2KpTmZmq934Olb6e5A==", + "dependencies": [ + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/is-array-buffer@2.2.0": { + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": [ + "tslib" + ] + }, + "@smithy/is-array-buffer@3.0.0": { + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": [ + "tslib" + ] + }, + "@smithy/is-array-buffer@4.2.0": { + "integrity": "sha512-DZZZBvC7sjcYh4MazJSGiWMI2L7E0oCiRHREDzIxi/M2LY79/21iXt6aPLHge82wi5LsuRF5A06Ds3+0mlh6CQ==", + "dependencies": [ + "tslib" + ] + }, + "@smithy/md5-js@4.2.5": { + "integrity": "sha512-Bt6jpSTMWfjCtC0s79gZ/WZ1w90grfmopVOWqkI2ovhjpD5Q2XRXuecIPB9689L2+cCySMbaXDhBPU56FKNDNg==", + "dependencies": [ + "@smithy/types@4.12.0", + "@smithy/util-utf8@4.2.0", + "tslib" + ] + }, + "@smithy/middleware-content-length@4.2.5": { + "integrity": "sha512-Y/RabVa5vbl5FuHYV2vUCwvh/dqzrEY/K2yWPSqvhFUwIY0atLqO4TienjBXakoy4zrKAMCZwg+YEqmH7jaN7A==", + "dependencies": [ + "@smithy/protocol-http@5.3.8", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/middleware-endpoint@4.4.11": { + "integrity": "sha512-/WqsrycweGGfb9sSzME4CrsuayjJF6BueBmkKlcbeU5q18OhxRrvvKlmfw3tpDsK5ilx2XUJvoukwxHB0nHs/Q==", + "dependencies": [ + "@smithy/core", + "@smithy/middleware-serde", + "@smithy/node-config-provider", + "@smithy/shared-ini-file-loader", + "@smithy/types@4.12.0", + "@smithy/url-parser", + "@smithy/util-middleware@4.2.8", + "tslib" + ] + }, + "@smithy/middleware-retry@4.4.12": { + "integrity": "sha512-S4kWNKFowYd0lID7/DBqWHOQxmxlsf0jBaos9chQZUWTVOjSW1Ogyh8/ib5tM+agFDJ/TCxuCTvrnlc+9cIBcQ==", + "dependencies": [ + "@smithy/node-config-provider", + "@smithy/protocol-http@5.3.8", + "@smithy/service-error-classification", + "@smithy/smithy-client", + "@smithy/types@4.12.0", + "@smithy/util-middleware@4.2.8", + "@smithy/util-retry", + "@smithy/uuid", + "tslib" + ] + }, + "@smithy/middleware-serde@4.2.9": { + "integrity": "sha512-eMNiej0u/snzDvlqRGSN3Vl0ESn3838+nKyVfF2FKNXFbi4SERYT6PR392D39iczngbqqGG0Jl1DlCnp7tBbXQ==", + "dependencies": [ + "@smithy/protocol-http@5.3.8", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/middleware-stack@4.2.8": { + "integrity": "sha512-w6LCfOviTYQjBctOKSwy6A8FIkQy7ICvglrZFl6Bw4FmcQ1Z420fUtIhxaUZZshRe0VCq4kvDiPiXrPZAe8oRA==", + "dependencies": [ + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/node-config-provider@4.3.8": { + "integrity": "sha512-aFP1ai4lrbVlWjfpAfRSL8KFcnJQYfTl5QxLJXY32vghJrDuFyPZ6LtUL+JEGYiFRG1PfPLHLoxj107ulncLIg==", + "dependencies": [ + "@smithy/property-provider", + "@smithy/shared-ini-file-loader", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/node-http-handler@4.4.8": { + "integrity": "sha512-q9u+MSbJVIJ1QmJ4+1u+cERXkrhuILCBDsJUBAW1MPE6sFonbCNaegFuwW9ll8kh5UdyY3jOkoOGlc7BesoLpg==", + "dependencies": [ + "@smithy/abort-controller", + "@smithy/protocol-http@5.3.8", + "@smithy/querystring-builder", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/property-provider@4.2.8": { + "integrity": "sha512-EtCTbyIveCKeOXDSWSdze3k612yCPq1YbXsbqX3UHhkOSW8zKsM9NOJG5gTIya0vbY2DIaieG8pKo1rITHYL0w==", + "dependencies": [ + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/protocol-http@4.1.8": { + "integrity": "sha512-hmgIAVyxw1LySOwkgMIUN0kjN8TG9Nc85LJeEmEE/cNEe2rkHDUWhnJf2gxcSRFLWsyqWsrZGw40ROjUogg+Iw==", + "dependencies": [ + "@smithy/types@3.7.2", + "tslib" + ] + }, + "@smithy/protocol-http@5.3.8": { + "integrity": "sha512-QNINVDhxpZ5QnP3aviNHQFlRogQZDfYlCkQT+7tJnErPQbDhysondEjhikuANxgMsZrkGeiAxXy4jguEGsDrWQ==", + "dependencies": [ + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/querystring-builder@4.2.8": { + "integrity": "sha512-Xr83r31+DrE8CP3MqPgMJl+pQlLLmOfiEUnoyAlGzzJIrEsbKsPy1hqH0qySaQm4oWrCBlUqRt+idEgunKB+iw==", + "dependencies": [ + "@smithy/types@4.12.0", + "@smithy/util-uri-escape@4.2.0", + "tslib" + ] + }, + "@smithy/querystring-parser@4.2.8": { + "integrity": "sha512-vUurovluVy50CUlazOiXkPq40KGvGWSdmusa3130MwrR1UNnNgKAlj58wlOe61XSHRpUfIIh6cE0zZ8mzKaDPA==", + "dependencies": [ + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/service-error-classification@4.2.5": { + "integrity": "sha512-8fEvK+WPE3wUAcDvqDQG1Vk3ANLR8Px979te96m84CbKAjBVf25rPYSzb4xU4hlTyho7VhOGnh5i62D/JVF0JQ==", + "dependencies": [ + "@smithy/types@4.12.0" + ] + }, + "@smithy/shared-ini-file-loader@4.4.3": { + "integrity": "sha512-DfQjxXQnzC5UbCUPeC3Ie8u+rIWZTvuDPAGU/BxzrOGhRvgUanaP68kDZA+jaT3ZI+djOf+4dERGlm9mWfFDrg==", + "dependencies": [ + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/signature-v4@4.2.4": { + "integrity": "sha512-5JWeMQYg81TgU4cG+OexAWdvDTs5JDdbEZx+Qr1iPbvo91QFGzjy0IkXAKaXUHqmKUJgSHK0ZxnCkgZpzkeNTA==", + "dependencies": [ + "@smithy/is-array-buffer@3.0.0", + "@smithy/protocol-http@4.1.8", + "@smithy/types@3.7.2", + "@smithy/util-hex-encoding@3.0.0", + "@smithy/util-middleware@3.0.11", + "@smithy/util-uri-escape@3.0.0", + "@smithy/util-utf8@3.0.0", + "tslib" + ] + }, + "@smithy/signature-v4@5.3.5": { + "integrity": "sha512-xSUfMu1FT7ccfSXkoLl/QRQBi2rOvi3tiBZU2Tdy3I6cgvZ6SEi9QNey+lqps/sJRnogIS+lq+B1gxxbra2a/w==", + "dependencies": [ + "@smithy/is-array-buffer@4.2.0", + "@smithy/protocol-http@5.3.8", + "@smithy/types@4.12.0", + "@smithy/util-hex-encoding@4.2.0", + "@smithy/util-middleware@4.2.8", + "@smithy/util-uri-escape@4.2.0", + "@smithy/util-utf8@4.2.0", + "tslib" + ] + }, + "@smithy/smithy-client@4.10.12": { + "integrity": "sha512-VKO/HKoQ5OrSHW6AJUmEnUKeXI1/5LfCwO9cwyao7CmLvGnZeM1i36Lyful3LK1XU7HwTVieTqO1y2C/6t3qtA==", + "dependencies": [ + "@smithy/core", + "@smithy/middleware-endpoint", + "@smithy/middleware-stack", + "@smithy/protocol-http@5.3.8", + "@smithy/types@4.12.0", + "@smithy/util-stream", + "tslib" + ] + }, + "@smithy/types@3.7.2": { + "integrity": "sha512-bNwBYYmN8Eh9RyjS1p2gW6MIhSO2rl7X9QeLM8iTdcGRP+eDiIWDt66c9IysCc22gefKszZv+ubV9qZc7hdESg==", + "dependencies": [ + "tslib" + ] + }, + "@smithy/types@4.12.0": { + "integrity": "sha512-9YcuJVTOBDjg9LWo23Qp0lTQ3D7fQsQtwle0jVfpbUHy9qBwCEgKuVH4FqFB3VYu0nwdHKiEMA+oXz7oV8X1kw==", + "dependencies": [ + "tslib" + ] + }, + "@smithy/url-parser@4.2.8": { + "integrity": "sha512-NQho9U68TGMEU639YkXnVMV3GEFFULmmaWdlu1E9qzyIePOHsoSnagTGSDv1Zi8DCNN6btxOSdgmy5E/hsZwhA==", + "dependencies": [ + "@smithy/querystring-parser", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/util-base64@4.3.0": { + "integrity": "sha512-GkXZ59JfyxsIwNTWFnjmFEI8kZpRNIBfxKjv09+nkAWPt/4aGaEWMM04m4sxgNVWkbt2MdSvE3KF/PfX4nFedQ==", + "dependencies": [ + "@smithy/util-buffer-from@4.2.0", + "@smithy/util-utf8@4.2.0", + "tslib" + ] + }, + "@smithy/util-body-length-browser@4.2.0": { + "integrity": "sha512-Fkoh/I76szMKJnBXWPdFkQJl2r9SjPt3cMzLdOB6eJ4Pnpas8hVoWPYemX/peO0yrrvldgCUVJqOAjUrOLjbxg==", + "dependencies": [ + "tslib" + ] + }, + "@smithy/util-body-length-node@4.2.1": { + "integrity": "sha512-h53dz/pISVrVrfxV1iqXlx5pRg3V2YWFcSQyPyXZRrZoZj4R4DeWRDo1a7dd3CPTcFi3kE+98tuNyD2axyZReA==", + "dependencies": [ + "tslib" + ] + }, + "@smithy/util-buffer-from@2.2.0": { + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": [ + "@smithy/is-array-buffer@2.2.0", + "tslib" + ] + }, + "@smithy/util-buffer-from@3.0.0": { + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": [ + "@smithy/is-array-buffer@3.0.0", + "tslib" + ] + }, + "@smithy/util-buffer-from@4.2.0": { + "integrity": "sha512-kAY9hTKulTNevM2nlRtxAG2FQ3B2OR6QIrPY3zE5LqJy1oxzmgBGsHLWTcNhWXKchgA0WHW+mZkQrng/pgcCew==", + "dependencies": [ + "@smithy/is-array-buffer@4.2.0", + "tslib" + ] + }, + "@smithy/util-config-provider@4.2.0": { + "integrity": "sha512-YEjpl6XJ36FTKmD+kRJJWYvrHeUvm5ykaUS5xK+6oXffQPHeEM4/nXlZPe+Wu0lsgRUcNZiliYNh/y7q9c2y6Q==", + "dependencies": [ + "tslib" + ] + }, + "@smithy/util-defaults-mode-browser@4.3.11": { + "integrity": "sha512-yHv+r6wSQXEXTPVCIQTNmXVWs7ekBTpMVErjqZoWkYN75HIFN5y9+/+sYOejfAuvxWGvgzgxbTHa/oz61YTbKw==", + "dependencies": [ + "@smithy/property-provider", + "@smithy/smithy-client", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/util-defaults-mode-node@4.2.14": { + "integrity": "sha512-ljZN3iRvaJUgulfvobIuG97q1iUuCMrvXAlkZ4msY+ZuVHQHDIqn7FKZCEj+bx8omz6kF5yQXms/xhzjIO5XiA==", + "dependencies": [ + "@smithy/config-resolver", + "@smithy/credential-provider-imds", + "@smithy/node-config-provider", + "@smithy/property-provider", + "@smithy/smithy-client", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/util-endpoints@3.2.5": { + "integrity": "sha512-3O63AAWu2cSNQZp+ayl9I3NapW1p1rR5mlVHcF6hAB1dPZUQFfRPYtplWX/3xrzWthPGj5FqB12taJJCfH6s8A==", + "dependencies": [ + "@smithy/node-config-provider", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/util-hex-encoding@3.0.0": { + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "dependencies": [ + "tslib" + ] + }, + "@smithy/util-hex-encoding@4.2.0": { + "integrity": "sha512-CCQBwJIvXMLKxVbO88IukazJD9a4kQ9ZN7/UMGBjBcJYvatpWk+9g870El4cB8/EJxfe+k+y0GmR9CAzkF+Nbw==", + "dependencies": [ + "tslib" + ] + }, + "@smithy/util-middleware@3.0.11": { + "integrity": "sha512-dWpyc1e1R6VoXrwLoLDd57U1z6CwNSdkM69Ie4+6uYh2GC7Vg51Qtan7ITzczuVpqezdDTKJGJB95fFvvjU/ow==", + "dependencies": [ + "@smithy/types@3.7.2", + "tslib" + ] + }, + "@smithy/util-middleware@4.2.8": { + "integrity": "sha512-PMqfeJxLcNPMDgvPbbLl/2Vpin+luxqTGPpW3NAQVLbRrFRzTa4rNAASYeIGjRV9Ytuhzny39SpyU04EQreF+A==", + "dependencies": [ + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/util-retry@4.2.5": { + "integrity": "sha512-GBj3+EZBbN4NAqJ/7pAhsXdfzdlznOh8PydUijy6FpNIMnHPSMO2/rP4HKu+UFeikJxShERk528oy7GT79YiJg==", + "dependencies": [ + "@smithy/service-error-classification", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/util-stream@4.5.10": { + "integrity": "sha512-jbqemy51UFSZSp2y0ZmRfckmrzuKww95zT9BYMmuJ8v3altGcqjwoV1tzpOwuHaKrwQrCjIzOib499ymr2f98g==", + "dependencies": [ + "@smithy/fetch-http-handler", + "@smithy/node-http-handler", + "@smithy/types@4.12.0", + "@smithy/util-base64", + "@smithy/util-buffer-from@4.2.0", + "@smithy/util-hex-encoding@4.2.0", + "@smithy/util-utf8@4.2.0", + "tslib" + ] + }, + "@smithy/util-uri-escape@3.0.0": { + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", + "dependencies": [ + "tslib" + ] + }, + "@smithy/util-uri-escape@4.2.0": { + "integrity": "sha512-igZpCKV9+E/Mzrpq6YacdTQ0qTiLm85gD6N/IrmyDvQFA4UnU3d5g3m8tMT/6zG/vVkWSU+VxeUyGonL62DuxA==", + "dependencies": [ + "tslib" + ] + }, + "@smithy/util-utf8@2.3.0": { + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": [ + "@smithy/util-buffer-from@2.2.0", + "tslib" + ] + }, + "@smithy/util-utf8@3.0.0": { + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "dependencies": [ + "@smithy/util-buffer-from@3.0.0", + "tslib" + ] + }, + "@smithy/util-utf8@4.2.0": { + "integrity": "sha512-zBPfuzoI8xyBtR2P6WQj63Rz8i3AmfAaJLuNG8dWsfvPe8lO4aCPYLn879mEgHndZH1zQ2oXmG8O1GGzzaoZiw==", + "dependencies": [ + "@smithy/util-buffer-from@4.2.0", + "tslib" + ] + }, + "@smithy/util-waiter@4.2.5": { + "integrity": "sha512-Dbun99A3InifQdIrsXZ+QLcC0PGBPAdrl4cj1mTgJvyc9N2zf7QSxg8TBkzsCmGJdE3TLbO9ycwpY0EkWahQ/g==", + "dependencies": [ + "@smithy/abort-controller", + "@smithy/types@4.12.0", + "tslib" + ] + }, + "@smithy/uuid@1.1.0": { + "integrity": "sha512-4aUIteuyxtBUhVdiQqcDhKFitwfd9hqoSDYY2KRXiWtgoWJ9Bmise+KfEPDiVHWeJepvF8xJO9/9+WDIciMFFw==", + "dependencies": [ + "tslib" + ] + }, + "@standard-schema/spec@1.1.0": { + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==" + }, + "@types/node@24.10.1": { + "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", + "dependencies": [ + "undici-types" + ] + }, + "ansi-styles@4.3.0": { + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": [ + "color-convert" + ] + }, + "ansi-styles@5.2.0": { + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==" + }, + "bowser@2.13.0": { + "integrity": "sha512-yHAbSRuT6LTeKi6k2aS40csueHqgAsFEgmrOsfRyFpJnFv5O2hl9FYmWEUZ97gZ/dG17U4IQQcTx4YAFYPuWRQ==" + }, + "braces@3.0.3": { + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dependencies": [ + "fill-range" + ] + }, + "chalk@4.1.2": { + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dependencies": [ + "ansi-styles@4.3.0", + "supports-color" + ] + }, + "color-convert@2.0.1": { + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": [ + "color-name" + ] + }, + "color-name@1.1.4": { + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "detect-libc@1.0.3": { + "integrity": "sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==", + "bin": true + }, + "detect-libc@2.1.2": { + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==" + }, + "diff-sequences@29.6.3": { + "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==" + }, + "effect@3.19.14": { + "integrity": "sha512-3vwdq0zlvQOxXzXNKRIPKTqZNMyGCdaFUBfMPqpsyzZDre67kgC1EEHDV4EoQTovJ4w5fmJW756f86kkuz7WFA==", + "dependencies": [ + "@standard-schema/spec", + "fast-check" + ] + }, + "fast-check@3.23.2": { + "integrity": "sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A==", + "dependencies": [ + "pure-rand" + ] + }, + "fast-xml-parser@5.2.5": { + "integrity": "sha512-pfX9uG9Ki0yekDHx2SiuRIyFdyAr1kMIMitPvb0YBo8SUfKvia7w7FIyd/l6av85pFYRhZscS75MwMnbvY+hcQ==", + "dependencies": [ + "strnum" + ], + "bin": true + }, + "fill-range@7.1.1": { + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dependencies": [ + "to-regex-range" + ] + }, + "find-my-way-ts@0.1.6": { + "integrity": "sha512-a85L9ZoXtNAey3Y6Z+eBWW658kO/MwR7zIafkIUPUMf3isZG0NCs2pjW2wtjxAKuJPxMAsHUIP4ZPGv0o5gyTA==" + }, + "has-flag@4.0.0": { + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + }, + "is-extglob@2.1.1": { + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==" + }, + "is-glob@4.0.3": { + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dependencies": [ + "is-extglob" + ] + }, + "is-number@7.0.0": { + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" + }, + "jest-diff@29.7.0": { + "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", + "dependencies": [ + "chalk", + "diff-sequences", + "jest-get-type", + "pretty-format" + ] + }, + "jest-get-type@29.6.3": { + "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==" + }, + "long@5.3.2": { + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==" + }, + "micromatch@4.0.8": { + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dependencies": [ + "braces", + "picomatch" + ] + }, + "mime@3.0.0": { + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "bin": true + }, + "msgpackr-extract@3.0.3": { + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "dependencies": [ + "node-gyp-build-optional-packages" + ], + "optionalDependencies": [ + "@msgpackr-extract/msgpackr-extract-darwin-arm64", + "@msgpackr-extract/msgpackr-extract-darwin-x64", + "@msgpackr-extract/msgpackr-extract-linux-arm", + "@msgpackr-extract/msgpackr-extract-linux-arm64", + "@msgpackr-extract/msgpackr-extract-linux-x64", + "@msgpackr-extract/msgpackr-extract-win32-x64" + ], + "scripts": true, + "bin": true + }, + "msgpackr@1.11.8": { + "integrity": "sha512-bC4UGzHhVvgDNS7kn9tV8fAucIYUBuGojcaLiz7v+P63Lmtm0Xeji8B/8tYKddALXxJLpwIeBmUN3u64C4YkRA==", + "optionalDependencies": [ + "msgpackr-extract" + ] + }, + "multipasta@0.2.7": { + "integrity": "sha512-KPA58d68KgGil15oDqXjkUBEBYc00XvbPj5/X+dyzeo/lWm9Nc25pQRlf1D+gv4OpK7NM0J1odrbu9JNNGvynA==" + }, + "node-addon-api@7.1.1": { + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==" + }, + "node-gyp-build-optional-packages@5.2.2": { + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "dependencies": [ + "detect-libc@2.1.2" + ], + "bin": true + }, + "npm@11.7.0": { + "integrity": "sha512-wiCZpv/41bIobCoJ31NStIWKfAxxYyD1iYnWCtiyns8s5v3+l8y0HCP/sScuH6B5+GhIfda4HQKiqeGZwJWhFw==", + "bin": true + }, + "picomatch@2.3.1": { + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" + }, + "pretty-format@29.7.0": { + "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", + "dependencies": [ + "@jest/schemas", + "ansi-styles@5.2.0", + "react-is" + ] + }, + "protobufjs@7.5.4": { + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "dependencies": [ + "@protobufjs/aspromise", + "@protobufjs/base64", + "@protobufjs/codegen", + "@protobufjs/eventemitter", + "@protobufjs/fetch", + "@protobufjs/float", + "@protobufjs/inquire", + "@protobufjs/path", + "@protobufjs/pool", + "@protobufjs/utf8", + "@types/node", + "long" + ], + "scripts": true + }, + "pure-rand@6.1.0": { + "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==" + }, + "react-is@18.3.1": { + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==" + }, + "sax@1.4.4": { + "integrity": "sha512-1n3r/tGXO6b6VXMdFT54SHzT9ytu9yr7TaELowdYpMqY/Ao7EnlQGmAQ1+RatX7Tkkdm6hONI2owqNx2aZj5Sw==" + }, + "strnum@2.1.1": { + "integrity": "sha512-7ZvoFTiCnGxBtDqJ//Cu6fWtZtc7Y3x+QOirG15wztbdngGSkht27o2pyGWrVy0b4WAy3jbKmnoK6g5VlVNUUw==" + }, + "supports-color@7.2.0": { + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dependencies": [ + "has-flag" + ] + }, + "to-regex-range@5.0.1": { + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dependencies": [ + "is-number" + ] + }, + "tslib@2.8.1": { + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" + }, + "undici-types@7.16.0": { + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==" + }, + "undici@7.18.2": { + "integrity": "sha512-y+8YjDFzWdQlSE9N5nzKMT3g4a5UBX1HKowfdXh0uvAnTaqqwqB92Jt4UXBAeKekDs5IaDKyJFR4X1gYVCgXcw==" + }, + "uuid@11.1.0": { + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", + "bin": true + }, + "ws@8.19.0": { + "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==" + }, + "xml2js@0.6.2": { + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "dependencies": [ + "sax", + "xmlbuilder" + ] + }, + "xmlbuilder@11.0.1": { + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" + } + }, + "remote": { + "https://deno.land/std@0.196.0/fmt/colors.ts": "a7eecffdf3d1d54db890723b303847b6e0a1ab4b528ba6958b8f2e754cf1b3bc", + "https://deno.land/x/cliffy@v1.0.0-rc.3/ansi/colors.ts": "328916ea1627c202b39f2ed0f1ca65a573cfb75fa8986aa3dbcc0b7463911005" + }, + "workspace": { + "dependencies": [ + "jsr:@david/dax@~0.44.2", + "jsr:@std/assert@1", + "jsr:@std/fmt@^1.0.3", + "jsr:@std/path@^1.0.8", + "jsr:@std/testing@1", + "jsr:@std/yaml@^1.0.5", + "npm:@aws-crypto/sha256-js@^5.2.0", + "npm:@aws-sdk/client-s3@3", + "npm:@effect/opentelemetry@~0.56.2", + "npm:@effect/platform-node@0.96", + "npm:@effect/platform@~0.90.3", + "npm:@opentelemetry/exporter-trace-otlp-http@0.203", + "npm:@opentelemetry/sdk-trace-base@^2.0.1", + "npm:@opentelemetry/sdk-trace-node@^2.0.1", + "npm:@smithy/node-http-handler@^4.4.8", + "npm:@smithy/signature-v4@^4.2.0", + "npm:@smithy/types@^3.7.0", + "npm:effect@^3.17.7", + "npm:jest-diff@^29.7.0", + "npm:xml2js@0.6.2" + ] + } +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..ded6c35 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1768135262, + "narHash": "sha256-PVvu7OqHBGWN16zSi6tEmPwwHQ4rLPU9Plvs8/1TUBY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "80daad04eddbbf5a4d883996a73f3f542fa437ac", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1769433173, + "narHash": "sha256-Gf1dFYgD344WZ3q0LPlRoWaNdNQq8kSBDLEWulRQSEs=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "13b0f9e6ac78abbbb736c635d87845c4f4bee51b", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1765674936, + "narHash": "sha256-k00uTP4JNfmejrCLJOwdObYC9jHRrr/5M/a/8L2EIdo=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "2075416fcb47225d9b68ac469a5c4801a9c4dd85", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..a1e07ab --- /dev/null +++ b/flake.nix @@ -0,0 +1,71 @@ +{ + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; + flake-parts.url = "github:hercules-ci/flake-parts"; + }; + outputs = { flake-parts, ... } @ inputs: flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ + # ./module.nix + ]; + + perSystem = { config, self', inputs', pkgs, system, ... }: { + # Allows definition of system-specific attributes + # without needing to declare the system explicitly! + # + # Quick rundown of the provided arguments: + # - config is a reference to the full configuration, lazily evaluated + # - self' is the outputs as provided here, without system. (self'.packages.default) + # - inputs' is the input without needing to specify system (inputs'.foo.packages.bar) + # - pkgs is an instance of nixpkgs for your specific system + # - system is the system this configuration is for + + devShells.default = pkgs.mkShell { + buildInputs = with pkgs; [ + # inputs'.hk.packages.hk + # corepack + # nodejs_24 + # biome + deno + uv + prek + + # For systems that do not ship with Python by default (required by `node-gyp`) + # python3 + + infisical + openstack-rs + # + # opentofu + # terragrunt + # awscli2 + ]; + shellHook = '' + export PATH=$PATH:$PWD/x/ + if [[ -t 0 ]]; then + exec $(getent passwd $USER | cut -d: -f7) + fi + ''; + }; + + packages = let + # Import tools/default.nix - flake-parts will handle source filtering + webTools = import ./tools/default.nix { inherit pkgs; }; + in { + # Web app build output + webApp = webTools.webApp; + + # Docker/OCI image for the web app + webImage = webTools.webImage; + }; + }; + + flake = { + # The usual flake attributes can be defined here, including + # system-agnostic and/or arbitrary outputs. + }; + + + # Declared systems that your flake supports. These will be enumerated in perSystem + systems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; + }; +} diff --git a/herald.yaml b/herald.yaml new file mode 100644 index 0000000..90d2b7d --- /dev/null +++ b/herald.yaml @@ -0,0 +1,9 @@ +backends: + minio: + protocol: s3 + endpoint: http://127.0.0.1:9000 + region: us-east-1 + credentials: + accessKeyId: minioadmin + secretAccessKey: minioadmin + buckets: "*" diff --git a/s3-tests b/s3-tests new file mode 160000 index 0000000..9e60e5e --- /dev/null +++ b/s3-tests @@ -0,0 +1 @@ +Subproject commit 9e60e5e578c42d2d206733c58beece860c28f9ec diff --git a/src/Api.ts b/src/Api.ts new file mode 100644 index 0000000..b7a62ec --- /dev/null +++ b/src/Api.ts @@ -0,0 +1,11 @@ +import { HttpApi, OpenApi } from "@effect/platform"; +import { HealthHttpApi } from "./Frontend/Health/Api.ts"; +import { HttpS3Api } from "./Frontend/Api.ts"; + +// the http interface is declared first and separately +// and the impl is to adhere to it +// used for openAPI +export class HttpHeraldApi extends HttpApi.make("HeraldHttpApi") + .add(HealthHttpApi) + .add(HttpS3Api) + .annotate(OpenApi.Title, "Herald API") {} diff --git a/src/Backends/S3/Backend.ts b/src/Backends/S3/Backend.ts new file mode 100644 index 0000000..c2fb53f --- /dev/null +++ b/src/Backends/S3/Backend.ts @@ -0,0 +1,65 @@ +import { Effect } from "effect"; +import { HeraldConfig } from "../../Config/Layer.ts"; +import type { MaterializedBucket } from "../../Domain/Config.ts"; +import { Backend } from "../../Services/Backend.ts"; +import { makeNoopKeyValueStore } from "../../Services/NoopKeyValueStore.ts"; +import { makeBucketOps } from "./Buckets.ts"; +import { S3ClientFactory } from "./Client.ts"; +import { makeObjectOps } from "./Objects.ts"; +import { makeMultipartOps } from "./Multipart.ts"; +import { mapS3Error } from "./Utils.ts"; +import { S3HeaderService } from "../../Services/S3HeaderService.ts"; +import { Checksum } from "../../Services/Checksum.ts"; + +/** + * Creates an S3-specific Backend implementation for a given configuration context. + * Composes bucket and object operations modularly. + * Resolves the target once per backend creation (request-scoped). + */ +export const makeS3Backend = ( + bucket: MaterializedBucket | { backend_id: string }, +) => + Effect.gen(function* () { + const clientFactory = yield* S3ClientFactory; + const config = yield* HeraldConfig; + const headerService = yield* S3HeaderService; + const checksumService = yield* Checksum; + + const resolveTargetBucket = (): MaterializedBucket => { + if ("bucket_name" in bucket) return bucket as MaterializedBucket; + + const backendConfig = config.raw.backends[bucket.backend_id]; + if (backendConfig && backendConfig.protocol === "s3") { + return { + name: "", + backend_id: bucket.backend_id, + protocol: "s3" as const, + endpoint: backendConfig.endpoint, + region: backendConfig.region, + bucket_name: "", + credentials: backendConfig.credentials, + }; + } + throw new Error(`Backend ${bucket.backend_id} is not an S3 backend`); + }; + + const targetBucket = resolveTargetBucket(); + const client = yield* clientFactory.getClient(targetBucket).pipe( + Effect.mapError((e) => mapS3Error(e, targetBucket.name)), + ); + + const multipartMetadataStore = makeNoopKeyValueStore(); + const target = { + client, + bucketName: targetBucket.bucket_name, + name: targetBucket.name, + headerService, + multipartMetadataStore, + checksumService, + }; + return Backend.of({ + ...makeBucketOps(target), + ...makeObjectOps(target), + ...makeMultipartOps(target), + }); + }); diff --git a/src/Backends/S3/Buckets.ts b/src/Backends/S3/Buckets.ts new file mode 100644 index 0000000..1754867 --- /dev/null +++ b/src/Backends/S3/Buckets.ts @@ -0,0 +1,74 @@ +import { + CreateBucketCommand, + DeleteBucketCommand, + HeadBucketCommand, + ListBucketsCommand, +} from "@aws-sdk/client-s3"; +import { Effect } from "effect"; +import type { BucketInfo, ListBucketsResult } from "../../Services/Backend.ts"; +import { mapS3Error, type S3Target } from "./Utils.ts"; + +export const makeBucketOps = ( + { client, bucketName: _bucketName }: S3Target, +) => ({ + listBuckets: () => + Effect.gen(function* () { + const result = yield* Effect.tryPromise({ + try: () => client.send(new ListBucketsCommand({})), + catch: (e) => mapS3Error(e, "*"), + }); + + return { + buckets: (result.Buckets ?? []).map((b): BucketInfo => ({ + name: b.Name ?? "", + creationDate: b.CreationDate ?? new Date(), + })), + owner: { + id: result.Owner?.ID ?? "unknown", + displayName: result.Owner?.DisplayName ?? "unknown", + }, + } satisfies ListBucketsResult; + }), + + createBucket: ( + name: string, + _headers: Record, + ) => + Effect.gen(function* () { + yield* Effect.tryPromise({ + try: () => + client.send( + new CreateBucketCommand({ + Bucket: name, + }), + ), + catch: (e) => mapS3Error(e, name), + }); + }), + + deleteBucket: (name: string) => + Effect.gen(function* () { + yield* Effect.tryPromise({ + try: () => + client.send( + new DeleteBucketCommand({ + Bucket: name, + }), + ), + catch: (e) => mapS3Error(e, name), + }); + }), + + headBucket: (name: string) => + Effect.gen(function* () { + yield* Effect.tryPromise({ + try: () => + client.send( + new HeadBucketCommand({ + Bucket: name, + }), + ), + catch: (e) => mapS3Error(e, name), + }); + }), +}); diff --git a/src/Backends/S3/Client.ts b/src/Backends/S3/Client.ts new file mode 100644 index 0000000..90b03eb --- /dev/null +++ b/src/Backends/S3/Client.ts @@ -0,0 +1,147 @@ +import { S3Client as S3ClientSDK } from "@aws-sdk/client-s3"; +import { NodeHttpHandler } from "@smithy/node-http-handler"; +import { Cache, Effect } from "effect"; +import { HeraldConfig } from "../../Config/Layer.ts"; +import type { MaterializedBucket } from "../../Domain/Config.ts"; + +/** + * Generate a stable cache key from MaterializedBucket configuration. + * The key is based on the fields that determine S3 client configuration: + * backend_id, endpoint, region, and credentials. + */ +const getCacheKey = (resolved: MaterializedBucket): string => { + let accessKeyId: string | undefined; + if (resolved.credentials) { + const creds = resolved.credentials; + if ("accessKeyId" in creds) { + accessKeyId = creds.accessKeyId; + } else if ("username" in creds) { + accessKeyId = creds.username; + } + } + // Create a stable key from the configuration that determines the S3 client + return JSON.stringify({ + backend_id: resolved.backend_id, + endpoint: resolved.endpoint ?? null, + region: resolved.region ?? null, + accessKeyId: accessKeyId ?? null, + }); +}; + +export class S3ClientFactory + extends Effect.Service()("S3ClientFactory", { + effect: Effect.gen(function* () { + const appConfig = yield* HeraldConfig; + + const cache = yield* Cache.make({ + capacity: 100, + timeToLive: "24 hours", // S3 clients can live a long time + lookup: (cacheKey: string) => + Effect.gen(function* () { + // Parse the cache key to get the configuration + const config = JSON.parse(cacheKey) as { + backend_id: string; + endpoint: string | null; + region: string | null; + accessKeyId: string | null; + }; + + if (config.endpoint === null) { + return yield* Effect.fail( + new Error( + `Missing endpoint for backend ${config.backend_id}`, + ), + ); + } + + if (config.region === null) { + return yield* Effect.fail( + new Error(`Missing region for backend ${config.backend_id}`), + ); + } + + // Get credentials from the backend config + const backendConfig = appConfig.raw.backends[config.backend_id]; + let accessKeyId: string | undefined; + let secretAccessKey: string | undefined; + + if (backendConfig?.credentials) { + const creds = backendConfig.credentials; + if ("accessKeyId" in creds) { + accessKeyId = creds.accessKeyId; + secretAccessKey = creds.secretAccessKey; + } else if ("username" in creds) { + accessKeyId = creds.username; + secretAccessKey = creds.password; + } + + if (accessKeyId === undefined) { + return yield* Effect.fail( + new Error( + `Missing accessKeyId/username for backend ${config.backend_id}`, + ), + ); + } + if (secretAccessKey === undefined) { + return yield* Effect.fail( + new Error( + `Missing secretAccessKey/password for backend ${config.backend_id}`, + ), + ); + } + } + + return new S3ClientSDK({ + endpoint: config.endpoint, + region: config.region, + credentials: accessKeyId && secretAccessKey + ? { + accessKeyId, + secretAccessKey, + } + : undefined, + forcePathStyle: true, + // we must rely on the node impl due to https://github.com/aws/aws-sdk-js-v3/issues/6770 + requestHandler: new NodeHttpHandler(), + // requestStreamBufferSize: 64 * 1024, + // requestHandler: new NodeHttpHandler(), + // requestChecksumCalculation: "WHEN_REQUIRED", + // responseChecksumValidation: "WHEN_REQUIRED", + }); + }), + }); + + return { + getClient: (bucket: MaterializedBucket | { backend_id: string }) => { + // Resolve full bucket if only backend_id provided + let resolved: MaterializedBucket; + if ("bucket_name" in bucket) { + resolved = bucket; + } else { + const backendConfig = appConfig.raw.backends[bucket.backend_id]; + if (backendConfig && backendConfig.protocol === "s3") { + resolved = { + name: "", + backend_id: bucket.backend_id, + protocol: "s3" as const, + endpoint: backendConfig.endpoint, + region: backendConfig.region, + bucket_name: "", + credentials: backendConfig.credentials, + }; + } else { + return Effect.fail( + new Error( + `Backend ${bucket.backend_id} is not an S3 backend or not found`, + ), + ); + } + } + + // Use stable cache key instead of the object itself + const cacheKey = getCacheKey(resolved); + return cache.get(cacheKey); + }, + }; + }), + }) {} diff --git a/src/Backends/S3/Multipart.ts b/src/Backends/S3/Multipart.ts new file mode 100644 index 0000000..2d6965b --- /dev/null +++ b/src/Backends/S3/Multipart.ts @@ -0,0 +1,406 @@ +import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CreateMultipartUploadCommand, + ListMultipartUploadsCommand, + ListPartsCommand, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import { Effect, Stream } from "effect"; +import { Readable } from "node-stream"; +import type sweb from "node-stream/web"; +import { + BadDigest, + type CompleteMultipartUploadResult, + InternalError, + InvalidRequest, + type ListMultipartUploadsResult, + type ListPartsResult, + type MultipartUploadResult, + type UploadPartResult, +} from "../../Services/Backend.ts"; +import { normalizeHeaders } from "../../Services/S3HeaderService.ts"; +import type { + ChecksumAlgorithm, + ChecksumType, +} from "../../Services/S3Schema.ts"; +import { mapS3Error, type S3Target } from "./Utils.ts"; + +interface S3ChecksumFields { + readonly ChecksumCRC32?: string; + readonly ChecksumCRC32C?: string; + readonly ChecksumCRC64NVME?: string; + readonly ChecksumSHA1?: string; + readonly ChecksumSHA256?: string; + readonly ChecksumAlgorithm?: string; + readonly ChecksumType?: string; +} + +export const makeMultipartOps = ( + { client, bucketName, headerService, checksumService }: S3Target, +) => ({ + createMultipartUpload: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { checksums, metadata } = headerService.fromRequestHeaders(headers); + const normalized = normalizeHeaders(headers); + + // Don't pass ChecksumAlgorithm to avoid SDK enabling checksum validation for uploadPart + // The SDK's checksum middleware converts Buffer to ReadableStream for validation, + // causing "Received an instance of ReadableStream" errors with Node.js crypto. + // We'll validate checksums ourselves and return them in the response headers. + const command = new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + Metadata: metadata, + ContentType: normalized["content-type"] as string, + // Intentionally NOT passing ChecksumAlgorithm or ChecksumType to avoid SDK validation + }); + + if (checksums.algorithm) { + command.middlewareStack.add( + (next) => (args) => { + const request = args.request as { headers: Record }; + request.headers["x-amz-checksum-algorithm"] = checksums.algorithm! + .toUpperCase(); + return next(args); + }, + { step: "build", name: "ManualAlgorithmInjection" }, + ); + } + + const response = yield* Effect.tryPromise({ + try: () => client.send(command), + catch: (e) => mapS3Error(e, bucketName), + }); + return { + uploadId: response.UploadId!, + checksumAlgorithm: response.ChecksumAlgorithm, + checksumType: response.ChecksumType, + } satisfies MultipartUploadResult; + }), + + uploadPart: ( + key: string, + uploadId: string, + partNumber: number, + bodyStream: Stream.Stream, + headers: Record, + ) => + Effect.gen(function* () { + const { checksums, s3Params } = headerService.fromRequestHeaders(headers); + + const contentLength = s3Params.contentLength; + + const validatedStream = yield* checksumService.validate( + bodyStream, + checksums, + ); + + const body = Readable.fromWeb( + Stream.toReadableStream(validatedStream.pipe( + Stream.mapError((e) => { + if (e instanceof BadDigest) return e; + if (e instanceof InvalidRequest) return e; + return new InternalError({ message: String(e) }); + }), + )) as sweb.ReadableStream, + ); + + // Build command WITHOUT any checksum parameters to avoid SDK's internal checksum validation + // The SDK's checksum middleware converts the body to a ReadableStream for validation, + // which causes "Received an instance of ReadableStream" errors with Node.js crypto. + // Since we've already validated checksums, we don't need the SDK to validate them. + const commandInput = { + Bucket: bucketName, + Key: key, + UploadId: uploadId, + PartNumber: partNumber, + Body: body, // Use Node Readable + ContentLength: contentLength, + // Intentionally NOT passing any checksum-related parameters to avoid SDK validation + }; + + const result = yield* Effect.tryPromise({ + try: () => { + const command = new UploadPartCommand(commandInput); + + // If it's a Node stream, add an error handler to prevent uncaught exceptions + // from the stream itself, as we handle failures through the send() promise. + if (body instanceof Readable) { + body.on("error", (err: unknown) => { + // Log at debug level for debugging purposes, but don't throw + // as we handle failures through the send() promise + Effect.logDebug("Stream error", { + operation: "uploadPart", + context: "handled by send() promise", + error: String(err), + }).pipe( + Effect.runPromise, + ).catch(() => { + // Ignore logging errors + }); + }); + } + + // Remove checksum middlewares to prevent them from trying to hash the stream twice + command.middlewareStack.remove("flexibleChecksumsMiddleware"); + command.middlewareStack.remove("getChecksumMiddleware"); + + // Manually inject validated checksums + command.middlewareStack.add( + (next) => (args) => { + const request = args.request as { + headers: Record; + duplex?: string; + }; + request.duplex = "half"; + request.headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; + if (contentLength !== undefined) { + request.headers["content-length"] = String(contentLength); + } + if (checksums.sha256) { + request.headers["x-amz-checksum-sha256"] = checksums.sha256; + } + if (checksums.sha1) { + request.headers["x-amz-checksum-sha1"] = checksums.sha1; + } + if (checksums.crc32) { + request.headers["x-amz-checksum-crc32"] = checksums.crc32; + } + if (checksums.crc32c) { + request.headers["x-amz-checksum-crc32c"] = checksums.crc32c; + } + if (checksums.crc64nvme) { + request.headers["x-amz-checksum-crc64nvme"] = + checksums.crc64nvme; + } + return next(args); + }, + { step: "build", name: "ManualChecksumInjection" }, + ); + + return client.send(command); + }, + catch: (e) => mapS3Error(e, bucketName, uploadId), + }); + + if (!result.ETag) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned empty ETag for UploadPart", + }), + ); + } + // Return checksums we calculated (since we didn't pass them to SDK to avoid validation issues) + // The SDK might return some checksums, but we prefer our validated ones + const s3Result = result as S3ChecksumFields; + return { + etag: result.ETag, + checksumAlgorithm: checksums.algorithm || + s3Result.ChecksumAlgorithm as ChecksumAlgorithm, + checksumType: checksums.type || s3Result.ChecksumType as ChecksumType, + checksumCRC32: checksums.crc32 || s3Result.ChecksumCRC32, + checksumCRC32C: checksums.crc32c || s3Result.ChecksumCRC32C, + checksumCRC64NVME: checksums.crc64nvme || s3Result.ChecksumCRC64NVME, + checksumSHA1: checksums.sha1 || s3Result.ChecksumSHA1, + checksumSHA256: checksums.sha256 || s3Result.ChecksumSHA256, + } satisfies UploadPartResult; + }), + + completeMultipartUpload: ( + key: string, + uploadId: string, + parts: readonly { + etag: string; + partNumber: number; + checksumCRC32?: string; + checksumCRC32C?: string; + checksumCRC64NVME?: string; + checksumSHA1?: string; + checksumSHA256?: string; + }[], + _metadata: Record, + headers: Record, + ) => + Effect.gen(function* () { + const { checksums } = headerService.fromRequestHeaders(headers); + + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: parts.map((p) => ({ + ETag: p.etag, + PartNumber: p.partNumber, + ChecksumCRC32: p.checksumCRC32, + ChecksumCRC32C: p.checksumCRC32C, + ChecksumCRC64NVME: p.checksumCRC64NVME, + ChecksumSHA1: p.checksumSHA1, + ChecksumSHA256: p.checksumSHA256, + })), + }, + ChecksumCRC32: checksums.crc32, + ChecksumCRC32C: checksums.crc32c, + ChecksumCRC64NVME: checksums.crc64nvme, + ChecksumSHA1: checksums.sha1, + ChecksumSHA256: checksums.sha256, + ChecksumType: checksums.type, + }), + ), + catch: (e) => mapS3Error(e, bucketName, uploadId), + }); + + if ( + !result.Location || !result.Bucket || !result.Key || + !result.ETag + ) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned incomplete CompleteMultipartUploadResult", + }), + ); + } + const checksumResult = result as S3ChecksumFields; + return { + location: result.Location, + bucket: result.Bucket, + key: result.Key, + etag: result.ETag, + versionId: result.VersionId, + checksumAlgorithm: checksumResult.ChecksumAlgorithm, + checksumType: checksumResult.ChecksumType, + checksumCRC32: result.ChecksumCRC32, + checksumCRC32C: result.ChecksumCRC32C, + checksumCRC64NVME: result.ChecksumCRC64NVME, + checksumSHA1: result.ChecksumSHA1, + checksumSHA256: result.ChecksumSHA256, + } satisfies CompleteMultipartUploadResult; + }), + + abortMultipartUpload: (key: string, uploadId: string) => + Effect.gen(function* () { + yield* Effect.tryPromise({ + try: () => + client.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + }), + ), + catch: (e) => mapS3Error(e, bucketName, uploadId), + }); + }), + + listMultipartUploads: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + uploadIdMarker?: string; + maxUploads?: number; + encodingType?: string; + }) => + Effect.gen(function* () { + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListMultipartUploadsCommand({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + KeyMarker: args.keyMarker, + UploadIdMarker: args.uploadIdMarker, + MaxUploads: args.maxUploads, + EncodingType: args.encodingType as "url" | undefined, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + bucket: result.Bucket ?? bucketName, + prefix: result.Prefix, + keyMarker: result.KeyMarker, + uploadIdMarker: result.UploadIdMarker, + nextKeyMarker: result.NextKeyMarker, + nextUploadIdMarker: result.NextUploadIdMarker, + maxUploads: result.MaxUploads ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: result.EncodingType ?? args.encodingType, + uploads: (result.Uploads ?? []).map((u) => ({ + key: u.Key ?? "", + uploadId: u.UploadId ?? "", + owner: { + id: u.Owner?.ID ?? "", + displayName: u.Owner?.DisplayName ?? "", + }, + initiator: { + id: u.Initiator?.ID ?? "", + displayName: u.Initiator?.DisplayName ?? "", + }, + storageClass: u.StorageClass ?? "STANDARD", + initiated: u.Initiated ?? new Date(), + })), + commonPrefixes: (result.CommonPrefixes ?? []).map((cp) => ({ + prefix: cp.Prefix ?? "", + })), + } satisfies ListMultipartUploadsResult; + }), + + listParts: (key: string, uploadId: string) => + Effect.gen(function* () { + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListPartsCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + }), + ), + catch: (e) => mapS3Error(e, bucketName, uploadId), + }); + + return { + bucket: result.Bucket ?? bucketName, + key: result.Key ?? key, + uploadId: result.UploadId ?? uploadId, + owner: { + id: result.Owner?.ID ?? "", + displayName: result.Owner?.DisplayName ?? "", + }, + initiator: { + id: result.Initiator?.ID ?? "", + displayName: result.Initiator?.DisplayName ?? "", + }, + storageClass: result.StorageClass ?? "STANDARD", + partNumberMarker: result.PartNumberMarker + ? parseInt(String(result.PartNumberMarker), 10) || 0 + : 0, + nextPartNumberMarker: result.NextPartNumberMarker + ? parseInt(String(result.NextPartNumberMarker), 10) || 0 + : 0, + maxParts: result.MaxParts ?? 1000, + isTruncated: result.IsTruncated ?? false, + parts: (result.Parts ?? []).map((p) => ({ + partNumber: p.PartNumber ?? 0, + lastModified: p.LastModified ?? new Date(), + etag: p.ETag ?? "", + size: p.Size ?? 0, + checksumCRC32: p.ChecksumCRC32, + checksumCRC32C: p.ChecksumCRC32C, + checksumCRC64NVME: p.ChecksumCRC64NVME, + checksumSHA1: p.ChecksumSHA1, + checksumSHA256: p.ChecksumSHA256, + })), + } satisfies ListPartsResult; + }), +}); diff --git a/src/Backends/S3/Objects.ts b/src/Backends/S3/Objects.ts new file mode 100644 index 0000000..879f561 --- /dev/null +++ b/src/Backends/S3/Objects.ts @@ -0,0 +1,652 @@ +import { + DeleteObjectCommand, + DeleteObjectsCommand, + GetObjectAttributesCommand, + GetObjectCommand, + HeadObjectCommand, + ListObjectsCommand, + type ListObjectsCommandOutput, + ListObjectsV2Command, + type ListObjectsV2CommandOutput, + ListObjectVersionsCommand, + type ObjectAttributes as S3ObjectAttributes, + PutObjectCommand, +} from "@aws-sdk/client-s3"; +import { Chunk, Effect, Option, Stream } from "effect"; +import { Readable } from "node-stream"; +import type sweb from "node-stream/web"; +import { + type BackendError, + BadDigest, + type CommonPrefix, + type HeadObjectResult, + InternalError, + InvalidRequest, + type ListObjectsResult, + type ObjectInfo, + type ObjectResponse, +} from "../../Services/Backend.ts"; +import { normalizeHeaders } from "../../Services/S3HeaderService.ts"; +import type { + ChecksumAlgorithm, + ChecksumType, +} from "../../Services/S3Schema.ts"; +import { mapS3Error, type S3Target, stripMinioMetadata } from "./Utils.ts"; + +interface S3ChecksumFields { + readonly ChecksumCRC32?: string; + readonly ChecksumCRC32C?: string; + readonly ChecksumCRC64NVME?: string; + readonly ChecksumSHA1?: string; + readonly ChecksumSHA256?: string; + readonly ChecksumAlgorithm?: string; + readonly ChecksumType?: string; +} + +const mapS3ChecksumsToResult = (result: S3ChecksumFields) => ({ + checksumAlgorithm: result.ChecksumAlgorithm as ChecksumAlgorithm, + checksumType: result.ChecksumType as ChecksumType, + checksumCRC32: result.ChecksumCRC32, + checksumCRC32C: result.ChecksumCRC32C, + checksumCRC64NVME: result.ChecksumCRC64NVME, + checksumSHA1: result.ChecksumSHA1, + checksumSHA256: result.ChecksumSHA256, +}); + +export const makeObjectOps = ( + { client, bucketName, headerService, checksumService }: S3Target, +) => ({ + listObjects: (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + encodingType?: string; + continuationToken?: string; + startAfter?: string; + listType?: 1 | 2; + }) => + Effect.gen(function* () { + if (args.listType === 2) { + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + MaxKeys: args.maxKeys, + ContinuationToken: args.continuationToken, + StartAfter: args.startAfter, + }), + ) as Promise, + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + name: result.Name ?? bucketName, + prefix: result.Prefix, + maxKeys: result.MaxKeys ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: args.encodingType, + continuationToken: result.ContinuationToken, + nextContinuationToken: result.NextContinuationToken, + keyCount: result.KeyCount, + listType: 2, + contents: (result.Contents ?? []).map((c): ObjectInfo => ({ + key: stripMinioMetadata(c.Key ?? ""), + lastModified: c.LastModified ?? new Date(), + etag: c.ETag ?? "", + size: c.Size ?? 0, + storageClass: c.StorageClass, + owner: c.Owner + ? { + id: c.Owner.ID ?? "unknown", + displayName: c.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + commonPrefixes: (result.CommonPrefixes ?? []).map(( + cp, + ): CommonPrefix => ({ + prefix: stripMinioMetadata(cp.Prefix ?? ""), + })), + } satisfies ListObjectsResult; + } else { + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListObjectsCommand({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + Marker: args.marker, + MaxKeys: args.maxKeys, + }), + ) as Promise, + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + name: result.Name ?? bucketName, + prefix: result.Prefix, + marker: result.Marker, + nextMarker: result.NextMarker, + maxKeys: result.MaxKeys ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: args.encodingType, + listType: 1, + contents: (result.Contents ?? []).map((c): ObjectInfo => ({ + key: stripMinioMetadata(c.Key ?? ""), + lastModified: c.LastModified ?? new Date(), + etag: c.ETag ?? "", + size: c.Size ?? 0, + storageClass: c.StorageClass, + owner: c.Owner + ? { + id: c.Owner.ID ?? "unknown", + displayName: c.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + commonPrefixes: (result.CommonPrefixes ?? []).map(( + cp, + ): CommonPrefix => ({ + prefix: stripMinioMetadata(cp.Prefix ?? ""), + })), + } satisfies ListObjectsResult; + } + }), + + listVersions: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + versionIdMarker?: string; + maxKeys?: number; + encodingType?: string; + }) => + Effect.gen(function* () { + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListObjectVersionsCommand({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + KeyMarker: args.keyMarker, + VersionIdMarker: args.versionIdMarker, + MaxKeys: args.maxKeys, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + name: result.Name ?? bucketName, + prefix: result.Prefix, + marker: result.KeyMarker, + nextMarker: result.NextKeyMarker, + maxKeys: result.MaxKeys ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: args.encodingType, + listType: 1, + contents: [ + ...(result.Versions ?? []).map((v): ObjectInfo => ({ + key: stripMinioMetadata(v.Key ?? ""), + lastModified: v.LastModified ?? new Date(), + etag: v.ETag ?? "", + size: v.Size ?? 0, + storageClass: v.StorageClass, + versionId: v.VersionId, + isDeleteMarker: false, + isLatest: v.IsLatest, + owner: v.Owner + ? { + id: v.Owner.ID ?? "unknown", + displayName: v.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + ...(result.DeleteMarkers ?? []).map((dm): ObjectInfo => ({ + key: stripMinioMetadata(dm.Key ?? ""), + lastModified: dm.LastModified ?? new Date(), + etag: "", + size: 0, + versionId: dm.VersionId, + isDeleteMarker: true, + isLatest: dm.IsLatest, + owner: dm.Owner + ? { + id: dm.Owner.ID ?? "unknown", + displayName: dm.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + ], + commonPrefixes: (result.CommonPrefixes ?? []).map(( + cp, + ): CommonPrefix => ({ + prefix: stripMinioMetadata(cp.Prefix ?? ""), + })), + } satisfies ListObjectsResult; + }), + + getObject: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const normalized = normalizeHeaders(headers); + const { s3Params } = headerService.fromRequestHeaders(headers); + + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: key, + Range: normalized["range"], + PartNumber: s3Params.partNumber, + ChecksumMode: s3Params.checksumMode as "ENABLED", + IfMatch: normalized["if-match"], + IfNoneMatch: normalized["if-none-match"], + IfModifiedSince: normalized["if-modified-since"] + ? new Date(normalized["if-modified-since"] as string) + : undefined, + IfUnmodifiedSince: normalized["if-unmodified-since"] + ? new Date(normalized["if-unmodified-since"] as string) + : undefined, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + const body = result.Body; + if (!body) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned empty body for GetObject", + }), + ); + } + + const getWebStream = (): ReadableStream => { + if ( + body && typeof body === "object" && + "transformToWebStream" in body + ) { + const b = body as { transformToWebStream: unknown }; + if (typeof b.transformToWebStream === "function") { + return b.transformToWebStream() as ReadableStream< + Uint8Array + >; + } + } + return body as ReadableStream; + }; + + const webStream = getWebStream(); + const stream: Stream.Stream = Stream + .fromReadableStream( + () => webStream, + (e) => new Error(String(e)), + ); + + const metadata: Record = {}; + if (result.Metadata) { + for (const [k, v] of Object.entries(result.Metadata)) { + metadata[k] = v.includes("%") + ? Option.liftThrowable(decodeURIComponent)(v).pipe( + Option.getOrElse(() => v), + ) + : v; + } + } + + const responseResult: ObjectResponse = { + stream, + nativeStream: webStream, + contentType: result.ContentType, + contentLength: result.ContentLength, + etag: result.ETag, + lastModified: result.LastModified, + metadata, + partsCount: result.PartsCount, + headers: headerService.toResponseHeaders({ + ...mapS3ChecksumsToResult(result as S3ChecksumFields), + metadata, + headers: {}, + partsCount: result.PartsCount, + contentLength: result.ContentLength, + contentType: result.ContentType, + etag: result.ETag, + lastModified: result.LastModified, + }), + ...mapS3ChecksumsToResult(result as S3ChecksumFields), + }; + + return responseResult; + }), + + headObject: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { s3Params } = headerService.fromRequestHeaders(headers); + + const commandInput = { + Bucket: bucketName, + Key: key, + PartNumber: s3Params.partNumber, + ChecksumMode: s3Params.checksumMode as "ENABLED", + }; + const result = yield* Effect.tryPromise({ + try: () => client.send(new HeadObjectCommand(commandInput)), + catch: (e) => mapS3Error(e, bucketName), + }); + + const metadata: Record = {}; + if (result.Metadata) { + for (const [k, v] of Object.entries(result.Metadata)) { + metadata[k] = v.includes("%") + ? Option.liftThrowable(decodeURIComponent)(v).pipe( + Option.getOrElse(() => v), + ) + : v; + } + } + + return { + contentType: result.ContentType, + contentLength: result.ContentLength, + etag: result.ETag, + lastModified: result.LastModified, + metadata, + partsCount: result.PartsCount, + headers: headerService.toResponseHeaders({ + ...mapS3ChecksumsToResult(result as S3ChecksumFields), + metadata, + headers: {}, + partsCount: result.PartsCount, + contentLength: result.ContentLength, + contentType: result.ContentType, + etag: result.ETag, + lastModified: result.LastModified, + }), + ...mapS3ChecksumsToResult(result as S3ChecksumFields), + } satisfies HeadObjectResult; + }), + + putObject: ( + key: string, + bodyStream: Stream.Stream, + headers: Record, + ) => + Effect.gen(function* () { + const { checksums, metadata, s3Params } = headerService + .fromRequestHeaders(headers); + const normalized = normalizeHeaders(headers); + + const contentType = normalized["content-type"]!; + const contentLength = s3Params.contentLength; + + const validatedStream = (yield* checksumService.validate( + bodyStream, + checksums, + )).pipe( + Stream.catchAll((e) => { + // Preserve BadDigest and InvalidRequest errors from checksum validation + if (e instanceof BadDigest || e instanceof InvalidRequest) { + return Stream.fail(e as BackendError); + } + return Stream.fail( + new InternalError({ + message: `error on checksum stream: ${String(e)}`, + }), + ); + }), + ); + + const isSmall = contentLength !== undefined && + contentLength < 1024 * 1024; + + const body = isSmall + ? yield* Stream.runCollect(validatedStream).pipe( + Effect.map((chunks) => { + const total = Chunk.reduce(chunks, 0, (acc, c) => acc + c.length); + const res = new Uint8Array(total); + let off = 0; + for (const c of chunks) { + res.set(c, off); + off += c.length; + } + return res; + }), + Effect.mapError((e) => { + if (e instanceof InvalidRequest) return e; + if (e instanceof BadDigest) return e; + return new InternalError({ + message: `error collecting body stream into memory: ${String(e)}`, + }); + }), + ) + : Readable.fromWeb( + Stream.toReadableStream(validatedStream) as sweb.ReadableStream, + ); + + const result = yield* Effect.tryPromise({ + try: () => { + const command = new PutObjectCommand({ + Bucket: bucketName, + Key: key, + Body: body, + ContentType: contentType, + ContentLength: contentLength, + Metadata: metadata, + }); + + // If it's a Node stream, add an error handler to prevent uncaught exceptions + // from the stream itself, as we handle failures through the send() promise. + if (body instanceof Readable) { + body.on("error", (err: unknown) => { + // Log at debug level for debugging purposes, but don't throw + // as we handle failures through the send() promise + Effect.logDebug("Stream error", { + operation: "putObject", + context: "handled by send() promise", + error: String(err), + }).pipe( + Effect.runPromise, + ).catch(() => { + // Ignore logging errors + }); + }); + } + + // Remove checksum middlewares to prevent them from trying to hash the stream twice + command.middlewareStack.remove("flexibleChecksumsMiddleware"); + command.middlewareStack.remove("getChecksumMiddleware"); + + // Manually inject validated checksums + if ( + checksums.sha256 || checksums.sha1 || checksums.crc32 || + checksums.crc32c || checksums.crc64nvme || !isSmall + ) { + command.middlewareStack.add( + (next) => (args) => { + const request = args.request as { + headers: Record; + duplex?: string; + }; + if (!isSmall) { + request.duplex = "half"; + request.headers["x-amz-content-sha256"] = "UNSIGNED-PAYLOAD"; + if (contentLength !== undefined) { + request.headers["content-length"] = String(contentLength); + } + } + if (checksums.sha256) { + request.headers["x-amz-checksum-sha256"] = checksums.sha256; + } + if (checksums.sha1) { + request.headers["x-amz-checksum-sha1"] = checksums.sha1; + } + if (checksums.crc32) { + request.headers["x-amz-checksum-crc32"] = checksums.crc32; + } + if (checksums.crc32c) { + request.headers["x-amz-checksum-crc32c"] = checksums.crc32c; + } + if (checksums.crc64nvme) { + request.headers["x-amz-checksum-crc64nvme"] = + checksums.crc64nvme; + } + return next(args); + }, + { step: "build", name: "ManualChecksumInjection" }, + ); + } + + return client.send(command); + }, + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + etag: result.ETag, + versionId: result.VersionId, + ...mapS3ChecksumsToResult(result as S3ChecksumFields), + }; + }), + + deleteObject: (key: string) => + Effect.gen(function* () { + yield* Effect.tryPromise({ + try: () => + client.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: key, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + }), + + deleteObjects: (objects: readonly { key: string; versionId?: string }[]) => + Effect.gen(function* () { + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects.map((o) => ({ + Key: o.key, + VersionId: o.versionId === "null" ? undefined : o.versionId, + })), + }, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + deleted: (result.Deleted ?? []).map((d) => d.Key ?? ""), + errors: (result.Errors ?? []).map((e) => ({ + key: e.Key ?? "unknown", + code: e.Code ?? "InternalError", + message: e.Message ?? "Unknown error", + })), + }; + }), + + getObjectAttributes: ( + key: string, + attributes: readonly string[], + headers: Record, + ) => + Effect.gen(function* () { + const { s3Params } = headerService.fromRequestHeaders(headers); + + // Map attribute names to what S3 SDK expects (case-sensitive) + const s3Attributes = attributes + .map((a) => { + const lower = a.toLowerCase(); + if (lower === "etag") return "ETag"; + if (lower === "checksum") return "Checksum"; + if (lower === "objectparts") return "ObjectParts"; + if (lower === "objectsize") return "ObjectSize"; + if (lower === "storageclass") return "StorageClass"; + return undefined; + }) + .filter((a): a is S3ObjectAttributes => a !== undefined); + + if (s3Attributes.length === 0) { + // If no recognized attributes, return a sensible default or fail? + // S3 requires at least one. + return yield* Effect.fail(mapS3Error({ + name: "InvalidArgument", + message: "At least one valid attribute must be specified.", + }, bucketName)); + } + + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new GetObjectAttributesCommand({ + Bucket: bucketName, + Key: key, + ObjectAttributes: s3Attributes, + VersionId: s3Params.versionId, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + etag: result.ETag, + checksum: result.Checksum + ? { + checksumCRC32: result.Checksum.ChecksumCRC32, + checksumCRC32C: result.Checksum.ChecksumCRC32C, + checksumCRC64NVME: result.Checksum.ChecksumCRC64NVME, + checksumSHA1: result.Checksum.ChecksumSHA1, + checksumSHA256: result.Checksum.ChecksumSHA256, + checksumType: result.Checksum.ChecksumType, + } + : undefined, + objectParts: result.ObjectParts + ? { + totalPartsCount: result.ObjectParts.TotalPartsCount, + partNumberMarker: result.ObjectParts.PartNumberMarker + ? parseInt(String(result.ObjectParts.PartNumberMarker)) + : undefined, + nextPartNumberMarker: result.ObjectParts.NextPartNumberMarker + ? parseInt(String(result.ObjectParts.NextPartNumberMarker)) + : undefined, + maxParts: result.ObjectParts.MaxParts, + isTruncated: result.ObjectParts.IsTruncated, + parts: (result.ObjectParts.Parts ?? []).map((p) => ({ + partNumber: p.PartNumber ?? 0, + etag: "", // GetObjectAttributes doesn't return ETag for parts + size: p.Size ?? 0, + lastModified: undefined, + checksumCRC32: p.ChecksumCRC32, + checksumCRC32C: p.ChecksumCRC32C, + checksumCRC64NVME: p.ChecksumCRC64NVME, + checksumSHA1: p.ChecksumSHA1, + checksumSHA256: p.ChecksumSHA256, + })), + } + : undefined, + objectSize: result.ObjectSize, + storageClass: result.StorageClass, + }; + }), +}); diff --git a/src/Backends/S3/Utils.ts b/src/Backends/S3/Utils.ts new file mode 100644 index 0000000..ca27a5e --- /dev/null +++ b/src/Backends/S3/Utils.ts @@ -0,0 +1,125 @@ +import { + AccessDenied, + BadDigest, + BucketAlreadyExists, + BucketAlreadyOwnedByYou, + BucketNotEmpty, + EntityTooSmall, + InternalError, + InvalidArgument, + InvalidBucketName, + InvalidPart, + InvalidPartOrder, + InvalidRequest, + MalformedXML, + NoSuchBucket, + NoSuchKey, + NoSuchUpload, +} from "../../Services/Backend.ts"; +import type { S3Client } from "@aws-sdk/client-s3"; +import type { S3HeaderService } from "../../Services/S3HeaderService.ts"; +import type { Checksum } from "../../Services/Checksum.ts"; + +export interface S3Target { + readonly client: S3Client; + readonly bucketName: string; + readonly name: string; + readonly headerService: S3HeaderService; + readonly checksumService: Checksum; +} + +export const mapS3Error = ( + e: unknown, + bucket: string, + uploadId?: string, +) => { + if (e instanceof BadDigest) return e; + + const error = e as { + name?: string; + Code?: string; + message?: string; + Message?: string; + Key?: string; + cause?: unknown; + }; + + // Check for BadDigest in the error message or cause + const errorStr = String(e); + if ( + errorStr.includes("BadDigest") || errorStr.includes("checksum mismatch") || + errorStr.includes("Checksum mismatch") + ) { + return new BadDigest({ message: errorStr }); + } + if (error.cause) { + if (error.cause instanceof BadDigest) return error.cause; + const causeStr = String(error.cause); + if ( + causeStr.includes("BadDigest") || + causeStr.includes("checksum mismatch") || + causeStr.includes("Checksum mismatch") + ) { + return new BadDigest({ message: causeStr }); + } + } + + const name = error.name || error.Code || "InternalError"; + const message = error.message || error.Message || "Internal S3 Error"; + + switch (name) { + case "NoSuchBucket": + case "NotFound": // S3 sometimes returns NotFound for HEAD requests on non-existent buckets + return new NoSuchBucket({ bucket, message }); + case "NoSuchKey": + return new NoSuchKey({ + bucket, + key: error.Key || "unknown", + message, + }); + case "AccessDenied": + return new AccessDenied({ message }); + case "BucketAlreadyExists": + return new BucketAlreadyExists({ bucket, message }); + case "BucketAlreadyOwnedByYou": + return new BucketAlreadyOwnedByYou({ bucket, message }); + case "BucketNotEmpty": + return new BucketNotEmpty({ bucket, message }); + case "InvalidBucketName": + return new InvalidBucketName({ + message: `Invalid bucket name: ${bucket}`, + }); + case "InvalidArgument": + return new InvalidArgument({ message }); + case "NoSuchUpload": + return new NoSuchUpload({ + uploadId: uploadId || "unknown", + message, + }); + case "InvalidRequest": + return new InvalidRequest({ message }); + case "MalformedXML": + return new MalformedXML({ message }); + case "InvalidPart": + return new InvalidPart({ message }); + case "InvalidPartOrder": + return new InvalidPartOrder({ message }); + case "EntityTooSmall": + return new EntityTooSmall({ message }); + default: + return new InternalError({ + message: `S3 Error [${name}]: ${message}`, + }); + } +}; + +/** + * Minio sometimes adds metadata prefixes like 'X-Amz-Meta-' to keys in listings. + * This helper strips them if present. + */ +export const stripMinioMetadata = (key: string): string => { + // if (key.startsWith("X-Amz-Meta-")) { + // return key.substring("X-Amz-Meta-".length); + // } + return key; +}; diff --git a/src/Backends/Swift/Backend.ts b/src/Backends/Swift/Backend.ts new file mode 100644 index 0000000..189c07c --- /dev/null +++ b/src/Backends/Swift/Backend.ts @@ -0,0 +1,81 @@ +import { HttpClient } from "@effect/platform"; +import type { Stream } from "effect"; +import { Effect } from "effect"; +import type { MaterializedBucket } from "../../Domain/Config.ts"; +import { Backend, InternalError } from "../../Services/Backend.ts"; +import { makeBackendKeyValueStore } from "../../Services/BackendKeyValueStore.ts"; +import { Checksum } from "../../Services/Checksum.ts"; +import { S3HeaderService } from "../../Services/S3HeaderService.ts"; +import { makeBucketOps } from "./Buckets.ts"; +import { SwiftClient } from "./Client.ts"; +import { makeObjectOps } from "./Objects.ts"; +import { makeMultipartOps } from "./Multipart.ts"; +import { MP_META_PREFIX } from "./Utils.ts"; + +/** + * Creates a Swift-specific Backend implementation for a given configuration context. + * Composes bucket and object operations modularly. + * Resolves the target and client once per backend creation (request-scoped). + */ +export const makeSwiftBackend = ( + bucket: MaterializedBucket | { backend_id: string }, +) => + Effect.gen(function* () { + const swiftClient = yield* SwiftClient; + const client = yield* HttpClient.HttpClient; + const headerService = yield* S3HeaderService; + const checksumService = yield* Checksum; + const auth = yield* swiftClient.getAuthMeta(bucket).pipe( + Effect.mapError((e) => new InternalError({ message: e.message })), + ); + const container = "bucket_name" in bucket ? bucket.bucket_name : ""; + const encodedContainer = container ? encodeURIComponent(container) : ""; + const target = { + storageUrl: auth.storageUrl, + token: auth.token, + container, + url: encodedContainer + ? `${auth.storageUrl}/${encodedContainer}` + : auth.storageUrl, + client, + headerService, + checksumService, + }; + + // Create a temporary objectOps to satisfy the store's requirement + // But we need the real one for the backend. + // In Swift, the store just uses listObjects/getObject/putObject/deleteObject. + + // deno-lint-ignore prefer-const + let objectOps: ReturnType; + const multipartMetadataStore = makeBackendKeyValueStore( + { + getObject: ( + key: string, + headers: Record, + ) => objectOps.getObject(key, headers), + putObject: ( + key: string, + stream: Stream.Stream, + headers: Record, + ) => objectOps.putObject(key, stream, headers), + deleteObject: (key: string) => objectOps.deleteObject(key), + }, + MP_META_PREFIX, + ); + + const objectOpsReal = makeObjectOps(target); + objectOps = objectOpsReal; + const bucketOps = makeBucketOps(target, objectOpsReal); + const multipartOps = makeMultipartOps( + target, + multipartMetadataStore, + objectOpsReal, + ); + + return Backend.of({ + ...bucketOps, + ...objectOpsReal, + ...multipartOps, + }); + }); diff --git a/src/Backends/Swift/Buckets.ts b/src/Backends/Swift/Buckets.ts new file mode 100644 index 0000000..82a725e --- /dev/null +++ b/src/Backends/Swift/Buckets.ts @@ -0,0 +1,197 @@ +import { HttpClientRequest } from "@effect/platform"; +import { Effect } from "effect"; +import type { + BackendError, + BucketInfo, + ListBucketsResult, + ListObjectsResult, +} from "../../Services/Backend.ts"; +import { BucketAlreadyOwnedByYou } from "../../Services/Backend.ts"; +import { + formatSwiftTransportError, + mapError, + MP_META_PREFIX, + MP_SEGMENTS_PREFIX, + type SwiftTarget, +} from "./Utils.ts"; + +export interface SwiftContainer { + readonly name: string; + readonly count: number; + readonly bytes: number; + readonly last_modified?: string; +} + +export const makeBucketOps = ( + { client, container, storageUrl, token, url: _url }: SwiftTarget, + objectOps: { + listObjects: (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + }) => Effect.Effect; + deleteObject: (key: string) => Effect.Effect; + }, +) => { + return { + listBuckets: () => + Effect.gen(function* () { + const response = yield* client.execute( + HttpClientRequest.get(`${storageUrl}?format=json`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", container, "GET"), + ); + } + + const containers = (yield* response.json.pipe( + Effect.mapError((e) => + mapError(500, `Failed to parse Swift response: ${e}`, container) + ), + )) as readonly SwiftContainer[]; + + const bucketInfos: BucketInfo[] = containers.map((b) => ({ + name: b.name, + creationDate: b.last_modified + ? new Date(b.last_modified) + : new Date(), + })); + + return { + buckets: bucketInfos, + owner: { id: "swift", displayName: "Swift User" }, + } satisfies ListBucketsResult; + }), + + createBucket: ( + _name: string, + _headers: Record, + ) => + Effect.gen(function* () { + // Use container from target (which is bucket_name from MaterializedBucket) + // Don't URL-encode container name - Swift handles it natively (unlike object keys) + const requestUrl = `${storageUrl}/${container}`; + const response = yield* client.execute( + HttpClientRequest.put(requestUrl).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + + // Swift returns 201 (Created) for new containers, 202/204 for existing containers + if (response.status === 201) { + // Successfully created + return; + } + + if (response.status === 202 || response.status === 204) { + return yield* Effect.fail( + new BucketAlreadyOwnedByYou({ + bucket: container, + message: + "The bucket you tried to create already exists, and you already own it.", + }), + ); + } + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", container, "PUT"), + ); + } + }), + + deleteBucket: (_name: string) => + Effect.gen(function* () { + // 1. Delete all segments and metadata first + for (const prefix of [MP_SEGMENTS_PREFIX, MP_META_PREFIX]) { + let marker: string | undefined = undefined; + while (true) { + const listResult: ListObjectsResult = yield* objectOps.listObjects({ + prefix, + marker, + }); + // Delete objects in parallel with concurrency limit + yield* Effect.all( + listResult.contents.map((obj) => + objectOps.deleteObject(obj.key).pipe(Effect.ignore) + ), + { concurrency: 10 }, + ); + if (!listResult.isTruncated || !listResult.nextMarker) break; + marker = listResult.nextMarker; + } + } + + // 2. Delete the container itself + const response = yield* client.execute( + HttpClientRequest.del(`${storageUrl}/${container}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "DELETE", + ), + ); + } + }), + + headBucket: (_name: string) => + Effect.gen(function* () { + const response = yield* client.execute( + HttpClientRequest.head(`${storageUrl}/${container}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "HEAD", + ), + ); + } + }), + }; +}; diff --git a/src/Backends/Swift/Client.ts b/src/Backends/Swift/Client.ts new file mode 100644 index 0000000..b55d6af --- /dev/null +++ b/src/Backends/Swift/Client.ts @@ -0,0 +1,220 @@ +import { HttpClient, HttpClientRequest } from "@effect/platform"; +import { Cache, Effect, Schema } from "effect"; +import { HeraldConfig } from "../../Config/Layer.ts"; +import type { MaterializedBucket, SwiftConfig } from "../../Domain/Config.ts"; + +export interface SwiftAuthMeta { + readonly token: string; + readonly storageUrl: string; +} + +const SwiftEndpoint = Schema.Struct({ + region: Schema.String, + interface: Schema.Literal("public", "internal", "admin"), + url: Schema.String, +}); + +const SwiftService = Schema.Struct({ + type: Schema.String, + endpoints: Schema.Array(SwiftEndpoint), +}); + +const SwiftTokenResponse = Schema.Struct({ + token: Schema.Struct({ + catalog: Schema.Array(SwiftService), + }), +}); + +export class SwiftClient extends Effect.Service()("SwiftClient", { + effect: Effect.gen(function* () { + const appConfig = yield* HeraldConfig; + const client = yield* HttpClient.HttpClient; + + const fetchAuthMeta = ( + config: Schema.Schema.Type, + ): Effect.Effect => { + const { auth_url, credentials, region } = config; + + if (!credentials || !("username" in credentials)) { + return Effect.fail( + new Error( + "Swift credentials (username, password, etc.) are required", + ), + ); + } + + const { + username, + password, + project_name, + user_domain_name = "Default", + project_domain_name = "Default", + } = credentials; + + const isV1 = auth_url.endsWith("/v1.0") || !project_name; + + if (isV1) { + return Effect.gen(function* () { + const request = HttpClientRequest.get(auth_url).pipe( + HttpClientRequest.setHeaders({ + "X-Auth-User": username || "", + "X-Auth-Key": password || "", + }), + ); + const response = yield* client.execute(request).pipe( + Effect.mapError((e) => new Error(String(e))), + ); + + if (response.status < 200 || response.status >= 300) { + const msg = yield* response.text.pipe( + Effect.orElseSucceed(() => "Unknown error"), + ); + return yield* Effect.fail( + new Error(`Failed to authenticate with Swift v1.0: ${msg}`), + ); + } + + const token = response.headers["x-auth-token"]; + const storageUrl = response.headers["x-storage-url"]; + + const tokenStr = Array.isArray(token) ? token[0] : (token || ""); + const storageUrlStr = Array.isArray(storageUrl) + ? storageUrl[0] + : (storageUrl || ""); + + if (!tokenStr || !storageUrlStr) { + return yield* Effect.fail( + new Error( + "X-Auth-Token or X-Storage-Url header missing from Swift v1.0 response", + ), + ); + } + + return { + token: tokenStr, + storageUrl: storageUrlStr, + }; + }).pipe( + Effect.mapError((e) => e instanceof Error ? e : new Error(String(e))), + ); + } + + const requestBody = { + auth: { + identity: { + methods: ["password"], + password: { + user: { + name: username, + domain: { name: user_domain_name }, + password: password, + }, + }, + }, + scope: { + project: { + domain: { name: project_domain_name }, + name: project_name, + }, + }, + }, + }; + + return Effect.gen(function* () { + const request = yield* HttpClientRequest.post(`${auth_url}/auth/tokens`) + .pipe( + HttpClientRequest.bodyJson(requestBody), + Effect.mapError((e) => new Error(String(e))), + ); + const response = yield* client.execute(request).pipe( + Effect.mapError((e) => new Error(String(e))), + ); + + if (response.status < 200 || response.status >= 300) { + const msg = yield* response.text.pipe( + Effect.orElseSucceed(() => "Unknown error"), + ); + return yield* Effect.fail( + new Error(`Failed to authenticate with Swift: ${msg}`), + ); + } + + const token = response.headers["x-subject-token"]; + const tokenStr = Array.isArray(token) ? token[0] : token; + + if (!tokenStr) { + return yield* Effect.fail( + new Error( + "X-Subject-Token header missing from Swift response", + ), + ); + } + + const json = yield* response.json.pipe( + Effect.mapError((e) => new Error(String(e))), + ); + const body = yield* Schema.decodeUnknown(SwiftTokenResponse)(json).pipe( + Effect.mapError((e) => + new Error(`Failed to parse Swift token response: ${e}`) + ), + ); + + const catalog = body.token.catalog; + const storageService = catalog.find((s) => s.type === "object-store"); + + if (!storageService) { + return yield* Effect.fail( + new Error( + "Object Store service not found in Swift catalog", + ), + ); + } + + const endpoint = storageService.endpoints.find( + (e) => + (region ? e.region === region : true) && + e.interface === "public", + ); + + if (!endpoint) { + return yield* Effect.fail( + new Error( + `Public Swift endpoint not found (region: ${region ?? "any"})`, + ), + ); + } + + return { + token: tokenStr, + storageUrl: endpoint.url, + }; + }); + }; + + const cache = yield* Cache.make({ + capacity: 100, + lookup: (config: Schema.Schema.Type) => + fetchAuthMeta(config), + timeToLive: "50 minutes", // Swift tokens usually last 1h + }); + + return { + getAuthMeta: ( + bucket: MaterializedBucket | { backend_id: string }, + ): Effect.Effect => { + const backend_id = bucket.backend_id; + const config = appConfig.raw.backends[backend_id] as Schema.Schema.Type< + typeof SwiftConfig + >; + + if (!config || config.protocol !== "swift") { + return Effect.fail( + new Error(`Backend ${backend_id} is not a Swift backend`), + ); + } + + return cache.get(config); + }, + }; + }), +}) {} diff --git a/src/Backends/Swift/Multipart.ts b/src/Backends/Swift/Multipart.ts new file mode 100644 index 0000000..1c926ca --- /dev/null +++ b/src/Backends/Swift/Multipart.ts @@ -0,0 +1,617 @@ +import { HttpClientRequest, type HttpClientResponse } from "@effect/platform"; +import { Effect, Option, Schedule, Stream } from "effect"; +import { + type BackendError, + type CompleteMultipartUploadResult, + type HeadObjectResult, + InternalError, + InvalidPart, + type ListMultipartUploadsResult, + type ListObjectsResult, + type ListPartsResult, + type MultipartUploadInfo, + type MultipartUploadResult, + NoSuchUpload, + type ObjectInfo, + type PartInfo, + type UploadPartResult, +} from "../../Services/Backend.ts"; +import { + encodeObjectKeyForSwift, + formatSwiftTransportError, + mapError, + MP_META_PREFIX, + MP_SEGMENTS_PREFIX, + type SwiftTarget, +} from "./Utils.ts"; +import type { KeyValueStore } from "@effect/platform"; + +export const makeMultipartOps = ( + target: SwiftTarget, + multipartMetadataStore: KeyValueStore.KeyValueStore, + objectOps: { + listObjects: (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + }) => Effect.Effect; + headObject: ( + key: string, + headers: Record, + ) => Effect.Effect; + }, +) => { + const { url, token, client, headerService, checksumService, container } = + target; + + return { + createMultipartUpload: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const uploadId = yield* Effect.try({ + try: () => crypto.randomUUID(), + catch: (e) => new InternalError({ message: String(e) }), + }); + const { checksums } = headerService.fromRequestHeaders(headers); + + // Save metadata for later use in CompleteMultipartUpload + const metadata: Record = {}; + for (const [k, v] of Object.entries(headers)) { + const lowK = k.toLowerCase(); + if ( + lowK.startsWith("x-amz-meta-") || + lowK === "content-type" || + lowK.startsWith("x-amz-checksum-") || + lowK === "x-amz-sdk-checksum-algorithm" + ) { + metadata[lowK] = String(v); + } + } + + const finalChecksumAlgorithm = ( + checksums.algorithm ?? + metadata["x-amz-checksum-algorithm"] ?? + metadata["x-amz-sdk-checksum-algorithm"] + )?.toUpperCase(); + const finalChecksumType = ( + checksums.type ?? + metadata["x-amz-checksum-type"] + )?.toUpperCase(); + + if (finalChecksumAlgorithm) { + metadata["x-amz-checksum-algorithm"] = finalChecksumAlgorithm; + } + if (finalChecksumType) { + metadata["x-amz-checksum-type"] = finalChecksumType; + } + + yield* multipartMetadataStore.set( + `${key}/${uploadId}`, + JSON.stringify(metadata), + ).pipe( + Effect.tapError((e) => + Effect.logError("Metadata store set failed", { + operation: "createMultipartUpload", + error: String(e), + key, + uploadId, + }) + ), + Effect.mapError((e) => + new InternalError({ + message: `Failed to persist multipart upload metadata: ${ + String(e) + }`, + }) + ), + ); + + return { + uploadId, + checksumAlgorithm: finalChecksumAlgorithm, + checksumType: finalChecksumType, + } satisfies MultipartUploadResult; + }), + + uploadPart: ( + _key: string, + uploadId: string, + partNumber: number, + body: Stream.Stream, + headers: Record, + ) => + Effect.gen(function* () { + const { checksums, metadata } = headerService.fromRequestHeaders( + headers, + ); + const segmentKey = `${MP_SEGMENTS_PREFIX}${uploadId}/${partNumber}`; + const encodedSegmentKey = encodeObjectKeyForSwift(segmentKey); + + const swiftHeaders: Record = { + "X-Auth-Token": token, + ...headerService.toSwiftHeaders(metadata, checksums), + }; + + const validatedStream = yield* checksumService.validate( + body, + checksums, + ); + + const request = HttpClientRequest.put(`${url}/${encodedSegmentKey}`) + .pipe( + HttpClientRequest.setHeaders(swiftHeaders), + HttpClientRequest.bodyStream(validatedStream.pipe( + Stream.mapError((e) => { + return e; + }), + )), + ); + + const response: HttpClientResponse.HttpClientResponse = yield* client + .execute(request).pipe( + Effect.retry({ + while: (e) => { + const s = String(e); + return (s.includes("Transport error") || + s.includes("ECONNRESET")); + }, + schedule: Schedule.exponential("100 millis").pipe( + Schedule.compose(Schedule.recurs(3)), + ), + }), + Effect.catchAll((e) => { + const s = String(e); + if ( + s.includes("NoSuchKey") || s.includes("NoSuchBucket") || + s.includes("InvalidRequest") || s.includes("BadDigest") + ) return Effect.fail(e as BackendError); + return Effect.fail( + mapError( + 500, + formatSwiftTransportError(e), + container, + "PUT", + _key, + ), + ); + }), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "PUT", + segmentKey, + ), + ); + } + + const etagHeader = response.headers["etag"]; + const etagValue = Array.isArray(etagHeader) + ? etagHeader[0] + : etagHeader; + + return { + etag: etagValue || "", + checksumAlgorithm: checksums.algorithm, + checksumType: checksums.type, + checksumCRC32: checksums.crc32, + checksumCRC32C: checksums.crc32c, + checksumCRC64NVME: checksums.crc64nvme, + checksumSHA1: checksums.sha1, + checksumSHA256: checksums.sha256, + } satisfies UploadPartResult; + }), + + completeMultipartUpload: ( + key: string, + uploadId: string, + parts: readonly { + etag: string; + partNumber: number; + checksumCRC32?: string; + checksumCRC32C?: string; + checksumCRC64NVME?: string; + checksumSHA1?: string; + checksumSHA256?: string; + }[], + _metadataArg: Record, + headers: Record, + ) => + Effect.gen(function* () { + if (parts.length === 0) { + return yield* Effect.fail( + new InvalidPart({ + message: "At least one part must be specified.", + }), + ); + } + + // Retrieve metadata from store + const metadataOpt = yield* multipartMetadataStore.get( + `${key}/${uploadId}`, + ).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + ); + let metadata: Record = {}; + if (Option.isSome(metadataOpt)) { + try { + metadata = JSON.parse(metadataOpt.value); + } catch (e) { + yield* Effect.logError("Parse multipart metadata failed", { + operation: "completeMultipartUpload", + key, + uploadId, + error: String(e), + }); + } + } + + const encodedKey = encodeObjectKeyForSwift(key); + + // Fetch segment info to get sizes + const segmentMap = new Map(); + const buildSegmentMap = Effect.gen(function* () { + segmentMap.clear(); + let segmentMarker: string | undefined = undefined; + while (true) { + const segmentsResult: ListObjectsResult = yield* objectOps + .listObjects({ + prefix: `${MP_SEGMENTS_PREFIX}${uploadId}/`, + marker: segmentMarker, + }); + for (const c of segmentsResult.contents) { + segmentMap.set(c.key, c); + } + if (!segmentsResult.isTruncated || !segmentsResult.nextMarker) { + break; + } + segmentMarker = segmentsResult.nextMarker; + } + + // Verify all parts are present + for (const p of parts) { + const segmentKey = + `${MP_SEGMENTS_PREFIX}${uploadId}/${p.partNumber}`; + if (!segmentMap.has(segmentKey)) { + return yield* Effect.fail( + new NoSuchUpload({ + uploadId, + message: `Part ${p.partNumber} not found in segment listing`, + }), + ); + } + } + }); + + // Retry with exponential backoff for eventual consistency + yield* buildSegmentMap.pipe( + Effect.retry({ + while: (e) => e instanceof NoSuchUpload, + schedule: Schedule.exponential("100 millis").pipe( + Schedule.compose(Schedule.recurs(4)), + ), + }), + ); + + // 1. Build SLO manifest (Swift requires each segment >= 1 byte) + const manifest = []; + for (const p of parts) { + const segmentKey = `${MP_SEGMENTS_PREFIX}${uploadId}/${p.partNumber}`; + const info = segmentMap.get(segmentKey)!; + if (info.size < 1) { + return yield* Effect.fail( + new InvalidPart({ + message: + `Part ${p.partNumber} has size 0; each part must be at least 1 byte`, + }), + ); + } + manifest.push({ + path: `/${container}/${segmentKey}`, + etag: p.etag.replace(/"/g, ""), + size_bytes: info.size, + }); + } + + // 2. PUT SLO manifest + const { checksums } = headerService.fromRequestHeaders(headers); + const swiftHeaders: Record = { + "X-Auth-Token": token, + "Content-Type": (metadata["content-type"] || + "application/octet-stream") as string, + ...headerService.toSwiftHeaders(metadata, checksums), + }; + + const body = new TextEncoder().encode(JSON.stringify(manifest)); + + const request = HttpClientRequest.put(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setUrlParams({ "multipart-manifest": "put" }), + HttpClientRequest.bodyUint8Array(body), + HttpClientRequest.setHeaders({ + ...swiftHeaders, + "X-Static-Large-Object": "true", + "Content-Length": String(body.length), + }), + ); + + const response: HttpClientResponse.HttpClientResponse = yield* client + .execute(request).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "PUT", + key, + ), + ); + } + + const etagHeader = response.headers["etag"]; + const etagValue = Array.isArray(etagHeader) + ? etagHeader[0] + : etagHeader; + + // 3. Cleanup metadata + yield* multipartMetadataStore.remove(`${key}/${uploadId}`).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + Effect.ignore, + ); + + // 4. Cleanup segments metadata object if it exists (for compatibility) + const metaKey = `${MP_META_PREFIX}${key}/${uploadId}`; + const encodedMetaKey = encodeObjectKeyForSwift(metaKey); + yield* client.execute( + HttpClientRequest.del(`${url}/${encodedMetaKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe(Effect.ignore); + + return { + location: `${url}/${encodedKey}`, + bucket: container, + key, + etag: etagValue || "", + checksumAlgorithm: checksums.algorithm, + checksumType: checksums.type || "COMPOSITE", + checksumCRC32: checksums.crc32, + checksumCRC32C: checksums.crc32c, + checksumCRC64NVME: checksums.crc64nvme, + checksumSHA1: checksums.sha1, + checksumSHA256: checksums.sha256, + } satisfies CompleteMultipartUploadResult; + }), + + abortMultipartUpload: ( + key: string, + uploadId: string, + ) => + Effect.gen(function* () { + // 1. Delete the segments + let marker: string | undefined = undefined; + while (true) { + const segmentsResult: ListObjectsResult = yield* objectOps + .listObjects({ + prefix: `${MP_SEGMENTS_PREFIX}${uploadId}/`, + marker, + }); + + yield* Effect.all( + segmentsResult.contents.map((content) => { + const encodedKey = encodeObjectKeyForSwift(content.key); + return client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe(Effect.ignore); + }), + { concurrency: 10 }, + ); + + if (!segmentsResult.isTruncated || !segmentsResult.nextMarker) { + break; + } + marker = segmentsResult.nextMarker; + } + + // 2. Delete metadata from store + yield* multipartMetadataStore.remove(`${key}/${uploadId}`).pipe( + Effect.ignore, + ); + + // 3. Delete metadata object (compatibility) + const metaKey = `${MP_META_PREFIX}${key}/${uploadId}`; + const encodedMetaKey = encodeObjectKeyForSwift(metaKey); + yield* client.execute( + HttpClientRequest.del(`${url}/${encodedMetaKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe(Effect.ignore); + }), + + listMultipartUploads: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + uploadIdMarker?: string; + maxUploads?: number; + encodingType?: string; + }) => + Effect.gen(function* () { + const prefix = `${MP_META_PREFIX}${args.prefix ?? ""}`; + const marker = args.keyMarker + ? `${MP_META_PREFIX}${args.keyMarker}/${args.uploadIdMarker ?? ""}` + : undefined; + + const metaResult = yield* objectOps.listObjects({ + prefix, + delimiter: args.delimiter, + maxKeys: args.maxUploads, + marker, + }); + + const uploads: MultipartUploadInfo[] = []; + for (const c of metaResult.contents) { + // Remove prefix and split by "/" + const keyWithoutPrefix = c.key.substring(MP_META_PREFIX.length); + // Skip keys that end with "/" or are empty after prefix removal + if (!keyWithoutPrefix || keyWithoutPrefix.endsWith("/")) { + yield* Effect.logWarning("Skipping malformed metadata key", { + operation: "listMultipartUploads", + key: c.key, + }); + continue; + } + + const parts = keyWithoutPrefix.split("/"); + const uploadId = parts.pop(); + // Validate uploadId: must be present and non-empty + if (!uploadId || uploadId === "") { + yield* Effect.logWarning( + "Skipping metadata key with missing uploadId", + { + operation: "listMultipartUploads", + key: c.key, + }, + ); + continue; + } + + const key = parts.join("/"); + uploads.push({ + key, + uploadId, + owner: { id: "swift", displayName: "Swift User" }, + initiator: { id: "swift", displayName: "Swift User" }, + storageClass: "STANDARD", + initiated: c.lastModified ?? new Date(), + }); + } + + return { + bucket: container, + prefix: args.prefix, + keyMarker: args.keyMarker, + uploadIdMarker: args.uploadIdMarker, + maxUploads: args.maxUploads ?? 1000, + delimiter: args.delimiter, + isTruncated: metaResult.isTruncated, + uploads, + commonPrefixes: metaResult.commonPrefixes.map((cp) => ({ + prefix: cp.prefix.substring(MP_META_PREFIX.length), + })), + encodingType: args.encodingType, + } satisfies ListMultipartUploadsResult; + }), + + listParts: ( + key: string, + uploadId: string, + ) => + Effect.gen(function* () { + // Check if upload exists by checking for metadata in store or object + const metadataOpt = yield* multipartMetadataStore.get( + `${key}/${uploadId}`, + ).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + ); + if (Option.isNone(metadataOpt)) { + const metaKey = `${MP_META_PREFIX}${key}/${uploadId}`; + const encodedMetaKey = encodeObjectKeyForSwift(metaKey); + const metaResponse: HttpClientResponse.HttpClientResponse = + yield* client.execute( + HttpClientRequest.head(`${url}/${encodedMetaKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + + if (metaResponse.status === 200) { + // Metadata object exists, continue processing + } else if (metaResponse.status === 404) { + return yield* Effect.fail( + new NoSuchUpload({ + uploadId, + message: + `The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.`, + }), + ); + } else { + // Non-200/non-404 status: fail with descriptive error + const errorMessage = yield* metaResponse.text.pipe( + Effect.orElseSucceed(() => "Unknown error"), + ); + return yield* Effect.fail( + mapError( + metaResponse.status, + `Metadata HEAD failed for upload ${uploadId} in container ${container}: ${errorMessage}`, + container, + "HEAD", + encodedMetaKey, + ), + ); + } + } + + const segmentsResult = yield* objectOps.listObjects({ + prefix: `${MP_SEGMENTS_PREFIX}${uploadId}/`, + }); + + const parts: PartInfo[] = []; + for (const c of segmentsResult.contents) { + const keySegment = c.key.split("/").pop() || "0"; + const partNumber = parseInt(keySegment, 10); + if (isNaN(partNumber) || partNumber <= 0) { + yield* Effect.logWarning("Invalid part number in segment key", { + operation: "listParts", + key: c.key, + parsed: keySegment, + }); + continue; + } + parts.push({ + partNumber, + lastModified: c.lastModified, + etag: c.etag, + size: c.size, + }); + } + + return { + bucket: container, + key, + uploadId, + owner: { id: "swift", displayName: "Swift User" }, + initiator: { id: "swift", displayName: "Swift User" }, + storageClass: "STANDARD", + partNumberMarker: 0, + nextPartNumberMarker: 0, + maxParts: 1000, + isTruncated: false, + parts, + } satisfies ListPartsResult; + }), + }; +}; diff --git a/src/Backends/Swift/Objects.ts b/src/Backends/Swift/Objects.ts new file mode 100644 index 0000000..6cfacd4 --- /dev/null +++ b/src/Backends/Swift/Objects.ts @@ -0,0 +1,744 @@ +import { HttpClientRequest, type HttpClientResponse } from "@effect/platform"; +import { type Chunk, Effect, Stream } from "effect"; +import type { + BackendError, + CommonPrefix, + DeleteObjectsResult, + HeadObjectResult, + ListObjectsResult, + ObjectAttributes, + ObjectInfo, + ObjectResponse, + PutObjectResult, +} from "../../Services/Backend.ts"; +import { + BadDigest, + InternalError, + InvalidRequest, +} from "../../Services/Backend.ts"; +import { normalizeHeaders } from "../../Services/S3HeaderService.ts"; +import { + encodeObjectKeyForSwift, + formatSwiftTransportError, + mapError, + type SwiftTarget, +} from "./Utils.ts"; + +export interface SwiftObject { + readonly name?: string; + readonly hash?: string; + readonly bytes?: number; + readonly content_type?: string; + readonly last_modified?: string; + readonly subdir?: string; +} + +export const makeObjectOps = ( + { + container, + storageUrl: _, + token, + url, + client, + headerService, + checksumService, + }: SwiftTarget, +) => { + const listObjects = (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + encodingType?: string; + continuationToken?: string; + startAfter?: string; + listType?: 1 | 2; + }): Effect.Effect => + Effect.gen(function* () { + const limit = args.maxKeys ?? 1000; + const query = new URLSearchParams({ format: "json" }); + if (args.prefix) query.set("prefix", args.prefix); + if (args.delimiter) query.set("delimiter", args.delimiter); + if (args.marker) query.set("marker", args.marker); + query.set("limit", String(limit + 1)); + if (args.continuationToken) query.set("marker", args.continuationToken); + if (args.startAfter) query.set("marker", args.startAfter); + + const response: HttpClientResponse.HttpClientResponse = yield* client + .execute( + HttpClientRequest.get(`${url}?${query.toString()}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", container, "GET"), + ); + } + + const rawObjects = (yield* response.json.pipe( + Effect.mapError((e) => + mapError(500, `Failed to parse Swift response: ${e}`, container) + ), + )) as readonly SwiftObject[]; + + const isTruncated = rawObjects.length > limit; + const objects = isTruncated ? rawObjects.slice(0, limit) : rawObjects; + + const contents: ObjectInfo[] = []; + const commonPrefixes: CommonPrefix[] = []; + + for (const obj of objects) { + if (obj.subdir) { + commonPrefixes.push({ prefix: obj.subdir }); + } else if (obj.name) { + contents.push({ + key: obj.name, + lastModified: obj.last_modified + ? new Date(obj.last_modified) + : new Date(), + etag: obj.hash ? `"${obj.hash}"` : "", + size: obj.bytes ?? 0, + storageClass: "STANDARD", + owner: { id: "swift", displayName: "Swift User" }, + }); + } + } + + const nextMarker = isTruncated && objects.length > 0 + ? objects[objects.length - 1].name || + objects[objects.length - 1].subdir + : undefined; + + return { + name: container, + prefix: args.prefix, + maxKeys: limit, + delimiter: args.delimiter, + isTruncated, + marker: args.marker, + nextMarker, + contents, + commonPrefixes, + encodingType: args.encodingType, + listType: args.listType ?? 1, + nextContinuationToken: args.listType === 2 ? nextMarker : undefined, + keyCount: contents.length + commonPrefixes.length, + } satisfies ListObjectsResult; + }); + const headObject = ( + key: string, + headers: Record, + ): Effect.Effect => + Effect.gen(function* () { + const encodedKey = encodeObjectKeyForSwift(key); + const swiftHeaders: Record = { + "X-Auth-Token": token, + }; + const response: HttpClientResponse.HttpClientResponse = yield* client + .execute( + HttpClientRequest.head(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders(swiftHeaders), + ), + ).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "HEAD", + key, + ), + ); + } + + const { metadata, s3Headers, checksums, partsCount } = headerService + .fromSwiftHeaders(response.headers); + + const contentLengthHeader = response.headers["content-length"]; + const contentLength = Array.isArray(contentLengthHeader) + ? parseInt(contentLengthHeader[0] || "0") + : parseInt(contentLengthHeader || "0"); + + const etagHeader = response.headers["etag"]; + const etag = Array.isArray(etagHeader) ? etagHeader[0] : etagHeader; + + const lastModifiedHeader = response.headers["last-modified"]; + const lastModified = Array.isArray(lastModifiedHeader) + ? lastModifiedHeader[0] + : lastModifiedHeader; + + const { s3Params } = headerService.fromRequestHeaders(headers); + const checksumMode = s3Params.checksumMode === "ENABLED"; + + if (checksumMode) { + Object.assign( + s3Headers, + headerService.toResponseHeaders({ + checksumAlgorithm: checksums.algorithm, + checksumCRC32: checksums.crc32, + checksumCRC32C: checksums.crc32c, + checksumCRC64NVME: checksums.crc64nvme, + checksumSHA1: checksums.sha1, + checksumSHA256: checksums.sha256, + checksumType: checksums.type, + metadata: {}, + headers: {}, + partsCount, + }), + ); + } + + return { + contentType: (Array.isArray(response.headers["content-type"]) + ? response.headers["content-type"][0] + : response.headers["content-type"]) || undefined, + contentLength, + etag: etag || undefined, + lastModified: lastModified ? new Date(lastModified) : undefined, + metadata, + headers: s3Headers, + checksumAlgorithm: checksums.algorithm, + checksumCRC32: checksums.crc32, + checksumCRC32C: checksums.crc32c, + checksumCRC64NVME: checksums.crc64nvme, + checksumSHA1: checksums.sha1, + checksumSHA256: checksums.sha256, + checksumType: checksums.type, + partsCount, + } satisfies HeadObjectResult; + }); + + return { + listObjects, + + listVersions: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + versionIdMarker?: string; + maxKeys?: number; + encodingType?: string; + }) => + Effect.gen(function* () { + const result = yield* listObjects({ + prefix: args.prefix, + delimiter: args.delimiter, + marker: args.keyMarker, + maxKeys: args.maxKeys, + }); + return { + ...result, + contents: result.contents.map((c) => ({ + ...c, + versionId: "null", + isLatest: true, + })), + }; + }), + + getObject: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const encodedKey = encodeObjectKeyForSwift(key); + const swiftHeaders: Record = { + "X-Auth-Token": token, + }; + const { s3Params } = headerService.fromRequestHeaders(headers); + + if (headers["range"] || headers["Range"]) { + swiftHeaders["Range"] = String(headers["range"] || headers["Range"]); + } + if (headers["if-match"] || headers["If-Match"]) { + swiftHeaders["If-Match"] = String( + headers["if-match"] || headers["If-Match"], + ); + } + if (headers["if-none-match"] || headers["If-None-Match"]) { + swiftHeaders["If-None-Match"] = String( + headers["if-none-match"] || headers["If-None-Match"], + ); + } + if (headers["if-modified-since"] || headers["If-Modified-Since"]) { + swiftHeaders["If-Modified-Since"] = String( + headers["if-modified-since"] || headers["If-Modified-Since"], + ); + } + if (headers["if-unmodified-since"] || headers["If-Unmodified-Since"]) { + swiftHeaders["If-Unmodified-Since"] = String( + headers["if-unmodified-since"] || headers["If-Unmodified-Since"], + ); + } + + const response: HttpClientResponse.HttpClientResponse = yield* client + .execute( + HttpClientRequest.get(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders(swiftHeaders), + ), + ).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "GET", + key, + ), + ); + } + + const { metadata, s3Headers, checksums, partsCount } = headerService + .fromSwiftHeaders(response.headers); + + const contentLengthHeader = response.headers["content-length"]; + const contentLength = Array.isArray(contentLengthHeader) + ? parseInt(contentLengthHeader[0] || "0") + : parseInt(contentLengthHeader || "0"); + + const etagHeader = response.headers["etag"]; + const etag = Array.isArray(etagHeader) ? etagHeader[0] : etagHeader; + + const lastModifiedHeader = response.headers["last-modified"]; + const lastModified = Array.isArray(lastModifiedHeader) + ? lastModifiedHeader[0] + : lastModifiedHeader; + + const checksumMode = s3Params.checksumMode === "ENABLED"; + + if (checksumMode) { + Object.assign( + s3Headers, + headerService.toResponseHeaders({ + checksumAlgorithm: checksums.algorithm, + checksumCRC32: checksums.crc32, + checksumCRC32C: checksums.crc32c, + checksumCRC64NVME: checksums.crc64nvme, + checksumSHA1: checksums.sha1, + checksumSHA256: checksums.sha256, + checksumType: checksums.type, + metadata: {}, + headers: {}, + partsCount, + }), + ); + } + + // Try to get the native stream to avoid Effect <-> WebStream conversion overhead + const nativeStream = + (response as unknown as { source?: unknown }).source instanceof + Response + ? (response as unknown as { source: Response }).source.body + : undefined; + + return { + stream: response.stream, + nativeStream: nativeStream || undefined, + contentType: (Array.isArray(response.headers["content-type"]) + ? response.headers["content-type"][0] + : response.headers["content-type"]) || undefined, + contentLength, + etag: etag || undefined, + lastModified: lastModified ? new Date(lastModified) : undefined, + metadata, + headers: s3Headers, + checksumAlgorithm: checksums.algorithm, + checksumCRC32: checksums.crc32, + checksumCRC32C: checksums.crc32c, + checksumCRC64NVME: checksums.crc64nvme, + checksumSHA1: checksums.sha1, + checksumSHA256: checksums.sha256, + checksumType: checksums.type, + partsCount, + } satisfies ObjectResponse; + }), + + headObject, + + putObject: ( + key: string, + stream: Stream.Stream, + headers: Record, + ) => { + const encodedKey = encodeObjectKeyForSwift(key); + + return Effect.gen(function* () { + const { checksums, metadata } = headerService.fromRequestHeaders( + headers, + ); + const normalized = normalizeHeaders(headers); + + const swiftHeaders: Record = { + "X-Auth-Token": token, + "Content-Type": (normalized["content-type"] || + "application/octet-stream") as string, + ...headerService.toSwiftHeaders(metadata, checksums), + }; + + const contentLength = normalized["content-length"] + ? parseInt(normalized["content-length"]) + : undefined; + if (contentLength !== undefined) { + swiftHeaders["Content-Length"] = String(contentLength); + } + + const validatedStream = (yield* checksumService.validate( + stream, + checksums, + )).pipe( + Stream.catchAll((e) => { + // Preserve BadDigest and InvalidRequest errors from checksum validation + if (e instanceof BadDigest || e instanceof InvalidRequest) { + return Stream.fail(e as BackendError); + } + return Stream.fail( + new InternalError({ + message: `error on checksum stream: ${String(e)}`, + }), + ); + }), + ); + + // Align with S3: buffer small files (< 1MB) and validate before HTTP request + const bodyStream = + (contentLength !== undefined && contentLength < 1024 * 1024) + ? yield* Effect.gen(function* () { + // Buffer small files: consume stream to trigger validation BEFORE HTTP request + const chunks: Chunk.Chunk = yield* Stream.runCollect( + validatedStream, + ).pipe( + Effect.mapError((e) => { + // Preserve BadDigest and InvalidRequest errors + if (e instanceof BadDigest || e instanceof InvalidRequest) { + return e; + } + return new InternalError({ message: String(e) }); + }), + ); + // Recreate stream from chunks for HTTP request + return Stream.fromIterable(chunks); + }) + : validatedStream; + + const request = HttpClientRequest.put(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders(swiftHeaders), + HttpClientRequest.bodyStream(bodyStream), + ); + + const response: HttpClientResponse.HttpClientResponse = yield* client + .execute(request).pipe( + Effect.catchAll( + ( + e, + ): Effect.Effect< + HttpClientResponse.HttpClientResponse, + BackendError + > => { + // Check for BadDigest in the error message or cause + const errorStr = String(e); + if ( + errorStr.includes("BadDigest") || + errorStr.includes("checksum mismatch") || + errorStr.includes("Checksum mismatch") + ) { + return Effect.fail(new BadDigest({ message: errorStr })); + } + if (e && typeof e === "object" && "cause" in e) { + const cause = (e as { cause?: unknown }).cause; + if ( + cause instanceof BadDigest || + cause instanceof InvalidRequest + ) { + return Effect.fail(cause); + } + const causeStr = String(cause); + if ( + causeStr.includes("BadDigest") || + causeStr.includes("checksum mismatch") || + causeStr.includes("Checksum mismatch") + ) { + return Effect.fail(new BadDigest({ message: causeStr })); + } + } + return Effect.fail( + mapError(500, formatSwiftTransportError(e), container), + ); + }, + ), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "PUT", + key, + ), + ); + } + + const etagHeader = response.headers["etag"]; + const etagValue = Array.isArray(etagHeader) + ? etagHeader[0] + : etagHeader; + + return { + etag: etagValue || undefined, + checksumAlgorithm: checksums.algorithm, + checksumCRC32: checksums.crc32, + checksumCRC32C: checksums.crc32c, + checksumCRC64NVME: checksums.crc64nvme, + checksumSHA1: checksums.sha1, + checksumSHA256: checksums.sha256, + } satisfies PutObjectResult; + }); + }, + + deleteObject: (key: string) => + Effect.gen(function* () { + const encodedKey = encodeObjectKeyForSwift(key); + + // Try SLO delete first (recursive) + const response: HttpClientResponse.HttpClientResponse = yield* client + .execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ + "X-Auth-Token": token, + "X-Static-Large-Object": "true", + }), + HttpClientRequest.setUrlParams({ + "multipart-manifest": "delete", + }), + ), + ).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + + const responseBody = yield* response.text.pipe( + Effect.orElseSucceed(() => ""), + ); + + if ( + response.status === 400 || + (response.status === 200 && responseBody.includes("Not an SLO")) + ) { + // Not an SLO, try regular delete + const regResponse: HttpClientResponse.HttpClientResponse = + yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + + if (regResponse.status < 200 || regResponse.status >= 300) { + if (regResponse.status === 404) return; + const regResponseBody = yield* regResponse.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + regResponse.status, + regResponseBody, + container, + "DELETE", + key, + ), + ); + } + return; + } + + if (response.status < 200 || response.status >= 300) { + if (response.status === 404) { + return; + } + // Reuse the already-read responseBody instead of reading response.text again + const message = responseBody || "Error"; + return yield* Effect.fail( + mapError( + response.status, + message, + container, + "DELETE", + key, + ), + ); + } + }), + + deleteObjects: ( + objects: readonly { key: string; versionId?: string }[], + ) => + Effect.gen(function* () { + const results = yield* Effect.all( + objects.map((obj) => + Effect.gen(function* () { + const encodedKey = encodeObjectKeyForSwift(obj.key); + let response: HttpClientResponse.HttpClientResponse = + yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ + "X-Auth-Token": token, + "X-Static-Large-Object": "true", + }), + HttpClientRequest.setUrlParams({ + "multipart-manifest": "delete", + }), + ), + ).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + + let responseBody = yield* response.text.pipe( + Effect.orElseSucceed(() => ""), + ); + + if ( + response.status === 400 || + (response.status === 200 && responseBody.includes("Not an SLO")) + ) { + // Not an SLO, try regular delete + response = yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + // Refresh responseBody cache for the new response + responseBody = yield* response.text.pipe( + Effect.orElseSucceed(() => ""), + ); + } + + if ( + (response.status >= 200 && response.status < 300) || + response.status === 204 || response.status === 404 + ) { + return { key: obj.key, error: null }; + } else { + // Reuse the cached responseBody instead of reading response.text again + const errorBody = responseBody || "Unknown error"; + return { + key: obj.key, + error: { + code: String(response.status), + message: errorBody, + }, + }; + } + }) + ), + { concurrency: 10 }, + ); + + const deleted: string[] = []; + const errors: { key: string; code: string; message: string }[] = []; + + for (const res of results) { + if (res.error) { + errors.push({ key: res.key, ...res.error }); + } else { + deleted.push(res.key); + } + } + + return { deleted, errors } satisfies DeleteObjectsResult; + }), + + getObjectAttributes: ( + key: string, + attributes: readonly string[], + headers: Record, + ) => + Effect.gen(function* () { + const head = yield* headObject( + key, + { "x-amz-checksum-mode": "ENABLED", ...headers }, + ); + + const lowerAttrs = attributes.map((a) => a.toLowerCase()); + const isSLO = + head.headers["x-static-large-object"]?.toLowerCase() === "true"; + const result: ObjectAttributes = { + ...(lowerAttrs.includes("etag") ? { etag: head.etag } : {}), + ...(lowerAttrs.includes("checksum") + ? { + checksum: { + checksumCRC32: head.checksumCRC32, + checksumCRC32C: head.checksumCRC32C, + checksumCRC64NVME: head.checksumCRC64NVME, + checksumSHA1: head.checksumSHA1, + checksumSHA256: head.checksumSHA256, + checksumType: head.checksumAlgorithm + ? (isSLO ? "COMPOSITE" : "FULL_OBJECT") + : undefined, + }, + } + : {}), + ...(lowerAttrs.includes("objectsize") + ? { objectSize: head.contentLength } + : {}), + ...(lowerAttrs.includes("storageclass") + ? { storageClass: "STANDARD" } + : {}), + ...(lowerAttrs.includes("objectparts") + ? { + objectParts: { + totalPartsCount: 0, // Placeholder + partNumberMarker: 0, + nextPartNumberMarker: 0, + maxParts: 1000, + isTruncated: false, + parts: [], + }, + } + : {}), + }; + + return result; + }), + }; +}; diff --git a/src/Backends/Swift/Utils.ts b/src/Backends/Swift/Utils.ts new file mode 100644 index 0000000..d72c57d --- /dev/null +++ b/src/Backends/Swift/Utils.ts @@ -0,0 +1,101 @@ +import { + AccessDenied, + BucketAlreadyExists, + BucketAlreadyOwnedByYou, + BucketNotEmpty, + InternalError, + NoSuchBucket, + NoSuchKey, +} from "../../Services/Backend.ts"; +import type { HttpClient } from "@effect/platform"; +import type { S3HeaderService } from "../../Services/S3HeaderService.ts"; +import type { Checksum } from "../../Services/Checksum.ts"; + +export interface SwiftTarget { + readonly client: HttpClient.HttpClient; + readonly container: string; + readonly storageUrl: string; + readonly token: string; + readonly url: string; + readonly headerService: S3HeaderService; + readonly checksumService: Checksum; +} + +export const MP_META_PREFIX = ".mp_meta/"; +export const MP_SEGMENTS_PREFIX = ".mp_segments/"; + +/** + * Encodes an object key for use in Swift URL paths. Decodes each segment first + * to avoid double-encoding when the key already contains percent-encoded chars + * (e.g. %2F from the client). + */ +export function encodeObjectKeyForSwift(key: string): string { + return key.split("/").map((seg) => { + try { + return encodeURIComponent(decodeURIComponent(seg)); + } catch { + return encodeURIComponent(seg); + } + }).join("/"); +} + +/** + * Format an unknown error from Swift HTTP client for logging. Extracts + * cause/reason when present so transport failures can be diagnosed. + */ +export function formatSwiftTransportError(e: unknown): string { + const base = String(e); + if (e === null || typeof e !== "object") return base; + const parts = [base]; + if ("cause" in e && (e as { cause?: unknown }).cause !== undefined) { + parts.push( + `cause=${String((e as { cause: unknown }).cause)}`, + ); + } + if ("reason" in e && (e as { reason?: unknown }).reason !== undefined) { + parts.push( + `reason=${String((e as { reason: unknown }).reason)}`, + ); + } + return parts.join(" "); +} + +export const mapError = ( + status: number, + message: string, + bucket: string, + method?: string, + key?: string, +) => { + if (status === 404) { + if (key) { + return new NoSuchKey({ bucket, key, message }); + } + return new NoSuchBucket({ bucket, message }); + } + if (status === 409) { + if (message.includes("not empty")) { + return new BucketNotEmpty({ bucket, message }); + } + if (message.includes("already exists")) { + return new BucketAlreadyExists({ bucket, message }); + } + // For bucket operations (no key), default to BucketAlreadyOwnedByYou + // For object operations (has key), 409 likely indicates a conflict (e.g., concurrent writes) + // Use InternalError to avoid misleading bucket ownership error + if (key) { + return new InternalError({ + message: `Swift Conflict [409] on ${ + method ?? "UNKNOWN" + } for object ${key}: ${message}`, + }); + } + return new BucketAlreadyOwnedByYou({ bucket, message }); + } + if (status === 403) { + return new AccessDenied({ message }); + } + return new InternalError({ + message: `Swift Error [${status}] on ${method ?? "UNKNOWN"}: ${message}`, + }); +}; diff --git a/src/Config/Layer.ts b/src/Config/Layer.ts new file mode 100644 index 0000000..a4eb51c --- /dev/null +++ b/src/Config/Layer.ts @@ -0,0 +1,397 @@ +import { Config, Context, Data, Effect, Layer, Option, Schema } from "effect"; +import { parse } from "@std/yaml"; +import { + type BackendConfig, + GlobalConfig, + lookupBucket, + type MaterializedBucket, + resolveAuthConfig, +} from "../Domain/Config.ts"; +import { + type AuthCredentials, + resolveAuthCredentials, +} from "../Services/Auth.ts"; + +export class HeraldConfig extends Context.Tag("HeraldConfig")< + HeraldConfig, + { + readonly raw: GlobalConfig; + readonly lookupBucket: (name: string) => Option.Option; + readonly resolveAuth: ( + bucketName: string, + ) => Option.Option; + readonly resolveAuthForBackendId: ( + backendId: string, + ) => Option.Option; + } +>() {} + +export class ConfigValidationError + extends Data.TaggedError("ConfigValidationError")<{ + readonly messages: readonly string[]; + }> {} + +/** + * Validates global config at startup. Fails with ConfigValidationError if any + * backend or auth config is invalid (e.g. Swift without credentials, S3 + * without endpoint/region, auth refs empty string). + */ +export function validateConfig( + config: GlobalConfig, +): Effect.Effect { + const messages: string[] = []; + + for (const [backendId, backend] of Object.entries(config.backends)) { + if (backend.protocol === "swift") { + const creds = backend.credentials; + if (!creds || !("username" in creds)) { + messages.push( + `Swift backend "${backendId}": credentials (username, password) are required`, + ); + } else { + if (!creds.username?.trim()) { + messages.push( + `Swift backend "${backendId}": credentials.username is required`, + ); + } + if (!creds.password?.trim()) { + messages.push( + `Swift backend "${backendId}": credentials.password is required`, + ); + } + } + } + + if (backend.protocol === "s3") { + if ( + backend.endpoint === undefined || backend.endpoint === null || + String(backend.endpoint).trim() === "" + ) { + messages.push( + `S3 backend "${backendId}": endpoint is required`, + ); + } + if ( + backend.region === undefined || backend.region === null || + String(backend.region).trim() === "" + ) { + messages.push( + `S3 backend "${backendId}": region is required`, + ); + } + const creds = backend.credentials; + if (creds && "accessKeyId" in creds) { + if (!creds.accessKeyId?.trim()) { + messages.push( + `S3 backend "${backendId}": credentials.accessKeyId is required when credentials are set`, + ); + } + if (!creds.secretAccessKey?.trim()) { + messages.push( + `S3 backend "${backendId}": credentials.secretAccessKey is required when credentials are set`, + ); + } + } + } + + const auth = backend.auth; + if (auth?.accessKeysRefs) { + for (let i = 0; i < auth.accessKeysRefs.length; i++) { + const ref = auth.accessKeysRefs[i]; + if (typeof ref !== "string" || ref.trim() === "") { + messages.push( + `Backend "${backendId}": auth.accessKeysRefs[${i}] must be a non-empty string`, + ); + } + } + } + + const buckets = backend.buckets; + if (buckets && typeof buckets !== "string") { + for (const [bucketKey, override] of Object.entries(buckets)) { + const bucketAuth = override?.auth; + if (bucketAuth?.accessKeysRefs) { + for (let i = 0; i < bucketAuth.accessKeysRefs.length; i++) { + const ref = bucketAuth.accessKeysRefs[i]; + if (typeof ref !== "string" || ref.trim() === "") { + messages.push( + `Backend "${backendId}" bucket "${bucketKey}": auth.accessKeysRefs[${i}] must be a non-empty string`, + ); + } + } + } + } + } + } + + if (config.auth?.accessKeysRefs) { + for (let i = 0; i < config.auth.accessKeysRefs.length; i++) { + const ref = config.auth.accessKeysRefs[i]; + if (typeof ref !== "string" || ref.trim() === "") { + messages.push( + `Global auth.accessKeysRefs[${i}] must be a non-empty string`, + ); + } + } + } + + if (messages.length > 0) { + return Effect.fail(new ConfigValidationError({ messages })); + } + return Effect.void; +} + +function toConfigKey(str: string): string { + const mapping: Record = { + "AUTH_URL": "auth_url", + "PROJECT_NAME": "project_name", + "USER_DOMAIN_NAME": "user_domain_name", + "PROJECT_DOMAIN_NAME": "project_domain_name", + "ACCESS_KEY_ID": "accessKeyId", + "SECRET_ACCESS_KEY": "secretAccessKey", + }; + return mapping[str] || str.toLowerCase(); +} + +export function parseConfig( + yamlConfig: unknown, + env: Record, +): GlobalConfig { + const yamlBackends = + (yamlConfig && typeof yamlConfig === "object" && "backends" in yamlConfig) + ? (yamlConfig as { backends: Record> }) + .backends + : {}; + + const backends: Record> = {}; + for (const [k, v] of Object.entries(yamlBackends)) { + backends[k] = { ...v }; + } + + const commonKeys = [ + "PROTOCOL", + "ENDPOINT", + "REGION", + "BUCKETS", + "ACCESS_KEY_ID", + "SECRET_ACCESS_KEY", + "AUTH_URL", + "CONTAINER", + "USERNAME", + "PASSWORD", + "PROJECT_NAME", + "USER_DOMAIN_NAME", + "PROJECT_DOMAIN_NAME", + "CORS_ALLOWED_ORIGINS", + "CORS_ALLOWED_METHODS", + "CORS_ALLOWED_HEADERS", + "CORS_EXPOSED_HEADERS", + "CORS_MAX_AGE", + "CORS_CREDENTIALS", + "AUTH_ACCESS_KEYS_REFS", + ]; + + const credentialKeys = [ + "accessKeyId", + "secretAccessKey", + "username", + "password", + "project_name", + "user_domain_name", + "project_domain_name", + ]; + const validConfigKeys = new Set([ + ...credentialKeys, + ...commonKeys.map((k) => toConfigKey(k)), + ]); + validConfigKeys.add("auth_access_keys_refs"); + + for (const [key, value] of Object.entries(env)) { + if (!key.startsWith("HERALD_")) continue; + if (key === "HERALD_CONFIG_PATH") continue; + if (key === "HERALD_LOG_LEVEL") continue; + + const parts = key.substring(7).split("_"); + let backendName: string; + let configParts: string[]; + let configKey: string; + + if (parts.length === 1 || commonKeys.includes(parts[0])) { + backendName = "default"; + configParts = parts; + configKey = toConfigKey(parts.join("_")); + } else { + // Backend id can contain underscores (e.g. openstack_swift). + // Use longest prefix that leaves a known config key. + backendName = parts[0].toLowerCase(); + configParts = parts.slice(1); + configKey = toConfigKey(configParts.join("_")); + for (let i = parts.length - 1; i >= 1; i--) { + const candidateParts = parts.slice(i); + const candidateKey = toConfigKey(candidateParts.join("_")); + if ( + validConfigKeys.has(candidateKey) || + candidateKey.startsWith("cors_") + ) { + backendName = parts.slice(0, i).join("_").toLowerCase(); + configParts = candidateParts; + configKey = candidateKey; + break; + } + } + } + if (!backends[backendName]) backends[backendName] = {}; + const backend = backends[backendName]; + + if (credentialKeys.includes(configKey)) { + if (!backend.credentials) { + backend.credentials = {} as Record; + } + (backend.credentials as Record)[configKey] = value; + } else if (configKey.startsWith("cors_")) { + if (!backend.cors) { + backend.cors = {} as Record; + } + const corsKey = configKey.substring(5); + const camelCorsKey = corsKey.replace( + /_([a-z])/g, + (_, g) => g.toUpperCase(), + ); + + if ( + camelCorsKey === "allowedOrigins" || + camelCorsKey === "allowedMethods" || + camelCorsKey === "allowedHeaders" || camelCorsKey === "exposedHeaders" + ) { + (backend.cors as Record)[camelCorsKey] = value.split( + ",", + ).map((s) => s.trim()); + } else if (camelCorsKey === "maxAge") { + const parsed = parseInt(value, 10); + if (Number.isInteger(parsed) && Number.isFinite(parsed)) { + (backend.cors as Record)[camelCorsKey] = parsed; + } + } else if (camelCorsKey === "credentials") { + (backend.cors as Record)[camelCorsKey] = + value.toLowerCase() === "true"; + } + } else if (configKey === "auth_access_keys_refs") { + backend.auth = { + accessKeysRefs: value.split(",").map((s) => s.trim()), + }; + } else { + backend[configKey] = value; + } + } + + // Handle global CORS from env + const globalCors: Record = (yamlConfig && + typeof yamlConfig === "object" && "cors" in yamlConfig) + ? { ...(yamlConfig as { cors: Record }).cors } + : {}; + + for (const [key, value] of Object.entries(env)) { + if (!key.startsWith("HERALD_CORS_")) continue; + const corsKey = key.substring(12).toLowerCase(); + const camelCorsKey = corsKey.replace( + /_([a-z])/g, + (_, g) => g.toUpperCase(), + ); + + if ( + camelCorsKey === "allowedOrigins" || camelCorsKey === "allowedMethods" || + camelCorsKey === "allowedHeaders" || camelCorsKey === "exposedHeaders" + ) { + globalCors[camelCorsKey] = value.split(",").map((s) => s.trim()); + } else if (camelCorsKey === "maxAge") { + const parsed = parseInt(value, 10); + if (Number.isInteger(parsed) && Number.isFinite(parsed)) { + globalCors[camelCorsKey] = parsed; + } + } else if (camelCorsKey === "credentials") { + globalCors[camelCorsKey] = value.toLowerCase() === "true"; + } + } + + // Handle global AUTH from env + const globalAuth: Record = (yamlConfig && + typeof yamlConfig === "object" && "auth" in yamlConfig) + ? { ...(yamlConfig as { auth: Record }).auth } + : {}; + + if (env["HERALD_AUTH_ACCESS_KEYS_REFS"]) { + globalAuth["accessKeysRefs"] = env["HERALD_AUTH_ACCESS_KEYS_REFS"] + .split(",") + .map((s) => s.trim()); + } + + // Default backend fallback if no backends defined at all + if (Object.keys(backends).length === 0) { + backends["default"] = { + protocol: "s3", + buckets: "*", + }; + } + + const validatedBackends: Record = {}; + for (const [id, b] of Object.entries(backends)) { + if (b.protocol === "s3" || b.protocol === "swift") { + validatedBackends[id] = b as BackendConfig; + } + } + + return Schema.decodeUnknownSync(GlobalConfig)({ + backends: validatedBackends, + cors: Object.keys(globalCors).length > 0 ? globalCors : undefined, + auth: Object.keys(globalAuth).length > 0 ? globalAuth : undefined, + }); +} + +export const HeraldConfigLive = Layer.effect( + HeraldConfig, + Effect.gen(function* () { + const configPath = yield* Config.string("HERALD_CONFIG_PATH").pipe( + Config.orElse(() => Config.string("CONFIG_PATH")), + Config.withDefault("herald.yaml"), + ); + + const yamlConfig = yield* Effect.tryPromise({ + try: () => Deno.readTextFile(configPath), + catch: () => new Error("Config file missing"), + }).pipe( + Effect.flatMap((content) => + Effect.try({ + try: () => parse(content), + catch: (e) => new Error(`YAML parse error: ${e}`), + }) + ), + Effect.orElseSucceed(() => ({ backends: {} })), + ); + + // Discovery needs the full environment. In Deno we use Deno.env.toObject(). + // We can wrap this in an Effect to be more idiomatic. + const env = yield* Effect.sync(() => Deno.env.toObject()); + + const raw = parseConfig(yamlConfig, env); + yield* validateConfig(raw); + + return { + raw, + lookupBucket: (name: string) => lookupBucket(raw, name), + resolveAuth: (bucketName: string) => { + const authConfig = resolveAuthConfig(raw, bucketName); + if (!authConfig) return Option.none(); + const creds = resolveAuthCredentials(authConfig.accessKeysRefs, env); + return Option.some(creds); + }, + resolveAuthForBackendId: (backendId: string) => { + const backend = raw.backends[backendId]; + if (!backend) return Option.none(); + const authConfig = backend.auth ?? raw.auth; + if (!authConfig) return Option.none(); + const creds = resolveAuthCredentials(authConfig.accessKeysRefs, env); + return Option.some(creds); + }, + }; + }), +); diff --git a/src/Domain/Config.ts b/src/Domain/Config.ts new file mode 100644 index 0000000..4b02f6b --- /dev/null +++ b/src/Domain/Config.ts @@ -0,0 +1,322 @@ +import { Option, Schema } from "effect"; + +export const S3Credentials = Schema.Struct({ + accessKeyId: Schema.optional(Schema.String), + secretAccessKey: Schema.optional(Schema.String), +}); + +export const SwiftCredentials = Schema.Struct({ + username: Schema.optional(Schema.String), + password: Schema.optional(Schema.String), + project_name: Schema.optional(Schema.String), + user_domain_name: Schema.optional(Schema.String), + project_domain_name: Schema.optional(Schema.String), +}); + +export const Credentials = Schema.Union(S3Credentials, SwiftCredentials); + +export const CorsConfig = Schema.Struct({ + allowedOrigins: Schema.optional(Schema.Array(Schema.String)), + allowedMethods: Schema.optional(Schema.Array(Schema.String)), + allowedHeaders: Schema.optional(Schema.Array(Schema.String)), + exposedHeaders: Schema.optional(Schema.Array(Schema.String)), + maxAge: Schema.optional(Schema.Number), + credentials: Schema.optional(Schema.Boolean), +}).pipe( + Schema.filter((c) => { + if (c.allowedOrigins?.includes("*") && c.credentials) { + return "CORS configuration cannot have allowedOrigins: ['*'] when credentials: true"; + } + return true; + }), +); + +export type CorsConfig = Schema.Schema.Type; + +export const AuthConfig = Schema.Struct({ + accessKeysRefs: Schema.Array(Schema.String), +}); + +export type AuthConfig = Schema.Schema.Type; + +export const BucketOverride = Schema.Struct({ + endpoint: Schema.optional(Schema.String), + bucket_name: Schema.optional(Schema.String), + region: Schema.optional(Schema.String), + cors: Schema.optional(CorsConfig), + auth: Schema.optional(AuthConfig), +}); + +export type BucketOverride = Schema.Schema.Type; + +export const BucketsConfig = Schema.optionalWith( + Schema.Union( + Schema.Record({ key: Schema.String, value: BucketOverride }), + Schema.String, + ), + { default: () => "*" }, +); + +export const S3Config = Schema.Struct({ + protocol: Schema.Literal("s3"), + endpoint: Schema.optional(Schema.String), + region: Schema.optional(Schema.String), + credentials: Schema.optional(S3Credentials), + buckets: BucketsConfig, + cors: Schema.optional(CorsConfig), + auth: Schema.optional(AuthConfig), +}); + +export const SwiftConfig = Schema.Struct({ + protocol: Schema.Literal("swift"), + auth_url: Schema.String, + region: Schema.optional(Schema.String), + container: Schema.optional(Schema.String), + credentials: Schema.optional(SwiftCredentials), + buckets: BucketsConfig, + cors: Schema.optional(CorsConfig), + auth: Schema.optional(AuthConfig), +}); + +export const BackendConfig = Schema.Union(S3Config, SwiftConfig); + +export type BackendConfig = Schema.Schema.Type; + +export const GlobalConfig = Schema.Struct({ + backends: Schema.Record({ key: Schema.String, value: BackendConfig }), + cors: Schema.optional(CorsConfig), + auth: Schema.optional(AuthConfig), +}); + +export type GlobalConfig = Schema.Schema.Type; + +export const MaterializedBucket = Schema.Struct({ + name: Schema.String, + backend_id: Schema.String, + protocol: Schema.Literal("s3", "swift"), + endpoint: Schema.optional(Schema.String), + region: Schema.optional(Schema.String), + bucket_name: Schema.String, + credentials: Schema.optional(Credentials), + // Swift specific + auth_url: Schema.optional(Schema.String), + container: Schema.optional(Schema.String), +}); + +export type MaterializedBucket = Schema.Schema.Type; + +/** + * Utility to convert simple glob (*) to RegExp + */ +export const globToRegex = (glob: string) => { + const regexStr = glob.split("*").map((s) => + s.replace(/[.+^${}()|[\]\\]/g, "\\$&") + ).join(".*"); + return new RegExp(`^${regexStr}$`); +}; + +export const lookupBucket = ( + config: GlobalConfig, + bucketName: string, +): Option.Option => { + // 1. Direct hit in any backend's bucket record + for (const [backend_id, backend] of Object.entries(config.backends)) { + const buckets = backend.buckets; + if (buckets && typeof buckets !== "string" && buckets[bucketName]) { + const override = buckets[bucketName]; + const base: MaterializedBucket = { + name: bucketName, + backend_id, + protocol: backend.protocol, + endpoint: override.endpoint ?? + (backend.protocol === "s3" ? backend.endpoint : undefined), + region: override.region ?? backend.region, + bucket_name: override.bucket_name ?? bucketName, + credentials: backend.credentials, + auth_url: backend.protocol === "swift" ? backend.auth_url : undefined, + container: backend.protocol === "swift" ? backend.container : undefined, + }; + + return Option.some(base); + } + } + + // 2. Glob match in any backend's bucket record keys + for (const [backend_id, backend] of Object.entries(config.backends)) { + const buckets = backend.buckets; + if (buckets && typeof buckets !== "string") { + for (const [key, override] of Object.entries(buckets)) { + if (globToRegex(key).test(bucketName)) { + const base: MaterializedBucket = { + name: bucketName, + backend_id, + protocol: backend.protocol, + endpoint: (override as BucketOverride).endpoint ?? + (backend.protocol === "s3" ? backend.endpoint : undefined), + region: (override as BucketOverride).region ?? backend.region, + bucket_name: (override as BucketOverride).bucket_name ?? + bucketName, + credentials: backend.credentials, + auth_url: backend.protocol === "swift" + ? backend.auth_url + : undefined, + container: backend.protocol === "swift" + ? backend.container + : undefined, + }; + + return Option.some(base); + } + } + } + } + + // 3. Glob match if backend.buckets is a string + for (const [backend_id, backend] of Object.entries(config.backends)) { + const buckets = backend.buckets; + if (buckets && typeof buckets === "string") { + if (globToRegex(buckets).test(bucketName)) { + const base: MaterializedBucket = { + name: bucketName, + backend_id, + protocol: backend.protocol, + endpoint: backend.protocol === "s3" ? backend.endpoint : undefined, + region: backend.region, + bucket_name: bucketName, + credentials: backend.credentials, + auth_url: backend.protocol === "swift" ? backend.auth_url : undefined, + container: backend.protocol === "swift" + ? backend.container + : undefined, + }; + + return Option.some(base); + } + } + } + + return Option.none(); +}; + +export const resolveCorsConfig = ( + config: GlobalConfig, + bucketName: string, +): CorsConfig | undefined => { + // 1. Find the backend and bucket override + let bucketCors: CorsConfig | undefined; + let backendCors: CorsConfig | undefined; + + for (const backend of Object.values(config.backends)) { + const buckets = backend.buckets; + if (buckets && typeof buckets !== "string" && buckets[bucketName]) { + bucketCors = buckets[bucketName].cors; + backendCors = backend.cors; + break; + } + } + + // If not found by direct hit, try glob match (similar to lookupBucket) + if (!bucketCors && !backendCors) { + for (const backend of Object.values(config.backends)) { + const buckets = backend.buckets; + if (buckets && typeof buckets !== "string") { + let foundMatch = false; + for (const [key, override] of Object.entries(buckets)) { + if (globToRegex(key).test(bucketName)) { + bucketCors = (override as BucketOverride).cors; + backendCors = backend.cors; + foundMatch = true; + break; + } + } + if (foundMatch) break; + } + } + } + + // If still not found, check if it's a general backend match + if (!bucketCors && !backendCors) { + for (const backend of Object.values(config.backends)) { + const buckets = backend.buckets; + if ( + typeof buckets === "string" && globToRegex(buckets).test(bucketName) + ) { + backendCors = backend.cors; + break; + } + } + } + + const globalCors = config.cors; + + if (!bucketCors && !backendCors && !globalCors) { + return undefined; + } + + // Merge with precedence: bucket > backend > global + return { + ...globalCors, + ...backendCors, + ...bucketCors, + }; +}; + +export const resolveAuthConfig = ( + config: GlobalConfig, + bucketName: string, +): AuthConfig | undefined => { + // 1. Find the backend and bucket override + let bucketAuth: AuthConfig | undefined; + let backendAuth: AuthConfig | undefined; + + for (const backend of Object.values(config.backends)) { + const buckets = backend.buckets; + if (buckets && typeof buckets !== "string" && buckets[bucketName]) { + bucketAuth = buckets[bucketName].auth; + backendAuth = backend.auth; + break; + } + } + + // If not found by direct hit, try glob match (similar to lookupBucket) + if (!bucketAuth && !backendAuth) { + for (const backend of Object.values(config.backends)) { + const buckets = backend.buckets; + if (buckets && typeof buckets !== "string") { + let foundMatch = false; + for (const [key, override] of Object.entries(buckets)) { + if (globToRegex(key).test(bucketName)) { + bucketAuth = (override as BucketOverride).auth; + backendAuth = backend.auth; + foundMatch = true; + break; + } + } + if (foundMatch) break; + } + } + } + + // If still not found, check if it's a general backend match + if (!bucketAuth && !backendAuth) { + for (const backend of Object.values(config.backends)) { + const buckets = backend.buckets; + if ( + typeof buckets === "string" && globToRegex(buckets).test(bucketName) + ) { + backendAuth = backend.auth; + break; + } + } + } + + const globalAuth = config.auth; + + if (!bucketAuth && !backendAuth && !globalAuth) { + return undefined; + } + + // Merge with precedence: bucket > backend > global + // For accessKeysRefs, we take the most specific one, not merge arrays + return bucketAuth ?? backendAuth ?? globalAuth; +}; diff --git a/src/Frontend/Api.ts b/src/Frontend/Api.ts new file mode 100644 index 0000000..f4a7fdc --- /dev/null +++ b/src/Frontend/Api.ts @@ -0,0 +1,68 @@ +import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "@effect/platform"; +import { Schema } from "effect"; + +export class BadGateway extends Schema.TaggedError()("BadGateway", { + message: Schema.String, +}) {} + +export const HttpS3Api = HttpApiGroup.make("s3") + .add( + HttpApiEndpoint.post("postRoot", "/") + .addError(BadGateway, { status: 502 }), + ) + .add( + HttpApiEndpoint.get("listBuckets", "/") + .addError(BadGateway, { status: 502 }), + ) + .add( + HttpApiEndpoint.get("listObjects", "/:bucket") + .setPath(Schema.Struct({ bucket: Schema.String })) + .addError(BadGateway, { status: 502 }), + ) + .add( + HttpApiEndpoint.put("createBucket", "/:bucket") + .setPath(Schema.Struct({ bucket: Schema.String })) + .addError(BadGateway, { status: 502 }), + ) + .add( + HttpApiEndpoint.del("deleteBucket", "/:bucket") + .setPath(Schema.Struct({ bucket: Schema.String })) + .addError(BadGateway, { status: 502 }), + ) + .add( + HttpApiEndpoint.head("headBucket", "/:bucket") + .setPath(Schema.Struct({ bucket: Schema.String })) + .addError(BadGateway, { status: 502 }), + ) + .add( + HttpApiEndpoint.post("postBucket", "/:bucket") + .setPath(Schema.Struct({ bucket: Schema.String })) + .addError(BadGateway, { status: 502 }), + ) + // Object operations with wildcards to support slashes in keys + .add( + HttpApiEndpoint.get("getObject", "/:bucket/*") + .setPath(Schema.Struct({ bucket: Schema.String })) + .addError(BadGateway, { status: 502 }), + ) + .add( + HttpApiEndpoint.put("putObject", "/:bucket/*") + .setPath(Schema.Struct({ bucket: Schema.String })) + .addError(BadGateway, { status: 502 }), + ) + .add( + HttpApiEndpoint.post("postObject", "/:bucket/*") + .setPath(Schema.Struct({ bucket: Schema.String })) + .addError(BadGateway, { status: 502 }), + ) + .add( + HttpApiEndpoint.del("deleteObject", "/:bucket/*") + .setPath(Schema.Struct({ bucket: Schema.String })) + .addError(BadGateway, { status: 502 }), + ) + .add( + HttpApiEndpoint.head("headObject", "/:bucket/*") + .setPath(Schema.Struct({ bucket: Schema.String })) + .addError(BadGateway, { status: 502 }), + ) + .annotate(OpenApi.Title, "S3 Compatibility"); diff --git a/src/Frontend/Buckets/Create.ts b/src/Frontend/Buckets/Create.ts new file mode 100644 index 0000000..b928844 --- /dev/null +++ b/src/Frontend/Buckets/Create.ts @@ -0,0 +1,38 @@ +import { Effect } from "effect"; +import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; +import { RequestContext, S3RequestParser } from "../Utils.ts"; +import { Backend } from "../../Services/Backend.ts"; + +export const createBucket = Effect.gen(function* () { + const backend = yield* Backend; + const request = yield* HttpServerRequest.HttpServerRequest; + const parser = yield* S3RequestParser; + const { bucket } = yield* RequestContext; + + const { s3Params } = parser; + + if (s3Params.acl !== undefined) { + // PutBucketAcl + // Check for canned ACL validity if present + const cannedAcl = request.headers["x-amz-acl"]; + const validCannedAcls = [ + "private", + "public-read", + "public-read-write", + "authenticated-read", + ]; + if (cannedAcl && !validCannedAcls.includes(cannedAcl)) { + return HttpServerResponse.text( + `InvalidArgumentArgument x-amz-acl is invalid.`, + { status: 400, headers: { "Content-Type": "application/xml" } }, + ); + } + + // For now, we just return 200 OK if the bucket exists + yield* backend.headBucket(bucket); + return HttpServerResponse.text("", { status: 200 }); + } + + yield* backend.createBucket(bucket, request.headers); + return HttpServerResponse.text("", { status: 200 }); +}); diff --git a/src/Frontend/Buckets/Delete.ts b/src/Frontend/Buckets/Delete.ts new file mode 100644 index 0000000..7a2cbad --- /dev/null +++ b/src/Frontend/Buckets/Delete.ts @@ -0,0 +1,11 @@ +import { Effect } from "effect"; +import { HttpServerResponse } from "@effect/platform"; +import { Backend } from "../../Services/Backend.ts"; +import { RequestContext } from "../Utils.ts"; + +export const deleteBucket = Effect.gen(function* () { + const backend = yield* Backend; + const { bucket } = yield* RequestContext; + yield* backend.deleteBucket(bucket); + return HttpServerResponse.empty({ status: 204 }); +}); diff --git a/src/Frontend/Buckets/Head.ts b/src/Frontend/Buckets/Head.ts new file mode 100644 index 0000000..9ba340a --- /dev/null +++ b/src/Frontend/Buckets/Head.ts @@ -0,0 +1,11 @@ +import { Effect } from "effect"; +import { HttpServerResponse } from "@effect/platform"; +import { Backend } from "../../Services/Backend.ts"; +import { RequestContext } from "../Utils.ts"; + +export const headBucket = Effect.gen(function* () { + const backend = yield* Backend; + const { bucket } = yield* RequestContext; + yield* backend.headBucket(bucket); + return HttpServerResponse.empty({ status: 200 }); +}); diff --git a/src/Frontend/Buckets/List.ts b/src/Frontend/Buckets/List.ts new file mode 100644 index 0000000..5ab618f --- /dev/null +++ b/src/Frontend/Buckets/List.ts @@ -0,0 +1,29 @@ +import { Effect } from "effect"; +import { HeraldConfig } from "../../Config/Layer.ts"; +import { BackendResolver } from "../../Services/BackendResolver.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; + +export const listBuckets = Effect.gen(function* () { + const config = yield* HeraldConfig; + const resolver = yield* BackendResolver; + + // For ListBuckets, we need to decide which backend to proxy to. + // We prefer an S3 backend if available, otherwise we take the first one. + const backendId = Object.keys(config.raw.backends).find((id) => + config.raw.backends[id].protocol === "s3" + ) ?? Object.keys(config.raw.backends)[0]; + + const s3xml = yield* S3Xml; + if (!backendId) { + return s3xml.formatError("No backend configured"); + } + return yield* resolver.getLayerForBackend(backendId).pipe( + Effect.andThen((backend) => + backend.listBuckets() + ), + Effect.andThen(({ buckets, owner }) => + s3xml.formatListBuckets(buckets, owner) + ), + Effect.catchAll((error) => Effect.succeed(s3xml.formatError(error))), + ); +}); diff --git a/src/Frontend/Cors.ts b/src/Frontend/Cors.ts new file mode 100644 index 0000000..ad2e7dc --- /dev/null +++ b/src/Frontend/Cors.ts @@ -0,0 +1,138 @@ +import { + HttpMiddleware, + HttpServerRequest, + HttpServerResponse, +} from "@effect/platform"; +import { Effect } from "effect"; +import { HeraldConfig } from "../Config/Layer.ts"; +import { resolveCorsConfig } from "../Domain/Config.ts"; + +/** + * Extracts the bucket name from the request URL path. + * Assumes path format like /:bucket or /:bucket/* + */ +function extractBucketFromPath(url: string): string | undefined { + try { + const path = new URL(url, "http://localhost").pathname; + const parts = path.split("/").filter((p) => p.length > 0); + return parts.length > 0 ? parts[0] : undefined; + } catch { + return undefined; + } +} + +/** + * Adds CORS headers to a response based on the provided config. + */ +function addCorsHeaders( + response: HttpServerResponse.HttpServerResponse, + cors: NonNullable>, + request: HttpServerRequest.HttpServerRequest, +): HttpServerResponse.HttpServerResponse { + const origin = request.headers["origin"]; + const headers = { ...response.headers }; + + if (cors.allowedOrigins) { + if (cors.allowedOrigins.includes("*")) { + if (cors.credentials && origin) { + headers["access-control-allow-origin"] = origin; + headers["vary"] = headers["vary"] + ? `${headers["vary"]}, Origin` + : "Origin"; + } else if (!cors.credentials) { + headers["access-control-allow-origin"] = "*"; + } + } else if (origin && cors.allowedOrigins.includes(origin)) { + headers["access-control-allow-origin"] = origin; + headers["vary"] = headers["vary"] + ? `${headers["vary"]}, Origin` + : "Origin"; + } + } + + if (cors.credentials) { + headers["access-control-allow-credentials"] = "true"; + } + + if (cors.exposedHeaders) { + headers["access-control-expose-headers"] = cors.exposedHeaders.join(", "); + } + + return HttpServerResponse.setHeaders(response, headers); +} + +/** + * Creates a 204 No Content response for OPTIONS preflight requests. + */ +function makePreflightResponse( + cors: NonNullable>, + request: HttpServerRequest.HttpServerRequest, +): HttpServerResponse.HttpServerResponse { + const origin = request.headers["origin"]; + const headers: Record = { + "access-control-max-age": String(cors.maxAge ?? 3600), + }; + + if (cors.allowedOrigins) { + if (cors.allowedOrigins.includes("*")) { + if (cors.credentials && origin) { + headers["access-control-allow-origin"] = origin; + headers["vary"] = "Origin"; + } else if (!cors.credentials) { + headers["access-control-allow-origin"] = "*"; + } + } else if (origin && cors.allowedOrigins.includes(origin)) { + headers["access-control-allow-origin"] = origin; + headers["vary"] = "Origin"; + } + } + + if (cors.credentials) { + headers["access-control-allow-credentials"] = "true"; + } + + if (cors.allowedMethods) { + headers["access-control-allow-methods"] = cors.allowedMethods.join(", "); + } else { + // Default to common S3 methods if not specified + headers["access-control-allow-methods"] = + "GET, PUT, POST, DELETE, HEAD, OPTIONS"; + } + + if (cors.allowedHeaders) { + headers["access-control-allow-headers"] = cors.allowedHeaders.join(", "); + } else { + const requestedHeaders = request.headers["access-control-request-headers"]; + if (requestedHeaders) { + headers["access-control-allow-headers"] = requestedHeaders; + } + } + + return HttpServerResponse.empty({ status: 204, headers }); +} + +/** + * Custom CORS middleware that resolves configuration per-request based on the bucket. + */ +export const corsMiddleware = HttpMiddleware.make((app) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const config = yield* HeraldConfig; + + const bucket = extractBucketFromPath(request.url); + const corsConfig = bucket + ? resolveCorsConfig(config.raw, bucket) + : config.raw.cors; + + if (!corsConfig) { + return yield* app; + } + + if (request.method === "OPTIONS") { + return makePreflightResponse(corsConfig, request); + } + + const response = yield* app; + return addCorsHeaders(response, corsConfig, request); + }) +); diff --git a/src/Frontend/Health/Api.ts b/src/Frontend/Health/Api.ts new file mode 100644 index 0000000..829ae5f --- /dev/null +++ b/src/Frontend/Health/Api.ts @@ -0,0 +1,10 @@ +import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "@effect/platform"; +import { Schema } from "effect"; + +export class HealthHttpApi extends HttpApiGroup.make("health") + .add( + HttpApiEndpoint.get("getStatus", "/health") + .addSuccess(Schema.Struct({ status: Schema.Literal("ok") })), + ) + .annotate(OpenApi.Title, "Health") + .annotate(OpenApi.Description, "Health check endpoint") {} diff --git a/src/Frontend/Health/Http.ts b/src/Frontend/Health/Http.ts new file mode 100644 index 0000000..0f186df --- /dev/null +++ b/src/Frontend/Health/Http.ts @@ -0,0 +1,13 @@ +import { HttpApiBuilder } from "@effect/platform"; +import { Effect } from "effect"; +import { HttpHeraldApi } from "../../Api.ts"; + +export const HttpHealthLive = HttpApiBuilder.group( + HttpHeraldApi, + "health", + (handlers) => + handlers.handle( + "getStatus", + () => Effect.succeed({ status: "ok" as const }), + ), +); diff --git a/src/Frontend/Http.ts b/src/Frontend/Http.ts new file mode 100644 index 0000000..e004a1d --- /dev/null +++ b/src/Frontend/Http.ts @@ -0,0 +1,273 @@ +import { + HttpApiBuilder, + HttpRouter, + HttpServerResponse, +} from "@effect/platform"; +import { Effect, Layer } from "effect"; +import { Backend, MethodNotAllowed } from "../Services/Backend.ts"; +import { BackendResolver } from "../Services/BackendResolver.ts"; +import { S3Xml } from "../Services/S3Xml.ts"; +import { RequestContext } from "./Utils.ts"; +import { listObjects } from "./Objects/List.ts"; +import { postObject } from "./Objects/Post.ts"; +import { getObject } from "./Objects/Get.ts"; +import { putObject } from "./Objects/Put.ts"; +import { deleteObject } from "./Objects/Delete.ts"; +import { headObject } from "./Objects/Head.ts"; +import { createBucket } from "./Buckets/Create.ts"; +import { deleteBucket } from "./Buckets/Delete.ts"; +import { headBucket } from "./Buckets/Head.ts"; +import { HttpHeraldApi } from "../Api.ts"; +import { BadGateway } from "./Api.ts"; +import * as HttpServerRequest from "@effect/platform/HttpServerRequest"; + +/** Build annotations and log 5xx as error, 4xx as warning; return response. */ +function logRequestFailureAndReturn( + err: unknown, + response: HttpServerResponse.HttpServerResponse, + bucket: string, + method: string, +): Effect.Effect { + const status = response.status ?? 500; + const errorType = + err != null && typeof err === "object" && "constructor" in err && + typeof (err as { constructor: { name?: string } }).constructor?.name === + "string" + ? (err as { constructor: { name: string } }).constructor.name + : "Unknown"; + const message = err instanceof Error ? err.message : String(err); + const annotations: Record = { + status, + errorType, + message, + bucket, + method, + }; + if ( + err != null && typeof err === "object" && "key" in err && + typeof (err as { key: unknown }).key === "string" + ) { + annotations.key = (err as { key: string }).key; + } + if ( + err != null && typeof err === "object" && "uploadId" in err && + typeof (err as { uploadId: unknown }).uploadId === "string" + ) { + annotations.uploadId = (err as { uploadId: string }).uploadId; + } + return Effect.gen(function* () { + if (status >= 500) { + yield* Effect.logError("Request failed", annotations); + } else if (status >= 400) { + yield* Effect.logWarning("Request failed", annotations); + } + return response; + }); +} + +/** + * Main HTTP Router for the S3 Proxy. + */ +export const makeS3Router = (prefix = "") => + Effect.gen(function* () { + const s3Xml = yield* S3Xml; + const resolver = yield* BackendResolver; + + const frontHandler = ( + handler: Effect.Effect, + ) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + // Extract bucket name from URL path + // request.url might be a full URL or just a pathname + const pathname = request.url.startsWith("http") + ? new URL(request.url).pathname + : request.url.split("?")[0]; // Remove query string if present + + // Remove prefix from pathname before extracting bucket + let pathWithoutPrefix = pathname; + if (prefix) { + // Normalize prefix: ensure it starts with / and remove trailing / + const normalizedPrefix = prefix.startsWith("/") + ? prefix + : `/${prefix}`; + const cleanPrefix = normalizedPrefix.endsWith("/") + ? normalizedPrefix.slice(0, -1) + : normalizedPrefix; + + // Check if pathname starts with the prefix (exact match) + if (pathname.startsWith(cleanPrefix)) { + pathWithoutPrefix = pathname.substring(cleanPrefix.length); + // Ensure it starts with / after prefix removal + if (!pathWithoutPrefix.startsWith("/")) { + pathWithoutPrefix = `/${pathWithoutPrefix}`; + } + } + } + + const bucket = pathWithoutPrefix.split("/").filter(Boolean)[0] || ""; + const isHead = request.method === "HEAD"; + const method = request.method ?? "UNKNOWN"; + + const backend = yield* resolver.getLayerForBucket(bucket); + const backendLayer = Layer.succeed(Backend, backend); + + const attrs = { bucket, method }; + return yield* handler.pipe( + Effect.provideService(RequestContext, { bucket }), + Effect.provide(backendLayer), + // convert the frontend errors to xml and log failure details + Effect.catchAll((err: unknown) => { + const response = s3Xml.formatError(err, isHead); + return logRequestFailureAndReturn(err, response, bucket, method); + }), + ).pipe( + Effect.annotateLogs(attrs), + Effect.withSpan("herald.s3.request", { attributes: attrs }), + ); + }); + + const router = HttpRouter.empty + .pipe( + HttpRouter.get( + "/health", + HttpServerResponse.json({ status: "ok" }), + ), + // List Buckets (GET /) + HttpRouter.get( + "/", + Effect.gen(function* () { + const backendInstance = yield* resolver.getLayerForBucket(""); + const backendLayer = Layer.succeed(Backend, backendInstance); + const result = yield* Effect.gen(function* () { + const backend = yield* Backend; + return yield* backend.listBuckets(); + }).pipe(Effect.provide(backendLayer)); + return s3Xml.formatListBuckets(result.buckets, result.owner); + }).pipe( + Effect.catchAll((err: unknown) => { + const response = s3Xml.formatError(err); + return logRequestFailureAndReturn(err, response, "", "GET"); + }), + ), + ), + // Bucket/Object operations + HttpRouter.all( + "/:bucket", + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + if (request.method === "GET") { + return yield* frontHandler(listObjects); + } + if (request.method === "PUT") { + return yield* frontHandler(createBucket); + } + if (request.method === "DELETE") { + return yield* frontHandler(deleteBucket); + } + if (request.method === "HEAD") { + return yield* frontHandler(headBucket); + } + if (request.method === "POST") { + return yield* frontHandler(postObject); + } + return yield* Effect.fail( + new MethodNotAllowed({ + message: + `Method ${request.method} not implemented for bucket operations`, + }), + ); + }), + ), + HttpRouter.all( + "/:bucket/*", + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + if (request.method === "GET") return yield* frontHandler(getObject); + if (request.method === "PUT") return yield* frontHandler(putObject); + if (request.method === "POST") { + return yield* frontHandler(postObject); + } + if (request.method === "DELETE") { + return yield* frontHandler(deleteObject); + } + if (request.method === "HEAD") { + return yield* frontHandler(headObject); + } + return yield* Effect.fail( + new MethodNotAllowed({ + message: `Method ${request.method} not implemented`, + }), + ); + }), + ), + ); + + return prefix + ? HttpRouter.empty.pipe(HttpRouter.mount( + prefix.startsWith("/") + ? prefix as `/${string}` + : `/${prefix}` as `/${string}`, + router, + )) + : router; + }); + +export const HttpS3Live = Layer.unwrapEffect( + Effect.gen(function* () { + const router = yield* makeS3Router(); + return HttpApiBuilder.group(HttpHeraldApi, "s3", (handlers) => { + const handler = ( + req: { readonly request: HttpServerRequest.HttpServerRequest }, + ) => + router.pipe( + Effect.provideService( + HttpServerRequest.HttpServerRequest, + req.request, + ), + Effect.catchAll((err: unknown) => { + const request = req.request; + const method = request.method ?? "UNKNOWN"; + const url = request.url.startsWith("http") + ? new URL(request.url).pathname + : request.url.split("?")[0]; + const errorType = + err != null && typeof err === "object" && "constructor" in err && + typeof (err as { constructor: { name?: string } }) + .constructor?.name === "string" + ? (err as { constructor: { name: string } }).constructor.name + : "Unknown"; + const message = err instanceof Error ? err.message : String(err); + return Effect.gen(function* () { + yield* Effect.logError("Request failed", { + status: 502, + errorType, + message, + method, + url, + }); + return yield* Effect.fail( + new BadGateway({ message: String(err) }), + ); + }); + }), + ) as Effect.Effect< + HttpServerResponse.HttpServerResponse, + BadGateway, + never + >; + return handlers.handleRaw("postRoot", handler) + .handleRaw("listBuckets", handler) + .handleRaw("listObjects", handler) + .handleRaw("createBucket", handler) + .handleRaw("deleteBucket", handler) + .handleRaw("headBucket", handler) + .handleRaw("postBucket", handler) + .handleRaw("getObject", handler) + .handleRaw("putObject", handler) + .handleRaw("postObject", handler) + .handleRaw("deleteObject", handler) + .handleRaw("headObject", handler); + }); + }), +); diff --git a/src/Frontend/Multipart/Delete.ts b/src/Frontend/Multipart/Delete.ts new file mode 100644 index 0000000..0d56017 --- /dev/null +++ b/src/Frontend/Multipart/Delete.ts @@ -0,0 +1,23 @@ +import { Effect } from "effect"; +import { HttpServerResponse } from "@effect/platform"; +import { S3RequestParser } from "../Utils.ts"; +import { Backend, InvalidRequest } from "../../Services/Backend.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; + +export const abortMultipartUpload = Effect.gen(function* () { + const backend = yield* Backend; + const { key, s3Params } = yield* S3RequestParser; + const s3Xml = yield* S3Xml; + + // Validate required parameters before calling backend + if (!s3Params.uploadId || typeof s3Params.uploadId !== "string") { + return s3Xml.formatError( + new InvalidRequest({ + message: "Missing or invalid uploadId parameter", + }), + ); + } + + yield* backend.abortMultipartUpload(key, s3Params.uploadId); + return HttpServerResponse.empty({ status: 204 }); +}); diff --git a/src/Frontend/Multipart/Get.ts b/src/Frontend/Multipart/Get.ts new file mode 100644 index 0000000..09cf33d --- /dev/null +++ b/src/Frontend/Multipart/Get.ts @@ -0,0 +1,22 @@ +import { Effect } from "effect"; +import { S3RequestParser } from "../Utils.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; +import { Backend, InvalidRequest } from "../../Services/Backend.ts"; + +export const listParts = Effect.gen(function* () { + const backend = yield* Backend; + const { key, s3Params } = yield* S3RequestParser; + const s3Xml = yield* S3Xml; + + // Validate required parameters before calling backend + if (!s3Params.uploadId || typeof s3Params.uploadId !== "string") { + return s3Xml.formatError( + new InvalidRequest({ + message: "Missing or invalid uploadId parameter", + }), + ); + } + + const result = yield* backend.listParts(key, s3Params.uploadId); + return s3Xml.formatListParts(result); +}); diff --git a/src/Frontend/Multipart/List.ts b/src/Frontend/Multipart/List.ts new file mode 100644 index 0000000..71de443 --- /dev/null +++ b/src/Frontend/Multipart/List.ts @@ -0,0 +1,20 @@ +import { Effect } from "effect"; +import { S3RequestParser } from "../Utils.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; +import { Backend } from "../../Services/Backend.ts"; + +export const listMultipartUploads = Effect.gen(function* () { + const backend = yield* Backend; + const { s3Params } = yield* S3RequestParser; + const s3Xml = yield* S3Xml; + + const result = yield* backend.listMultipartUploads({ + prefix: s3Params.prefix, + delimiter: s3Params.delimiter, + keyMarker: s3Params["key-marker"], + uploadIdMarker: s3Params["upload-id-marker"], + maxUploads: s3Params["max-uploads"], + encodingType: s3Params["encoding-type"], + }); + return s3Xml.formatListMultipartUploads(result); +}); diff --git a/src/Frontend/Multipart/Post.ts b/src/Frontend/Multipart/Post.ts new file mode 100644 index 0000000..8f75bd6 --- /dev/null +++ b/src/Frontend/Multipart/Post.ts @@ -0,0 +1,46 @@ +import { Effect } from "effect"; +import { HttpServerRequest } from "@effect/platform"; +import { RequestContext, S3RequestParser } from "../Utils.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; +import { parseCompleteMultipartUploadRequest } from "../../Services/XmlParser.ts"; +import { Backend, InvalidRequest } from "../../Services/Backend.ts"; + +export const initiateMultipartUpload = Effect.gen(function* () { + const backend = yield* Backend; + const request = yield* HttpServerRequest.HttpServerRequest; + const { key } = yield* S3RequestParser; + const { bucket } = yield* RequestContext; + const s3Xml = yield* S3Xml; + + const result = yield* backend.createMultipartUpload(key, request.headers); + return s3Xml.formatInitiateMultipartUpload(bucket, key, result); +}); + +export const completeMultipartUpload = Effect.gen(function* () { + const backend = yield* Backend; + const request = yield* HttpServerRequest.HttpServerRequest; + const { key, s3Params } = yield* S3RequestParser; + const s3Xml = yield* S3Xml; + + // Validate required parameters before calling backend + if (!s3Params.uploadId || typeof s3Params.uploadId !== "string") { + return s3Xml.formatError( + new InvalidRequest({ + message: "Missing or invalid uploadId parameter", + }), + ); + } + + const bodyText = yield* request.text; + const parts = yield* parseCompleteMultipartUploadRequest(bodyText); + + const result = yield* backend.completeMultipartUpload( + key, + s3Params.uploadId, + parts, + {}, // Metadata handled by backend + request.headers, + ); + + return s3Xml.formatCompleteMultipartUpload(result); +}); diff --git a/src/Frontend/Multipart/Put.ts b/src/Frontend/Multipart/Put.ts new file mode 100644 index 0000000..204af67 --- /dev/null +++ b/src/Frontend/Multipart/Put.ts @@ -0,0 +1,57 @@ +import { Effect } from "effect"; +import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; +import { S3RequestParser } from "../Utils.ts"; +import { Backend, InvalidRequest } from "../../Services/Backend.ts"; +import { S3HeaderService } from "../../Services/S3HeaderService.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; + +export const uploadPart = Effect.gen(function* () { + const backend = yield* Backend; + const request = yield* HttpServerRequest.HttpServerRequest; + const { key, s3Params } = yield* S3RequestParser; + const headerService = yield* S3HeaderService; + const s3Xml = yield* S3Xml; + + // Validate required parameters before calling backend + if (!s3Params.uploadId || typeof s3Params.uploadId !== "string") { + return s3Xml.formatError( + new InvalidRequest({ + message: "Missing or invalid uploadId parameter", + }), + ); + } + + if ( + s3Params.partNumber === undefined || + s3Params.partNumber === null || + typeof s3Params.partNumber !== "number" || + !Number.isInteger(s3Params.partNumber) || + s3Params.partNumber < 1 + ) { + return s3Xml.formatError( + new InvalidRequest({ + message: "Missing or invalid partNumber parameter", + }), + ); + } + + // S3 allows 0-byte for the last part; no Frontend rejection here. + // Swift backend rejects 0-byte segments at CompleteMultipartUpload (SLO manifest requirement). + + const result = yield* backend.uploadPart( + key, + s3Params.uploadId, + s3Params.partNumber, + request.stream, + request.headers, + ).pipe( + Effect.catchAll((e) => { + return Effect.fail(e); + }), + ); + + return HttpServerResponse.empty({ + status: 200, + headers: headerService.toResponseHeaders(result), + }); +}); diff --git a/src/Frontend/Objects/Delete.ts b/src/Frontend/Objects/Delete.ts new file mode 100644 index 0000000..379d74b --- /dev/null +++ b/src/Frontend/Objects/Delete.ts @@ -0,0 +1,20 @@ +import { HttpServerResponse } from "@effect/platform"; +import { Effect } from "effect"; +import { Backend } from "../../Services/Backend.ts"; +import { S3RequestParser } from "../Utils.ts"; +import { abortMultipartUpload } from "../Multipart/Delete.ts"; + +/** + * Handler for DeleteObject (DELETE /:bucket/*) + */ +export const deleteObject = Effect.gen(function* () { + const backend = yield* Backend; + const { key, s3Params } = yield* S3RequestParser; + + if (s3Params.uploadId) { + return yield* abortMultipartUpload; + } + + yield* backend.deleteObject(key); + return HttpServerResponse.empty({ status: 204 }); +}); diff --git a/src/Frontend/Objects/Get.ts b/src/Frontend/Objects/Get.ts new file mode 100644 index 0000000..2fd74ed --- /dev/null +++ b/src/Frontend/Objects/Get.ts @@ -0,0 +1,86 @@ +import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; +import { Effect } from "effect"; +import { Backend, InvalidRequest } from "../../Services/Backend.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; +import { S3RequestParser } from "../Utils.ts"; +import { listParts } from "../Multipart/Get.ts"; + +/** + * Handler for GetObjectAttributes (GET /:bucket/*?attributes) + */ +export const getObjectAttributes = () => + Effect.gen(function* () { + const backend = yield* Backend; + const request = yield* HttpServerRequest.HttpServerRequest; + const { key, headers, s3Params } = yield* S3RequestParser; + + // Attributes can come from query parameter ?attributes=... or header x-amz-object-attributes + const attributesFromQuery = s3Params.attributes + ? s3Params.attributes.split(",").map((a) => a.trim()).filter((a) => + a !== "" + ) + : []; + const attributesFromHeader = headers.objectAttributes; + // Deduplicate attributes + const allAttributes = Array.from( + new Set([...attributesFromQuery, ...attributesFromHeader]), + ); + + const s3Xml = yield* S3Xml; + + if (allAttributes.length === 0) { + return s3Xml.formatError( + new InvalidRequest({ + message: "At least one attribute must be specified.", + }), + ); + } + + const result = yield* backend.getObjectAttributes( + key, + allAttributes, + request.headers, + ); + return s3Xml.formatObjectAttributes(result); + }); + +/** + * Handler for GetObject (GET /:bucket/*) + * Also handles ListParts (?uploadId=...). + */ +export const getObject = Effect.gen(function* () { + const backend = yield* Backend; + const { key, s3Params, headers } = yield* S3RequestParser; + const request = yield* HttpServerRequest.HttpServerRequest; + + // Route to getObjectAttributes if attributes are specified in query or header + if ( + s3Params.attributes !== undefined || + (headers.objectAttributes && headers.objectAttributes.length > 0) + ) { + return yield* getObjectAttributes(); + } + + if (s3Params.uploadId) { + return yield* listParts; + } + + const result = yield* backend.getObject(key, request.headers); + const status = (request.headers["range"] || request.headers["Range"]) + ? 206 + : 200; + + if (result.nativeStream) { + return HttpServerResponse.raw(result.nativeStream, { + status, + headers: result.headers, + contentType: result.contentType, + }); + } + + return HttpServerResponse.stream(result.stream, { + status, + headers: result.headers, + contentType: result.contentType, + }); +}); diff --git a/src/Frontend/Objects/Head.ts b/src/Frontend/Objects/Head.ts new file mode 100644 index 0000000..7cfd398 --- /dev/null +++ b/src/Frontend/Objects/Head.ts @@ -0,0 +1,24 @@ +import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; +import { Effect } from "effect"; +import { Backend } from "../../Services/Backend.ts"; +import { S3RequestParser } from "../Utils.ts"; + +/** + * Handler for HeadObject (HEAD /:bucket/*) + */ +export const headObject = Effect.gen(function* () { + const backend = yield* Backend; + const request = yield* HttpServerRequest.HttpServerRequest; + const { key, s3Params } = yield* S3RequestParser; + + const combinedHeaders = { ...request.headers }; + if (s3Params.partNumber) { + combinedHeaders["x-amz-part-number"] = String(s3Params.partNumber); + } + + const result = yield* backend.headObject(key, combinedHeaders); + return HttpServerResponse.empty({ + status: 200, + headers: result.headers, + }); +}); diff --git a/src/Frontend/Objects/List.ts b/src/Frontend/Objects/List.ts new file mode 100644 index 0000000..a48e195 --- /dev/null +++ b/src/Frontend/Objects/List.ts @@ -0,0 +1,43 @@ +import { Effect } from "effect"; +import { Backend } from "../../Services/Backend.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; +import { S3RequestParser } from "../Utils.ts"; +import { listMultipartUploads } from "../Multipart/List.ts"; + +/** + * Handler for ListObjects (GET /:bucket) + */ +export const listObjects = Effect.gen(function* () { + const backend = yield* Backend; + const { s3Params } = yield* S3RequestParser; + const s3Xml = yield* S3Xml; + + if (s3Params.versions !== undefined) { + const result = yield* backend.listVersions({ + prefix: s3Params.prefix, + delimiter: s3Params.delimiter, + keyMarker: s3Params["key-marker"], + versionIdMarker: s3Params["version-id-marker"], + maxKeys: s3Params["max-keys"], + encodingType: s3Params["encoding-type"], + }); + return s3Xml.formatListVersions(result); + } + + if (s3Params.uploads !== undefined) { + return yield* listMultipartUploads; + } + + const result = yield* backend.listObjects({ + prefix: s3Params.prefix, + delimiter: s3Params.delimiter, + marker: s3Params.marker, + maxKeys: s3Params["max-keys"], + encodingType: s3Params["encoding-type"], + continuationToken: s3Params["continuation-token"], + startAfter: s3Params["start-after"], + listType: s3Params["list-type"] === "2" ? 2 : 1, + }); + + return s3Xml.formatListObjects(result); +}); diff --git a/src/Frontend/Objects/Post.ts b/src/Frontend/Objects/Post.ts new file mode 100644 index 0000000..ab5958a --- /dev/null +++ b/src/Frontend/Objects/Post.ts @@ -0,0 +1,216 @@ +import { Effect, Option, Stream } from "effect"; +import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; +import { RequestContext, S3RequestParser } from "../Utils.ts"; +import { S3HeaderService } from "../../Services/S3HeaderService.ts"; +import { parseDeleteObjectsRequest } from "../../Services/XmlParser.ts"; +import { Backend } from "../../Services/Backend.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; +import { HeraldConfig } from "../../Config/Layer.ts"; +import { + completeMultipartUpload, + initiateMultipartUpload, +} from "../Multipart/Post.ts"; +import { parseMultipartFormData } from "../../Services/MultipartForm.ts"; +import { + getSecretForAccessKey, + parsePolicyJson, + validatePolicyConditions, + verifyPolicySignatureV2, +} from "./PostObject.ts"; +import { + AccessDenied, + InvalidRequest, + NoSuchBucket, +} from "../../Services/Backend.ts"; + +/** + * Handler for POST requests on buckets or objects. + * Primarily used for Multi-Object Delete (POST /:bucket?delete). + * Also handles InitiateMultipartUpload (?uploads), CompleteMultipartUpload (?uploadId=...), + * and S3 PostObject (multipart/form-data with policy + signature). + */ +export const postObject = Effect.gen(function* () { + const backend = yield* Backend; + const request = yield* HttpServerRequest.HttpServerRequest; + const { s3Params, key: pathKey } = yield* S3RequestParser; + const { bucket } = yield* RequestContext; + const s3Xml = yield* S3Xml; + + if (s3Params.delete !== undefined) { + // Multi-Object Delete + const bodyText = yield* request.text; + const objects = yield* parseDeleteObjectsRequest(bodyText); + + if (objects.length > 0) { + const deleteResult = yield* backend.deleteObjects(objects); + return s3Xml.formatDeleteObjects(deleteResult); + } + // If no keys, still return empty result + return HttpServerResponse.text( + ``, + { headers: { "Content-Type": "application/xml" } }, + ); + } + + if (s3Params.uploads !== undefined) { + return yield* initiateMultipartUpload; + } + + if (s3Params.uploadId) { + return yield* completeMultipartUpload; + } + + // PostObject: multipart/form-data with policy + signature + const contentType = request.headers["content-type"] ?? + request.headers["Content-Type"]; + const contentTypeStr = Array.isArray(contentType) + ? contentType[0] + : contentType; + if ( + typeof contentTypeStr === "string" && + contentTypeStr.toLowerCase().startsWith("multipart/form-data") + ) { + const bodyText = yield* request.text; + const parsed = yield* parseMultipartFormData(bodyText, contentTypeStr).pipe( + Effect.catchAll((e) => Effect.fail(e)), + ); + const { fields, filePart } = parsed; + // S3 PostObject allows case-insensitive condition field names (e.g. pOLICy) + const field = (name: string) => { + const lower = name.toLowerCase(); + const key = Object.keys(fields).find((k) => k.toLowerCase() === lower); + return key ? fields[key] : undefined; + }; + const policyB64 = field("policy"); + const signatureVal = field("signature") ?? fields["x-amz-signature"]; + const hasSignature = !!signatureVal; + if (policyB64 && hasSignature) { + const keyFromForm = field("key") ?? pathKey; + if (!keyFromForm || keyFromForm.trim() === "") { + return yield* Effect.fail( + new InvalidRequest({ message: "Missing key in form" }), + ); + } + if (!filePart) { + return yield* Effect.fail( + new InvalidRequest({ message: "Missing file or content part" }), + ); + } + let objectKey = keyFromForm.trim(); + if (objectKey === "${filename}" && filePart.filename) { + objectKey = filePart.filename; + } else if (objectKey === "${filename}") { + return yield* Effect.fail( + new InvalidRequest({ + message: "Missing filename for ${filename} key", + }), + ); + } + const config = yield* HeraldConfig; + const materializedOpt = config.lookupBucket(bucket); + if (Option.isNone(materializedOpt)) { + return yield* Effect.fail( + new NoSuchBucket({ + bucket, + message: "The specified bucket does not exist", + }), + ); + } + const materialized = materializedOpt.value; + const accessKeyId = field("AWSAccessKeyId") ?? + fields["x-amz-credential"]?.split("/")[0]; + if (!accessKeyId) { + return yield* Effect.fail( + new AccessDenied({ message: "Access Denied" }), + ); + } + const signature = signatureVal; + if (!signature) { + return yield* Effect.fail( + new AccessDenied({ message: "Access Denied" }), + ); + } + const policy = yield* parsePolicyJson(policyB64); + // Normalize form keys to lowercase for condition matching (S3 allows case-insensitive field names) + const fieldsNorm: Record = {}; + for (const [k, v] of Object.entries(fields)) { + fieldsNorm[k.toLowerCase()] = v; + } + // When key is ${filename}, validate policy against the resolved key so starts-with "foo" matches "foo.txt" + const fieldsForValidation = objectKey !== keyFromForm.trim() + ? { ...fieldsNorm, key: objectKey } + : fieldsNorm; + yield* validatePolicyConditions( + policy, + bucket, + fieldsForValidation, + filePart.body.length, + ); + // Resolve secret: use proxy auth (resolveAuth) first so Swift and multi-user configs work + let secretOpt = Option.none(); + const authCreds = config.resolveAuth(bucket); + if (Option.isSome(authCreds)) { + const cred = authCreds.value.find( + (c) => c.accessKeyId === accessKeyId, + ); + if (cred?.secretAccessKey) { + secretOpt = Option.some(cred.secretAccessKey); + } + } + if (Option.isNone(secretOpt)) { + secretOpt = getSecretForAccessKey( + materialized.credentials, + accessKeyId, + ); + } + if (Option.isNone(secretOpt)) { + return yield* Effect.fail( + new AccessDenied({ message: "Access Denied" }), + ); + } + yield* verifyPolicySignatureV2( + policyB64, + signature, + secretOpt.value, + ); + const headerService = yield* S3HeaderService; + const putHeaders = headerService.formFieldsToPutHeaders( + fieldsNorm, + filePart.body.length, + ); + const result = yield* backend.putObject( + objectKey, + Stream.fromIterable([filePart.body]), + putHeaders, + ); + const successActionStatus = field("success_action_status"); + if (successActionStatus === "201") { + const baseUrl = request.url.startsWith("http") + ? new URL(request.url).origin + : `http://${request.headers["host"] ?? "localhost"}`; + const location = `${baseUrl}/${bucket}/${ + encodeURIComponent(objectKey) + }`; + return s3Xml.formatPostResponse({ + location, + bucket, + key: objectKey, + etag: result.etag ?? "", + }); + } + if (successActionStatus === "200") { + return HttpServerResponse.empty({ status: 200 }); + } + return HttpServerResponse.empty({ status: 204 }); + } + if (policyB64 && !hasSignature) { + return yield* Effect.fail( + new InvalidRequest({ message: "Missing signature in form" }), + ); + } + } + + return yield* Effect.fail( + new Error(`Method POST not implemented for this request`), + ); +}); diff --git a/src/Frontend/Objects/PostObject.ts b/src/Frontend/Objects/PostObject.ts new file mode 100644 index 0000000..c2cc504 --- /dev/null +++ b/src/Frontend/Objects/PostObject.ts @@ -0,0 +1,257 @@ +/** + * S3 PostObject: policy document shape and condition matching. + * Policy is base64-encoded JSON with "expiration" and "conditions". + */ + +import { Effect, Option } from "effect"; +import { createHmac } from "node-crypto"; +import { AccessDenied, InvalidRequest } from "../../Services/Backend.ts"; +import type { MaterializedBucket } from "../../Domain/Config.ts"; + +export interface PostObjectPolicy { + readonly expiration: string; + readonly conditions: readonly PostObjectCondition[]; +} + +export type PostObjectCondition = + | { readonly type: "eq"; key: string; value: string } + | { readonly type: "starts-with"; key: string; value: string } + | { readonly type: "content-length-range"; min: number; max: number }; + +export function parsePolicyJson(raw: string): Effect.Effect< + PostObjectPolicy, + InvalidRequest | AccessDenied +> { + return Effect.gen(function* () { + const decoded = yield* Effect.try({ + try: () => atob(raw), + catch: () => + new InvalidRequest({ message: "Policy is not valid base64" }), + }); + const parsed = yield* Effect.try({ + try: () => JSON.parse(decoded) as Record, + catch: () => new InvalidRequest({ message: "Policy is not valid JSON" }), + }); + if (!("expiration" in parsed) || !("conditions" in parsed)) { + return yield* Effect.fail( + new InvalidRequest({ + message: "Policy must contain expiration and conditions", + }), + ); + } + if ( + typeof parsed.expiration !== "string" || + !Array.isArray(parsed.conditions) + ) { + return yield* Effect.fail( + new InvalidRequest({ + message: "Policy must contain expiration and conditions", + }), + ); + } + const conditions: PostObjectCondition[] = []; + for (const c of parsed.conditions) { + if (typeof c === "object" && c !== null && Array.isArray(c)) { + const [op, key, value] = c as [string, string, unknown]; + if ( + op === "starts-with" && typeof key === "string" && + typeof value === "string" + ) { + conditions.push({ type: "starts-with", key, value }); + } else if ( + op === "eq" && typeof key === "string" && typeof value === "string" + ) { + conditions.push({ type: "eq", key, value }); + } else if ( + op === "content-length-range" && Array.isArray(c) && c.length >= 3 + ) { + const min = Number((c as [string, number, number])[1]); + const max = Number((c as [string, number, number])[2]); + if (!Number.isNaN(min) && !Number.isNaN(max)) { + conditions.push({ type: "content-length-range", min, max }); + } + } + } else if (typeof c === "object" && c !== null && !Array.isArray(c)) { + const obj = c as Record; + const keys = Object.keys(obj); + if (keys.length === 1) { + const k = keys[0]; + const v = obj[k]; + if (typeof v === "string") { + conditions.push({ type: "eq", key: k, value: v }); + } + } + } + } + return { + expiration: parsed.expiration as string, + conditions, + }; + }); +} + +function getSecretForAccessKey( + credentials: MaterializedBucket["credentials"], + accessKeyId: string, +): Option.Option { + if (!credentials) return Option.none(); + if ("accessKeyId" in credentials && credentials.accessKeyId === accessKeyId) { + return credentials.secretAccessKey + ? Option.some(credentials.secretAccessKey) + : Option.none(); + } + if ("username" in credentials && credentials.username === accessKeyId) { + return credentials.password + ? Option.some(credentials.password) + : Option.none(); + } + return Option.none(); +} + +export function validatePolicyConditions( + policy: PostObjectPolicy, + bucket: string, + fields: Record, + fileSize: number, +): Effect.Effect { + return Effect.gen(function* () { + const expDate = new Date(policy.expiration); + if (Number.isNaN(expDate.getTime())) { + return yield* Effect.fail( + new AccessDenied({ message: "Invalid policy expiration date" }), + ); + } + if (expDate.getTime() <= Date.now()) { + return yield* Effect.fail( + new AccessDenied({ message: "Policy has expired" }), + ); + } + + // Case-insensitive form field lookup (caller may pass normalized lowercase keys) + const getField = (key: string) => + fields[key.toLowerCase()] ?? fields[key] ?? ""; + + for (const c of policy.conditions) { + if (c.type === "eq") { + const keyLower = c.key.toLowerCase(); + if (keyLower === "bucket") { + if (c.value !== bucket) { + return yield* Effect.fail( + new AccessDenied({ + message: "Policy bucket does not match request", + }), + ); + } + continue; + } + const formKey = c.key.startsWith("$") ? c.key.slice(1) : c.key; + const formValue = getField(formKey); + const expected = c.value; + const actual = formValue ?? ""; + if ( + actual !== expected && actual.toLowerCase() !== expected.toLowerCase() + ) { + return yield* Effect.fail( + new AccessDenied({ + message: `Policy condition failed: ${c.key} must be ${expected}`, + }), + ); + } + } else if (c.type === "starts-with") { + const formKey = c.key.startsWith("$") ? c.key.slice(1) : c.key; + const formValue = getField(formKey); + const prefix = c.value; + if (formKey.toLowerCase() === "key") { + if (!formValue.startsWith(prefix)) { + return yield* Effect.fail( + new AccessDenied({ + message: + `Policy condition failed: key must start with ${prefix}`, + }), + ); + } + } else if (formKey.toLowerCase() === "content-type") { + if (!formValue.toLowerCase().startsWith(prefix.toLowerCase())) { + return yield* Effect.fail( + new AccessDenied({ + message: + `Policy condition failed: Content-Type must start with ${prefix}`, + }), + ); + } + } + } else if (c.type === "content-length-range") { + if (fileSize < c.min || fileSize > c.max) { + return yield* Effect.fail( + new AccessDenied({ + message: + `Policy condition failed: content-length-range ${c.min}-${c.max}`, + }), + ); + } + } + } + + // Strict policy: every form field must appear in the policy (S3/MinIO/s3-tests) + const allowedKeys = new Set([ + "policy", + "signature", + "awsaccesskeyid", + "x-amz-signature", + "file", + ]); + const allowedPrefixes: string[] = []; + for (const c of policy.conditions) { + if (c.type === "eq" || c.type === "starts-with") { + const formKey = c.key.startsWith("$") ? c.key.slice(1) : c.key; + const formKeyLower = formKey.toLowerCase(); + if (formKeyLower !== "bucket") { + allowedKeys.add(formKeyLower); + if (c.type === "starts-with" && c.value === "") { + allowedPrefixes.push(formKeyLower); + } + } + } + } + for (const key of Object.keys(fields)) { + const keyLower = key.toLowerCase(); + if (allowedKeys.has(keyLower)) continue; + if (allowedPrefixes.some((p) => keyLower.startsWith(p))) continue; + // Allow x-amz-checksum-* without requiring them in the policy (s3-tests, reliability) + if (keyLower.startsWith("x-amz-checksum-")) continue; + return yield* Effect.fail( + new AccessDenied({ + message: + `Each form field that you specify in a form must appear in the list of policy conditions. "${key}" not specified in the policy.`, + }), + ); + } + }); +} + +/** + * Verify PostObject policy signature (HMAC-SHA1 of base64 policy). + * Caller must resolve the secret (e.g. from resolveAuth or backend credentials). + */ +export function verifyPolicySignatureV2( + policyBase64: string, + signatureBase64: string, + secret: string, +): Effect.Effect { + return Effect.gen(function* () { + const expectedSig = yield* Effect.try({ + try: () => + createHmac("sha1", secret) + .update(policyBase64, "utf8") + .digest("base64"), + catch: () => new AccessDenied({ message: "Access Denied" }), + }); + if (expectedSig !== signatureBase64) { + return yield* Effect.fail( + new AccessDenied({ message: "Access Denied" }), + ); + } + }); +} + +export { getSecretForAccessKey }; diff --git a/src/Frontend/Objects/Put.ts b/src/Frontend/Objects/Put.ts new file mode 100644 index 0000000..b0f035e --- /dev/null +++ b/src/Frontend/Objects/Put.ts @@ -0,0 +1,31 @@ +import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; +import { Effect } from "effect"; +import { Backend } from "../../Services/Backend.ts"; +import { S3RequestParser } from "../Utils.ts"; +import { S3HeaderService } from "../../Services/S3HeaderService.ts"; +import { uploadPart } from "../Multipart/Put.ts"; + +/** + * Handler for PutObject (PUT /:bucket/*) + */ +export const putObject = Effect.gen(function* () { + const backend = yield* Backend; + const request = yield* HttpServerRequest.HttpServerRequest; + const { key, s3Params } = yield* S3RequestParser; + const headerService = yield* S3HeaderService; + + if (s3Params.partNumber && s3Params.uploadId) { + return yield* uploadPart; + } + + const result = yield* backend.putObject( + key, + request.stream, + request.headers, + ); + + return HttpServerResponse.empty({ + status: 200, + headers: headerService.toResponseHeaders(result), + }); +}); diff --git a/src/Frontend/Utils.ts b/src/Frontend/Utils.ts new file mode 100644 index 0000000..4513242 --- /dev/null +++ b/src/Frontend/Utils.ts @@ -0,0 +1,111 @@ +import { HttpServerRequest, Url } from "@effect/platform"; +import { Context, Effect, Either, Schema } from "effect"; +import { InternalError } from "../Services/Backend.ts"; +import { S3HeaderService } from "../Services/S3HeaderService.ts"; + +/** + * Context for S3 operations (bucket or object). + */ +export class RequestContext extends Context.Tag("RequestContext")< + RequestContext, + { + readonly bucket: string; + } +>() {} + +export interface S3RequestData { + readonly s3Params: S3QueryParams & Record; + readonly headers: ReturnType< + typeof S3HeaderService.Service.fromRequestHeaders + >; + readonly key: string; +} + +export const S3RequestParser = Effect.gen(function* () { + const { bucket } = yield* RequestContext; + const request = yield* HttpServerRequest.HttpServerRequest; + const urlResult = Url.fromString(request.url, "http://localhost"); + if (Either.isLeft(urlResult)) { + return yield* Effect.fail( + new InternalError({ message: String(urlResult.left) }), + ); + } + const url = urlResult.right; + const headerService = yield* S3HeaderService; + + const paramsRecord: Record = {}; + url.searchParams.forEach((value, key) => { + paramsRecord[key] = value; + }); + + const s3Params = yield* Schema.decodeUnknown(S3QueryParams)(paramsRecord, { + onExcessProperty: "ignore", + }).pipe( + Effect.mapError((e) => { + return new InternalError({ message: String(e) }); + }), + ); + + const parsedHeaders = headerService.fromRequestHeaders(request.headers); + + // url.pathname from a parsed URL object does not include the query string + const pathOnly = url.pathname; + const bucketPrefixWithSlash = `/${bucket}/`; + const bucketPrefixNoSlash = `/${bucket}`; + + let key = ""; + if (pathOnly.startsWith(bucketPrefixWithSlash)) { + key = decodeURIComponent( + pathOnly.substring(bucketPrefixWithSlash.length), + ); + } else if (pathOnly === bucketPrefixNoSlash) { + key = ""; + } + + // Explicitly type the merged s3Params to make the type relationship clear + const mergedS3Params: S3QueryParams & Record = { + ...s3Params, + ...(parsedHeaders.s3Params.uploadId + ? { uploadId: parsedHeaders.s3Params.uploadId } + : {}), + ...(parsedHeaders.s3Params.partNumber + ? { partNumber: parsedHeaders.s3Params.partNumber } + : {}), + ...(parsedHeaders.s3Params.contentLength !== undefined + ? { contentLength: parsedHeaders.s3Params.contentLength } + : {}), + }; + + return { + s3Params: mergedS3Params, + headers: parsedHeaders, + key, + }; +}); + +/** + * Common S3 Query Parameters Schema + */ +export const S3QueryParams = Schema.Struct({ + uploadId: Schema.optional(Schema.String), + partNumber: Schema.optional(Schema.NumberFromString), + prefix: Schema.optional(Schema.String), + delimiter: Schema.optional(Schema.String), + marker: Schema.optional(Schema.String), + "max-keys": Schema.optional(Schema.NumberFromString), + "max-uploads": Schema.optional(Schema.NumberFromString), + "encoding-type": Schema.optional(Schema.String), + "continuation-token": Schema.optional(Schema.String), + "start-after": Schema.optional(Schema.String), + "list-type": Schema.optional(Schema.String), + "version-id-marker": Schema.optional(Schema.String), + "key-marker": Schema.optional(Schema.String), + "upload-id-marker": Schema.optional(Schema.String), + versions: Schema.optional(Schema.String), + uploads: Schema.optional(Schema.String), + delete: Schema.optional(Schema.String), + acl: Schema.optional(Schema.String), + attributes: Schema.optional(Schema.String), +}); + +export type S3QueryParams = Schema.Schema.Type; diff --git a/src/Http.ts b/src/Http.ts new file mode 100644 index 0000000..51b2748 --- /dev/null +++ b/src/Http.ts @@ -0,0 +1,58 @@ +import { + HttpApiBuilder, + HttpApiSwagger, + HttpMiddleware, + HttpServer, +} from "@effect/platform"; +import { NodeHttpServer } from "@effect/platform-node"; +import { Config, Effect, flow, Layer } from "effect"; +import { createServer } from "node-http"; + +export { HttpHeraldApi as HeraldHttpApi } from "./Api.ts"; +export { HttpHealthLive } from "./Frontend/Health/Http.ts"; +export { HttpS3Live } from "./Frontend/Http.ts"; +import { HeraldConfigLive } from "./Config/Layer.ts"; +import { HttpHealthLive } from "./Frontend/Health/Http.ts"; +import { HttpS3Live } from "./Frontend/Http.ts"; +import { HttpHeraldApi } from "./Api.ts"; +import { corsMiddleware } from "./Frontend/Cors.ts"; +import { heraldHttpMetricsMiddleware } from "./Instrumentation.ts"; +import { S3XmlLive } from "./Services/S3Xml.ts"; +import { S3ClientFactory } from "./Backends/S3/Client.ts"; +import { SwiftClient } from "./Backends/Swift/Client.ts"; +import { BackendResolver } from "./Services/BackendResolver.ts"; +import { S3HeaderService } from "./Services/S3HeaderService.ts"; +import { Checksum } from "./Services/Checksum.ts"; + +export const HttpHeraldLive = HttpApiBuilder.api(HttpHeraldApi).pipe( + Layer.provide(HttpHealthLive), + Layer.provide(HttpS3Live), +); + +export const HttpServerHeraldLive = Layer.unwrapEffect( + Effect.gen(function* () { + const port = yield* Config.withDefault( + Config.integer("PORT"), + 3000, + ); + const middleware = flow( + corsMiddleware, + heraldHttpMetricsMiddleware, + HttpMiddleware.logger, + ); + return HttpApiBuilder.serve(middleware).pipe( + Layer.provide(HttpApiSwagger.layer()), + Layer.provide(HttpApiBuilder.middlewareOpenApi()), + Layer.provide(HttpHeraldLive), + Layer.provide(S3XmlLive), + Layer.provide(BackendResolver.Default), + Layer.provide(S3ClientFactory.Default), + Layer.provide(SwiftClient.Default), + Layer.provide(S3HeaderService.Default), + Layer.provide(Checksum.Default), + HttpServer.withLogAddress, + Layer.provide(NodeHttpServer.layer(createServer, { port })), + Layer.provide(HeraldConfigLive), + ); + }), +); diff --git a/src/Instrumentation.ts b/src/Instrumentation.ts new file mode 100644 index 0000000..15a3c0c --- /dev/null +++ b/src/Instrumentation.ts @@ -0,0 +1,49 @@ +import { HttpMiddleware, HttpServerRequest } from "@effect/platform"; +import { Duration, Effect, Metric, pipe } from "effect"; + +/** HTTP request count by method and status class. Low cardinality. */ +export const heraldHttpRequestsTotal = Metric.counter( + "herald_http_requests_total", + { description: "Total HTTP requests" }, +); + +/** HTTP request duration in milliseconds. */ +export const heraldHttpRequestDurationMs = Metric.timer( + "herald_http_request_duration_ms", + "Request duration in milliseconds", +); + +function statusClass(status: number): string { + if (status >= 100 && status < 200) return "1xx"; + if (status >= 200 && status < 300) return "2xx"; + if (status >= 300 && status < 400) return "3xx"; + if (status >= 400 && status < 500) return "4xx"; + if (status >= 500) return "5xx"; + return "unknown"; +} + +/** + * Middleware that records herald_http_requests_total (method, status_class) and + * herald_http_request_duration_ms for every request. Single place, DRY. + */ +export const heraldHttpMetricsMiddleware = HttpMiddleware.make((app) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const start = Date.now(); + const response = yield* app; + const status = response.status ?? 0; + const statusClassTag = statusClass(status); + const method = request.method ?? "UNKNOWN"; + const taggedCounter = pipe( + heraldHttpRequestsTotal, + Metric.tagged("method", method), + Metric.tagged("status_class", statusClassTag), + ); + yield* Metric.increment(taggedCounter); + yield* Metric.update( + heraldHttpRequestDurationMs, + Duration.millis(Date.now() - start), + ); + return response; + }) +); diff --git a/src/Logging/Layer.ts b/src/Logging/Layer.ts new file mode 100644 index 0000000..b253f15 --- /dev/null +++ b/src/Logging/Layer.ts @@ -0,0 +1,48 @@ +import { Config, Effect, Layer, Logger, LogLevel, Option } from "effect"; + +export const LoggingLive = Layer.mergeAll( + Layer.unwrapEffect( + Effect.gen(function* () { + const logLevelStr = yield* Config.option( + Config.string("HERALD_LOG_LEVEL"), + ); + + if (Option.isNone(logLevelStr)) { + return Logger.minimumLogLevel(LogLevel.Info); + } + + const level = logLevelStr.value.toUpperCase(); + switch (level) { + case "ALL": + return Logger.minimumLogLevel(LogLevel.All); + case "TRACE": + return Logger.minimumLogLevel(LogLevel.Trace); + case "DEBUG": + return Logger.minimumLogLevel(LogLevel.Debug); + case "INFO": + return Logger.minimumLogLevel(LogLevel.Info); + case "WARN": + return Logger.minimumLogLevel(LogLevel.Warning); + case "ERROR": + return Logger.minimumLogLevel(LogLevel.Error); + case "FATAL": + return Logger.minimumLogLevel(LogLevel.Fatal); + case "NONE": + return Logger.minimumLogLevel(LogLevel.None); + default: + return Logger.minimumLogLevel(LogLevel.Info); + } + }), + ), +); + +/** Annotation key names used consistently across logs and spans. */ +export const HERALD_KEYS = { + algorithm: "herald_algorithm", + bucket: "herald_bucket", + key: "herald_key", + uploadId: "herald_uploadId", + error: "herald_error", + method: "herald_method", + operation: "herald_operation", +} as const; diff --git a/src/Services/Auth.ts b/src/Services/Auth.ts new file mode 100644 index 0000000..6c1d9a1 --- /dev/null +++ b/src/Services/Auth.ts @@ -0,0 +1,300 @@ +import { Effect, Either, Schema } from "effect"; +import { SignatureV4 } from "@smithy/signature-v4"; +import { Sha256 } from "@aws-crypto/sha256"; +// deno-lint-ignore no-external-import +import { timingSafeEqual } from "node:crypto"; +import type { HttpRequest } from "@smithy/types"; +import type { HttpServerRequest } from "@effect/platform"; + +export const AuthCredentials = Schema.Struct({ + accessKeyId: Schema.String, + secretAccessKey: Schema.String, +}); + +export type AuthCredentials = Schema.Schema.Type; + +export class AuthError extends Schema.TaggedError()("AuthError", { + message: Schema.String, +}) {} + +/** + * Resolves authentication credentials from environment variables based on refs. + */ +export function resolveAuthCredentials( + refs: readonly string[], + env: Record, +): AuthCredentials[] { + const credentials: AuthCredentials[] = []; + for (const ref of refs) { + const accessKeyId = env[`HERALD_AUTH_${ref.toUpperCase()}_ACCESS_KEY_ID`]; + const secretAccessKey = env[`HERALD_AUTH_${ref.toUpperCase()}_SECRET_KEY`]; + if (accessKeyId && secretAccessKey) { + credentials.push({ accessKeyId, secretAccessKey }); + } + } + return credentials; +} + +/** + * Verifies a SigV4 signature for an incoming request. + */ +export function verifyIncomingSigV4( + request: HttpServerRequest.HttpServerRequest, + credentials: AuthCredentials[], + region: string, +): Effect.Effect { + return Effect.gen(function* () { + if (credentials.length === 0) { + return false; + } + + const headers: Record = {}; + for (const [k, v] of Object.entries(request.headers)) { + if (typeof v === "string") { + headers[k.toLowerCase()] = v; + } + } + + const host = headers["host"] || "localhost"; + const protocol = request.url.startsWith("https") ? "https:" : "http:"; + const url = new URL(request.url, `${protocol}//${host}`); + const queryParams = url.searchParams; + const hasSigInQuery = queryParams.has("X-Amz-Signature"); + + const authHeader = headers["authorization"]; + if (!authHeader && !hasSigInQuery) { + return false; + } + + let requestAccessKeyId: string | undefined; + let signedHeadersList: string[] = []; + let headerRegion: string | undefined; + + if (authHeader?.startsWith("AWS4-HMAC-SHA256")) { + const match = authHeader.match(/Credential=([^, ]+)/); + if (match && match[1]) { + const parts = match[1].split("/"); + requestAccessKeyId = parts[0]; + if (parts.length >= 4) { + headerRegion = parts[2]; + } + } + + const headersMatch = authHeader.match(/SignedHeaders=([^, ]+)/); + if (headersMatch && headersMatch[1]) { + signedHeadersList = headersMatch[1].split(";"); + } + } else if (hasSigInQuery) { + const credential = queryParams.get("X-Amz-Credential"); + if (credential && typeof credential === "string") { + const parts = credential.split("/"); + requestAccessKeyId = parts[0]; + if (parts.length >= 4) { + headerRegion = parts[2]; + } + } + + const signedHeaders = queryParams.get("X-Amz-SignedHeaders"); + if (signedHeaders && typeof signedHeaders === "string") { + signedHeadersList = signedHeaders.split(";"); + } + } + + if (!requestAccessKeyId) { + return false; + } + + // Use region from header if available, otherwise use provided region + const effectiveRegion = headerRegion ?? region; + + const matchingCreds = credentials.filter( + (c) => c.accessKeyId === requestAccessKeyId, + ); + if (matchingCreds.length === 0) { + return false; + } + + // Filter headers to only those that were signed + const filteredHeaders: Record = {}; + for (const h of signedHeadersList) { + const val = headers[h]; + if (val !== undefined) { + filteredHeaders[h] = val; + } + } + + const encoder = new TextEncoder(); + + for (const cred of matchingCreds) { + const signer = new SignatureV4({ + credentials: { + accessKeyId: cred.accessKeyId, + secretAccessKey: cred.secretAccessKey, + }, + region: effectiveRegion, + service: "s3", + sha256: Sha256, + uriEscapePath: false, // Path is already encoded in rawPath + }); + + // Extract signing date from request if possible + const amzDate = headers["x-amz-date"]; + const dateHeader = headers["date"]; + let signingDate: Date | undefined; + + if (amzDate) { + // format: YYYYMMDDTHHMMSSZ (minimum 15 characters needed for extraction) + if (amzDate.length >= 15) { + const year = amzDate.substring(0, 4); + const month = amzDate.substring(4, 6); + const day = amzDate.substring(6, 8); + const hour = amzDate.substring(9, 11); + const min = amzDate.substring(11, 13); + const sec = amzDate.substring(13, 15); + signingDate = new Date( + `${year}-${month}-${day}T${hour}:${min}:${sec}Z`, + ); + } + } else if (dateHeader) { + signingDate = new Date(dateHeader); + } else if (hasSigInQuery) { + const amzDateQuery = queryParams.get("X-Amz-Date"); + if ( + amzDateQuery && typeof amzDateQuery === "string" && + amzDateQuery.length >= 15 + ) { + const year = amzDateQuery.substring(0, 4); + const month = amzDateQuery.substring(4, 6); + const day = amzDateQuery.substring(6, 8); + const hour = amzDateQuery.substring(9, 11); + const min = amzDateQuery.substring(11, 13); + const sec = amzDateQuery.substring(13, 15); + signingDate = new Date( + `${year}-${month}-${day}T${hour}:${min}:${sec}Z`, + ); + } + } + + if (signingDate && isNaN(signingDate.getTime())) { + signingDate = undefined; + } + + // Validate signingDate: reject if missing or outside allowed windows + if (!signingDate) { + return false; + } + + const now = new Date(); + const timeDiffMs = Math.abs(now.getTime() - signingDate.getTime()); + const timeDiffMinutes = timeDiffMs / (1000 * 60); + + if (hasSigInQuery) { + // For query-presigned requests: validate X-Amz-Expires + const expiresParam = queryParams.get("X-Amz-Expires"); + if (!expiresParam) { + return false; + } + + // Type-check X-Amz-Expires: must be a valid integer + const expires = parseInt(expiresParam, 10); + if (isNaN(expires) || expiresParam !== String(expires) || expires < 0) { + return false; + } + + // Reject if expired: now > signingDate + expires + const expirationTime = new Date(signingDate.getTime() + expires * 1000); + if (now > expirationTime) { + return false; + } + } else { + // For header-signed requests: enforce ±15 minutes clock skew + if (timeDiffMinutes > 15) { + return false; + } + } + + // Convert query params to smithy format (Record) + const queryBag: Record = {}; + queryParams.forEach((v, k) => { + const existing = queryBag[k]; + if (existing !== undefined) { + if (Array.isArray(existing)) { + existing.push(v); + } else { + queryBag[k] = [existing, v]; + } + } else { + queryBag[k] = v; + } + }); + + // Use raw path from request.url to avoid URL constructor decoding + // We want the part between the host and the query string, as-is. + const urlString = request.url; + const queryIndex = urlString.indexOf("?"); + const withoutQuery = queryIndex === -1 + ? urlString + : urlString.substring(0, queryIndex); + + // Remove protocol and host if present + const rawPath = withoutQuery.replace(/^[a-z]+:\/\/[^/]+/, ""); + + const signableReq: HttpRequest = { + method: request.method, + protocol: url.protocol, + hostname: url.hostname, + port: url.port ? parseInt(url.port) : undefined, + path: rawPath, + query: queryBag, + headers: filteredHeaders, + }; + + const signedResult = yield* Effect.tryPromise({ + try: async () => { + return await signer.sign(signableReq, { + signingDate, + }); + }, + catch: (e) => e, + }).pipe(Effect.either); + + if (Either.isLeft(signedResult)) { + continue; + } + const signed = signedResult.right; + + if (authHeader) { + const expectedAuth = signed.headers["authorization"]; + if ( + !expectedAuth || typeof expectedAuth !== "string" || + authHeader.length !== expectedAuth.length + ) { + continue; + } + const isValid = timingSafeEqual( + encoder.encode(authHeader), + encoder.encode(expectedAuth), + ); + if (isValid) return true; + } else { + const expectedSig = (signed.query as Record)[ + "X-Amz-Signature" + ]; + const actualSig = queryParams.get("X-Amz-Signature"); + if ( + !actualSig || !expectedSig || typeof expectedSig !== "string" || + actualSig.length !== expectedSig.length + ) { + continue; + } + const isValid = timingSafeEqual( + encoder.encode(actualSig), + encoder.encode(expectedSig), + ); + if (isValid) return true; + } + } + + return false; + }); +} diff --git a/src/Services/Backend.ts b/src/Services/Backend.ts new file mode 100644 index 0000000..00214c6 --- /dev/null +++ b/src/Services/Backend.ts @@ -0,0 +1,396 @@ +import type { HttpClientError } from "@effect/platform"; +import { Context, Data } from "effect"; +import type { Effect, Stream } from "effect"; + +export class NoSuchBucket extends Data.TaggedError("NoSuchBucket")<{ + readonly bucket: string; + readonly message: string; +}> {} + +export class NoSuchKey extends Data.TaggedError("NoSuchKey")<{ + readonly bucket: string; + readonly key: string; + readonly message: string; +}> {} + +export class BucketAlreadyExists + extends Data.TaggedError("BucketAlreadyExists")<{ + readonly bucket: string; + readonly message: string; + }> {} + +export class BucketAlreadyOwnedByYou extends Data.TaggedError( + "BucketAlreadyOwnedByYou", +)<{ + readonly bucket: string; + readonly message: string; +}> {} + +export class BucketNotEmpty extends Data.TaggedError("BucketNotEmpty")<{ + readonly bucket: string; + readonly message: string; +}> {} + +export class InternalError extends Data.TaggedError("InternalError")<{ + readonly message: string; +}> {} + +export class AccessDenied extends Data.TaggedError("AccessDenied")<{ + readonly message: string; +}> {} + +export class BadGateway extends Data.TaggedError("BadGateway")<{ + readonly message: string; +}> {} + +export class NoSuchUpload extends Data.TaggedError("NoSuchUpload")<{ + readonly uploadId: string; + readonly message: string; +}> {} + +export class InvalidPart extends Data.TaggedError("InvalidPart")<{ + readonly message: string; +}> {} + +export class InvalidPartOrder extends Data.TaggedError("InvalidPartOrder")<{ + readonly message: string; +}> {} + +export class EntityTooSmall extends Data.TaggedError("EntityTooSmall")<{ + readonly message: string; +}> {} + +export class InvalidRequest extends Data.TaggedError("InvalidRequest")<{ + readonly message: string; +}> {} + +export class BadDigest extends Data.TaggedError("BadDigest")<{ + readonly message: string; +}> {} + +export class InvalidBucketName extends Data.TaggedError("InvalidBucketName")<{ + readonly message: string; +}> {} + +export class InvalidArgument extends Data.TaggedError("InvalidArgument")<{ + readonly message: string; +}> {} + +export class MalformedXML extends Data.TaggedError("MalformedXML")<{ + readonly message: string; +}> {} + +export class MethodNotAllowed extends Data.TaggedError("MethodNotAllowed")<{ + readonly message: string; +}> {} + +export class DeleteObjectsError extends Data.TaggedError("DeleteObjectsError")<{ + readonly errors: readonly { + readonly key: string; + readonly code: string; + readonly message: string; + }[]; +}> {} + +export type BackendError = + | NoSuchBucket + | NoSuchKey + | BucketAlreadyExists + | BucketAlreadyOwnedByYou + | BucketNotEmpty + | InternalError + | AccessDenied + | BadGateway + | NoSuchUpload + | InvalidPart + | InvalidPartOrder + | EntityTooSmall + | InvalidRequest + | BadDigest + | InvalidBucketName + | InvalidArgument + | MalformedXML + | MethodNotAllowed + | HttpClientError.HttpClientError + | DeleteObjectsError; + +export interface BucketInfo { + readonly name: string; + readonly creationDate: Date; +} + +export interface OwnerInfo { + readonly id: string; + readonly displayName: string; +} + +export interface ListBucketsResult { + readonly buckets: readonly BucketInfo[]; + readonly owner: OwnerInfo; +} + +export interface ObjectInfo { + readonly key: string; + readonly lastModified: Date; + readonly etag: string; + readonly size: number; + readonly storageClass?: string; + readonly owner?: OwnerInfo; + readonly versionId?: string; + readonly isLatest?: boolean; + readonly isDeleteMarker?: boolean; +} + +export interface CommonPrefix { + readonly prefix: string; +} + +export interface ListObjectsResult { + readonly name: string; + readonly prefix?: string; + readonly marker?: string; + readonly nextMarker?: string; + readonly maxKeys: number; + readonly delimiter?: string; + readonly isTruncated: boolean; + readonly contents: readonly ObjectInfo[]; + readonly commonPrefixes: readonly CommonPrefix[]; + readonly encodingType?: string; + readonly listType: 1 | 2; + readonly continuationToken?: string; + readonly nextContinuationToken?: string; + readonly keyCount?: number; + readonly startAfter?: string; +} + +export interface ChecksumInfo { + readonly checksumAlgorithm?: string; + readonly checksumCRC32?: string; + readonly checksumCRC32C?: string; + readonly checksumCRC64NVME?: string; + readonly checksumSHA1?: string; + readonly checksumSHA256?: string; + readonly checksumType?: string; +} + +export interface ObjectResponse extends ChecksumInfo { + readonly stream: Stream.Stream; + readonly nativeStream?: ReadableStream; + readonly contentType?: string; + readonly contentLength?: number; + readonly etag?: string; + readonly lastModified?: Date; + readonly metadata: Record; + readonly headers: Record; + readonly partsCount?: number; +} + +export interface HeadObjectResult extends ChecksumInfo { + readonly contentType?: string; + readonly contentLength?: number; + readonly etag?: string; + readonly lastModified?: Date; + readonly metadata: Record; + readonly headers: Record; + readonly partsCount?: number; +} + +export interface PutObjectResult extends ChecksumInfo { + readonly etag?: string; + readonly versionId?: string; +} + +export interface MultipartUploadResult extends ChecksumInfo { + readonly uploadId: string; +} + +export interface UploadPartResult extends ChecksumInfo { + readonly etag: string; +} + +export interface CompleteMultipartUploadResult extends ChecksumInfo { + readonly location: string; + readonly bucket: string; + readonly key: string; + readonly etag: string; + readonly versionId?: string; +} + +export interface ObjectAttributes { + readonly etag?: string; + readonly checksum?: ChecksumInfo; + readonly objectParts?: { + readonly totalPartsCount?: number; + readonly partNumberMarker?: number; + readonly nextPartNumberMarker?: number; + readonly maxParts?: number; + readonly isTruncated?: boolean; + readonly parts?: readonly PartInfo[]; + }; + readonly objectSize?: number; + readonly storageClass?: string; +} + +export interface PartInfo extends ChecksumInfo { + readonly partNumber: number; + readonly lastModified?: Date; + readonly etag: string; + readonly size: number; +} + +export interface DeleteObjectsResult { + readonly deleted: readonly string[]; + readonly errors: readonly { + readonly key: string; + readonly code: string; + readonly message: string; + }[]; +} + +export interface MultipartUploadInfo { + readonly key: string; + readonly uploadId: string; + readonly owner: OwnerInfo; + readonly initiator: OwnerInfo; + readonly storageClass: string; + readonly initiated: Date; +} + +export interface ListMultipartUploadsResult { + readonly bucket: string; + readonly keyMarker?: string; + readonly uploadIdMarker?: string; + readonly nextKeyMarker?: string; + readonly nextUploadIdMarker?: string; + readonly maxUploads: number; + readonly isTruncated: boolean; + readonly uploads: readonly MultipartUploadInfo[]; + readonly commonPrefixes: readonly CommonPrefix[]; + readonly prefix?: string; + readonly delimiter?: string; + readonly encodingType?: string; +} + +export interface ListPartsResult { + readonly bucket: string; + readonly key: string; + readonly uploadId: string; + readonly partNumberMarker: number; + readonly nextPartNumberMarker: number; + readonly maxParts: number; + readonly isTruncated: boolean; + readonly parts: readonly PartInfo[]; + readonly initiator: OwnerInfo; + readonly owner: OwnerInfo; + readonly storageClass: string; +} + +export class Backend extends Context.Tag("Backend")< + Backend, + { + listBuckets: () => Effect.Effect; + createBucket: ( + name: string, + headers: Record, + ) => Effect.Effect; + deleteBucket: (name: string) => Effect.Effect; + headBucket: (name: string) => Effect.Effect; + + listObjects: (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + encodingType?: string; + continuationToken?: string; + startAfter?: string; + listType?: 1 | 2; + }) => Effect.Effect; + + listVersions: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + versionIdMarker?: string; + maxKeys?: number; + encodingType?: string; + }) => Effect.Effect; + + getObject: ( + key: string, + headers: Record, + ) => Effect.Effect; + + headObject: ( + key: string, + headers: Record, + ) => Effect.Effect; + + putObject: ( + key: string, + stream: Stream.Stream, + headers: Record, + ) => Effect.Effect; + + deleteObject: (key: string) => Effect.Effect; + + deleteObjects: ( + objects: readonly { key: string; versionId?: string }[], + ) => Effect.Effect; + + getObjectAttributes: ( + key: string, + attributes: readonly string[], + headers: Record, + ) => Effect.Effect; + + // Multipart Upload + createMultipartUpload: ( + key: string, + headers: Record, + ) => Effect.Effect; + + uploadPart: ( + key: string, + uploadId: string, + partNumber: number, + body: Stream.Stream, + headers: Record, + ) => Effect.Effect; + + completeMultipartUpload: ( + key: string, + uploadId: string, + parts: readonly { + etag: string; + partNumber: number; + checksumCRC32?: string; + checksumCRC32C?: string; + checksumCRC64NVME?: string; + checksumSHA1?: string; + checksumSHA256?: string; + }[], + metadata: Record, + headers: Record, + ) => Effect.Effect; + + abortMultipartUpload: ( + key: string, + uploadId: string, + ) => Effect.Effect; + + listMultipartUploads: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + uploadIdMarker?: string; + maxUploads?: number; + encodingType?: string; + }) => Effect.Effect; + + listParts: ( + key: string, + uploadId: string, + ) => Effect.Effect; + } +>() {} diff --git a/src/Services/BackendKeyValueStore.ts b/src/Services/BackendKeyValueStore.ts new file mode 100644 index 0000000..e6b4558 --- /dev/null +++ b/src/Services/BackendKeyValueStore.ts @@ -0,0 +1,151 @@ +import { Chunk, Effect, Option, Stream } from "effect"; +import { KeyValueStore } from "@effect/platform"; +import { SystemError } from "@effect/platform/Error"; +import type { + BackendError, + ObjectResponse, + PutObjectResult, +} from "./Backend.ts"; + +const collectChunks = (chunks: Chunk.Chunk) => { + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const all = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + all.set(chunk, offset); + offset += chunk.length; + } + return all; +}; + +/** + * A KeyValueStore that persists its data as objects in a BackendService. + * This is used by backends like Swift that don't natively support S3 multipart metadata + * persistence during the upload lifecycle. + */ +export const makeBackendKeyValueStore = ( + ops: { + getObject: ( + key: string, + headers: Record, + ) => Effect.Effect; + putObject: ( + key: string, + stream: Stream.Stream, + headers: Record, + ) => Effect.Effect; + deleteObject: (key: string) => Effect.Effect; + }, + prefix: string, +): KeyValueStore.KeyValueStore => + KeyValueStore.make({ + get: (key) => { + return ops.getObject(`${prefix}${key}`, {}).pipe( + Effect.flatMap((res) => Stream.runCollect(res.stream)), + Effect.map((chunks: Chunk.Chunk) => { + const all = collectChunks(chunks); + return Option.some(new TextDecoder().decode(all)); + }), + Effect.catchIf( + (e) => (e as { _tag?: string })._tag === "NoSuchKey", + () => Effect.succeed(Option.none()), + ), + Effect.catchAll((e) => + Effect.fail( + new SystemError({ + module: "KeyValueStore", + method: "get", + reason: "Unknown", + syscall: "getObject", + description: String(e), + cause: e, + }), + ) + ), + ); + }, + getUint8Array: (key) => { + return ops.getObject(`${prefix}${key}`, {}).pipe( + Effect.flatMap((res) => Stream.runCollect(res.stream)), + Effect.map((chunks: Chunk.Chunk) => { + const all = collectChunks(chunks); + return Option.some(all); + }), + Effect.catchIf( + (e) => (e as { _tag?: string })._tag === "NoSuchKey", + () => Effect.succeed(Option.none()), + ), + Effect.catchAll((e) => + Effect.fail( + new SystemError({ + module: "KeyValueStore", + method: "getUint8Array", + reason: "Unknown", + syscall: "getObject", + description: String(e), + cause: e, + }), + ) + ), + ); + }, + set: (key, value) => { + const encodedValue = typeof value === "string" + ? new TextEncoder().encode(value) + : value; + return ops.putObject( + `${prefix}${key}`, + Stream.succeed(encodedValue), + { "Content-Type": "application/json" }, + ).pipe( + Effect.asVoid, + Effect.catchAll((e) => + Effect.fail( + new SystemError({ + module: "KeyValueStore", + method: "set", + reason: "Unknown", + syscall: "putObject", + description: String(e), + cause: e, + }), + ) + ), + ); + }, + remove: (key) => + ops.deleteObject(`${prefix}${key}`).pipe( + Effect.catchAll((e) => + Effect.fail( + new SystemError({ + module: "KeyValueStore", + method: "remove", + reason: "Unknown", + syscall: "deleteObject", + description: String(e), + cause: e, + }), + ) + ), + ), + clear: Effect.fail( + new SystemError({ + module: "KeyValueStore", + method: "clear", + reason: "Unknown", + description: "Clear not supported in BackendKeyValueStore", + }), + ), + size: Effect.fail( + new SystemError({ + module: "KeyValueStore", + method: "size", + reason: "Unknown", + description: "Size not supported in BackendKeyValueStore", + }), + ), + }); diff --git a/src/Services/BackendResolver.ts b/src/Services/BackendResolver.ts new file mode 100644 index 0000000..6825564 --- /dev/null +++ b/src/Services/BackendResolver.ts @@ -0,0 +1,72 @@ +import { Cache, Effect, Option } from "effect"; +import { makeS3Backend } from "../Backends/S3/Backend.ts"; +import { makeSwiftBackend } from "../Backends/Swift/Backend.ts"; +import { HeraldConfig } from "../Config/Layer.ts"; +import type { MaterializedBucket } from "../Domain/Config.ts"; + +export class BackendResolver + extends Effect.Service()("BackendResolver", { + effect: Effect.gen(function* () { + const config = yield* HeraldConfig; + + const makeBackend = ( + bucketConfig: MaterializedBucket | { backend_id: string }, + ) => + Effect.gen(function* () { + const protocol = "protocol" in bucketConfig + ? bucketConfig.protocol + : config.raw.backends[bucketConfig.backend_id]?.protocol; + + if (protocol === "s3") { + return yield* makeS3Backend(bucketConfig); + } else if (protocol === "swift") { + return yield* makeSwiftBackend(bucketConfig); + } else { + return yield* Effect.fail( + new Error(`Unsupported protocol: ${protocol}`), + ); + } + }); + + const bucketCache = yield* Cache.make({ + capacity: 100, + timeToLive: "24 hours", + lookup: (bucketName: string) => + Effect.gen(function* () { + const matched = config.lookupBucket(bucketName); + if (Option.isNone(matched)) { + return yield* Effect.fail( + new Error(`No configuration found for bucket: ${bucketName}`), + ); + } + return yield* makeBackend(matched.value); + }), + }); + + const backendCache = yield* Cache.make({ + capacity: 100, + timeToLive: "24 hours", + lookup: (backendId: string) => + Effect.gen(function* () { + const backendConfig = config.raw.backends[backendId]; + if (!backendConfig) { + return yield* Effect.fail( + new Error(`No configuration found for backend: ${backendId}`), + ); + } + return yield* makeBackend({ backend_id: backendId }); + }), + }); + + return { + getLayerForBucket: (bucketName: string) => + Effect.gen(function* () { + return yield* bucketCache.get(bucketName); + }), + getLayerForBackend: (backendId: string) => + Effect.gen(function* () { + return yield* backendCache.get(backendId); + }), + }; + }), + }) {} diff --git a/src/Services/Checksum.ts b/src/Services/Checksum.ts new file mode 100644 index 0000000..954edc8 --- /dev/null +++ b/src/Services/Checksum.ts @@ -0,0 +1,190 @@ +import { Effect, Stream } from "effect"; +import { Buffer } from "node-buffer"; +import { createHash } from "node-crypto"; +import { BadDigest, type InvalidRequest } from "./Backend.ts"; +import type { ChecksumAlgorithm, ChecksumHeaders } from "./S3Schema.ts"; + +/** + * CRC32 implementation for S3 (IEEE 802.3) + */ +const CRC32_TABLE = new Int32Array(256); +for (let i = 0; i < 256; i++) { + let c = i; + for (let j = 0; j < 8; j++) { + c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1); + } + CRC32_TABLE[i] = c; +} + +function crc32(data: Uint8Array, previous = 0) { + let crc = previous ^ -1; + for (let i = 0; i < data.length; i++) { + crc = (crc >>> 8) ^ CRC32_TABLE[(crc ^ data[i]) & 0xFF]; + } + return (crc ^ -1) >>> 0; +} + +/** + * CRC32C (Castagnoli) + */ +const CRC32C_TABLE = new Int32Array(256); +for (let i = 0; i < 256; i++) { + let c = i; + for (let j = 0; j < 8; j++) { + c = (c & 1) ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1); + } + CRC32C_TABLE[i] = c; +} + +function crc32c(data: Uint8Array, previous = 0) { + let crc = previous ^ -1; + for (let i = 0; i < data.length; i++) { + crc = (crc >>> 8) ^ CRC32C_TABLE[(crc ^ data[i]) & 0xFF]; + } + return (crc ^ -1) >>> 0; +} + +export class Checksum extends Effect.Service()("Checksum", { + succeed: { + calculate: ( + stream: Stream.Stream, + algorithm: ChecksumAlgorithm, + ): Effect.Effect => + Effect.gen(function* () { + const algo = algorithm.toUpperCase(); + let sha256: ReturnType | undefined; + let sha1: ReturnType | undefined; + let currentCRC32: number | undefined; + let currentCRC32C: number | undefined; + + yield* Stream.runForEach(stream, (chunk) => + Effect.sync(() => { + if (algo === "SHA256") { + if (!sha256) sha256 = createHash("sha256"); + sha256.update(chunk); + } else if (algo === "SHA1") { + if (!sha1) sha1 = createHash("sha1"); + sha1.update(chunk); + } else if (algo === "CRC32") { + if (currentCRC32 === undefined) currentCRC32 = 0; + currentCRC32 = crc32(chunk, currentCRC32); + } else if (algo === "CRC32C") { + if (currentCRC32C === undefined) currentCRC32C = 0; + currentCRC32C = crc32c(chunk, currentCRC32C); + } + })); + + if (algo === "SHA256") { + if (!sha256) sha256 = createHash("sha256"); + return sha256.digest("base64"); + } + if (algo === "SHA1") { + if (!sha1) sha1 = createHash("sha1"); + return sha1.digest("base64"); + } + if (algo === "CRC32") { + const buf = Buffer.alloc(4); + buf.writeUInt32BE(currentCRC32 ?? 0, 0); + return buf.toString("base64"); + } + if (algo === "CRC32C") { + const buf = Buffer.alloc(4); + buf.writeUInt32BE(currentCRC32C ?? 0, 0); + return buf.toString("base64"); + } + return yield* Effect.fail( + new Error(`Unsupported checksum algorithm: ${algorithm}`), + ); + }), + + validate: ( + stream: Stream.Stream, + expected: ChecksumHeaders, + ): Effect.Effect< + Stream.Stream, + BadDigest | InvalidRequest + > => + Effect.sync(function () { + const algo = expected.algorithm; + if (!algo) return stream; + + const algoUpper = algo.toUpperCase(); + let expectedValue: string | undefined; + switch (algoUpper) { + case "SHA256": + expectedValue = expected.sha256; + break; + case "SHA1": + expectedValue = expected.sha1; + break; + case "CRC32": + expectedValue = expected.crc32; + break; + case "CRC32C": + expectedValue = expected.crc32c; + break; + case "CRC64NVME": + expectedValue = expected.crc64nvme; + break; + default: + return stream; + } + + if (!expectedValue) { + return stream; + } + + let sha256: ReturnType | undefined; + let sha1: ReturnType | undefined; + let currentCRC32: number | undefined; + let currentCRC32C: number | undefined; + + return stream.pipe( + Stream.tap((chunk) => + Effect.sync(() => { + if (algoUpper === "SHA256") { + if (!sha256) sha256 = createHash("sha256"); + sha256.update(chunk); + } else if (algoUpper === "SHA1") { + if (!sha1) sha1 = createHash("sha1"); + sha1.update(chunk); + } else if (algoUpper === "CRC32") { + if (currentCRC32 === undefined) currentCRC32 = 0; + currentCRC32 = crc32(chunk, currentCRC32); + } else if (algoUpper === "CRC32C") { + if (currentCRC32C === undefined) currentCRC32C = 0; + currentCRC32C = crc32c(chunk, currentCRC32C); + } + }) + ), + Stream.onEnd(Effect.gen(function* () { + let calculated = ""; + if (algoUpper === "SHA256") { + if (!sha256) sha256 = createHash("sha256"); + calculated = sha256.digest("base64"); + } else if (algoUpper === "SHA1") { + if (!sha1) sha1 = createHash("sha1"); + calculated = sha1.digest("base64"); + } else if (algoUpper === "CRC32") { + const buf = Buffer.alloc(4); + buf.writeUInt32BE(currentCRC32 ?? 0, 0); + calculated = buf.toString("base64"); + } else if (algoUpper === "CRC32C") { + const buf = Buffer.alloc(4); + buf.writeUInt32BE(currentCRC32C ?? 0, 0); + calculated = buf.toString("base64"); + } + + if (calculated && calculated !== expectedValue) { + yield* Effect.fail( + new BadDigest({ + message: + `Checksum mismatch. Expected ${expectedValue}, calculated ${calculated}`, + }), + ); + } + })), + ); + }), + }, +}) {} diff --git a/src/Services/MultipartForm.ts b/src/Services/MultipartForm.ts new file mode 100644 index 0000000..112dc28 --- /dev/null +++ b/src/Services/MultipartForm.ts @@ -0,0 +1,139 @@ +/** + * Parse multipart/form-data body for S3 PostObject. + * Extracts form fields and the file/content part. + */ + +import { Effect } from "effect"; +import { InvalidRequest } from "./Backend.ts"; + +export interface ParsedFilePart { + readonly name: string; + readonly filename?: string; + readonly contentType?: string; + readonly body: Uint8Array; +} + +export interface ParsedMultipartForm { + readonly fields: Record; + readonly filePart: ParsedFilePart | null; +} + +function getBoundary(contentType: string): string | null { + const match = contentType.match(/boundary\s*=\s*"?([^";\s]+)"?/i); + return match ? match[1].trim() : null; +} + +function parsePartHeaders(headerBlock: string): { + name: string | null; + filename: string | null; + contentType: string | null; +} { + let name: string | null = null; + let filename: string | null = null; + let contentType: string | null = null; + // name= can be quoted (name="key") or unquoted (name=key); support both + const dispositionMatch = headerBlock.match( + /Content-Disposition\s*:\s*form-data\s*;\s*name\s*=\s*(?:"([^"]*)"|'([^']*)'|([^;\r\n\s]+))/i, + ); + if (dispositionMatch) { + name = + (dispositionMatch[1] ?? dispositionMatch[2] ?? dispositionMatch[3] ?? "") + .trim() || null; + const filenameMatch = headerBlock.match( + /filename\s*=\s*(?:"([^"]*)"|'([^']*)'|([^;\r\n\s]+))/i, + ); + if (filenameMatch) { + const f = filenameMatch[1] ?? filenameMatch[2] ?? filenameMatch[3]; + filename = (f ?? "").trim() || null; + } + } + const contentTypeMatch = headerBlock.match(/Content-Type\s*:\s*([^\r\n]+)/i); + if (contentTypeMatch) { + contentType = contentTypeMatch[1].trim(); + } + return { name, filename, contentType }; +} + +/** + * Parse a multipart/form-data body string. + * Requires Content-Type header to extract the boundary. + */ +export function parseMultipartFormData( + body: string, + contentType: string, +): Effect.Effect { + return Effect.try({ + try: () => { + const boundary = getBoundary(contentType); + if (!boundary) { + throw new InvalidRequest({ + message: "Missing or invalid multipart boundary in Content-Type", + }); + } + const fields: Record = {}; + let filePart: ParsedFilePart | null = null; + + // Normalize line endings and split by boundary + const normalized = body.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); + const boundaryLine = `--${boundary}`; + const boundaryEnd = `--${boundary}--`; + const parts = normalized.split(`\n${boundaryLine}`); + + for (let i = 0; i < parts.length; i++) { + let part = parts[i]; + if (i === 0 && part.startsWith(boundaryLine)) { + part = part.slice(boundaryLine.length); + } + if ( + part.endsWith("\n" + boundaryEnd) || + part.endsWith("\n" + boundaryEnd + "\n") + ) { + part = part.slice(0, part.indexOf("\n" + boundaryEnd)); + } + // Strip leading CRLF so header block starts at first line (fixes first part) + part = part.replace(/^[\r\n]+/, ""); + if (!part.trim()) continue; + + const headerEnd = part.indexOf("\n\n"); + if (headerEnd === -1) continue; + let headerBlock = part.slice(0, headerEnd); + const bodyPart = part.slice(headerEnd + 2).replace(/\n?$/, ""); + const bodyBytes = new TextEncoder().encode(bodyPart); + + // Normalize folded headers (RFC 2231) into one line so regex matches + headerBlock = headerBlock.replace(/\r?\n\s+/g, " "); + const { name, filename, contentType: partContentType } = + parsePartHeaders(headerBlock); + if (!name) continue; + + // Python requests sends form fields as name="key"; filename="key" (same value). + // Treat as form field when filename equals name, except for the object body parts. + const isObjectBodyPart = name === "file" || name === "content"; + const isFormFieldDisguisedAsFile = !isObjectBodyPart && + filename !== null && filename === name; + const isFilePart = isObjectBodyPart || + (!isFormFieldDisguisedAsFile && filename !== null); + if (isFilePart) { + filePart = { + name, + filename: filename ?? undefined, + contentType: partContentType ?? undefined, + body: bodyBytes, + }; + } else { + fields[name] = bodyPart; + } + } + + return { fields, filePart }; + }, + catch: (e) => { + if (e instanceof InvalidRequest) return e; + return new InvalidRequest({ + message: e instanceof Error + ? e.message + : "Failed to parse multipart form", + }); + }, + }); +} diff --git a/src/Services/NoopKeyValueStore.ts b/src/Services/NoopKeyValueStore.ts new file mode 100644 index 0000000..2f3ce59 --- /dev/null +++ b/src/Services/NoopKeyValueStore.ts @@ -0,0 +1,12 @@ +import { Effect, Option } from "effect"; +import { KeyValueStore } from "@effect/platform"; + +export const makeNoopKeyValueStore = (): KeyValueStore.KeyValueStore => + KeyValueStore.make({ + get: (_key) => Effect.succeed(Option.none()), + getUint8Array: (_key) => Effect.succeed(Option.none()), + set: (_key, _value) => Effect.void, + remove: (_key) => Effect.void, + clear: Effect.void, + size: Effect.succeed(0), + }); diff --git a/src/Services/S3HeaderService.ts b/src/Services/S3HeaderService.ts new file mode 100644 index 0000000..c8d0bcd --- /dev/null +++ b/src/Services/S3HeaderService.ts @@ -0,0 +1,324 @@ +import { Effect, Schema } from "effect"; +import type { + CompleteMultipartUploadResult, + HeadObjectResult, + ObjectResponse, + PutObjectResult, + UploadPartResult, +} from "./Backend.ts"; +import { ChecksumHeaders } from "./S3Schema.ts"; + +export const normalizeHeaders = ( + raw: Record, +): Record => { + const normalized: Record = {}; + for (const [key, value] of Object.entries(raw)) { + normalized[key.toLowerCase()] = Array.isArray(value) ? value[0] : value; + } + return normalized; +}; + +export class S3HeaderService + extends Effect.Service()("S3HeaderService", { + succeed: { + toResponseHeaders: ( + result: + | PutObjectResult + | ObjectResponse + | HeadObjectResult + | UploadPartResult + | CompleteMultipartUploadResult, + ): Record => { + const headers: Record = {}; + + if ("etag" in result && result.etag) headers["ETag"] = result.etag; + if ("versionId" in result && result.versionId) { + headers["x-amz-version-id"] = result.versionId; + } + if ("lastModified" in result && result.lastModified) { + headers["Last-Modified"] = result.lastModified.toUTCString(); + } + if ("contentLength" in result && result.contentLength !== undefined) { + headers["Content-Length"] = String(result.contentLength); + } + if ("contentType" in result && result.contentType) { + headers["Content-Type"] = result.contentType; + } + + // Metadata + if ("metadata" in result && result.metadata) { + for (const [key, value] of Object.entries(result.metadata)) { + const lowKey = key.toLowerCase(); + // Skip internal checksum metadata to avoid duplication in response + if (lowKey.startsWith("s3-checksum-")) { + continue; + } + const encodedValue = /[^\x20-\x7E]/.test(value) + ? encodeURIComponent(value) + : value; + headers[`x-amz-meta-${lowKey}`] = encodedValue; + } + } + + // Checksums + if (result.checksumAlgorithm) { + headers["x-amz-checksum-algorithm"] = result.checksumAlgorithm + .toUpperCase(); + } + if (result.checksumCRC32) { + headers["x-amz-checksum-crc32"] = result.checksumCRC32; + } + if (result.checksumCRC32C) { + headers["x-amz-checksum-crc32c"] = result.checksumCRC32C; + } + if (result.checksumCRC64NVME) { + headers["x-amz-checksum-crc64nvme"] = result.checksumCRC64NVME; + } + if (result.checksumSHA1) { + headers["x-amz-checksum-sha1"] = result.checksumSHA1; + } + if (result.checksumSHA256) { + headers["x-amz-checksum-sha256"] = result.checksumSHA256; + } + if (result.checksumType) { + headers["x-amz-checksum-type"] = result.checksumType.toUpperCase(); + } + if ("partsCount" in result && result.partsCount !== undefined) { + headers["x-amz-mp-parts-count"] = String(result.partsCount); + } + + return headers; + }, + + fromRequestHeaders: ( + raw: Record, + ): { + readonly checksums: ChecksumHeaders; + readonly metadata: Record; + readonly objectAttributes: string[]; + readonly s3Params: { + readonly partNumber?: number; + readonly uploadId?: string; + readonly versionId?: string; + readonly checksumMode?: string; + readonly contentLength?: number; + }; + } => { + const normalized = normalizeHeaders(raw); + + // Extract Checksums; infer algorithm when only a single checksum header is present (e.g. PostObject x-amz-checksum-sha256) + const explicitAlgo = normalized["x-amz-checksum-algorithm"] ?? + normalized["x-amz-sdk-checksum-algorithm"]; + const inferredAlgo = !explicitAlgo && + (normalized["x-amz-checksum-sha256"] != null + ? "SHA256" + : normalized["x-amz-checksum-sha1"] != null + ? "SHA1" + : normalized["x-amz-checksum-crc32"] != null + ? "CRC32" + : normalized["x-amz-checksum-crc32c"] != null + ? "CRC32C" + : normalized["x-amz-checksum-crc64nvme"] != null + ? "CRC64NVME" + : undefined); + const checksumInput = { + algorithm: explicitAlgo ?? inferredAlgo, + sha256: normalized["x-amz-checksum-sha256"], + sha1: normalized["x-amz-checksum-sha1"], + crc32: normalized["x-amz-checksum-crc32"], + crc32c: normalized["x-amz-checksum-crc32c"], + crc64nvme: normalized["x-amz-checksum-crc64nvme"], + type: normalized["x-amz-checksum-type"], + }; + + const checksums = Schema.decodeUnknownSync(ChecksumHeaders)( + checksumInput, + ); + + // Extract Metadata + const metadata: Record = {}; + for (const [k, v] of Object.entries(normalized)) { + if (k.startsWith("x-amz-meta-") && v !== undefined) { + const metaKey = k.substring("x-amz-meta-".length); + metadata[metaKey] = v.includes("%") ? decodeURIComponent(v) : v; + } + } + + // Extract Object Attributes + const attributesHeader = normalized["x-amz-object-attributes"]; + const objectAttributes = attributesHeader + ? attributesHeader.split(",").map((a) => a.trim()).filter((a) => + a !== "" + ) + : []; + + // Extract S3 Params + const s3Params = { + partNumber: normalized["x-amz-part-number"] + ? parseInt(normalized["x-amz-part-number"]) + : undefined, + uploadId: normalized["x-amz-upload-id"], + versionId: + (normalized["x-amz-version-id"] || normalized["versionid"]) || + undefined, + checksumMode: normalized["x-amz-checksum-mode"], + contentLength: normalized["content-length"] + ? parseInt(normalized["content-length"]) + : undefined, + }; + + return { checksums, metadata, objectAttributes, s3Params }; + }, + + /** + * Reconstructs S3 headers and metadata from raw Swift headers. + * Also handles internal checksum metadata correctly. + */ + fromSwiftHeaders: ( + raw: Record, + ): { + readonly metadata: Record; + readonly s3Headers: Record; + readonly checksums: ChecksumHeaders; + readonly partsCount?: number; + } => { + const normalized = normalizeHeaders(raw); + const metadata: Record = {}; + const s3Headers: Record = {}; + + for (const [k, v] of Object.entries(normalized)) { + if (v === undefined) continue; + + if (k.startsWith("x-object-meta-")) { + const metaKey = k.substring("x-object-meta-".length); + + // CRITICAL: Skip internal checksum metadata when reconstructing generic metadata + if (metaKey.startsWith("s3-checksum-")) { + continue; + } + + const decodedValue = v.includes("%") ? decodeURIComponent(v) : v; + metadata[metaKey] = decodedValue; + s3Headers[`x-amz-meta-${metaKey}`] = decodedValue; + } else if (k === "content-type") { + s3Headers["Content-Type"] = v; + } else if (k === "content-length") { + s3Headers["Content-Length"] = v; + } else if (k === "etag") { + s3Headers["ETag"] = v; + } else if (k === "last-modified") { + s3Headers["Last-Modified"] = v; + } else if (k === "x-static-large-object") { + s3Headers["x-static-large-object"] = v; + } else if (k === "x-amz-mp-parts-count") { + s3Headers["x-amz-mp-parts-count"] = v; + } + } + + const checksumInput = { + algorithm: normalized["x-object-meta-s3-checksum-algorithm"], + sha256: normalized["x-object-meta-s3-checksum-sha256"], + sha1: normalized["x-object-meta-s3-checksum-sha1"], + crc32: normalized["x-object-meta-s3-checksum-crc32"], + crc32c: normalized["x-object-meta-s3-checksum-crc32c"], + crc64nvme: normalized["x-object-meta-s3-checksum-crc64nvme"], + type: normalized["x-object-meta-s3-checksum-type"], + }; + + const checksums = Schema.decodeUnknownSync(ChecksumHeaders)( + checksumInput, + ); + const partsCount = normalized["x-amz-mp-parts-count"] + ? parseInt(normalized["x-amz-mp-parts-count"]) + : undefined; + + return { metadata, s3Headers, checksums, partsCount }; + }, + + /** + * Build putObject request headers from PostObject form fields. + * Returns the same header names that fromRequestHeaders reads, so backend + * logic (checksum validation, etc.) is shared with PUT object. + */ + formFieldsToPutHeaders: ( + fields: Record, + contentLength: number, + ): Record => { + const get = (name: string): string | undefined => { + const lower = name.toLowerCase(); + const key = Object.keys(fields).find((k) => + k.toLowerCase() === lower + ); + return key ? fields[key] : undefined; + }; + const headers: Record = { + "Content-Length": String(contentLength), + }; + const contentType = get("Content-Type") ?? get("content-type"); + if (contentType) headers["Content-Type"] = contentType; + const acl = get("acl"); + if (acl) headers["x-amz-acl"] = acl; + const sha256 = get("x-amz-checksum-sha256"); + if (sha256) headers["x-amz-checksum-sha256"] = sha256; + const sha1 = get("x-amz-checksum-sha1"); + if (sha1) headers["x-amz-checksum-sha1"] = sha1; + const crc32 = get("x-amz-checksum-crc32"); + if (crc32) headers["x-amz-checksum-crc32"] = crc32; + const crc32c = get("x-amz-checksum-crc32c"); + if (crc32c) headers["x-amz-checksum-crc32c"] = crc32c; + const crc64nvme = get("x-amz-checksum-crc64nvme"); + if (crc64nvme) headers["x-amz-checksum-crc64nvme"] = crc64nvme; + for (const [k, v] of Object.entries(fields)) { + if (k.toLowerCase().startsWith("x-amz-meta-") && v !== undefined) { + headers[k] = v; + } + } + return headers; + }, + + /** + * Maps S3 metadata and checksums to Swift headers. + */ + toSwiftHeaders: ( + metadata: Record, + checksums: ChecksumHeaders, + ): Record => { + const swiftHeaders: Record = {}; + + // S3 Metadata -> Swift Metadata + for (const [key, value] of Object.entries(metadata)) { + const encodedValue = /[^\x20-\x7E]/.test(value) + ? encodeURIComponent(value) + : value; + swiftHeaders[`X-Object-Meta-${key}`] = encodedValue; + } + + // S3 Checksums -> Swift Metadata (prefixed for later reconstruction) + if (checksums.algorithm) { + swiftHeaders["X-Object-Meta-S3-Checksum-Algorithm"] = + checksums.algorithm; + } + if (checksums.crc32) { + swiftHeaders["X-Object-Meta-S3-Checksum-CRC32"] = checksums.crc32; + } + if (checksums.crc32c) { + swiftHeaders["X-Object-Meta-S3-Checksum-CRC32C"] = checksums.crc32c; + } + if (checksums.crc64nvme) { + swiftHeaders["X-Object-Meta-S3-Checksum-CRC64NVME"] = + checksums.crc64nvme; + } + if (checksums.sha1) { + swiftHeaders["X-Object-Meta-S3-Checksum-SHA1"] = checksums.sha1; + } + if (checksums.sha256) { + swiftHeaders["X-Object-Meta-S3-Checksum-SHA256"] = checksums.sha256; + } + if (checksums.type) { + swiftHeaders["X-Object-Meta-S3-Checksum-Type"] = checksums.type; + } + + return swiftHeaders; + }, + }, + }) {} diff --git a/src/Services/S3Schema.ts b/src/Services/S3Schema.ts new file mode 100644 index 0000000..6170289 --- /dev/null +++ b/src/Services/S3Schema.ts @@ -0,0 +1,74 @@ +import { Schema } from "effect"; + +/** + * Checksum algorithm enum - parsed, not cast. + */ +export const ChecksumAlgorithm = Schema.Literal( + "SHA256", + "SHA1", + "CRC32", + "CRC32C", + "CRC64NVME", +); +export type ChecksumAlgorithm = Schema.Schema.Type; + +/** + * Checksum type enum. + */ +export const ChecksumType = Schema.Literal("COMPOSITE", "FULL_OBJECT"); +export type ChecksumType = Schema.Schema.Type; + +/** + * Header extraction schema - parses headers into typed structure. + */ +export const ChecksumHeaders = Schema.Struct({ + algorithm: Schema.optional(Schema.transform( + Schema.String, + ChecksumAlgorithm, + { + decode: (s) => s.toUpperCase() as ChecksumAlgorithm, + encode: (s) => s, + }, + )), + sha256: Schema.optional(Schema.String), + sha1: Schema.optional(Schema.String), + crc32: Schema.optional(Schema.String), + crc32c: Schema.optional(Schema.String), + crc64nvme: Schema.optional(Schema.String), + type: Schema.optional(ChecksumType), +}); +export type ChecksumHeaders = Schema.Schema.Type; + +/** + * XML body schema for DeleteObjects. + */ +export const DeleteObjectEntry = Schema.Struct({ + key: Schema.String, + versionId: Schema.optional(Schema.String), +}); +export type DeleteObjectEntry = Schema.Schema.Type; + +/** + * XML body schema for CompleteMultipartUpload part. + */ +export const CompleteMultipartPart = Schema.Struct({ + partNumber: Schema.Number, + etag: Schema.String, + checksumSHA256: Schema.optional(Schema.String), + checksumSHA1: Schema.optional(Schema.String), + checksumCRC32: Schema.optional(Schema.String), + checksumCRC32C: Schema.optional(Schema.String), + checksumCRC64NVME: Schema.optional(Schema.String), +}); +export type CompleteMultipartPart = Schema.Schema.Type< + typeof CompleteMultipartPart +>; + +/** + * Swift Token Response schema. + */ +export const SwiftTokenResponse = Schema.Struct({ + token: Schema.String, + storageUrl: Schema.String, +}); +export type SwiftTokenResponse = Schema.Schema.Type; diff --git a/src/Services/S3Xml.ts b/src/Services/S3Xml.ts new file mode 100644 index 0000000..6e5bff4 --- /dev/null +++ b/src/Services/S3Xml.ts @@ -0,0 +1,543 @@ +import { HttpServerResponse } from "@effect/platform"; +import { Context, Effect, Layer } from "effect"; +import { + AccessDenied, + BadDigest, + BadGateway, + BucketAlreadyExists, + BucketAlreadyOwnedByYou, + type BucketInfo, + BucketNotEmpty, + type CompleteMultipartUploadResult, + DeleteObjectsError, + type DeleteObjectsResult, + EntityTooSmall, + InternalError, + InvalidArgument, + InvalidBucketName, + InvalidPart, + InvalidPartOrder, + InvalidRequest, + type ListMultipartUploadsResult, + type ListObjectsResult, + type ListPartsResult, + MalformedXML, + MethodNotAllowed, + type MultipartUploadResult, + NoSuchBucket, + NoSuchKey, + NoSuchUpload, + type ObjectAttributes, + type OwnerInfo, +} from "./Backend.ts"; + +export class S3Xml extends Context.Tag("S3Xml")< + S3Xml, + { + formatError: ( + err: unknown, + isHead?: boolean, + ) => HttpServerResponse.HttpServerResponse; + formatListBuckets: ( + buckets: readonly BucketInfo[], + owner: OwnerInfo, + ) => HttpServerResponse.HttpServerResponse; + formatListObjects: ( + result: ListObjectsResult, + ) => HttpServerResponse.HttpServerResponse; + formatListVersions: ( + result: ListObjectsResult, + ) => HttpServerResponse.HttpServerResponse; + formatListParts: ( + result: ListPartsResult, + ) => HttpServerResponse.HttpServerResponse; + formatListMultipartUploads: ( + result: ListMultipartUploadsResult, + ) => HttpServerResponse.HttpServerResponse; + formatInitiateMultipartUpload: ( + bucket: string, + key: string, + result: MultipartUploadResult, + ) => HttpServerResponse.HttpServerResponse; + formatCompleteMultipartUpload: ( + result: CompleteMultipartUploadResult, + ) => HttpServerResponse.HttpServerResponse; + formatObjectAttributes: ( + result: ObjectAttributes, + ) => HttpServerResponse.HttpServerResponse; + formatDeleteObjects: ( + result: DeleteObjectsResult, + ) => HttpServerResponse.HttpServerResponse; + formatPostResponse: (args: { + location: string; + bucket: string; + key: string; + etag: string; + }) => HttpServerResponse.HttpServerResponse; + } +>() {} + +export const makeS3Xml = Effect.sync(() => { + const encode = (s: string) => + s.replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); + + return S3Xml.of({ + formatError: (err: unknown, isHead = false) => { + let code = "InternalError"; + let message = "An internal error occurred."; + let status = 500; + + if (err instanceof NoSuchBucket) { + // For HEAD requests, S3 returns NotFound instead of NoSuchBucket + code = isHead ? "NotFound" : "NoSuchBucket"; + message = err.message; + status = 404; + } else if (err instanceof NoSuchKey) { + code = "NoSuchKey"; + message = err.message; + status = 404; + } else if (err instanceof BucketAlreadyExists) { + code = "BucketAlreadyExists"; + message = err.message; + status = 409; + } else if (err instanceof BucketAlreadyOwnedByYou) { + code = "BucketAlreadyOwnedByYou"; + message = err.message; + status = 409; + } else if (err instanceof InternalError) { + code = "InternalError"; + message = err.message; + status = 500; + } else if (err instanceof AccessDenied) { + code = "AccessDenied"; + message = err.message; + status = 403; + } else if (err instanceof BadGateway) { + code = "BadGateway"; + message = err.message; + status = 502; + } else if (err instanceof BucketNotEmpty) { + code = "BucketNotEmpty"; + message = err.message; + status = 409; + } else if (err instanceof NoSuchUpload) { + code = "NoSuchUpload"; + message = err.message; + status = 404; + } else if (err instanceof InvalidPart) { + code = "InvalidPart"; + message = err.message; + status = 400; + } else if (err instanceof InvalidPartOrder) { + code = "InvalidPartOrder"; + message = err.message; + status = 400; + } else if (err instanceof EntityTooSmall) { + code = "EntityTooSmall"; + message = err.message; + status = 400; + } else if (err instanceof InvalidRequest) { + code = "InvalidRequest"; + message = err.message; + status = 400; + } else if (err instanceof BadDigest) { + code = "BadDigest"; + message = err.message; + status = 400; + } else if (err instanceof InvalidBucketName) { + code = "InvalidBucketName"; + message = err.message; + status = 400; + } else if (err instanceof InvalidArgument) { + code = "InvalidArgument"; + message = err.message; + status = 400; + } else if (err instanceof MalformedXML) { + code = "MalformedXML"; + message = err.message; + status = 400; + } else if (err instanceof MethodNotAllowed) { + code = "MethodNotAllowed"; + message = err.message; + status = 405; + } else if (err instanceof DeleteObjectsError) { + // Multi-object delete errors are returned in the body, but the response status is 200 + // Wait, S3 documentation says 200 OK even if some deletes fail. + // But if the request is malformed, it's 400. + // For now, we'll return 200 and format the errors in the body. + status = 200; + } + + if (isHead) { + return HttpServerResponse.empty({ status }); + } + + const xml = + `${code}${message}`; + return HttpServerResponse.text(xml, { + status, + headers: { "Content-Type": "application/xml" }, + }); + }, + + formatListBuckets: (buckets: readonly BucketInfo[], owner: OwnerInfo) => { + const bucketsXml = buckets.map((b) => + `${b.name}${b.creationDate.toISOString()}` + ).join(""); + + const xml = + `${owner.id}${owner.displayName}${bucketsXml}`; + + return HttpServerResponse.text(xml, { + headers: { + "Content-Type": "application/xml", + }, + }); + }, + + formatListObjects: (result: ListObjectsResult) => { + const contentsXml = result.contents.map((c) => + `${ + encode(c.key) + }${c.lastModified.toISOString()}${c.etag}${c.size}${ + c.storageClass || "STANDARD" + }${ + c.owner + ? `${c.owner.id}${c.owner.displayName}` + : "" + }` + ).join(""); + + const commonPrefixesXml = result.commonPrefixes.map((cp) => + `${encode(cp.prefix)}` + ).join(""); + + const isV2 = result.listType === 2; + + const xml = isV2 + ? `${result.name}${ + encode(result.prefix ?? "") + }${ + result.keyCount ?? 0 + }${result.maxKeys}${result.isTruncated}${ + result.continuationToken + ? `${ + encode(result.continuationToken) + }` + : "" + }${ + result.nextContinuationToken + ? `${ + encode(result.nextContinuationToken) + }` + : "" + }${ + result.startAfter + ? `${encode(result.startAfter)}` + : "" + }${contentsXml}${commonPrefixesXml}` + : `${result.name}${ + encode(result.prefix ?? "") + }${ + encode(result.marker ?? "") + }${result.maxKeys}${result.isTruncated}${ + result.nextMarker + ? `${encode(result.nextMarker)}` + : "" + }${contentsXml}${commonPrefixesXml}`; + + return HttpServerResponse.text(xml, { + headers: { "Content-Type": "application/xml" }, + }); + }, + + formatListVersions: (result: ListObjectsResult) => { + const versionsXml = result.contents.map((c) => { + const tag = c.isDeleteMarker ? "DeleteMarker" : "Version"; + return `<${tag}>${encode(c.key)}${ + c.versionId || "null" + }${ + c.isLatest || false + }${c.lastModified.toISOString()}${c.etag}${c.size}${ + c.storageClass || "STANDARD" + }${ + c.owner + ? `${c.owner.id}${c.owner.displayName}` + : "" + }`; + }).join(""); + + const commonPrefixesXml = result.commonPrefixes.map((cp) => + `${encode(cp.prefix)}` + ).join(""); + + const xml = + `${result.name}${ + encode(result.prefix ?? "") + }${ + encode(result.marker ?? "") + }${ + encode(result.continuationToken ?? "") + }${result.maxKeys}${result.isTruncated}${ + result.nextMarker + ? `${encode(result.nextMarker)}` + : "" + }${ + result.nextContinuationToken + ? `${ + encode(result.nextContinuationToken) + }` + : "" + }${versionsXml}${commonPrefixesXml}`; + + return HttpServerResponse.text(xml, { + headers: { "Content-Type": "application/xml" }, + }); + }, + + formatListParts: (result: ListPartsResult) => { + const partsXml = result.parts.map((p) => + `${p.partNumber}${ + p.lastModified !== undefined + ? `${p.lastModified.toISOString()}` + : "" + }${p.etag}${p.size}` + ).join(""); + + const xml = + `${result.bucket}${ + encode(result.key) + }${result.uploadId}${result.initiator.id}${result.initiator.displayName}${result.owner.id}${result.owner.displayName}${result.storageClass}${result.partNumberMarker}${result.nextPartNumberMarker}${result.maxParts}${result.isTruncated}${partsXml}`; + + return HttpServerResponse.text(xml, { + headers: { "Content-Type": "application/xml" }, + }); + }, + + formatListMultipartUploads: ( + result: ListMultipartUploadsResult, + ) => { + const uploadsXml = result.uploads.map((u) => + `${ + encode(u.key) + }${u.uploadId}${u.initiator.id}${u.initiator.displayName}${u.owner.id}${u.owner.displayName}${u.storageClass}${u.initiated.toISOString()}` + ).join(""); + + const commonPrefixesXml = result.commonPrefixes.map((cp) => + `${encode(cp.prefix)}` + ).join(""); + + const xml = + `${result.bucket}${ + encode(result.keyMarker ?? "") + }${ + encode(result.uploadIdMarker ?? "") + }${ + encode(result.nextKeyMarker ?? "") + }${ + encode(result.nextUploadIdMarker ?? "") + }${result.maxUploads}${result.isTruncated}${uploadsXml}${commonPrefixesXml}`; + + return HttpServerResponse.text(xml, { + headers: { "Content-Type": "application/xml" }, + }); + }, + formatInitiateMultipartUpload: ( + bucket: string, + key: string, + result: MultipartUploadResult, + ) => { + const checksumAlgorithmXml = result.checksumAlgorithm + ? `${result.checksumAlgorithm.toUpperCase()}` + : ""; + const checksumTypeXml = result.checksumType + ? `${result.checksumType.toUpperCase()}` + : ""; + const xml = + `${bucket}${ + encode(key) + }${result.uploadId}${checksumAlgorithmXml}${checksumTypeXml}`; + + return HttpServerResponse.text(xml, { + headers: { + "Content-Type": "application/xml", + ...(result.checksumAlgorithm + ? { + "x-amz-checksum-algorithm": result.checksumAlgorithm + .toUpperCase(), + } + : {}), + ...(result.checksumType + ? { "x-amz-checksum-type": result.checksumType.toUpperCase() } + : {}), + }, + }); + }, + formatCompleteMultipartUpload: ( + result: CompleteMultipartUploadResult, + ) => { + const checksumAlgorithmXml = result.checksumAlgorithm + ? `${result.checksumAlgorithm.toUpperCase()}` + : ""; + const checksumTypeXml = result.checksumType + ? `${result.checksumType.toUpperCase()}` + : ""; + const checksumCRC32Xml = result.checksumCRC32 + ? `${result.checksumCRC32}` + : ""; + const checksumCRC32CXml = result.checksumCRC32C + ? `${result.checksumCRC32C}` + : ""; + const checksumCRC64NVMEXml = result.checksumCRC64NVME + ? `${result.checksumCRC64NVME}` + : ""; + const checksumSHA1Xml = result.checksumSHA1 + ? `${result.checksumSHA1}` + : ""; + const checksumSHA256Xml = result.checksumSHA256 + ? `${result.checksumSHA256}` + : ""; + + const xml = + `${result.location}${result.bucket}${ + encode(result.key) + }${result.etag}${checksumAlgorithmXml}${checksumTypeXml}${checksumCRC32Xml}${checksumCRC32CXml}${checksumCRC64NVMEXml}${checksumSHA1Xml}${checksumSHA256Xml}`; + + return HttpServerResponse.text(xml, { + headers: { + "Content-Type": "application/xml", + }, + }); + }, + formatDeleteObjects: (result: DeleteObjectsResult) => { + const deletedXml = result.deleted.map((k) => + `${encode(k)}` + ).join(""); + const errorsXml = result.errors.map((e) => + `${encode(e.key)}${e.code}${ + encode(e.message) + }` + ).join(""); + + const xml = + `${deletedXml}${errorsXml}`; + return HttpServerResponse.text(xml, { + headers: { "Content-Type": "application/xml" }, + }); + }, + + formatPostResponse: (args) => { + const xml = + `${ + encode(args.location) + }${encode(args.bucket)}${ + encode(args.key) + }${encode(args.etag)}`; + return HttpServerResponse.text(xml, { + status: 201, + headers: { "Content-Type": "application/xml" }, + }); + }, + + formatObjectAttributes: (result: ObjectAttributes) => { + const checksumXml = result.checksum + ? `${ + result.checksum.checksumAlgorithm + ? `${result.checksum.checksumAlgorithm.toUpperCase()}` + : "" + }${ + result.checksum.checksumCRC32 + ? `${result.checksum.checksumCRC32}` + : "" + }${ + result.checksum.checksumCRC32C + ? `${result.checksum.checksumCRC32C}` + : "" + }${ + result.checksum.checksumCRC64NVME + ? `${result.checksum.checksumCRC64NVME}` + : "" + }${ + result.checksum.checksumSHA1 + ? `${result.checksum.checksumSHA1}` + : "" + }${ + result.checksum.checksumSHA256 + ? `${result.checksum.checksumSHA256}` + : "" + }` + : ""; + + const objectPartsXml = result.objectParts + ? `${ + result.objectParts.totalPartsCount !== undefined + ? `${result.objectParts.totalPartsCount}` + : "" + }${ + result.objectParts.partNumberMarker !== undefined + ? `${result.objectParts.partNumberMarker}` + : "" + }${ + result.objectParts.nextPartNumberMarker !== undefined + ? `${result.objectParts.nextPartNumberMarker}` + : "" + }${ + result.objectParts.maxParts !== undefined + ? `${result.objectParts.maxParts}` + : "" + }${ + result.objectParts.isTruncated !== undefined + ? `${result.objectParts.isTruncated}` + : "" + }${ + (result.objectParts.parts ?? []).map((p) => + `${p.partNumber}${p.size}${ + p.checksumCRC32 !== undefined + ? `${p.checksumCRC32}` + : "" + }${ + p.checksumCRC32C !== undefined + ? `${p.checksumCRC32C}` + : "" + }${ + p.checksumSHA1 !== undefined + ? `${p.checksumSHA1}` + : "" + }${ + p.checksumSHA256 !== undefined + ? `${p.checksumSHA256}` + : "" + }${ + p.checksumCRC64NVME !== undefined + ? `${p.checksumCRC64NVME}` + : "" + }` + ).join("") + }` + : ""; + + const xml = + `${ + result.etag ? `${result.etag}` : "" + }${checksumXml}${objectPartsXml}${ + result.objectSize + ? `${result.objectSize}` + : "" + }${ + result.storageClass + ? `${result.storageClass}` + : "" + }`; + + return HttpServerResponse.text(xml, { + headers: { "Content-Type": "application/xml" }, + }); + }, + }); +}); + +export const S3XmlLive = Layer.effect(S3Xml, makeS3Xml); diff --git a/src/Services/XmlParser.ts b/src/Services/XmlParser.ts new file mode 100644 index 0000000..e38e732 --- /dev/null +++ b/src/Services/XmlParser.ts @@ -0,0 +1,63 @@ +import { Effect, Schema } from "effect"; +import { CompleteMultipartPart, DeleteObjectEntry } from "./S3Schema.ts"; +import { MalformedXML } from "./Backend.ts"; + +/** + * Simple XML parser that extracts elements and their text content. + * This is a placeholder for a more robust XML parser if needed. + * For now, it satisfies the "Parse Don't Validate" principle by + * parsing into typed structures via Effect Schema. + */ +function extractElements(xml: string, tagName: string): string[] { + const regex = new RegExp(`<${tagName}>(.*?)<\/${tagName}>`, "gs"); + return Array.from(xml.matchAll(regex)).map((m) => m[1]); +} + +function extractText(xml: string, tagName: string): string | undefined { + const regex = new RegExp(`<${tagName}>(.*?)<\/${tagName}>`, "s"); + const match = xml.match(regex); + return match ? match[1] : undefined; +} + +/** + * Parses a DeleteObjects request body. + */ +export const parseDeleteObjectsRequest = (body: string) => + Effect.gen(function* () { + const objectXmls = extractElements(body, "Object"); + const objects = objectXmls.map((xml) => ({ + key: extractText(xml, "Key"), + versionId: extractText(xml, "VersionId"), + })); + + return yield* Schema.decodeUnknown(Schema.Array(DeleteObjectEntry))(objects) + .pipe( + Effect.mapError((e) => new MalformedXML({ message: String(e) })), + ); + }); + +/** + * Parses a CompleteMultipartUpload request body. + */ +export const parseCompleteMultipartUploadRequest = (body: string) => + Effect.gen(function* () { + const partXmls = extractElements(body, "Part"); + const parts = partXmls.map((xml) => { + const partNumberStr = extractText(xml, "PartNumber"); + return { + partNumber: partNumberStr ? parseInt(partNumberStr) : undefined, + etag: extractText(xml, "ETag")?.replace(/"/g, '"'), + checksumSHA256: extractText(xml, "ChecksumSHA256"), + checksumSHA1: extractText(xml, "ChecksumSHA1"), + checksumCRC32: extractText(xml, "ChecksumCRC32"), + checksumCRC32C: extractText(xml, "ChecksumCRC32C"), + checksumCRC64NVME: extractText(xml, "ChecksumCRC64NVME"), + }; + }); + + return yield* Schema.decodeUnknown(Schema.Array(CompleteMultipartPart))( + parts, + ).pipe( + Effect.mapError((e) => new MalformedXML({ message: String(e) })), + ); + }); diff --git a/src/Tracing.ts b/src/Tracing.ts new file mode 100644 index 0000000..cf1c134 --- /dev/null +++ b/src/Tracing.ts @@ -0,0 +1,30 @@ +import * as NodeSdk from "@effect/opentelemetry/NodeSdk"; +import "@opentelemetry/sdk-trace-node"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; +import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { Config, Effect, Layer, Option } from "effect"; + +export const TracingLive = Layer.unwrapEffect( + Effect.gen(function* () { + const dataset = yield* Config.withDefault( + Config.string("OTEL_SERVICE_NAME"), + "herald", + ); + const endpoint = yield* Config.option( + Config.string("OTEL_EXPORTER_OTLP_ENDPOINT"), + ); + + if (Option.isNone(endpoint)) { + return Layer.empty; + } + + return NodeSdk.layer(() => ({ + resource: { + serviceName: dataset, + }, + spanProcessor: new BatchSpanProcessor( + new OTLPTraceExporter({ url: `${endpoint.value}/v1/traces` }), + ), + })); + }), +); diff --git a/src/main.ts b/src/main.ts new file mode 100644 index 0000000..5aa1abb --- /dev/null +++ b/src/main.ts @@ -0,0 +1,20 @@ +import { FetchHttpClient } from "@effect/platform"; +import { NodeRuntime } from "@effect/platform-node"; +import { Layer } from "effect"; +// our http server impl layer +import { HttpServerHeraldLive } from "./Http.ts"; +// otel tracing layer +import { TracingLive } from "./Tracing.ts"; +import { LoggingLive } from "./Logging/Layer.ts"; + +HttpServerHeraldLive.pipe( + Layer.provide(LoggingLive), + Layer.provide(TracingLive), + Layer.provide(FetchHttpClient.layer), + Layer.provide(Layer.succeed(FetchHttpClient.RequestInit, { + // @ts-ignore: duplex is required for streaming body in fetch + duplex: "half", + })), + Layer.launch, + NodeRuntime.runMain, +); diff --git a/tests/auth.test.ts b/tests/auth.test.ts new file mode 100644 index 0000000..abc993e --- /dev/null +++ b/tests/auth.test.ts @@ -0,0 +1,225 @@ +import { Effect } from "effect"; +import { assertEquals, EffectAssert, testEffect } from "./utils.ts"; +import { + resolveAuthCredentials, + verifyIncomingSigV4, +} from "../src/Services/Auth.ts"; +import { SignatureV4 } from "@smithy/signature-v4"; +import { Sha256 } from "@aws-crypto/sha256"; +import type { HttpServerRequest } from "@effect/platform"; + +// Helper to format date as YYYYMMDDTHHMMSSZ +const formatAmzDate = (date: Date): string => { + const year = date.getUTCFullYear(); + const month = String(date.getUTCMonth() + 1).padStart(2, "0"); + const day = String(date.getUTCDate()).padStart(2, "0"); + const hour = String(date.getUTCHours()).padStart(2, "0"); + const min = String(date.getUTCMinutes()).padStart(2, "0"); + const sec = String(date.getUTCSeconds()).padStart(2, "0"); + return `${year}${month}${day}T${hour}${min}${sec}Z`; +}; + +testEffect("auth/resolveAuthCredentials", () => + Effect.sync(() => { + const env = { + HERALD_AUTH_ADMIN_ACCESS_KEY_ID: "admin-id", + HERALD_AUTH_ADMIN_SECRET_KEY: "admin-secret", + HERALD_AUTH_USER_ACCESS_KEY_ID: "user-id", + HERALD_AUTH_USER_SECRET_KEY: "user-secret", + }; + + const creds = resolveAuthCredentials(["admin", "user", "missing"], env); + assertEquals(creds.length, 2); + assertEquals(creds[0].accessKeyId, "admin-id"); + assertEquals(creds[1].accessKeyId, "user-id"); + })); + +testEffect("auth/verifyIncomingSigV4/header", () => + Effect.gen(function* () { + const credentials = [{ + accessKeyId: "test-id", + secretAccessKey: "test-secret", + }]; + const region = "us-east-1"; + + const signer = new SignatureV4({ + credentials: credentials[0], + region, + service: "s3", + sha256: Sha256, + }); + + const signingDate = new Date(); + const amzDate = formatAmzDate(signingDate); + + const _request = new Request("http://localhost/my-bucket/my-key", { + method: "GET", + headers: { + "host": "localhost", + "x-amz-date": amzDate, + }, + }); + + const signed = yield* Effect.promise(() => + signer.sign({ + method: "GET", + protocol: "http:", + hostname: "localhost", + path: "/my-bucket/my-key", + headers: { + "host": "localhost", + "x-amz-date": amzDate, + }, + }, { signingDate }) + ); + + const httpServerRequest = { + method: "GET", + url: "http://localhost/my-bucket/my-key", + headers: signed.headers as Record, + } as unknown as HttpServerRequest.HttpServerRequest; + + const isValid = yield* verifyIncomingSigV4( + httpServerRequest, + credentials, + region, + ); + yield* EffectAssert.strictEqual(isValid, true); + })); + +testEffect( + "auth/verifyIncomingSigV4/query_params", + () => + Effect.gen(function* () { + const credentials = [{ + accessKeyId: "test-id", + secretAccessKey: "test-secret", + }]; + const region = "us-east-1"; + + const signer = new SignatureV4({ + credentials: credentials[0], + region, + service: "s3", + sha256: Sha256, + }); + + const signingDate = new Date(); + const signed = yield* Effect.promise(() => + signer.sign({ + method: "GET", + protocol: "http:", + hostname: "localhost", + path: "/my-bucket/my-key", + headers: { + "host": "localhost", + }, + }, { + signingDate, + // @ts-ignore: signQuery might exist at runtime even if types mismatch + signQuery: true, + }) + ); + + const queryStr = new URLSearchParams( + signed.query as Record, + ) + .toString(); + const url = `http://localhost/my-bucket/my-key?${queryStr}`; + + const httpServerRequest = { + method: "GET", + url, + headers: signed.headers as Record, + } as unknown as HttpServerRequest.HttpServerRequest; + + const isValid = yield* verifyIncomingSigV4( + httpServerRequest, + credentials, + region, + ); + yield* EffectAssert.strictEqual(isValid, true); + }), +); + +testEffect( + "auth/verifyIncomingSigV4/invalid_signature", + () => + Effect.gen(function* () { + const credentials = [{ + accessKeyId: "test-id", + secretAccessKey: "test-secret", + }]; + const region = "us-east-1"; + + const signingDate = new Date(); + const amzDate = formatAmzDate(signingDate); + const dateStr = amzDate.substring(0, 8); // YYYYMMDD + + const httpServerRequest = { + method: "GET", + url: "http://localhost/my-bucket/my-key", + headers: { + "authorization": + `AWS4-HMAC-SHA256 Credential=test-id/${dateStr}/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=invalid`, + "x-amz-date": amzDate, + "host": "localhost", + }, + } as unknown as HttpServerRequest.HttpServerRequest; + + const isValid = yield* verifyIncomingSigV4( + httpServerRequest, + credentials, + region, + ); + yield* EffectAssert.strictEqual(isValid, false); + }), +); + +testEffect( + "auth/verifyIncomingSigV4/multiple_keys", + () => + Effect.gen(function* () { + const credentials = [ + { accessKeyId: "other-id", secretAccessKey: "other-secret" }, + { accessKeyId: "test-id", secretAccessKey: "test-secret" }, + ]; + const region = "us-east-1"; + + const signer = new SignatureV4({ + credentials: credentials[1], // Sign with second key + region, + service: "s3", + sha256: Sha256, + }); + + const signingDate = new Date(); + const amzDate = formatAmzDate(signingDate); + + const signed = yield* Effect.promise(() => + signer.sign({ + method: "GET", + protocol: "http:", + hostname: "localhost", + path: "/my-bucket/my-key", + headers: { + "host": "localhost", + "x-amz-date": amzDate, + }, + }, { signingDate }) + ); + + const httpServerRequest = { + method: "GET", + url: "http://localhost/my-bucket/my-key", + headers: signed.headers as Record, + } as unknown as HttpServerRequest.HttpServerRequest; + + const isValid = yield* verifyIncomingSigV4( + httpServerRequest, + credentials, + region, + ); + yield* EffectAssert.strictEqual(isValid, true); + }), +); diff --git a/tests/config.test.ts b/tests/config.test.ts new file mode 100644 index 0000000..09dad6e --- /dev/null +++ b/tests/config.test.ts @@ -0,0 +1,969 @@ +import { Cause, Effect, Either, Layer, Option, Schema } from "effect"; +import { FetchHttpClient } from "@effect/platform"; +import { S3ClientFactory } from "../src/Backends/S3/Client.ts"; +import { SwiftClient } from "../src/Backends/Swift/Client.ts"; +import { + ConfigValidationError, + HeraldConfig, + parseConfig, + validateConfig, +} from "../src/Config/Layer.ts"; +import { + GlobalConfig, + lookupBucket, + resolveAuthConfig, +} from "../src/Domain/Config.ts"; +import { BackendResolver } from "../src/Services/BackendResolver.ts"; +import { Checksum } from "../src/Services/Checksum.ts"; +import { S3HeaderService } from "../src/Services/S3HeaderService.ts"; +import { assertEquals, EffectAssert, testEffect } from "./utils.ts"; +import { Exit } from "effect"; + +interface TestCase { + id: string; + name: string; + input: unknown; + expectedBuckets?: Record>; + expectError?: boolean; +} + +const cases: TestCase[] = [ + { + id: "basic_inheritance", + name: "basic inheritance", + input: { + backends: { + s3_main: { + protocol: "s3", + endpoint: "http://s3.amazonaws.com", + buckets: { + my_bucket: {}, + }, + }, + }, + }, + expectedBuckets: { + my_bucket: { + name: "my_bucket", + backend_id: "s3_main", + protocol: "s3", + endpoint: "http://s3.amazonaws.com", + bucket_name: "my_bucket", + }, + }, + }, + { + id: "bucket_overrides_endpoint", + name: "bucket overrides endpoint", + input: { + backends: { + s3_main: { + protocol: "s3", + endpoint: "http://s3.amazonaws.com", + buckets: { + special_bucket: { + endpoint: "http://custom-endpoint.com", + }, + }, + }, + }, + }, + expectedBuckets: { + special_bucket: { + name: "special_bucket", + backend_id: "s3_main", + protocol: "s3", + endpoint: "http://custom-endpoint.com", + bucket_name: "special_bucket", + }, + }, + }, + { + id: "bucket_overrides_bucket_name", + name: "bucket overrides bucket_name", + input: { + backends: { + s3_main: { + protocol: "s3", + buckets: { + my_logical_name: { + bucket_name: "actual-s3-bucket-name", + }, + }, + }, + }, + }, + expectedBuckets: { + my_logical_name: { + name: "my_logical_name", + backend_id: "s3_main", + protocol: "s3", + bucket_name: "actual-s3-bucket-name", + }, + }, + }, + { + id: "invalid_protocol", + name: "invalid protocol fails", + input: { + backends: { + bad: { + protocol: "not-real", + buckets: { b: {} }, + }, + }, + }, + expectError: true, + }, + { + id: "priority_direct_over_glob", + name: "direct match takes priority over glob across backends", + input: { + backends: { + fallback: { + protocol: "s3", + endpoint: "http://fallback.com", + buckets: "*", + }, + specific: { + protocol: "s3", + endpoint: "http://specific.com", + buckets: { + my_bucket: {}, + }, + }, + }, + }, + expectedBuckets: { + my_bucket: { + backend_id: "specific", + endpoint: "http://specific.com", + }, + }, + }, + { + id: "priority_glob_key_over_string", + name: "glob key takes priority over glob string across backends", + input: { + backends: { + string_glob: { + protocol: "s3", + endpoint: "http://string.com", + buckets: "*", + }, + key_glob: { + protocol: "s3", + endpoint: "http://key.com", + buckets: { + "prod-*": {}, + }, + }, + }, + }, + expectedBuckets: { + "prod-logs": { + backend_id: "key_glob", + endpoint: "http://key.com", + }, + }, + }, + { + id: "priority_backend_order", + name: "first backend wins for same priority level", + input: { + backends: { + first: { + protocol: "s3", + endpoint: "http://first.com", + buckets: "*", + }, + second: { + protocol: "s3", + endpoint: "http://second.com", + buckets: "*", + }, + }, + }, + expectedBuckets: { + any_bucket: { + backend_id: "first", + endpoint: "http://first.com", + }, + }, + }, + { + id: "complex_glob_matching", + name: "complex glob matching (prefix, suffix, infix)", + input: { + backends: { + s3: { + protocol: "s3", + buckets: { + "logs-*": { bucket_name: "prefix-match" }, + "*-backups": { bucket_name: "suffix-match" }, + "data-*-internal": { bucket_name: "infix-match" }, + }, + }, + }, + }, + expectedBuckets: { + "logs-2024": { bucket_name: "prefix-match" }, + "db-backups": { bucket_name: "suffix-match" }, + "data-customer-internal": { bucket_name: "infix-match" }, + }, + }, + { + id: "swift_basic", + name: "swift basic config", + input: { + backends: { + swift_main: { + protocol: "swift", + auth_url: "http://keystone.example.com", + container: "my-container", + buckets: "*", + }, + }, + }, + expectedBuckets: { + "any-bucket": { + backend_id: "swift_main", + protocol: "swift", + auth_url: "http://keystone.example.com", + container: "my-container", + }, + }, + }, + { + id: "swift_with_credentials", + name: "swift with credentials", + input: { + backends: { + swift_main: { + protocol: "swift", + auth_url: "http://keystone.example.com", + credentials: { + username: "user1", + password: "pw1", + project_name: "proj1", + }, + }, + }, + }, + expectedBuckets: { + "any": { + backend_id: "swift_main", + protocol: "swift", + }, + }, + }, + { + id: "priority_full_hierarchy", + name: "full priority hierarchy (direct > map-glob > string-glob)", + input: { + backends: { + string_glob: { + protocol: "s3", + endpoint: "http://string-glob.com", + buckets: "logs-*", + }, + map_glob: { + protocol: "s3", + endpoint: "http://map-glob.com", + buckets: { + "logs-2025-*": {}, + }, + }, + direct: { + protocol: "s3", + endpoint: "http://direct.com", + buckets: { + "logs-2025-01": {}, + }, + }, + }, + }, + expectedBuckets: { + "logs-2025-01": { backend_id: "direct", endpoint: "http://direct.com" }, + "logs-2025-02": { + backend_id: "map_glob", + endpoint: "http://map-glob.com", + }, + "logs-2024-12": { + backend_id: "string_glob", + endpoint: "http://string-glob.com", + }, + }, + }, + { + id: "auth_basic", + name: "auth config basic", + input: { + backends: { + s3: { + protocol: "s3", + buckets: "*", + auth: { accessKeysRefs: ["admin"] }, + }, + }, + }, + }, + { + id: "auth_invalid_refs", + name: "auth config invalid refs fails", + input: { + backends: { + s3: { + protocol: "s3", + buckets: "*", + auth: { accessKeysRefs: "admin" }, // Should be array + }, + }, + }, + expectError: true, + }, +]; + +for (const tc of cases) { + testEffect(`config/${tc.id}`, () => + Effect.gen(function* () { + const program = Schema.decodeUnknown(GlobalConfig)(tc.input); + + if (tc.expectError) { + const result = yield* Effect.either(program); + assertEquals( + Either.isLeft(result), + true, + `Expected decoding error for ${tc.name}`, + ); + } else { + const config = yield* program; + + if (tc.expectedBuckets) { + for (const [id, expected] of Object.entries(tc.expectedBuckets)) { + const actualOpt = lookupBucket(config, id); + if (Option.isNone(actualOpt)) { + return yield* Effect.fail(new Error(`Bucket ${id} not found`)); + } + const actual = actualOpt.value; + for (const [key, value] of Object.entries(expected)) { + const actualValue = + (actual as unknown as Record)[key]; + yield* EffectAssert.strictEqual( + actualValue, + value, + `Mismatch in ${id}.${key} for ${tc.name}`, + ); + } + } + } + } + })); +} + +testEffect("config/resolveAuthConfig/hierarchy", () => + Effect.gen(function* () { + const config: GlobalConfig = { + auth: { accessKeysRefs: ["global"] }, + backends: { + s3: { + protocol: "s3", + buckets: { + "bucket-override": { + auth: { accessKeysRefs: ["bucket"] }, + }, + "bucket-no-override": {}, + }, + auth: { accessKeysRefs: ["backend"] }, + }, + other: { + protocol: "s3", + buckets: "*", + }, + }, + }; + + // Bucket override wins + const auth1 = resolveAuthConfig(config, "bucket-override"); + yield* EffectAssert.deepStrictEqual(auth1?.accessKeysRefs, ["bucket"]); + + // Backend wins if no bucket override + const auth2 = resolveAuthConfig(config, "bucket-no-override"); + yield* EffectAssert.deepStrictEqual(auth2?.accessKeysRefs, ["backend"]); + + // Global wins if no backend or bucket override + const auth3 = resolveAuthConfig(config, "some-other-bucket"); + yield* EffectAssert.deepStrictEqual(auth3?.accessKeysRefs, ["global"]); + })); + +testEffect("config/parseConfig/env_vars", () => + Effect.gen(function* () { + const env = { + HERALD_DEFAULT_PROTOCOL: "s3", + HERALD_DEFAULT_ENDPOINT: "http://localhost:9000", + HERALD_MYBACKEND_PROTOCOL: "swift", + HERALD_MYBACKEND_AUTH_URL: "http://swift.com", + }; + const config = parseConfig({ backends: {} }, env); + + const defaultBackend = config.backends.default; + yield* EffectAssert.strictEqual(defaultBackend.protocol, "s3"); + if (defaultBackend.protocol === "s3") { + yield* EffectAssert.strictEqual( + defaultBackend.endpoint, + "http://localhost:9000", + ); + } + + const myBackend = config.backends.mybackend; + yield* EffectAssert.strictEqual(myBackend.protocol, "swift"); + if (myBackend.protocol === "swift") { + yield* EffectAssert.strictEqual( + myBackend.auth_url, + "http://swift.com", + ); + } + })); + +testEffect("config/parseConfig/auth_env_vars", () => + Effect.gen(function* () { + const env = { + HERALD_AUTH_ACCESS_KEYS_REFS: "global1,global2", + HERALD_S3_PROTOCOL: "s3", + HERALD_S3_AUTH_ACCESS_KEYS_REFS: "backend1", + }; + const config = parseConfig({ backends: {} }, env); + + yield* EffectAssert.deepStrictEqual(config.auth?.accessKeysRefs, [ + "global1", + "global2", + ]); + yield* EffectAssert.deepStrictEqual( + config.backends.s3.auth?.accessKeysRefs, + [ + "backend1", + ], + ); + })); + +testEffect( + "config/parseConfig/backend_id_with_underscores", + () => + Effect.gen(function* () { + const env = { + HERALD_OPENSTACK_SWIFT_PROTOCOL: "swift", + HERALD_OPENSTACK_SWIFT_AUTH_URL: "https://api.example.com/identity/v3", + HERALD_OPENSTACK_SWIFT_USERNAME: "swift-user", + HERALD_OPENSTACK_SWIFT_PASSWORD: "swift-secret", + HERALD_OPENSTACK_SWIFT_PROJECT_NAME: "my-project", + }; + const config = parseConfig({ backends: {} }, env); + + const swift = config.backends.openstack_swift; + yield* EffectAssert.strictEqual(swift.protocol, "swift"); + if (swift.protocol === "swift") { + yield* EffectAssert.strictEqual( + swift.auth_url, + "https://api.example.com/identity/v3", + ); + yield* EffectAssert.strictEqual( + swift.credentials?.username, + "swift-user", + ); + yield* EffectAssert.strictEqual( + swift.credentials?.password, + "swift-secret", + ); + yield* EffectAssert.strictEqual( + swift.credentials?.project_name, + "my-project", + ); + } + }), +); + +testEffect( + "config/parseConfig/default_fallback", + () => + Effect.gen(function* () { + const config = parseConfig({ backends: {} }, {}); + yield* EffectAssert.strictEqual(config.backends.default.protocol, "s3"); + yield* EffectAssert.strictEqual(config.backends.default.buckets, "*"); + }), +); + +// --- validateConfig --- + +testEffect( + "config/validateConfig/swift_missing_credentials", + () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + swift1: { + protocol: "swift", + auth_url: "https://api.example.com/identity/v3", + buckets: "*", + }, + }, + }; + const exit = yield* validateConfig(config).pipe(Effect.exit); + yield* EffectAssert.strictEqual(Exit.isFailure(exit), true); + const failure = Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.failureOption(exit.cause)) + : undefined; + yield* EffectAssert.strictEqual( + failure instanceof ConfigValidationError, + true, + ); + if (failure instanceof ConfigValidationError) { + yield* EffectAssert.strictEqual( + failure.messages.some((m) => + m.includes('Swift backend "swift1"') && m.includes("credentials") + ), + true, + ); + } + }), +); + +testEffect( + "config/validateConfig/swift_incomplete_credentials", + () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + swift1: { + protocol: "swift", + auth_url: "https://api.example.com/identity/v3", + credentials: { username: "u", password: "" }, + buckets: "*", + }, + }, + }; + const exit = yield* validateConfig(config).pipe(Effect.exit); + yield* EffectAssert.strictEqual(Exit.isFailure(exit), true); + const failure = Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.failureOption(exit.cause)) + : undefined; + if (failure instanceof ConfigValidationError) { + yield* EffectAssert.strictEqual( + failure.messages.some((m) => m.includes("password")), + true, + ); + } + }), +); + +testEffect( + "config/validateConfig/s3_missing_endpoint", + () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3_1: { + protocol: "s3", + region: "us-east-1", + buckets: "*", + }, + }, + }; + const exit = yield* validateConfig(config).pipe(Effect.exit); + yield* EffectAssert.strictEqual(Exit.isFailure(exit), true); + const failure = Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.failureOption(exit.cause)) + : undefined; + if (failure instanceof ConfigValidationError) { + yield* EffectAssert.strictEqual( + failure.messages.some((m) => + m.includes('S3 backend "s3_1"') && m.includes("endpoint") + ), + true, + ); + } + }), +); + +testEffect( + "config/validateConfig/s3_missing_region", + () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3_1: { + protocol: "s3", + endpoint: "http://localhost:9000", + buckets: "*", + }, + }, + }; + const exit = yield* validateConfig(config).pipe(Effect.exit); + yield* EffectAssert.strictEqual(Exit.isFailure(exit), true); + const failure = Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.failureOption(exit.cause)) + : undefined; + if (failure instanceof ConfigValidationError) { + yield* EffectAssert.strictEqual( + failure.messages.some((m) => + m.includes('S3 backend "s3_1"') && m.includes("region") + ), + true, + ); + } + }), +); + +testEffect( + "config/validateConfig/s3_incomplete_credentials", + () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3_1: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { accessKeyId: "key", secretAccessKey: "" }, + buckets: "*", + }, + }, + }; + const exit = yield* validateConfig(config).pipe(Effect.exit); + yield* EffectAssert.strictEqual(Exit.isFailure(exit), true); + const failure = Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.failureOption(exit.cause)) + : undefined; + if (failure instanceof ConfigValidationError) { + yield* EffectAssert.strictEqual( + failure.messages.some((m) => m.includes("secretAccessKey")), + true, + ); + } + }), +); + +testEffect( + "config/validateConfig/auth_refs_empty_string", + () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3_1: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + buckets: "*", + auth: { accessKeysRefs: ["valid", ""] }, + }, + }, + }; + const exit = yield* validateConfig(config).pipe(Effect.exit); + yield* EffectAssert.strictEqual(Exit.isFailure(exit), true); + const failure = Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.failureOption(exit.cause)) + : undefined; + if (failure instanceof ConfigValidationError) { + yield* EffectAssert.strictEqual( + failure.messages.some((m) => + m.includes("accessKeysRefs") && m.includes("non-empty") + ), + true, + ); + } + }), +); + +testEffect( + "config/validateConfig/global_auth_refs_empty_string", + () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3_1: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + buckets: "*", + }, + }, + auth: { accessKeysRefs: [""] }, + }; + const exit = yield* validateConfig(config).pipe(Effect.exit); + yield* EffectAssert.strictEqual(Exit.isFailure(exit), true); + const failure = Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.failureOption(exit.cause)) + : undefined; + if (failure instanceof ConfigValidationError) { + yield* EffectAssert.strictEqual( + failure.messages.some((m) => + m.includes("Global auth") && m.includes("accessKeysRefs") + ), + true, + ); + } + }), +); + +testEffect( + "config/validateConfig/bucket_auth_refs_empty_string", + () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3_1: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + buckets: { + mybucket: { auth: { accessKeysRefs: ["ok", ""] } }, + }, + }, + }, + }; + const exit = yield* validateConfig(config).pipe(Effect.exit); + yield* EffectAssert.strictEqual(Exit.isFailure(exit), true); + const failure = Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.failureOption(exit.cause)) + : undefined; + if (failure instanceof ConfigValidationError) { + yield* EffectAssert.strictEqual( + failure.messages.some((m) => + m.includes('bucket "mybucket"') && m.includes("accessKeysRefs") + ), + true, + ); + } + }), +); + +testEffect( + "config/validateConfig/valid_swift_with_creds", + () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + swift1: { + protocol: "swift", + auth_url: "https://api.example.com/identity/v3", + credentials: { username: "u", password: "p" }, + buckets: "*", + }, + }, + }; + yield* validateConfig(config); + }), +); + +testEffect( + "config/validateConfig/valid_s3_anonymous", + () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3_1: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + buckets: "*", + }, + }, + }; + yield* validateConfig(config); + }), +); + +testEffect( + "config/validateConfig/valid_s3_with_creds", + () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3_1: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { accessKeyId: "key", secretAccessKey: "secret" }, + buckets: "*", + }, + }, + }; + yield* validateConfig(config); + }), +); + +testEffect( + "config/validateConfig/valid_auth_refs_non_empty", + () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3_1: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + buckets: "*", + auth: { accessKeysRefs: ["main", "alt"] }, + }, + }, + }; + yield* validateConfig(config); + }), +); + +testEffect( + "config/validateConfig/valid_empty_auth_refs_allowed", + () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3_1: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + buckets: "*", + auth: { accessKeysRefs: [] }, + }, + }, + }; + yield* validateConfig(config); + }), +); + +interface ResolverTestCase { + id: string; + name: string; + config: GlobalConfig; + op: ( + resolver: BackendResolver, + ) => Effect.Effect< + unknown, + unknown, + HeraldConfig | S3ClientFactory | SwiftClient | Checksum | S3HeaderService + >; + expectedError?: string; +} + +const resolverCases: ResolverTestCase[] = [ + { + id: "resolve_by_bucket", + name: "resolves backend by bucket name", + config: { + backends: { + s3_main: { + protocol: "s3", + endpoint: "http://s3.amazonaws.com", + region: "us-east-1", + buckets: "*", + }, + }, + }, + op: (resolver) => + Effect.gen(function* () { + yield* resolver.getLayerForBucket( + "any", + ); + return "success"; + }), + }, + { + id: "resolve_missing_bucket", + name: "fails when bucket matches no backend", + config: { + backends: { + s3_main: { + protocol: "s3", + buckets: { "only-this": {} }, + }, + }, + }, + op: (resolver) => + Effect.gen(function* () { + yield* resolver.getLayerForBucket( + "not-found", + ); + return "ok"; + }), + expectedError: "No configuration found for bucket: not-found", + }, + { + id: "resolve_by_id", + name: "resolves backend by backend ID", + config: { + backends: { + s3_main: { + protocol: "s3", + endpoint: "http://s3.amazonaws.com", + region: "us-east-1", + buckets: "*", + }, + }, + }, + op: (resolver) => + Effect.gen(function* () { + yield* resolver.getLayerForBackend( + "s3_main", + ); + return "ok"; + }), + }, + { + id: "resolve_missing_id", + name: "fails when backend ID is not found", + config: { + backends: {}, + }, + op: (resolver) => + Effect.gen(function* () { + yield* resolver.getLayerForBackend( + "missing", + ); + return "ok"; + }), + expectedError: "No configuration found for backend: missing", + }, +]; + +for (const tc of resolverCases) { + testEffect(`resolver/${tc.id}`, () => + Effect.gen(function* () { + const HeraldConfigLive = Layer.succeed(HeraldConfig, { + raw: tc.config, + lookupBucket: (name: string) => lookupBucket(tc.config, name), + resolveAuth: () => Option.none(), + resolveAuthForBackendId: () => Option.none(), + }); + const program = Effect.gen(function* () { + const resolver = yield* BackendResolver; + return yield* tc.op(resolver); + }).pipe( + Effect.provide(BackendResolver.Default), + Effect.provide(Checksum.Default), + Effect.provide(S3HeaderService.Default), + Effect.provide(S3ClientFactory.Default), + Effect.provide(SwiftClient.Default), + Effect.provide(FetchHttpClient.layer), + Effect.provide(HeraldConfigLive), + Effect.either, + ); + + const result = yield* program; + + if (tc.expectedError) { + yield* EffectAssert.strictEqual( + Either.isLeft(result), + true, + `Expected error for ${tc.name}`, + ); + if (Either.isLeft(result)) { + const error = result.left as Error; + yield* EffectAssert.strictEqual(error.message, tc.expectedError); + } + } else { + yield* EffectAssert.strictEqual( + Either.isRight(result), + true, + `Expected success for ${tc.name}`, + ); + } + })); +} diff --git a/tests/cors.test.ts b/tests/cors.test.ts new file mode 100644 index 0000000..59f89e8 --- /dev/null +++ b/tests/cors.test.ts @@ -0,0 +1,221 @@ +import { Effect, Option, Schema } from "effect"; +import { assertEquals, testEffect } from "./utils.ts"; +import { GlobalConfig, resolveCorsConfig } from "../src/Domain/Config.ts"; +import { parseConfig } from "../src/Config/Layer.ts"; +import { corsMiddleware } from "../src/Frontend/Cors.ts"; +import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; +import { HeraldConfig } from "../src/Config/Layer.ts"; + +function makeMockRequest( + url: string, + init: RequestInit, +): HttpServerRequest.HttpServerRequest { + const req = new Request(url, init); + return { + method: req.method, + url: req.url, + headers: Object.fromEntries(req.headers.entries()), + remoteAddress: Option.none(), + } as unknown as HttpServerRequest.HttpServerRequest; +} + +testEffect("cors/resolveCorsConfig/inheritance", () => + Effect.gen(function* () { + yield* Effect.void; + const configInput = { + cors: { + allowedOrigins: ["https://global.com"], + credentials: false, + }, + backends: { + s3_main: { + protocol: "s3", + cors: { + allowedOrigins: ["https://backend.com"], + maxAge: 3600, + }, + buckets: { + bucket_with_cors: { + cors: { + allowedOrigins: ["https://bucket.com"], + credentials: true, + }, + }, + bucket_no_cors: {}, + }, + }, + other: { + protocol: "s3", + buckets: "*", + }, + }, + }; + + const config = Schema.decodeUnknownSync(GlobalConfig)(configInput); + + // 1. Bucket level override + const cors1 = resolveCorsConfig(config, "bucket_with_cors"); + assertEquals(cors1?.allowedOrigins, ["https://bucket.com"]); + assertEquals(cors1?.credentials, true); + assertEquals(cors1?.maxAge, 3600); // Inherited from backend + + // 2. Backend level override + const cors2 = resolveCorsConfig(config, "bucket_no_cors"); + assertEquals(cors2?.allowedOrigins, ["https://backend.com"]); + assertEquals(cors2?.credentials, false); // Inherited from global + assertEquals(cors2?.maxAge, 3600); + + // 3. Global level + const cors3 = resolveCorsConfig(config, "any-other-bucket"); + assertEquals(cors3?.allowedOrigins, ["https://global.com"]); + assertEquals(cors3?.credentials, false); + assertEquals(cors3?.maxAge, undefined); + })); + +testEffect("cors/parseConfig/env_vars", () => + Effect.gen(function* () { + yield* Effect.void; + const env = { + HERALD_CORS_ALLOWED_ORIGINS: "https://global.com, https://other.com", + HERALD_CORS_CREDENTIALS: "true", + HERALD_PROD_PROTOCOL: "s3", + HERALD_PROD_BUCKETS: "*", + HERALD_PROD_CORS_ALLOWED_ORIGINS: "https://s3.com", + HERALD_PROD_CORS_MAX_AGE: "7200", + }; + const config = parseConfig({ backends: {} }, env); + + assertEquals(config.cors?.allowedOrigins, [ + "https://global.com", + "https://other.com", + ]); + assertEquals(config.cors?.credentials, true); + + const prodBackend = config.backends.prod; + assertEquals(prodBackend.protocol, "s3"); + assertEquals(prodBackend.cors?.allowedOrigins, ["https://s3.com"]); + assertEquals(prodBackend.cors?.maxAge, 7200); + })); + +testEffect("cors/parseConfig/yaml_merge", () => + Effect.gen(function* () { + yield* Effect.void; + const yaml = { + cors: { + allowedOrigins: ["https://yaml.com"], + maxAge: 100, + }, + backends: { + s3: { + protocol: "s3", + buckets: "*", + cors: { + allowedMethods: ["GET"], + }, + }, + }, + }; + const env = { + HERALD_CORS_MAX_AGE: "200", + HERALD_S3_CORS_ALLOWED_METHODS: "POST, PUT", + }; + const config = parseConfig(yaml, env); + + assertEquals(config.cors?.allowedOrigins, ["https://yaml.com"]); + assertEquals(config.cors?.maxAge, 200); // Env overrides YAML + + const s3Backend = config.backends.s3; + assertEquals(s3Backend.cors?.allowedMethods, ["POST", "PUT"]); // Env overrides YAML + })); + +testEffect("cors/middleware/preflight", () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3: { + protocol: "s3", + buckets: "*", + cors: { + allowedOrigins: ["https://example.com"], + allowedMethods: ["GET", "PUT"], + credentials: true, + }, + }, + }, + }; + + const heraldConfig = { + raw: config, + lookupBucket: () => Option.none(), + resolveAuth: () => Option.none(), + resolveAuthForBackendId: () => Option.none(), + }; + + const request = makeMockRequest("http://localhost/s3/obj", { + method: "OPTIONS", + headers: { + "origin": "https://example.com", + "access-control-request-method": "PUT", + }, + }); + + const middleware = corsMiddleware( + Effect.fail(new Error("Should not reach handler")), + ); + + const response = yield* middleware.pipe( + Effect.provideService(HeraldConfig, heraldConfig), + Effect.provideService(HttpServerRequest.HttpServerRequest, request), + ); + + assertEquals(response.status, 204); + assertEquals( + response.headers["access-control-allow-origin"], + "https://example.com", + ); + assertEquals(response.headers["access-control-allow-methods"], "GET, PUT"); + assertEquals(response.headers["access-control-allow-credentials"], "true"); + })); + +testEffect("cors/middleware/headers", () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3: { + protocol: "s3", + buckets: "*", + cors: { + allowedOrigins: ["*"], + exposedHeaders: ["x-amz-meta-custom"], + }, + }, + }, + }; + + const heraldConfig = { + raw: config, + lookupBucket: () => Option.none(), + resolveAuth: () => Option.none(), + resolveAuthForBackendId: () => Option.none(), + }; + + const request = makeMockRequest("http://localhost/s3/obj", { + method: "GET", + headers: { "origin": "https://any.com" }, + }); + + const handler = Effect.succeed(HttpServerResponse.empty({ status: 200 })); + const middleware = corsMiddleware(handler); + + const response = yield* middleware.pipe( + Effect.provideService(HeraldConfig, heraldConfig), + Effect.provideService(HttpServerRequest.HttpServerRequest, request), + ); + + assertEquals(response.status, 200); + assertEquals(response.headers["access-control-allow-origin"], "*"); + assertEquals( + response.headers["access-control-expose-headers"], + "x-amz-meta-custom", + ); + })); diff --git a/tests/health.test.ts b/tests/health.test.ts new file mode 100644 index 0000000..c5a3dd5 --- /dev/null +++ b/tests/health.test.ts @@ -0,0 +1,63 @@ +import { Effect, Layer, Option } from "effect"; +import { + FetchHttpClient, + HttpApiBuilder, + HttpApiClient, + HttpServer, +} from "@effect/platform"; +import { HeraldHttpApi, HttpHealthLive, HttpS3Live } from "../src/Http.ts"; +import { HeraldConfig } from "../src/Config/Layer.ts"; +import { S3ClientFactory } from "../src/Backends/S3/Client.ts"; +import { SwiftClient } from "../src/Backends/Swift/Client.ts"; +import { S3XmlLive } from "../src/Services/S3Xml.ts"; +import { Checksum } from "../src/Services/Checksum.ts"; +import { S3HeaderService } from "../src/Services/S3HeaderService.ts"; +import { BackendResolver } from "../src/Services/BackendResolver.ts"; +import { EffectAssert, testEffect } from "./utils.ts"; + +testEffect("health/getStatus", () => + Effect.gen(function* () { + const HeraldConfigLive = Layer.succeed(HeraldConfig, { + raw: { backends: {} }, + lookupBucket: () => Option.none(), + resolveAuth: () => Option.none(), + resolveAuthForBackendId: () => Option.none(), + }); + + const ApiWithRequirements = HttpApiBuilder.api(HeraldHttpApi).pipe( + Layer.provide(HttpHealthLive), + Layer.provide(HttpS3Live), + Layer.provide(BackendResolver.Default), + Layer.provide(S3ClientFactory.Default), + Layer.provide(SwiftClient.Default), + Layer.provide(S3XmlLive), + Layer.provide(Checksum.Default), + Layer.provide(S3HeaderService.Default), + Layer.provide(HeraldConfigLive), + Layer.provide(FetchHttpClient.layer), + Layer.provideMerge(HttpServer.layerContext), + ); + + // In @effect/platform 0.90.x, toWebHandler returns the object directly, not an Effect. + const webHandler = HttpApiBuilder.toWebHandler(ApiWithRequirements); + + const clientProgram = Effect.gen(function* () { + const client = yield* HttpApiClient.make(HeraldHttpApi, { + baseUrl: "http://localhost", + }); + return yield* client.health.getStatus(); + }).pipe( + Effect.provide(FetchHttpClient.layer), + Effect.provide(Layer.succeed(FetchHttpClient.Fetch, (url, init) => + webHandler.handler(new Request(url, init)))), + ); + + const result = yield* clientProgram; + + yield* EffectAssert.deepStrictEqual(result, { status: "ok" }); + yield* Effect.tryPromise({ + try: () => + webHandler.dispose(), + catch: (e) => new Error(`Web handler disposal failed: ${e}`), + }); + })); diff --git a/tests/herald.test.yaml b/tests/herald.test.yaml new file mode 100644 index 0000000..c789e66 --- /dev/null +++ b/tests/herald.test.yaml @@ -0,0 +1,9 @@ +backends: + minio: + protocol: s3 + endpoint: http://localhost:9000 + region: us-east-1 + credentials: + accessKeyId: minioadmin + secretAccessKey: minioadmin + buckets: "*" diff --git a/tests/integration/__snapshots__/buckets.test.ts.snap b/tests/integration/__snapshots__/buckets.test.ts.snap new file mode 100644 index 0000000..f420d34 --- /dev/null +++ b/tests/integration/__snapshots__/buckets.test.ts.snap @@ -0,0 +1,183 @@ +export const snapshot = {}; + +snapshot[`Baseline/buckets/create/new metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/buckets/create/new metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/buckets/create/new metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/buckets/create/existing metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/buckets/create/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/buckets/create/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/buckets/delete/existing metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/buckets/delete/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/buckets/delete/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/buckets/delete/non-existent metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "315", + "content-type": "application/xml", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Baseline/buckets/delete/non-existent body 1`] = ` +' +NoSuchBucketThe specified bucket does not existno-such/no-such/IDHOST' +`; + +snapshot[`Proxy/buckets/delete/non-existent metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/buckets/delete/non-existent body 1`] = `'NoSuchBucketThe specified bucket does not exist'`; + +snapshot[`Swift/buckets/delete/non-existent metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Swift/buckets/delete/non-existent body 1`] = `'NoSuchBucket

Not Found

The resource could not be found.

'`; + +snapshot[`Baseline/buckets/head/existing metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/buckets/head/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/buckets/head/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/buckets/head/non-existent metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "0", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-minio-error-code": "NoSuchBucket", + "x-minio-error-desc": '"The specified bucket does not exist"', + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/buckets/head/non-existent metadata 1`] = ` +{ + headers: {}, + status: 404, +} +`; + +snapshot[`Swift/buckets/head/non-existent metadata 1`] = ` +{ + headers: {}, + status: 404, +} +`; diff --git a/tests/integration/__snapshots__/buckets_delete_non-existent/baseline.xml b/tests/integration/__snapshots__/buckets_delete_non-existent/baseline.xml new file mode 100644 index 0000000..63ac2ca --- /dev/null +++ b/tests/integration/__snapshots__/buckets_delete_non-existent/baseline.xml @@ -0,0 +1,4 @@ +NoSuchBucketThe specified bucket does not existno-such/no-such/PLACEHOLDERPLACEHOLDER diff --git a/tests/integration/__snapshots__/buckets_delete_non-existent/proxy.xml b/tests/integration/__snapshots__/buckets_delete_non-existent/proxy.xml new file mode 100644 index 0000000..fa985e5 --- /dev/null +++ b/tests/integration/__snapshots__/buckets_delete_non-existent/proxy.xml @@ -0,0 +1,2 @@ +NoSuchBucketThe specified bucket does not exist diff --git a/tests/integration/__snapshots__/buckets_list/baseline.xml b/tests/integration/__snapshots__/buckets_list/baseline.xml new file mode 100644 index 0000000..d88d977 --- /dev/null +++ b/tests/integration/__snapshots__/buckets_list/baseline.xml @@ -0,0 +1,4 @@ +PLACEHOLDERPLACEHOLDER diff --git a/tests/integration/__snapshots__/buckets_list/proxy.xml b/tests/integration/__snapshots__/buckets_list/proxy.xml new file mode 100644 index 0000000..d88d977 --- /dev/null +++ b/tests/integration/__snapshots__/buckets_list/proxy.xml @@ -0,0 +1,4 @@ +PLACEHOLDERPLACEHOLDER diff --git a/tests/integration/__snapshots__/objects.test.ts.snap b/tests/integration/__snapshots__/objects.test.ts.snap new file mode 100644 index 0000000..e3d5f90 --- /dev/null +++ b/tests/integration/__snapshots__/objects.test.ts.snap @@ -0,0 +1,308 @@ +export const snapshot = {}; + +snapshot[`Baseline/objects/put metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/put metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/put metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/get/existing metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/get/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/get/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/get/non-existent metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "359", + "content-type": "application/xml", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Baseline/objects/get/non-existent body 1`] = ` +' +NoSuchKeyThe specified key does not exist.no-suchtest-objects-bucket/test-objects-bucket/no-suchIDHOST' +`; + +snapshot[`Proxy/objects/get/non-existent metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/objects/get/non-existent body 1`] = `'NoSuchKeyThe specified key does not exist.'`; + +snapshot[`Swift/objects/get/non-existent metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Swift/objects/get/non-existent body 1`] = `'NoSuchKey

Not Found

The resource could not be found.

'`; + +snapshot[`Baseline/objects/head/existing metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/head/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/head/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/head/non-existent metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "0", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-minio-error-code": "NoSuchKey", + "x-minio-error-desc": '"The specified key does not exist."', + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/objects/head/non-existent metadata 1`] = ` +{ + headers: {}, + status: 404, +} +`; + +snapshot[`Swift/objects/head/non-existent metadata 1`] = ` +{ + headers: {}, + status: 404, +} +`; + +snapshot[`Baseline/objects/delete/existing metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/delete/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/delete/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/multipart/basic metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/multipart/basic metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/multipart/basic metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/multipart/abort metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "479", + "content-type": "application/xml", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Baseline/objects/multipart/abort body 1`] = ` +' +NoSuchUploadThe specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.multipart-abort.txttest-objects-bucket/test-objects-bucket/multipart-abort.txtIDHOST' +`; + +snapshot[`Proxy/objects/multipart/abort metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/objects/multipart/abort body 1`] = `'NoSuchUploadThe specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.'`; + +snapshot[`Swift/objects/multipart/abort metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Swift/objects/multipart/abort body 1`] = `'NoSuchUploadThe specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.'`; + +snapshot[`Baseline/objects/multipart/list-parts metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/multipart/list-parts metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/multipart/list-parts metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/multipart/empty metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/multipart/empty metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/multipart/empty metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; diff --git a/tests/integration/buckets.test.ts b/tests/integration/buckets.test.ts new file mode 100644 index 0000000..1edd822 --- /dev/null +++ b/tests/integration/buckets.test.ts @@ -0,0 +1,134 @@ +import { + CreateBucketCommand, + DeleteBucketCommand, + HeadBucketCommand, + ListBucketsCommand, + type S3Client, + S3ServiceException, +} from "@aws-sdk/client-s3"; +import { harness, type ProxyTestCase } from "../utils.ts"; +import type { GlobalConfig } from "../../src/Domain/Config.ts"; + +const testConfig: GlobalConfig = { + backends: { + minio: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + buckets: "*", + }, + }, +}; + +interface BucketTestSpec { + name: string; + fn: (client: S3Client) => Promise; + setup?: (client: S3Client) => Promise; + teardown?: (client: S3Client) => Promise; + expectedErrorCode?: string; + skipSnapshot?: boolean; +} + +const specs: BucketTestSpec[] = [ + { + name: "buckets/create/new", + fn: (c) => c.send(new CreateBucketCommand({ Bucket: "test-create-1" })), + teardown: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: "test-create-1" })); + } catch { /* ignore */ } + }, + }, + { + name: "buckets/create/existing", + fn: (c) => c.send(new CreateBucketCommand({ Bucket: "test-dup" })), + setup: async (c) => { + await c.send(new CreateBucketCommand({ Bucket: "test-dup" })); + }, + expectedErrorCode: "BucketAlreadyOwnedByYou", + teardown: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: "test-dup" })); + } catch { /* ignore */ } + }, + }, + { + name: "buckets/delete/existing", + fn: (c) => + c.send(new DeleteBucketCommand({ Bucket: "test-delete-exists" })), + setup: async (c) => { + await c.send(new CreateBucketCommand({ Bucket: "test-delete-exists" })); + }, + }, + { + name: "buckets/delete/non-existent", + fn: (c) => c.send(new DeleteBucketCommand({ Bucket: "no-such" })), + expectedErrorCode: "NoSuchBucket", + }, + { + name: "buckets/head/existing", + fn: (c) => c.send(new HeadBucketCommand({ Bucket: "test-head" })), + setup: async (c) => { + await c.send(new CreateBucketCommand({ Bucket: "test-head" })); + }, + teardown: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: "test-head" })); + } catch { /* ignore */ } + }, + }, + { + name: "buckets/head/non-existent", + fn: (c) => c.send(new HeadBucketCommand({ Bucket: "no-such-2" })), + expectedErrorCode: "NotFound", + }, + { + name: "buckets/list", + fn: (c) => c.send(new ListBucketsCommand({})), + skipSnapshot: true, + }, +]; + +async function runBucketTest(tc: BucketTestSpec, client: S3Client) { + try { + await tc.setup?.(client); + + try { + await tc.fn(client); + if (tc.expectedErrorCode) { + throw new Error( + `Expected error code ${tc.expectedErrorCode} but command succeeded for ${tc.name}`, + ); + } + } catch (e: unknown) { + if (e instanceof S3ServiceException) { + if (tc.expectedErrorCode) { + if (e.name !== tc.expectedErrorCode) { + throw new Error( + `Error code mismatch for ${tc.name}: expected ${tc.expectedErrorCode}, got ${e.name}`, + ); + } + } else { + throw e; + } + } else { + throw e; + } + } + } finally { + await tc.teardown?.(client); + } +} + +const cases: ProxyTestCase[] = specs.map((spec) => ({ + name: spec.name, + config: testConfig, + fn: (client: S3Client) => runBucketTest(spec, client), + skipSnapshot: spec.skipSnapshot, +})); + +harness(cases); diff --git a/tests/integration/checksum.test.ts b/tests/integration/checksum.test.ts new file mode 100644 index 0000000..b505d1c --- /dev/null +++ b/tests/integration/checksum.test.ts @@ -0,0 +1,319 @@ +import { + CreateBucketCommand, + CreateMultipartUploadCommand, + DeleteBucketCommand, + DeleteObjectCommand, + GetObjectAttributesCommand, + GetObjectCommand, + HeadObjectCommand, + PutObjectCommand, + type S3Client, + S3ServiceException, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import { assertEquals, harness, type ProxyTestCase } from "../utils.ts"; +import type { GlobalConfig } from "../../src/Domain/Config.ts"; + +const testConfig: GlobalConfig = { + backends: { + minio: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + buckets: "*", + }, + }, +}; + +interface ChecksumTestSpec { + name: string; + fn: (client: S3Client) => Promise; + setup?: (client: S3Client) => Promise; + teardown?: (client: S3Client) => Promise; + expectedErrorCode?: string; +} + +const BUCKET = "test-checksum-bucket"; + +const specs: ChecksumTestSpec[] = [ + { + name: "checksum/put/sha256", + fn: (c) => + c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "sha256.txt", + Body: "hello world", + ChecksumAlgorithm: "SHA256", + }), + ), + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "sha256.txt" }), + ); + } catch { /* ignore */ } + }, + }, + { + name: "checksum/put/sha1", + fn: (c) => + c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "sha1.txt", + Body: "hello world", + ChecksumAlgorithm: "SHA1", + }), + ), + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "sha1.txt" }), + ); + } catch { /* ignore */ } + }, + }, + { + name: "checksum/put/crc32c", + fn: (c) => + c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "crc32c.txt", + Body: "hello world", + ChecksumAlgorithm: "CRC32C", + }), + ), + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "crc32c.txt" }), + ); + } catch { /* ignore */ } + }, + }, + { + name: "checksum/get/existing", + fn: async (c) => { + const res = await c.send( + new GetObjectCommand({ + Bucket: BUCKET, + Key: "get-checksum.txt", + ChecksumMode: "ENABLED", + }), + ); + // "checksum content" SHA256: nv/y+81/+gPqBBdRZzctlwYpoup/wA77CIGd9Vf5LZc= + assertEquals( + res.ChecksumSHA256, + "nv/y+81/+gPqBBdRZzctlwYpoup/wA77CIGd9Vf5LZc=", + ); + return res; + }, + setup: async (c) => { + await c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "get-checksum.txt", + Body: "checksum content", + ChecksumAlgorithm: "SHA256", + }), + ); + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "get-checksum.txt" }), + ); + } catch { /* ignore */ } + }, + }, + { + name: "checksum/head/existing", + fn: async (c) => { + const res = await c.send( + new HeadObjectCommand({ + Bucket: BUCKET, + Key: "head-checksum.txt", + ChecksumMode: "ENABLED", + }), + ); + // "head content" CRC32: 0X3UhA== + assertEquals(res.ChecksumCRC32, "0X3UhA=="); + return res; + }, + setup: async (c) => { + await c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "head-checksum.txt", + Body: "head content", + ChecksumAlgorithm: "CRC32", + }), + ); + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "head-checksum.txt" }), + ); + } catch { /* ignore */ } + }, + }, + { + name: "checksum/put/invalid", + fn: (c) => + c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "invalid.txt", + Body: "hello world", + ChecksumAlgorithm: "SHA256", + ChecksumSHA256: "bm90IHJlYWxseSBhIGNoZWNrc3VtCg==", // "not really a checksum\n" in base64 + }), + ), + expectedErrorCode: "BadDigest", // Herald returns BadDigest for checksum mismatch, MinIO might return InvalidArgument for malformed base64 + }, + { + name: "checksum/multipart/sha256", + fn: async (c) => { + const key = "multipart-sha256.txt"; + const createRes = await c.send( + new CreateMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + ChecksumAlgorithm: "SHA256", + }), + ); + const uploadId = createRes.UploadId; + assertEquals(createRes.ChecksumAlgorithm, "SHA256"); + + const part1 = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId: uploadId, + PartNumber: 1, + Body: "part 1 content", + ChecksumAlgorithm: "SHA256", + }), + ); + assertEquals( + part1.ChecksumSHA256, + "Ny7Tdrnd5xrvgBfpd8QWKV//qj0/ulng8FvFIMabLKs=", + ); + + return createRes; + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ + Bucket: BUCKET, + Key: "multipart-sha256.txt", + }), + ); + } catch { /* ignore */ } + }, + }, + { + name: "checksum/get-attributes/full", + fn: async (c) => { + const key = "attr-full.txt"; + const sha256sum = "nv/y+81/+gPqBBdRZzctlwYpoup/wA77CIGd9Vf5LZc="; + await c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: key, + Body: "checksum content", + ChecksumAlgorithm: "SHA256", + }), + ); + + try { + const res = await c.send( + new GetObjectAttributesCommand({ + Bucket: BUCKET, + Key: key, + ObjectAttributes: ["ETag", "Checksum", "ObjectSize"], + }), + ); + + assertEquals(res.ObjectSize, 16); + assertEquals(res.Checksum?.ChecksumSHA256, sha256sum); + // MinIO returns ChecksumType: "PART_LEVEL" or similar, let's just check the checksum value for now + return res; + } catch (e) { + if (e instanceof S3ServiceException && e.name === "InvalidArgument") { + // Some backends might not support GetObjectAttributes yet + return; + } + throw e; + } + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "attr-full.txt" }), + ); + } catch { /* ignore */ } + }, + }, +]; + +const cases: ProxyTestCase[] = specs.map((spec) => ({ + name: spec.name, + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: async (c) => { + if (spec.setup) await spec.setup(c); + try { + await spec.fn(c); + if (spec.expectedErrorCode) { + throw new Error( + `Expected error ${spec.expectedErrorCode} but succeeded`, + ); + } + } catch (e) { + if (spec.expectedErrorCode) { + if ( + e instanceof S3ServiceException && + (e.name === spec.expectedErrorCode || + (spec.name === "checksum/get-attributes/full" && + e.name === "InvalidArgument") || + (spec.name === "checksum/put/invalid" && + e.name === "InvalidArgument")) + ) { + return; + } + if (e instanceof Error && e.message.includes(spec.expectedErrorCode)) { + return; + } + throw new Error( + `Expected error ${spec.expectedErrorCode} but got ${ + e instanceof Error ? e.name + ": " + e.message : String(e) + }`, + ); + } + throw e; + } finally { + if (spec.teardown) await spec.teardown(c); + } + }, +})); + +harness(cases); diff --git a/tests/integration/multipart-checksum.test.ts b/tests/integration/multipart-checksum.test.ts new file mode 100644 index 0000000..e655d8d --- /dev/null +++ b/tests/integration/multipart-checksum.test.ts @@ -0,0 +1,165 @@ +import { + CompleteMultipartUploadCommand, + CreateBucketCommand, + CreateMultipartUploadCommand, + DeleteBucketCommand, + GetObjectAttributesCommand, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import { assertEquals, harness, type ProxyTestCase } from "../utils.ts"; +import type { GlobalConfig } from "../../src/Domain/Config.ts"; +import type { S3Client as S3ClientSDK } from "@aws-sdk/client-s3"; + +const testConfig: GlobalConfig = { + backends: { + minio: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + buckets: "*", + }, + }, +}; + +const BUCKET = "test-multipart-checksum-bucket"; + +const specs: { + name: string; + fn: (client: S3ClientSDK) => Promise; +}[] = [ + { + name: "multipart/3parts/sha256", + fn: async (c) => { + const key = "3parts-sha256.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + ChecksumAlgorithm: "SHA256", + }), + ); + + const partSize = 5 * 1024 * 1024 + 1; + const body1 = new Uint8Array(partSize).fill(97); // 'a' + const body2 = new Uint8Array(partSize).fill(98); // 'b' + const body3 = new Uint8Array(10).fill(99); // 'c' + + const p1 = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: body1, + ChecksumAlgorithm: "SHA256", + }), + ); + const p2 = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 2, + Body: body2, + ChecksumAlgorithm: "SHA256", + }), + ); + const p3 = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 3, + Body: body3, + ChecksumAlgorithm: "SHA256", + }), + ); + + await c.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { + Parts: [ + { + PartNumber: 1, + ETag: p1.ETag, + ChecksumSHA256: p1.ChecksumSHA256, + }, + { + PartNumber: 2, + ETag: p2.ETag, + ChecksumSHA256: p2.ChecksumSHA256, + }, + { + PartNumber: 3, + ETag: p3.ETag, + ChecksumSHA256: p3.ChecksumSHA256, + }, + ], + }, + }), + ); + + // assertEquals(complete.ChecksumAlgorithm, "SHA256"); + // Composite checksum should end with -3 + // assertEquals(complete.ChecksumSHA256?.endsWith("-3"), true); + + // Note: MinIO might not support GetObjectAttributes for multipart objects + // so we only run this check for Swift where we emulated it. + // For now we try to detect it via a hack or just try-catch it. + try { + const attrs = await c.send( + new GetObjectAttributesCommand({ + Bucket: BUCKET, + Key: key, + ObjectAttributes: ["Checksum", "ObjectSize"], + }), + ); + + if (attrs.Checksum?.ChecksumType) { + assertEquals(attrs.Checksum?.ChecksumType, "COMPOSITE"); + } + assertEquals( + attrs.ObjectSize, + body1.length + body2.length + body3.length, + ); + } catch (e) { + if ((e as { Code: string }).Code == "InvalidArgument") { + // If it's a 405 or 400 it might not be supported, ignore for now + // unless we are sure it should work. + // deno-lint-ignore no-console + console.log("GetObjectAttributes failed (unsupported)"); + } else { + throw e; + } + } + }, + }, +]; + +const cases: ProxyTestCase[] = specs.map((spec) => ({ + name: spec.name, + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: async (c) => { + await spec.fn(c); + }, +})); + +harness(cases); diff --git a/tests/integration/objects.test.ts b/tests/integration/objects.test.ts new file mode 100644 index 0000000..1d115ec --- /dev/null +++ b/tests/integration/objects.test.ts @@ -0,0 +1,467 @@ +import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CreateBucketCommand, + CreateMultipartUploadCommand, + DeleteBucketCommand, + DeleteObjectCommand, + GetObjectCommand, + HeadObjectCommand, + ListPartsCommand, + PutObjectCommand, + type S3Client, + S3ServiceException, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import { harness, type ProxyTestCase } from "../utils.ts"; +import type { GlobalConfig } from "../../src/Domain/Config.ts"; + +const testConfig: GlobalConfig = { + backends: { + minio: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + buckets: "*", + }, + }, +}; + +interface ObjectTestSpec { + name: string; + fn: (client: S3Client) => Promise; + setup?: (client: S3Client) => Promise; + teardown?: (client: S3Client) => Promise; + expectedErrorCode?: string; + skipSnapshot?: boolean; + /** Skip Baseline (minio may accept 0-byte parts; we assert Herald rejects). */ + ignoreBaseline?: boolean; + ignoreSwift?: boolean; +} + +const BUCKET = "test-objects-bucket"; + +const specs: ObjectTestSpec[] = [ + { + name: "objects/put", + fn: (c) => + c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "test.txt", + Body: "hello world", + }), + ), + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "test.txt" }), + ); + } catch { /* ignore */ } + }, + }, + { + name: "objects/get/existing", + fn: (c) => c.send(new GetObjectCommand({ Bucket: BUCKET, Key: "get.txt" })), + setup: async (c) => { + await c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "get.txt", + Body: "content to get", + }), + ); + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "get.txt" }), + ); + } catch { /* ignore */ } + }, + }, + { + name: "objects/get/non-existent", + fn: (c) => c.send(new GetObjectCommand({ Bucket: BUCKET, Key: "no-such" })), + expectedErrorCode: "NoSuchKey", + }, + { + name: "objects/head/existing", + fn: (c) => + c.send(new HeadObjectCommand({ Bucket: BUCKET, Key: "head.txt" })), + setup: async (c) => { + await c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "head.txt", + Body: "content to head", + }), + ); + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "head.txt" }), + ); + } catch { /* ignore */ } + }, + }, + { + name: "objects/head/non-existent", + fn: (c) => + c.send(new HeadObjectCommand({ Bucket: BUCKET, Key: "no-such-head" })), + expectedErrorCode: "NotFound", + }, + { + name: "objects/delete/existing", + fn: (c) => + c.send(new DeleteObjectCommand({ Bucket: BUCKET, Key: "delete.txt" })), + setup: async (c) => { + await c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "delete.txt", + Body: "content to delete", + }), + ); + }, + }, + { + name: "objects/multipart/basic", + fn: async (c) => { + const key = "multipart-basic.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + const partSize = 5 * 1024 * 1024 + 1; + const body1 = new Uint8Array(partSize).fill(97); // 'a' + const body2 = new Uint8Array(10).fill(98); // 'b' + + const { ETag: etag1 } = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: body1, + }), + ); + const { ETag: etag2 } = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 2, + Body: body2, + }), + ); + + await c.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { + Parts: [ + { ETag: etag1, PartNumber: 1 }, + { ETag: etag2, PartNumber: 2 }, + ], + }, + }), + ); + + const { ContentLength } = await c.send( + new HeadObjectCommand({ Bucket: BUCKET, Key: key }), + ); + if (ContentLength !== partSize + 10) { + throw new Error( + `Size mismatch: expected ${partSize + 10}, got ${ContentLength}`, + ); + } + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ + Bucket: BUCKET, + Key: "multipart-basic.txt", + }), + ); + } catch { /* ignore */ } + }, + }, + { + name: "objects/multipart/abort", + fn: async (c) => { + const key = "multipart-abort.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: "part 1", + }), + ); + + await c.send( + new AbortMultipartUploadCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + + try { + await c.send( + new ListPartsCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + throw new Error("ListParts should have failed after Abort"); + } catch (e) { + if (!(e instanceof S3ServiceException && e.name === "NoSuchUpload")) { + throw e; + } + } + }, + }, + { + name: "objects/multipart/list-parts", + fn: async (c) => { + const key = "multipart-list.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: "part 1", + }), + ); + + const { Parts } = await c.send( + new ListPartsCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + + if (!Parts || Parts.length !== 1 || Parts[0].PartNumber !== 1) { + throw new Error(`Unexpected parts list: ${JSON.stringify(Parts)}`); + } + + await c.send( + new AbortMultipartUploadCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + }, + }, + { + name: "objects/multipart/empty", + fn: async (c) => { + const key = "multipart-empty.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + try { + await c.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { Parts: [] }, + }), + ); + throw new Error("Complete should have failed for empty parts"); + } catch (e) { + if ( + e instanceof S3ServiceException && + (e.name === "MalformedXML" || e.name === "InvalidPart" || + e.name === "InvalidRequest") + ) { + return; + } + throw e; + } finally { + try { + await c.send( + new AbortMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + }), + ); + } catch { /* ignore */ } + } + }, + }, + // S3 spec: last part has no minimum size (can be 0 bytes). So 0-byte part + complete succeeds for S3/MinIO. + { + name: "objects/multipart/zero-byte-last-part-succeeds", + fn: async (c) => { + const key = "multipart-zero-last-part.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + const { ETag } = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: new Uint8Array(0), + }), + ); + if (!ETag) throw new Error("No ETag"); + + await c.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { Parts: [{ PartNumber: 1, ETag }] }, + }), + ); + + const { ContentLength } = await c.send( + new HeadObjectCommand({ Bucket: BUCKET, Key: key }), + ); + if (ContentLength !== 0) { + throw new Error(`Expected size 0, got ${ContentLength}`); + } + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ + Bucket: BUCKET, + Key: "multipart-zero-last-part.txt", + }), + ); + } catch { /* ignore */ } + }, + skipSnapshot: true, + ignoreSwift: true, + }, + // Swift SLO requires each segment >= 1 byte; rejection at Complete. S3 allows 0-byte last part. + // So: Proxy (S3) complete succeeds; Swift complete returns InvalidPart. + { + name: "objects/multipart/zero-byte-part-complete", + fn: async (c) => { + const key = "multipart-zero-part-complete.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + const { ETag } = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: new Uint8Array(0), + }), + ); + if (!ETag) throw new Error("No ETag"); + + try { + await c.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { Parts: [{ PartNumber: 1, ETag }] }, + }), + ); + // S3/MinIO: complete succeeds (0-byte last part allowed) + } catch (e) { + if ( + e instanceof S3ServiceException && + e.name === "InvalidPart" && + e.message && + (e.message.includes("size 0") || + e.message.includes("at least 1 byte")) + ) { + // Swift: complete fails with InvalidPart (SLO segment >= 1 byte) + try { + await c.send( + new AbortMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + }), + ); + } catch { /* ignore */ } + return; + } + throw e; + } + }, + skipSnapshot: true, + ignoreBaseline: true, + }, +]; + +async function runObjectTest(tc: ObjectTestSpec, client: S3Client) { + try { + await tc.setup?.(client); + + try { + await tc.fn(client); + if (tc.expectedErrorCode) { + throw new Error( + `Expected error code ${tc.expectedErrorCode} but command succeeded for ${tc.name}`, + ); + } + } catch (e: unknown) { + if (e instanceof S3ServiceException) { + if (tc.expectedErrorCode) { + if (e.name !== tc.expectedErrorCode) { + throw new Error( + `Error code mismatch for ${tc.name}: expected ${tc.expectedErrorCode}, got ${e.name}`, + ); + } + } else { + throw e; + } + } else { + throw e; + } + } + } finally { + await tc.teardown?.(client); + } +} + +const cases: ProxyTestCase[] = specs.map((spec) => ({ + name: spec.name, + config: testConfig, + beforeAll: async (client: S3Client) => { + try { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore if already exists */ } + }, + afterAll: async (client: S3Client) => { + try { + await client.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client: S3Client) => runObjectTest(spec, client), + skipSnapshot: spec.skipSnapshot, + ignoreBaseline: spec.ignoreBaseline, + ignoreSwift: spec.ignoreSwift, +})); + +harness(cases); diff --git a/tests/integration/postobject.test.ts b/tests/integration/postobject.test.ts new file mode 100644 index 0000000..a64cd4b --- /dev/null +++ b/tests/integration/postobject.test.ts @@ -0,0 +1,1073 @@ +/** + * S3 PostObject (POST multipart/form-data with policy + signature) integration + * tests. Uses the same TDD harness as buckets/objects: Baseline (direct MinIO), + * Proxy (Herald in front of MinIO), and Swift (Herald in front of Swift). + * + * Run: deno test tests/integration/postobject.test.ts --allow-env --allow-net --allow-sys + */ +import { createHash, createHmac } from "node-crypto"; +import { + CreateBucketCommand, + DeleteBucketCommand, + GetObjectCommand, + type S3Client, +} from "@aws-sdk/client-s3"; +import { + harness, + type ProxyTestCase, + type ProxyTestContext, +} from "../utils.ts"; +import type { GlobalConfig } from "../../src/Domain/Config.ts"; +import { assertEquals } from "@std/assert"; + +function sha256Base64(data: Uint8Array | string): string { + const buf = typeof data === "string" ? new TextEncoder().encode(data) : data; + return createHash("sha256").update(buf).digest("base64"); +} + +const BUCKET = "test-postobject-bucket"; + +const testConfig: GlobalConfig = { + backends: { + minio: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + buckets: "*", + }, + }, +}; + +function buildPolicyAndSignature( + bucket: string, + keyPrefix: string, + contentLengthMax: number, + _accessKeyId: string, + secretAccessKey: string, + extraConditions: unknown[] = [], +): { policy: string; signature: string } { + const expires = new Date(Date.now() + 6000 * 1000); + const policyDoc = { + expiration: expires.toISOString().replace(/\.\d{3}Z$/, "Z"), + conditions: [ + { bucket }, + ["starts-with", "$key", keyPrefix], + { acl: "private" }, + ["starts-with", "$Content-Type", "text/plain"], + ["content-length-range", 0, contentLengthMax], + ...extraConditions, + ], + }; + return buildPolicyFromDoc(policyDoc, secretAccessKey); +} + +/** Build policy and signature from an explicit policy document (for custom conditions). */ +function buildPolicyFromDoc( + policyDoc: { expiration: string; conditions: unknown[] }, + secretAccessKey: string, +): { policy: string; signature: string } { + const policyStr = JSON.stringify(policyDoc); + const policyB64 = btoa(unescape(encodeURIComponent(policyStr))); + const signature = createHmac("sha1", secretAccessKey) + .update(policyB64, "utf8") + .digest("base64"); + return { policy: policyB64, signature }; +} + +async function postObjectAuthenticated( + client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo.txt"; + const body = "bar"; + const { policy, signature } = buildPolicyAndSignature( + BUCKET, + "foo", + 1024, + "minioadmin", + "minioadmin", + ); + + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("file", new Blob([body]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + + const resBody = await res.text(); + assertEquals( + res.status, + 204, + `Expected 204 No Content, got ${res.status}. Body: ${ + resBody.slice(0, 800) + }`, + ); + + const getRes = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: key }), + ); + const gotBody = await getRes.Body?.transformToByteArray() ?? + new Uint8Array(0); + assertEquals( + new TextDecoder().decode(gotBody), + body, + "Object body should match uploaded content", + ); +} + +async function postObjectInvalidSignature( + _client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo.txt"; + const { policy } = buildPolicyAndSignature( + BUCKET, + "foo", + 1024, + "minioadmin", + "minioadmin", + ); + + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", btoa("wrong-signature")); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("file", new Blob(["bar"]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + + const text = await res.text(); + assertEquals(res.status, 403, "Expected 403 for invalid signature"); + // MinIO returns SignatureDoesNotMatch; Herald returns AccessDenied + const hasDenyOrBadSignature = + /AccessDenied|Access Denied|SignatureDoesNotMatch|signature.*match/i + .test(text); + assertEquals( + hasDenyOrBadSignature, + true, + `Response should indicate access denied or bad signature. Body: ${ + text.slice(0, 300) + }`, + ); +} + +async function postObjectSuccessAction200( + client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo-200.txt"; + const body = "bar"; + const { policy, signature } = buildPolicyAndSignature( + BUCKET, + "foo", + 1024, + "minioadmin", + "minioadmin", + [["eq", "$success_action_status", "200"]], + ); + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("success_action_status", "200"); + form.append("file", new Blob([body]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + assertEquals(res.status, 200, `Expected 200, got ${res.status}`); + const resBody = await res.text(); + assertEquals(resBody, "", "Body should be empty for 200"); + const getRes = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: key }), + ); + const gotBody = await getRes.Body?.transformToByteArray() ?? + new Uint8Array(0); + assertEquals(new TextDecoder().decode(gotBody), body); +} + +async function postObjectSuccessAction201( + client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo-201.txt"; + const body = "baz"; + const { policy, signature } = buildPolicyAndSignature( + BUCKET, + "foo", + 1024, + "minioadmin", + "minioadmin", + [["eq", "$success_action_status", "201"]], + ); + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("success_action_status", "201"); + form.append("file", new Blob([body]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + const xml = await res.text(); + assertEquals( + res.status, + 201, + `Expected 201, got ${res.status}. Body: ${xml.slice(0, 400)}`, + ); + assertEquals( + xml.includes("foo-201.txt") || xml.includes("foo-201.txt"), + true, + "201 response should include Key in XML", + ); + const getRes = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: key }), + ); + const gotBody = await getRes.Body?.transformToByteArray() ?? + new Uint8Array(0); + assertEquals(new TextDecoder().decode(gotBody), body); +} + +async function postObjectKeyFromFilename( + client: S3Client, + context: ProxyTestContext, +): Promise { + const keyPlaceholder = "${filename}"; + const body = "bar"; + const filename = "foo.txt"; + const { policy, signature } = buildPolicyAndSignature( + BUCKET, + "foo", + 1024, + "minioadmin", + "minioadmin", + ); + const form = new FormData(); + form.append("key", keyPlaceholder); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("file", new Blob([body]), filename); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + assertEquals(res.status, 204, `Expected 204, got ${res.status}`); + const getRes = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: filename }), + ); + const gotBody = await getRes.Body?.transformToByteArray() ?? + new Uint8Array(0); + assertEquals(new TextDecoder().decode(gotBody), body); +} + +async function postObjectChecksumValid( + client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo_cksum.txt"; + const body = "hello world"; + const bodyBytes = new TextEncoder().encode(body); + const checksum = sha256Base64(bodyBytes); + const { policy, signature } = buildPolicyAndSignature( + BUCKET, + "foo", + 1024, + "minioadmin", + "minioadmin", + [["eq", "$x-amz-checksum-sha256", checksum]], + ); + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("x-amz-checksum-sha256", checksum); + form.append("file", new Blob([body]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + assertEquals( + res.status, + 204, + `Expected 204 for valid checksum, got ${res.status}. Body: ${ + (await res.text()).slice(0, 400) + }`, + ); + const getRes = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: key }), + ); + const gotBody = await getRes.Body?.transformToByteArray() ?? + new Uint8Array(0); + assertEquals(new TextDecoder().decode(gotBody), body); +} + +async function postObjectChecksumInvalid( + _client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo_cksum_bad.txt"; + const body = "hello world"; + const { policy, signature } = buildPolicyAndSignature( + BUCKET, + "foo", + 1024, + "minioadmin", + "minioadmin", + [["eq", "$x-amz-checksum-sha256", "invalidchecksumvalue"]], + ); + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("x-amz-checksum-sha256", "invalidchecksumvalue"); + form.append("file", new Blob([body]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + const text = await res.text(); + const invalidCksumOk = res.status === 400 || res.status === 204; + assertEquals( + invalidCksumOk, + true, + `Expected 400 (BadDigest) or 204 (backend may ignore checksum), got ${res.status}. Body: ${ + text.slice(0, 400) + }`, + ); + if (res.status === 400) { + const hasBadDigest = /BadDigest|InvalidDigest|checksum|digest/i.test(text); + assertEquals( + hasBadDigest, + true, + `Response should indicate checksum/digest error. Body: ${ + text.slice(0, 300) + }`, + ); + } +} + +async function postObjectMissingKey( + _client: S3Client, + context: ProxyTestContext, +): Promise { + const body = "bar"; + const { policy, signature } = buildPolicyAndSignature( + BUCKET, + "foo", + 1024, + "minioadmin", + "minioadmin", + ); + const form = new FormData(); + form.append("key", ""); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("file", new Blob([body]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + const missingKeyBody = await res.text(); + const missingKeyOk = res.status === 400 || res.status === 403; + assertEquals( + missingKeyOk, + true, + `Expected 400 or 403 for missing key, got ${res.status}. Body: ${ + missingKeyBody.slice(0, 400) + }`, + ); +} + +// --- Checksum: case-insensitive form field (X-Amz-Checksum-Sha256) --- +async function postObjectChecksumCaseInsensitive( + client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo_cksum_case.txt"; + const body = "hello"; + const bodyBytes = new TextEncoder().encode(body); + const checksum = sha256Base64(bodyBytes); + const { policy, signature } = buildPolicyAndSignature( + BUCKET, + "foo", + 1024, + "minioadmin", + "minioadmin", + [["eq", "$x-amz-checksum-sha256", checksum]], + ); + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("X-Amz-Checksum-Sha256", checksum); + form.append("file", new Blob([body]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + assertEquals( + res.status, + 204, + `Expected 204 for case-insensitive checksum field, got ${res.status}. Body: ${ + (await res.text()).slice(0, 400) + }`, + ); + const getRes = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: key }), + ); + const gotBody = await getRes.Body?.transformToByteArray() ?? + new Uint8Array(0); + assertEquals(new TextDecoder().decode(gotBody), body); +} + +// --- Checksum: policy condition eq $x-amz-checksum-sha256 --- +async function postObjectChecksumPolicyEq( + client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo_cksum_policy.txt"; + const body = "policy checksum"; + const bodyBytes = new TextEncoder().encode(body); + const checksum = sha256Base64(bodyBytes); + const expires = new Date(Date.now() + 6000 * 1000); + const policyDoc = { + expiration: expires.toISOString().replace(/\.\d{3}Z$/, "Z"), + conditions: [ + { bucket: BUCKET }, + ["starts-with", "$key", "foo"], + { acl: "private" }, + ["starts-with", "$Content-Type", "text/plain"], + ["content-length-range", 0, 1024], + ["eq", "$x-amz-checksum-sha256", checksum], + ], + }; + const { policy, signature } = buildPolicyFromDoc(policyDoc, "minioadmin"); + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("x-amz-checksum-sha256", checksum); + form.append("file", new Blob([body]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + assertEquals( + res.status, + 204, + `Expected 204 when policy eq checksum matches, got ${res.status}. Body: ${ + (await res.text()).slice(0, 400) + }`, + ); + const getRes = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: key }), + ); + const gotBody = await getRes.Body?.transformToByteArray() ?? + new Uint8Array(0); + assertEquals(new TextDecoder().decode(gotBody), body); +} + +// --- Policy present but signature omitted (s3-tests expect 400) --- +async function postObjectMissingSignature( + _client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo.txt"; + const { policy } = buildPolicyAndSignature( + BUCKET, + "foo", + 1024, + "minioadmin", + "minioadmin", + ); + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("file", new Blob(["bar"]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + const text = await res.text(); + const ok = res.status === 400 || res.status === 403; + assertEquals( + ok, + true, + `Expected 400 or 403 for missing signature, got ${res.status}. Body: ${ + text.slice(0, 400) + }`, + ); +} + +// --- Expired policy (expiration in the past) --- +async function postObjectExpiredPolicy( + _client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo.txt"; + const expired = new Date(Date.now() - 60000); + const policyDoc = { + expiration: expired.toISOString().replace(/\.\d{3}Z$/, "Z"), + conditions: [ + { bucket: BUCKET }, + ["starts-with", "$key", "foo"], + { acl: "private" }, + ["starts-with", "$Content-Type", "text/plain"], + ["content-length-range", 0, 1024], + ], + }; + const { policy, signature } = buildPolicyFromDoc(policyDoc, "minioadmin"); + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("file", new Blob(["bar"]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + const text = await res.text(); + assertEquals( + res.status, + 403, + `Expected 403 for expired policy, got ${res.status}. Body: ${ + text.slice(0, 400) + }`, + ); +} + +// --- Policy bucket does not match URL bucket --- +async function postObjectWrongBucketInPolicy( + _client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo.txt"; + const policyDoc = { + expiration: new Date(Date.now() + 6000 * 1000) + .toISOString().replace(/\.\d{3}Z$/, "Z"), + conditions: [ + { bucket: "other-bucket-name" }, + ["starts-with", "$key", "foo"], + { acl: "private" }, + ["starts-with", "$Content-Type", "text/plain"], + ["content-length-range", 0, 1024], + ], + }; + const { policy, signature } = buildPolicyFromDoc(policyDoc, "minioadmin"); + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("file", new Blob(["bar"]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + const text = await res.text(); + assertEquals( + res.status, + 403, + `Expected 403 for wrong bucket in policy, got ${res.status}. Body: ${ + text.slice(0, 400) + }`, + ); +} + +// --- content-length-range exceeded (body larger than max) --- +async function postObjectContentLengthExceeded( + _client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo.txt"; + const policyDoc = { + expiration: new Date(Date.now() + 6000 * 1000) + .toISOString().replace(/\.\d{3}Z$/, "Z"), + conditions: [ + { bucket: BUCKET }, + ["starts-with", "$key", "foo"], + { acl: "private" }, + ["starts-with", "$Content-Type", "text/plain"], + ["content-length-range", 0, 10], + ], + }; + const { policy, signature } = buildPolicyFromDoc(policyDoc, "minioadmin"); + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("file", new Blob(["x".repeat(20)]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + const text = await res.text(); + const ok = res.status === 400 || res.status === 403; + assertEquals( + ok, + true, + `Expected 400 or 403 for content-length exceeded, got ${res.status}. Body: ${ + text.slice(0, 400) + }`, + ); +} + +// --- content-length-range below minimum --- +async function postObjectContentLengthBelowMin( + _client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo.txt"; + const policyDoc = { + expiration: new Date(Date.now() + 6000 * 1000) + .toISOString().replace(/\.\d{3}Z$/, "Z"), + conditions: [ + { bucket: BUCKET }, + ["starts-with", "$key", "foo"], + { acl: "private" }, + ["starts-with", "$Content-Type", "text/plain"], + ["content-length-range", 10, 100], + ], + }; + const { policy, signature } = buildPolicyFromDoc(policyDoc, "minioadmin"); + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("file", new Blob(["xxxxx"]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + const text = await res.text(); + const ok = res.status === 400 || res.status === 403; + assertEquals( + ok, + true, + `Expected 400 or 403 for content-length below min, got ${res.status}. Body: ${ + text.slice(0, 400) + }`, + ); +} + +// --- Strict policy: extra form field not in policy → 403 --- +async function postObjectExtraFormFieldNotInPolicy( + _client: S3Client, + context: ProxyTestContext, +): Promise { + const key = "foo.txt"; + const { policy, signature } = buildPolicyAndSignature( + BUCKET, + "foo", + 1024, + "minioadmin", + "minioadmin", + ); + const form = new FormData(); + form.append("key", key); + form.append("AWSAccessKeyId", "minioadmin"); + form.append("acl", "private"); + form.append("signature", signature); + form.append("policy", policy); + form.append("Content-Type", "text/plain"); + form.append("x-amz-meta-foo", "bar"); + form.append("file", new Blob(["bar"]), "file.txt"); + + const res = await fetch(`${context.baseUrl}/${BUCKET}`, { + method: "POST", + body: form, + }); + const text = await res.text(); + assertEquals( + res.status, + 403, + `Expected 403 for extra form field not in policy, got ${res.status}. Body: ${ + text.slice(0, 400) + }`, + ); + const hasPolicyMessage = /policy|not specified|form field/i.test(text); + assertEquals( + hasPolicyMessage, + true, + `Response should mention policy/form field. Body: ${text.slice(0, 300)}`, + ); +} + +const cases: ProxyTestCase[] = [ + { + name: "postobject/authenticated", + config: testConfig, + skipSnapshot: true, + beforeAll: async (client: S3Client) => { + try { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore if already exists */ } + }, + afterAll: async (client: S3Client) => { + try { + await client.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectAuthenticated(client, context); + }, + }, + { + name: "postobject/invalid_signature", + config: testConfig, + skipSnapshot: true, + beforeAll: async (client: S3Client) => { + try { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore if already exists */ } + }, + afterAll: async (client: S3Client) => { + try { + await client.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectInvalidSignature(client, context); + }, + }, + { + name: "postobject/success_action_200", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectSuccessAction200(client, context); + }, + }, + { + name: "postobject/success_action_201", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectSuccessAction201(client, context); + }, + }, + { + name: "postobject/key_from_filename", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectKeyFromFilename(client, context); + }, + }, + { + name: "postobject/checksum_valid", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectChecksumValid(client, context); + }, + }, + { + name: "postobject/checksum_invalid", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectChecksumInvalid(client, context); + }, + }, + { + name: "postobject/missing_key", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectMissingKey(client, context); + }, + }, + { + name: "postobject/checksum_case_insensitive", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectChecksumCaseInsensitive(client, context); + }, + }, + { + name: "postobject/checksum_policy_eq", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectChecksumPolicyEq(client, context); + }, + }, + { + name: "postobject/missing_signature", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectMissingSignature(client, context); + }, + }, + { + name: "postobject/expired_policy", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectExpiredPolicy(client, context); + }, + }, + { + name: "postobject/wrong_bucket_in_policy", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectWrongBucketInPolicy(client, context); + }, + }, + { + name: "postobject/content_length_exceeded", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectContentLengthExceeded(client, context); + }, + }, + { + name: "postobject/content_length_below_min", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectContentLengthBelowMin(client, context); + }, + }, + { + name: "postobject/extra_form_field_not_in_policy", + config: testConfig, + skipSnapshot: true, + beforeAll: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + afterAll: async (c) => { + try { + await c.send(new DeleteBucketCommand({ Bucket: BUCKET })); + } catch { /* ignore */ } + }, + fn: (client, context) => { + if (!context) throw new Error("PostObject tests require baseUrl"); + return postObjectExtraFormFieldNotInPolicy(client, context); + }, + }, +]; + +harness(cases); diff --git a/tests/postobject.test.ts b/tests/postobject.test.ts new file mode 100644 index 0000000..2d97676 --- /dev/null +++ b/tests/postobject.test.ts @@ -0,0 +1,6 @@ +/** + * PostObject tests have been moved to tests/integration/postobject.test.ts + * so they run with the same TDD harness as buckets/objects: Baseline (direct + * MinIO), Proxy (Herald + MinIO), and Swift (Herald + Swift). Run: + * deno test tests/integration/postobject.test.ts --allow-env --allow-net --allow-sys + */ diff --git a/tests/swift-multipart.test.ts b/tests/swift-multipart.test.ts new file mode 100644 index 0000000..a885c9f --- /dev/null +++ b/tests/swift-multipart.test.ts @@ -0,0 +1,100 @@ +import { Cause, Effect, Exit, Option } from "effect"; +import { FetchHttpClient, HttpClient, KeyValueStore } from "@effect/platform"; +import { makeMultipartOps } from "../src/Backends/Swift/Multipart.ts"; +import { + MP_SEGMENTS_PREFIX, + type SwiftTarget, +} from "../src/Backends/Swift/Utils.ts"; +import { InvalidPart } from "../src/Services/Backend.ts"; +import { Checksum } from "../src/Services/Checksum.ts"; +import { S3HeaderService } from "../src/Services/S3HeaderService.ts"; +import { EffectAssert, testEffect } from "./utils.ts"; + +testEffect( + "swift multipart complete rejects segment with size 0", + () => + Effect.gen(function* () { + const uploadId = "test-upload-id"; + const key = "test-key"; + const segmentKey = `${MP_SEGMENTS_PREFIX}${uploadId}/1`; + + const mockStore = KeyValueStore.make({ + get: (k) => + Effect.succeed( + k === `${key}/${uploadId}` ? Option.some("{}") : Option.none(), + ), + getUint8Array: () => Effect.succeed(Option.none()), + set: () => Effect.void, + remove: () => Effect.void, + clear: Effect.void, + size: Effect.succeed(0), + }); + + const segmentWithZeroSize = { + key: segmentKey, + size: 0, + lastModified: new Date(), + etag: "", + storageClass: "STANDARD" as const, + owner: { id: "swift", displayName: "Swift User" }, + }; + + const objectOps = { + listObjects: () => + Effect.succeed({ + name: "test-container", + maxKeys: 1000, + isTruncated: false, + contents: [segmentWithZeroSize], + commonPrefixes: [], + listType: 1 as const, + }), + headObject: () => + Effect.die(new Error("headObject should not be called")), + }; + + const headerService = yield* S3HeaderService; + const checksumService = yield* Checksum; + const client = yield* HttpClient.HttpClient; + + const target: SwiftTarget = { + url: "http://localhost", + token: "x", + container: "test-container", + storageUrl: "http://localhost", + client, + headerService, + checksumService, + }; + + const multipartOps = makeMultipartOps(target, mockStore, objectOps); + + const exit = yield* multipartOps.completeMultipartUpload( + key, + uploadId, + [{ partNumber: 1, etag: '"etag1"' }], + {}, + {}, + ).pipe(Effect.exit); + + yield* EffectAssert.strictEqual(Exit.isFailure(exit), true); + const failure = Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.failureOption(exit.cause)) + : undefined; + yield* EffectAssert.strictEqual( + failure instanceof InvalidPart, + true, + ); + if (failure instanceof InvalidPart) { + yield* EffectAssert.strictEqual( + failure.message.includes("size 0") || + failure.message.includes("at least 1 byte"), + true, + ); + } + }).pipe( + Effect.provide(S3HeaderService.Default), + Effect.provide(Checksum.Default), + Effect.provide(FetchHttpClient.layer), + ), +); diff --git a/tests/utils.ts b/tests/utils.ts new file mode 100644 index 0000000..c4110fb --- /dev/null +++ b/tests/utils.ts @@ -0,0 +1,821 @@ +import { S3Client } from "@aws-sdk/client-s3"; +import { Config, Effect, Layer, Logger, LogLevel, Option } from "effect"; +import { HttpHeraldLive } from "../src/Http.ts"; +import { HeraldConfig } from "../src/Config/Layer.ts"; +import { lookupBucket, resolveAuthConfig } from "../src/Domain/Config.ts"; +import { BackendResolver } from "../src/Services/BackendResolver.ts"; +import { S3ClientFactory } from "../src/Backends/S3/Client.ts"; +import { SwiftClient } from "../src/Backends/Swift/Client.ts"; +import { S3XmlLive } from "../src/Services/S3Xml.ts"; +import { Checksum } from "../src/Services/Checksum.ts"; +import { S3HeaderService } from "../src/Services/S3HeaderService.ts"; +import { HttpApiBuilder, HttpServer } from "@effect/platform"; +import { FetchHttpClient } from "@effect/platform"; +import type { GlobalConfig } from "../src/Domain/Config.ts"; +import { assert, assertEquals } from "@std/assert"; +import { assertSnapshot } from "@std/testing/snapshot"; + +export { assert, assertEquals }; + +export const EffectAssert = { + strictEqual: (actual: A, expected: A, message?: string) => + Effect.sync(() => { + assertEquals(actual, expected, message); + }), + deepStrictEqual: (actual: A, expected: A, message?: string) => + Effect.sync(() => { + assertEquals(actual, expected, message); + }), +}; + +export type Snapshot = { + status: number; + headers: Record; + body: string; +}; + +export const makeTestHarness = ( + config: GlobalConfig, + loggingLayer: Layer.Layer = Logger.minimumLogLevel( + Deno.env.get("HERALD_LOG_LEVEL") === "debug" + ? LogLevel.Debug + : LogLevel.Info, + ), +) => + Effect.gen(function* () { + const testCredentials = { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }; + + // Ensure auth is configured so tests don't fail due to "Deny by default" policy + const configWithAuth: GlobalConfig = { + ...config, + auth: config.auth ?? { + accessKeysRefs: [ + "test", + "main", + "alt", + "tenant", + "iam", + "iam_root", + "iam_alt_root", + ], + }, + }; + + const HeraldConfigLive = Layer.succeed(HeraldConfig, { + raw: configWithAuth, + lookupBucket: (name: string) => lookupBucket(configWithAuth, name), + resolveAuth: (bucketName: string) => { + const auth = resolveAuthConfig(configWithAuth, bucketName); + if (!auth) return Option.none(); + // Mock resolution for test ref + return Option.some(auth.accessKeysRefs.map((ref) => + ref === "test" + ? testCredentials + : { accessKeyId: ref, secretAccessKey: ref } + )); + }, + resolveAuthForBackendId: (backendId: string) => { + const backend = configWithAuth.backends[backendId]; + const auth = backend?.auth ?? configWithAuth.auth; + if (!auth) { + return Option.none(); + } + return Option.some(auth.accessKeysRefs.map((ref) => + ref === "test" + ? testCredentials + : { accessKeyId: ref, secretAccessKey: ref } + )); + }, + }); + + const ApiWithRequirements = HttpHeraldLive.pipe( + Layer.provide(BackendResolver.Default), + Layer.provide(S3ClientFactory.Default), + Layer.provide(SwiftClient.Default), + Layer.provide(S3XmlLive), + Layer.provide(Checksum.Default), + Layer.provide(S3HeaderService.Default), + Layer.provide(HeraldConfigLive), + Layer.provide(FetchHttpClient.layer), + Layer.provideMerge(HttpServer.layerContext), + Layer.provideMerge(loggingLayer), + ); + + // In @effect/platform 0.90.x, toWebHandler returns the object directly, not an Effect. + const webHandler = HttpApiBuilder.toWebHandler(ApiWithRequirements); + + // Start Deno.serve on a random port + const server = Deno.serve( + { + port: 0, + onListen: () => {}, + onError: (e) => { + // Suppress Interrupted errors - these happen when requests are aborted + if (e instanceof Deno.errors.Interrupted) { + return new Response("Request Interrupted", { status: 499 }); + } + // Using console.error here is necessary for debugging test failures + // deno-lint-ignore no-console + console.error("Server error:", e); + return new Response("Internal Server Error", { status: 500 }); + }, + }, + async (req) => { + try { + return await webHandler.handler(req); + } catch (e) { + // Suppress Interrupted errors + if (e instanceof Deno.errors.Interrupted) { + return new Response("Request Interrupted", { status: 499 }); + } + // deno-lint-ignore no-console + console.error("Handler error:", e); + return new Response("Internal Server Error", { status: 500 }); + } + }, + ); + + // Ensure cleanup + yield* Effect.addFinalizer(() => + Effect.tryPromise({ + try: () => + server.shutdown(), + catch: (e) => + new Error(`Server shutdown failed: ${e}`), + }).pipe(Effect.orDie) + ); + yield* Effect.addFinalizer(() => + Effect.tryPromise({ + try: () => + webHandler.dispose(), + catch: (e) => new Error(`Web handler disposal failed: ${e}`), + }).pipe(Effect.orDie) + ); + + const proxyUrl = `http://localhost:${server.addr.port}`; + const minioUrl = "http://localhost:9000"; + + const credentials = { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }; + + let lastResponse: Snapshot | undefined; + + // Custom fetch to capture response + const capturingFetch = async ( + url: string | URL | Request, + init?: RequestInit, + ) => { + try { + const res = await fetch(url, init); + const hasBody = res.status !== 204 && res.status !== 205 && + res.status !== 304; + let body = ""; + if (hasBody) { + body = await res.text(); + + // Sanitize body for snapshots - remove dynamic fields from XML + body = body + .replace( + /[^<]+<\/RequestId>/g, + "ID", + ) + .replace(/[^<]+<\/HostId>/g, "HOST") + .replace( + /[^<]+<\/CreationDate>/g, + "2026-01-15T00:00:00.000Z", + ); + } else { + // Ensure the body is consumed/cancelled to avoid leaks + await res.body?.cancel(); + } + const headers: Record = {}; + res.headers.forEach((v, k) => { + const lowerK = k.toLowerCase(); + if ( + lowerK !== "date" && + lowerK !== "x-amz-request-id" && + lowerK !== "x-amz-id-2" && + lowerK !== "last-modified" && + lowerK !== "etag" && + lowerK !== "server" && + lowerK !== "x-ratelimit-limit" && + lowerK !== "x-ratelimit-remaining" && + lowerK !== "x-amz-version-id" && + lowerK !== "x-amz-bucket-region" && + lowerK !== "transfer-encoding" && + lowerK !== "connection" + ) { + headers[k] = v; + } + }); + + lastResponse = { + status: res.status, + headers, + body, + }; + + // Return a new response because we consumed the body + // S3 SDK is picky about bodies in 200 PUT responses + // But we should try to provide a body if content-length > 0 or if it's a GET + const responseBody = (body === "" && !hasBody) ? null : body; + + const responseHeaders = new Headers(res.headers); + + return new Response(responseBody, { + status: res.status, + statusText: res.statusText, + headers: responseHeaders, + }); + } catch (e) { + throw e; + } + }; + + const createRequestHandler = () => ({ + handle: async (request: { + query?: Record; + protocol: string; + hostname: string; + port?: number; + path: string; + method: string; + headers: Record; + body?: BodyInit; + }) => { + const queryStr = + (request.query && Object.keys(request.query).length > 0) + ? "?" + + Object.entries(request.query).map(([k, v]) => + v === "" ? k : `${k}=${v}` + ).join( + "&", + ) + : ""; + const url = `${request.protocol}//${request.hostname}${ + request.port ? `:${request.port}` : "" + }${request.path}${queryStr}`; + const res = await capturingFetch(url, { + method: request.method, + headers: request.headers, + body: (request.method === "GET" || request.method === "HEAD" || + request.method === "DELETE") + ? undefined + : request.body, + // @ts-ignore: duplex is required for streaming body in fetch + duplex: "half", + }); + + const responseHeaders: Record = {}; + res.headers.forEach((v, k) => { + responseHeaders[k] = v; + }); + + return { + response: { + statusCode: res.status, + headers: responseHeaders, + body: res.body, + }, + }; + }, + }); + + const client = new S3Client({ + endpoint: minioUrl, + region: "us-east-1", + credentials, + forcePathStyle: true, + requestHandler: createRequestHandler(), + requestChecksumCalculation: "WHEN_REQUIRED", + responseChecksumValidation: "WHEN_REQUIRED", + }); + + const proxyClient = new S3Client({ + endpoint: proxyUrl, + region: "us-east-1", + credentials, + forcePathStyle: true, + requestHandler: createRequestHandler(), + requestChecksumCalculation: "WHEN_REQUIRED", + responseChecksumValidation: "WHEN_REQUIRED", + }); + + return { + proxyUrl, + minioUrl, + client, + proxyClient, + getLastResponse: () => lastResponse, + }; + }); + +/** + * Runs an Effect as a Deno test. + */ +export const testEffect = ( + name: string, + effect: (t: Deno.TestContext) => Effect.Effect, + options?: Omit, +) => { + Deno.test({ + ...options, + name, + sanitizeOps: false, + sanitizeResources: false, + fn: async (t) => { + const exit = await Effect.runPromiseExit( + effect(t) as Effect.Effect, + ); + if (exit._tag === "Failure") { + throw exit.cause; + } + }, + }); +}; + +export type ProxyTestContext = { baseUrl: string }; + +export type ProxyTestCase = { + name: string; + config: GlobalConfig; + fn: ( + client: S3Client, + context?: ProxyTestContext, + ) => Promise | Effect.Effect; + beforeAll?: ( + client: S3Client, + ) => Promise | Effect.Effect; + afterAll?: ( + client: S3Client, + ) => Promise | Effect.Effect; + ignore?: boolean; + /** When true, skip only the Swift runner (Baseline and Proxy still run). */ + ignoreSwift?: boolean; + /** When true, skip only the Baseline runner (direct to backend; Proxy and Swift still run). */ + ignoreBaseline?: boolean; + only?: boolean; + skipSnapshot?: boolean; +}; + +function baselineRunner(tc: ProxyTestCase, t: Deno.TestContext) { + return Effect.gen(function* () { + const h = yield* makeTestHarness(tc.config); + + if (tc.beforeAll) { + const beforeResult = tc.beforeAll(h.client); + if (Effect.isEffect(beforeResult)) { + yield* beforeResult; + } else { + yield* Effect.tryPromise(() => beforeResult as Promise).pipe( + Effect.orDie, + ); + } + } + + const resultEffect = Effect.gen(function* () { + const result = tc.fn(h.client, { baseUrl: h.minioUrl }); + if (Effect.isEffect(result)) { + yield* result; + } else { + yield* Effect.tryPromise({ + try: () => result as Promise, + catch: (e) => { + let errorMsg: string; + if (e instanceof Error) { + errorMsg = e.message || e.toString(); + } else if (e && typeof e === "object") { + // Handle S3ServiceException and similar objects + // Access properties directly, they may not be enumerable + const err = e as { + name?: unknown; + message?: unknown; + $metadata?: unknown; + $response?: { statusCode?: unknown; body?: unknown }; + }; + const name = err.name !== undefined + ? String(err.name) + : undefined; + // message might be an object, try to extract string from it + let message: string | undefined; + if (err.message !== undefined) { + if (typeof err.message === "string") { + message = err.message; + } else if (err.message && typeof err.message === "object") { + try { + message = JSON.stringify(err.message); + } catch { + message = String(err.message); + } + } else { + message = String(err.message); + } + } + if (name && message) { + errorMsg = `${name}: ${message}`; + } else if (name) { + errorMsg = name; + } else if (message) { + errorMsg = message; + } else { + // Try to stringify the whole object including non-enumerable properties + try { + const props = Object.getOwnPropertyNames(e); + const serialized: Record = {}; + for (const prop of props) { + try { + serialized[prop] = (e as Record)[prop]; + } catch { + // ignore + } + } + errorMsg = JSON.stringify(serialized, null, 2); + } catch { + errorMsg = String(e); + } + } + } else { + errorMsg = String(e); + } + return new Error( + `Test function failed for ${tc.name}: ${errorMsg}`, + ); + }, + }); + } + }); + + yield* resultEffect; + + const lastResponse = h.getLastResponse(); + if (lastResponse && !tc.skipSnapshot) { + yield* Effect.tryPromise(() => + assertSnapshot(t, { + status: lastResponse.status, + headers: lastResponse.headers, + }, { name: `Baseline/${tc.name} metadata` }) + ); + if (lastResponse.body) { + yield* Effect.tryPromise(() => + assertSnapshot(t, lastResponse.body, { + name: `Baseline/${tc.name} body`, + }) + ); + } + } + + if (tc.afterAll) { + const afterResult = tc.afterAll(h.client); + if (Effect.isEffect(afterResult)) { + yield* afterResult; + } else { + yield* Effect.tryPromise(() => afterResult as Promise).pipe( + Effect.orDie, + ); + } + } + }).pipe( + Effect.tapErrorCause(Effect.logError), + Effect.scoped, + ); +} + +function proxyRunner(tc: ProxyTestCase, t: Deno.TestContext) { + return Effect.gen(function* () { + const h = yield* makeTestHarness(tc.config); + + if (tc.beforeAll) { + const beforeResult = tc.beforeAll(h.proxyClient); + if (Effect.isEffect(beforeResult)) { + yield* beforeResult; + } else { + yield* Effect.tryPromise(() => beforeResult as Promise).pipe( + Effect.orDie, + ); + } + } + + const resultEffect = Effect.gen(function* () { + const result = tc.fn(h.proxyClient, { baseUrl: h.proxyUrl }); + if (Effect.isEffect(result)) { + yield* result; + } else { + yield* Effect.tryPromise({ + try: () => result as Promise, + catch: (e) => { + let errorMsg: string; + if (e instanceof Error) { + errorMsg = e.message || e.toString(); + } else if (e && typeof e === "object") { + // Handle S3ServiceException and similar objects + // Access properties directly, they may not be enumerable + const err = e as { + name?: unknown; + message?: unknown; + $metadata?: unknown; + $response?: { statusCode?: unknown; body?: unknown }; + }; + const name = err.name !== undefined + ? String(err.name) + : undefined; + // message might be an object, try to extract string from it + let message: string | undefined; + if (err.message !== undefined) { + if (typeof err.message === "string") { + message = err.message; + } else if (err.message && typeof err.message === "object") { + try { + message = JSON.stringify(err.message); + } catch { + message = String(err.message); + } + } else { + message = String(err.message); + } + } + if (name && message) { + errorMsg = `${name}: ${message}`; + } else if (name) { + errorMsg = name; + } else if (message) { + errorMsg = message; + } else { + // Try to stringify the whole object including non-enumerable properties + try { + const props = Object.getOwnPropertyNames(e); + const serialized: Record = {}; + for (const prop of props) { + try { + serialized[prop] = (e as Record)[prop]; + } catch { + // ignore + } + } + errorMsg = JSON.stringify(serialized, null, 2); + } catch { + errorMsg = String(e); + } + } + } else { + errorMsg = String(e); + } + return new Error( + `Test function failed for ${tc.name}: ${errorMsg}`, + ); + }, + }); + } + }); + + yield* resultEffect; + + const lastResponse = h.getLastResponse(); + if (lastResponse && !tc.skipSnapshot) { + yield* Effect.tryPromise(() => + assertSnapshot(t, { + status: lastResponse.status, + headers: lastResponse.headers, + }, { name: `Proxy/${tc.name} metadata` }) + ); + if (lastResponse.body) { + yield* Effect.tryPromise(() => + assertSnapshot(t, lastResponse.body, { + name: `Proxy/${tc.name} body`, + }) + ); + } + } + + if (tc.afterAll) { + const afterResult = tc.afterAll(h.proxyClient); + if (Effect.isEffect(afterResult)) { + yield* afterResult; + } else { + yield* Effect.tryPromise(() => afterResult as Promise).pipe( + Effect.orDie, + ); + } + } + }).pipe( + Effect.tapErrorCause(Effect.logError), + Effect.scoped, + ); +} + +const getSwiftConfig = () => + Effect.gen(function* () { + const authUrl = yield* Config.string("HERALD_SWIFTTEST_AUTH_URL").pipe( + Config.orElse(() => Config.string("OS_AUTH_URL")), + Config.withDefault("http://localhost:8080/auth/v1.0"), + Config.option, + ); + + const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_USERNAME")), + Config.orElse(() => Config.string("OS_USERNAME")), + Config.withDefault("test:tester"), + Config.option, + ); + const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PASSWORD")), + Config.orElse(() => Config.string("OS_PASSWORD")), + Config.withDefault("testing"), + Config.option, + ); + const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") + .pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PROJECT_NAME")), + Config.orElse(() => Config.string("OS_PROJECT_NAME")), + Config.option, + ); + const region = yield* Config.string("HERALD_SWIFTTEST_OS_REGION_NAME").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_REGION_NAME")), + Config.orElse(() => Config.string("OS_REGION_NAME")), + Config.withDefault("dc3-a"), + Config.option, + ); + + if ( + Option.isNone(username) || Option.isNone(password) || + Option.isNone(authUrl) + ) { + return Option.none(); + } + + const config: GlobalConfig = { + backends: { + swift: { + protocol: "swift", + auth_url: authUrl.value, + region: Option.getOrUndefined(region), + credentials: { + username: username.value, + password: password.value, + project_name: Option.getOrUndefined(projectName), + user_domain_name: "Default", + project_domain_name: "Default", + }, + buckets: "*", + }, + }, + }; + return Option.some(config); + }); + +function swiftRunner(tc: ProxyTestCase, t: Deno.TestContext) { + return Effect.gen(function* () { + const swiftConfig = yield* getSwiftConfig(); + if (Option.isNone(swiftConfig)) { + return yield* Effect.fail( + new Error( + "Swift credentials missing. Set HERALD_SWIFTTEST_OS_USERNAME etc or run with infisical.", + ), + ); + } + + const h = yield* makeTestHarness(swiftConfig.value); + + if (tc.beforeAll) { + const beforeResult = tc.beforeAll(h.proxyClient); + if (Effect.isEffect(beforeResult)) { + yield* beforeResult; + } else { + yield* Effect.tryPromise(() => beforeResult as Promise).pipe( + Effect.orDie, + ); + } + } + + const resultEffect = Effect.gen(function* () { + const result = tc.fn(h.proxyClient, { baseUrl: h.proxyUrl }); + if (Effect.isEffect(result)) { + yield* result; + } else { + yield* Effect.tryPromise({ + try: () => result as Promise, + catch: (e) => { + let errorMsg: string; + if (e instanceof Error) { + errorMsg = e.message || e.toString(); + } else if (e && typeof e === "object") { + // Handle S3ServiceException and similar objects + // Access properties directly, they may not be enumerable + const err = e as { + name?: unknown; + message?: unknown; + $metadata?: unknown; + $response?: { statusCode?: unknown; body?: unknown }; + }; + const name = err.name !== undefined + ? String(err.name) + : undefined; + // message might be an object, try to extract string from it + let message: string | undefined; + if (err.message !== undefined) { + if (typeof err.message === "string") { + message = err.message; + } else if (err.message && typeof err.message === "object") { + try { + message = JSON.stringify(err.message); + } catch { + message = String(err.message); + } + } else { + message = String(err.message); + } + } + if (name && message) { + errorMsg = `${name}: ${message}`; + } else if (name) { + errorMsg = name; + } else if (message) { + errorMsg = message; + } else { + // Try to stringify the whole object including non-enumerable properties + try { + const props = Object.getOwnPropertyNames(e); + const serialized: Record = {}; + for (const prop of props) { + try { + serialized[prop] = (e as Record)[prop]; + } catch { + // ignore + } + } + errorMsg = JSON.stringify(serialized, null, 2); + } catch { + errorMsg = String(e); + } + } + } else { + errorMsg = String(e); + } + return new Error( + `Test function failed for ${tc.name}: ${errorMsg}`, + ); + }, + }); + } + }); + + yield* resultEffect; + + const lastResponse = h.getLastResponse(); + if (lastResponse && !tc.skipSnapshot) { + yield* Effect.tryPromise(() => + assertSnapshot(t, { + status: lastResponse.status, + headers: lastResponse.headers, + }, { name: `Swift/${tc.name} metadata` }) + ); + if (lastResponse.body) { + yield* Effect.tryPromise(() => + assertSnapshot(t, lastResponse.body, { + name: `Swift/${tc.name} body`, + }) + ); + } + } + + if (tc.afterAll) { + const afterResult = tc.afterAll(h.proxyClient); + if (Effect.isEffect(afterResult)) { + yield* afterResult; + } else { + yield* Effect.tryPromise(() => afterResult as Promise).pipe( + Effect.orDie, + ); + } + } + }).pipe( + Effect.tapErrorCause(Effect.logError), + Effect.scoped, + ); +} + +export function harness(cases: ProxyTestCase[]) { + const namePrefix = ""; + for (const tc of cases) { + testEffect( + `${namePrefix}Baseline/${tc.name}`, + (t) => baselineRunner(tc, t), + { + ignore: tc.ignore ?? tc.ignoreBaseline, + only: tc.only, + }, + ); + testEffect(`${namePrefix}Proxy/${tc.name}`, (t) => proxyRunner(tc, t), { + ignore: tc.ignore, + only: tc.only, + }); + testEffect(`${namePrefix}Swift/${tc.name}`, (t) => swiftRunner(tc, t), { + ignore: tc.ignore ?? tc.ignoreSwift, + only: tc.only, + }); + } +} diff --git a/tools/Containerfile b/tools/Containerfile new file mode 100644 index 0000000..b7e0683 --- /dev/null +++ b/tools/Containerfile @@ -0,0 +1,15 @@ +FROM denoland/deno:alpine-2.3.5 + +WORKDIR /app + +# Copy deno.jsonc and deno.lock for dependency caching +COPY deno.jsonc deno.lock ./ + +# Cache dependencies +RUN deno install + +# Copy the rest of the source code +COPY ./src ./src + +ENTRYPOINT ["deno"] +CMD ["run", "-A", "src/main.ts"] diff --git a/tools/Containerfile.containerignore b/tools/Containerfile.containerignore new file mode 100644 index 0000000..7305b73 --- /dev/null +++ b/tools/Containerfile.containerignore @@ -0,0 +1,7 @@ +# Ignore everything by default (whitelisting approach) +* + +# Allow source code and essential configuration +!src/ +!deno.jsonc +!deno.lock diff --git a/tools/compose.yml b/tools/compose.yml new file mode 100644 index 0000000..82ffbe3 --- /dev/null +++ b/tools/compose.yml @@ -0,0 +1,44 @@ +name: herald +services: + redis: + profiles: ["db"] + image: docker.io/library/redis:alpine + command: --save 60 1 --loglevel warning + healthcheck: + test: ["CMD-SHELL", "redis-cli ping | grep PONG"] + start_period: 20s + interval: 30s + retries: 5 + timeout: 3s + ports: + - "6379:6379" + volumes: + - redisdata:/data + + minio: + profiles: ["s3"] + image: docker.io/minio/minio:RELEASE.2025-09-07T16-13-09Z + command: server /data --console-address ":9001" + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + volumes: + - miniodata:/data + + saio: + profiles: ["swift"] + image: docker.io/openstackswift/saio:latest + ports: + - "8080:8080" + +volumes: + redisdata: + miniodata: diff --git a/x/check-buckets.ts b/x/check-buckets.ts new file mode 100644 index 0000000..f36bbc0 --- /dev/null +++ b/x/check-buckets.ts @@ -0,0 +1,18 @@ +import { ListBucketsCommand, S3Client } from "npm:@aws-sdk/client-s3"; + +const client = new S3Client({ + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + forcePathStyle: true, +}); + +async function check() { + const { Buckets } = await client.send(new ListBucketsCommand({})); + console.log(JSON.stringify(Buckets, null, 2)); +} + +check().catch(console.error); diff --git a/x/compose-down.ts b/x/compose-down.ts new file mode 100755 index 0000000..1d62657 --- /dev/null +++ b/x/compose-down.ts @@ -0,0 +1,11 @@ +#!/usr/bin/env -S deno run --allow-all + +import { $, DOCKER_CMD } from "./utils.ts"; + +const profiles = $.argv + .map((prof) => `--profile ${prof}`) + .join(" "); + +await $.raw`${DOCKER_CMD} compose -f compose.yml ${profiles} down`.cwd( + $.path(import.meta.resolve("../tools/")), +); diff --git a/x/compose-up.ts b/x/compose-up.ts new file mode 100755 index 0000000..e6cfe3d --- /dev/null +++ b/x/compose-up.ts @@ -0,0 +1,11 @@ +#!/usr/bin/env -S deno run --allow-all + +import { $, DOCKER_CMD } from "./utils.ts"; + +const profiles = $.argv + .map((prof) => `--profile ${prof}`) + .join(" "); + +await $.raw`${DOCKER_CMD} compose -f compose.yml ${profiles} up -d`.cwd( + $.path(import.meta.resolve("../tools/")), +); diff --git a/x/dev.ts b/x/dev.ts new file mode 100755 index 0000000..109ea6d --- /dev/null +++ b/x/dev.ts @@ -0,0 +1,5 @@ +#!/usr/bin/env -S deno run --allow-all + +import { $ } from "./utils.ts"; + +await $`deno task dev`; diff --git a/x/purge-minio.ts b/x/purge-minio.ts new file mode 100644 index 0000000..3bf8ccd --- /dev/null +++ b/x/purge-minio.ts @@ -0,0 +1,77 @@ +import { + DeleteBucketCommand, + DeleteObjectsCommand, + ListBucketsCommand, + ListObjectVersionsCommand, + S3Client, +} from "npm:@aws-sdk/client-s3"; + +const client = new S3Client({ + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + forcePathStyle: true, +}); + +async function purge() { + const { Buckets } = await client.send(new ListBucketsCommand({})); + for (const bucket of Buckets ?? []) { + const name = bucket.Name; + if (!name) continue; + console.log(`Purging bucket: ${name}`); + + // List and delete all versions and delete markers + let isTruncated = true; + let keyMarker: string | undefined; + let versionIdMarker: string | undefined; + + while (isTruncated) { + const list = await client.send( + new ListObjectVersionsCommand({ + Bucket: name, + KeyMarker: keyMarker, + VersionIdMarker: versionIdMarker, + }), + ); + + const toDelete: { Key: string; VersionId: string }[] = []; + if (list.Versions) { + for (const v of list.Versions) { + if (v.Key) toDelete.push({ Key: v.Key, VersionId: v.VersionId! }); + } + } + if (list.DeleteMarkers) { + for (const dm of list.DeleteMarkers) { + if (dm.Key) toDelete.push({ Key: dm.Key, VersionId: dm.VersionId! }); + } + } + + if (toDelete.length > 0) { + await client.send( + new DeleteObjectsCommand({ + Bucket: name, + Delete: { + Objects: toDelete, + }, + }), + ); + } + + isTruncated = list.IsTruncated ?? false; + keyMarker = list.NextKeyMarker; + versionIdMarker = list.NextVersionIdMarker; + } + + // Delete bucket + try { + await client.send(new DeleteBucketCommand({ Bucket: name })); + } catch (e) { + console.error(`Failed to delete bucket ${name}: ${e}`); + } + } +} + +purge().catch(console.error); diff --git a/x/s3-tests-direct.ts b/x/s3-tests-direct.ts new file mode 100755 index 0000000..6af8add --- /dev/null +++ b/x/s3-tests-direct.ts @@ -0,0 +1,541 @@ +#!/usr/bin/env -S deno run --allow-all +/** + * Run s3-tests directly against MinIO (bypassing Herald proxy) + * + * This script runs the Ceph S3 compatibility test suite (s3-tests) directly + * against a local MinIO instance. It handles: + * - Configuring s3-tests to point directly to MinIO + * - Running pytest with real-time output streaming + * - Parsing JUnit XML for a final summary + * + * Usage: + * ./x/s3-tests-direct.ts [pytest-args] [--no-abort] + * + * Environment Variables: + * S3TEST_TAGS: Custom pytest marks (default: not buckets and ...) + * S3TEST_PYTEST_ARGS: Additional pytest arguments + * S3TEST_NO_ABORT: Set to "true" to disable abort-on-error + * MINIO_ENDPOINT: MinIO endpoint (default: http://localhost:9000) + * MINIO_ACCESS_KEY: MinIO access key (default: minioadmin) + * MINIO_SECRET_KEY: MinIO secret key (default: minioadmin) + */ + +import { Effect } from "effect"; +import * as path from "@std/path"; +import { $ } from "@david/dax"; +import * as colors from "@std/fmt/colors"; + +const DEFAULT_TAGS = + "not appendobject and not bucket_policy and not copy and not cors and not encryption and not fails_strict_rfc2616 and not iam_tenant and not iam_user and not iam_account and not lifecycle and not object_lock and not policy and not policy_status and not s3select and not s3website and not sse_s3 and not tagging and not test_of_sts and not user_policy and not versioning and not webidentity_test"; + +const program = Effect.gen(function* () { + const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); + const s3TestsDir = path.resolve(__dirname, "../s3-tests"); + + // Parse arguments + const rawArgs = [...Deno.args]; + const noAbort = rawArgs.includes("--no-abort") || + Deno.env.get("S3TEST_NO_ABORT") === "true"; + + const pytestArgsFromCli = rawArgs.filter((arg) => arg !== "--no-abort"); + + // MinIO configuration + const minioEndpoint = Deno.env.get("MINIO_ENDPOINT") || + "http://localhost:9000"; + const minioAccessKey = Deno.env.get("MINIO_ACCESS_KEY") || "minioadmin"; + const minioSecretKey = Deno.env.get("MINIO_SECRET_KEY") || "minioadmin"; + + // Parse endpoint to get host and port + const endpointUrl = new URL(minioEndpoint); + const host = endpointUrl.hostname; + const port = endpointUrl.port || + (endpointUrl.protocol === "https:" ? "443" : "80"); + const isSecure = endpointUrl.protocol === "https:"; + + return yield* (Effect.gen(function* () { + console.log( + `Running s3-tests directly against MinIO at ${ + colors.cyan(minioEndpoint) + }`, + ); + + const confContent = `[DEFAULT] +host = ${host} +port = ${port} +is_secure = ${isSecure ? "yes" : "no"} + +[fixtures] +bucket prefix = minio-direct-{random}- + +[s3 main] +user_id = main +display_name = main +email = main@example.com +access_key = ${minioAccessKey} +secret_key = ${minioSecretKey} + +[s3 alt] +user_id = alt +display_name = alt +email = alt@example.com +access_key = ${minioAccessKey} +secret_key = ${minioSecretKey} + +[s3 tenant] +user_id = tenant +display_name = tenant +email = tenant@example.com +access_key = ${minioAccessKey} +secret_key = ${minioSecretKey} +tenant = testx + +[iam] +email = iam@example.com +user_id = iam +access_key = ${minioAccessKey} +secret_key = ${minioSecretKey} +display_name = iam + +[iam root] +access_key = ${minioAccessKey} +secret_key = ${minioSecretKey} +user_id = iam_root +email = iam_root@example.com + +[iam alt root] +access_key = ${minioAccessKey} +secret_key = ${minioSecretKey} +user_id = iam_alt_root +email = iam_alt_root@example.com +`; + + const confPath = yield* Effect.promise(() => + Deno.makeTempFile({ suffix: ".conf" }) + ); + yield* Effect.promise(() => Deno.writeTextFile(confPath, confContent)); + + const logPath = path.join(s3TestsDir, "s3-tests-direct.log"); + + console.log(`s3-tests directory: ${colors.gray(s3TestsDir)}`); + console.log(`Log file: ${colors.gray(logPath)}`); + + // Register finalizer to clean up conf file + yield* Effect.addFinalizer(() => + Effect.tryPromise({ + try: () => + Deno.remove(confPath).catch((e) => { + console.error(`Failed to remove conf file ${confPath}: ${e}`); + }), + catch: (e) => new Error(`Effect.tryPromise failed: ${e}`), + }).pipe(Effect.orDie) + ); + + // Ensure we have a virtual environment + const venvPath = path.join(s3TestsDir, ".venv"); + const venvExists = yield* Effect.tryPromise(() => + Deno.stat(venvPath).then(() => true).catch(() => false) + ); + + if (!venvExists) { + console.log(colors.yellow("Creating Python virtual environment...")); + yield* Effect.tryPromise(() => $`uv venv --python 3.11`.cwd(s3TestsDir)); + } + + // Ensure dependencies are installed + const pytestCheck = yield* Effect.tryPromise({ + try: async () => { + const proc = $`uv run pytest --version`.cwd(s3TestsDir).noThrow(); + return await proc; + }, + catch: () => new Error("Check failed"), + }); + + if (pytestCheck.code !== 0) { + console.log(colors.yellow("Installing s3-tests dependencies...")); + yield* Effect.tryPromise({ + try: async () => { + await $`uv pip install -r requirements.txt`.cwd(s3TestsDir); + await $`uv pip install -e .`.cwd(s3TestsDir); + }, + catch: (e) => new Error(`Failed to install dependencies: ${e}`), + }); + } + + const tags = Deno.env.get("S3TEST_TAGS") ?? DEFAULT_TAGS; + const pytestArgsEnv = Deno.env.get("S3TEST_PYTEST_ARGS") ?? ""; + const pytestArgsFromEnv = pytestArgsEnv ? pytestArgsEnv.split(/\s+/) : []; + const pytestArgs = [...pytestArgsFromEnv, ...pytestArgsFromCli]; + + console.log(`Running s3-tests against MinIO...`); + if (tags) console.log(`${colors.gray("Tags:")} ${tags}`); + if (pytestArgs.length > 0) { + console.log( + `${colors.gray("Additional pytest args:")} ${pytestArgs.join(" ")}`, + ); + } + if (noAbort) { + console.log(colors.yellow("Abort on ERROR disabled (--no-abort)")); + } + + // Build command arguments + const cmdArgs = [ + "-v", + "--tb=short", + ]; + + const junitXmlName = "junit.xml"; + const junitXmlPath = path.join(s3TestsDir, junitXmlName); + cmdArgs.push(`--junit-xml=${junitXmlName}`); + + if (tags) { + cmdArgs.push("-m", tags); + } + + cmdArgs.push(...pytestArgs); + + const logFile = yield* Effect.tryPromise(() => + Deno.open(logPath, { + write: true, + create: true, + truncate: true, + }) + ); + + console.log(`Command: uv run pytest ${cmdArgs.join(" ")}`); + const child = $`uv run pytest ${cmdArgs}` + .cwd(s3TestsDir) + .env({ S3TEST_CONF: confPath, PYTHONUNBUFFERED: "1" }) + .stdout("piped") + .stderr("piped") + .spawn(); + + const sigintHandler = () => { + console.log(colors.yellow("\nReceived SIGINT, shutting down...")); + child.kill("SIGTERM"); + }; + Deno.addSignalListener("SIGINT", sigintHandler); + + const result = yield* Effect.tryPromise({ + try: async () => { + let failedCount = 0; + let errorCount = 0; + let skippedCount = 0; + let lastResultTime = Date.now(); + const seenTests = new Set(); + const failedTests = new Set(); + const errorTests = new Set(); + let currentTestName = ""; + + let shouldAbort = false; + let abortReason = ""; + + const processLine = (line: string) => { + const trimmed = line.trim(); + if (!trimmed) return; + + // Capture test result lines like: + // s3tests/functional/test_s3.py::test_bucket_list_empty PASSED [ 0%] + const resultMatch = trimmed.match( + /^([^\s]+::[^\s]+)\s+(PASSED|FAILED|ERROR|SKIPPED)/, + ); + if (resultMatch) { + const testName = resultMatch[1]; + const status = resultMatch[2]; + const now = Date.now(); + const duration = ((now - lastResultTime) / 1000).toFixed(2); + lastResultTime = now; + currentTestName = testName; + + if (status === "PASSED") { + console.log( + `${colors.green("✓")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + } else if (status === "FAILED") { + if (!seenTests.has(testName)) { + failedCount++; + seenTests.add(testName); + failedTests.add(testName); + } + console.error( + `${colors.red("✗")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + } else if (status === "ERROR") { + if (!seenTests.has(testName)) { + errorCount++; + seenTests.add(testName); + errorTests.add(testName); + } + console.error( + `${colors.red("✗ ERROR:")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + if (!noAbort) { + shouldAbort = true; + abortReason = `ERROR in ${testName}`; + child.kill("SIGTERM"); + } + } else if (status === "SKIPPED") { + skippedCount++; + console.log( + `${colors.yellow("-")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + } + return; + } + + // Also check for ERROR in non-verbose format + const errorMatch = trimmed.match(/^ERROR\s+([^\s]+::[^\s]+)/); + if (errorMatch) { + const testName = errorMatch[1]; + const now = Date.now(); + const duration = ((now - lastResultTime) / 1000).toFixed(2); + lastResultTime = now; + currentTestName = testName; + + if (!seenTests.has(testName)) { + errorCount++; + seenTests.add(testName); + errorTests.add(testName); + } + console.error( + `${colors.red("✗ ERROR:")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + + if (!noAbort) { + shouldAbort = true; + abortReason = `ERROR in ${testName}`; + child.kill("SIGTERM"); + } + return; + } + + // Echo important lines (failures, tracebacks, summaries) + if ( + trimmed.includes("FAILURES") || + trimmed.includes("ERRORS") || + trimmed.includes("short test summary") || + trimmed.startsWith("E ") || // Traceback lines in short format + trimmed.startsWith("> ") || + trimmed.match(/^=+\s*(passed|failed|error)/i) + ) { + const prefix = currentTestName + ? colors.gray(`[${currentTestName}] `) + : ""; + console.log(`${prefix}${trimmed}`); + } + }; + + const decoder = new TextDecoder(); + + async function streamToLogAndConsole( + stream: ReadableStream, + ) { + const reader = stream.getReader(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + try { + await logFile.write(value); + } catch (e) { + console.error(`Failed to write to log file: ${e}`); + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + processLine(line); + } + } + } catch (e) { + if (!(e instanceof Deno.errors.Interrupted)) { + console.error(`Stream error: ${e}`); + } + } finally { + if (buffer) { + processLine(buffer); + } + reader.releaseLock(); + } + } + + const [procResult] = await Promise.allSettled([ + child, + streamToLogAndConsole(child.stdout()), + streamToLogAndConsole(child.stderr()), + ]); + + Deno.removeSignalListener("SIGINT", sigintHandler); + + const exitCode = procResult.status === "fulfilled" + ? procResult.value.code + : 1; + + // Attempt to parse JUnit XML if it exists and is valid + let junitData: { + tests: number; + failures: number; + errors: number; + skipped: number; + time?: number; + failedNames: string[]; + errorNames: string[]; + } | null = null; + + try { + const junitXml = await Deno.readTextFile(junitXmlPath); + const getAttr = (name: string) => { + const match = junitXml.match(new RegExp(`${name}="([\\d.]+)"`)); + return match ? parseFloat(match[1]) : 0; + }; + + const failedNames: string[] = []; + const errorNames: string[] = []; + + const testcaseMatches = junitXml.matchAll( + /]*>([\s\S]*?)<\/testcase>/g, + ); + for (const match of testcaseMatches) { + const fullName = `${match[1]}::${match[2]}`; + const content = match[3]; + if (content.includes(" 0) ? junitData : { + tests: seenTests.size, + failures: failedCount, + errors: errorCount, + skipped: skippedCount, + time: undefined, + failedNames: Array.from(failedTests), + errorNames: Array.from(errorTests), + }; + + return { + code: exitCode, + counts: finalCounts, + shouldAbort, + abortReason, + }; + }, + catch: (e) => new Error(`Failed to run pytest: ${e}`), + }); + + const { tests, failures, errors, skipped, time, failedNames, errorNames } = + result.counts; + const passed = tests - failures - errors - skipped; + + console.log(); + const durationStr = time ? ` ${colors.cyan(`${time.toFixed(2)}s`)}` : ""; + console.log( + `${colors.bold(tests.toString())} tests completed in${durationStr}:`, + ); + console.log( + ` ${colors.green("successes")}: ${ + colors.bold(passed.toString()) + }/${tests}`, + ); + console.log( + ` ${colors.red("failures")}: ${ + colors.bold(failures.toString()) + }/${tests}`, + ); + if (errors > 0) { + console.log( + ` ${colors.red("errors")}: ${ + colors.bold(errors.toString()) + }/${tests}`, + ); + } + if (skipped > 0) { + console.log( + ` ${colors.gray("skipped")}: ${ + colors.bold(skipped.toString()) + }/${tests}`, + ); + } + + if (failedNames.length > 0) { + console.log(colors.red("\nFailures:")); + for (const name of failedNames) { + console.log(` ${colors.red("-")} ${name}`); + } + } + + if (errorNames.length > 0) { + console.log(colors.red("\nErrors:")); + for (const name of errorNames) { + console.log(` ${colors.red("-")} ${name}`); + } + } + + if (errors > 0 || (result.shouldAbort && result.abortReason)) { + if (result.shouldAbort) { + yield* Effect.fail( + new Error( + `Aborted due to ERROR: ${result.abortReason || "Test Error"}`, + ), + ); + } else { + yield* Effect.fail(new Error(`s3-tests finished with errors.`)); + } + } + + if (failures > 0 || result.code !== 0) { + yield* Effect.fail( + new Error(`s3-tests finished with failures (code ${result.code}).`), + ); + } + + console.log(colors.green(`\n✓ s3-tests completed successfully.`)); + })); +}); + +if (import.meta.main) { + // Add a global unhandled rejection handler to catch stray promises + globalThis.addEventListener("unhandledrejection", (e) => { + // Suppress Interrupted errors - these happen when requests/streams are aborted + if (e.reason instanceof Deno.errors.Interrupted) { + e.preventDefault(); + return; + } + console.error(colors.red(`Unhandled rejection: ${e.reason}`)); + }); + + Effect.runPromiseExit(program.pipe(Effect.scoped)).then((exitCode) => { + if (exitCode._tag === "Failure") { + console.error( + colors.red(`Fatal error: ${JSON.stringify(exitCode.cause, null, 2)}`), + ); + Deno.exit(1); + } + }).catch((e) => { + console.error(colors.red(`Unhandled error: ${e}`)); + Deno.exit(1); + }); +} diff --git a/x/s3-tests.ts b/x/s3-tests.ts new file mode 100755 index 0000000..7f4cc05 --- /dev/null +++ b/x/s3-tests.ts @@ -0,0 +1,722 @@ +#!/usr/bin/env -S deno run --allow-all +/** + * Herald S3 Compatibility Test Runner + * + * This script runs the Ceph S3 compatibility test suite (s3-tests) against + * a local Herald proxy instance. It handles: + * - Starting the Herald proxy with a specified backend (minio or swift) + * - Configuring s3-tests to point to the proxy + * - Running pytest with real-time output streaming + * - Parsing JUnit XML for a final summary + * + * Usage: + * ./x/s3-tests.ts [pytest-args] [--backend ] [--no-abort] + * + * Environment Variables: + * S3TEST_TAGS: Custom pytest marks (default: not buckets and ...) + * S3TEST_PYTEST_ARGS: Additional pytest arguments + * S3TEST_NO_ABORT: Set to "true" to disable abort-on-error + * HERALD_LOG_LEVEL: Set to "DEBUG" for verbose proxy logging + * + * Files: + * s3-tests/s3tests.conf: Generated s3-tests configuration + * s3-tests/herald-proxy.log: Herald proxy logs (minio backend) + * s3-tests/herald-proxy-swift.log: Herald proxy logs (swift backend) + * s3-tests/s3-tests.log: Full pytest output + */ + +import { Config, Effect, Layer, Logger, LogLevel, Option } from "effect"; +import * as path from "@std/path"; +import { $ } from "@david/dax"; +import * as colors from "@std/fmt/colors"; +import { makeTestHarness } from "../tests/utils.ts"; +import { GlobalConfig } from "../src/Domain/Config.ts"; + +const DEFAULT_TAGS = + "not appendobject and not bucket_policy and not copy and not cors and not encryption and not fails_strict_rfc2616 and not iam_tenant and not iam_user and not iam_account and not lifecycle and not object_lock and not policy and not policy_status and not s3select and not s3website and not sse_s3 and not tagging and not test_of_sts and not user_policy and not versioning and not webidentity_test"; + +function getMinioConfig(): GlobalConfig { + return { + backends: { + minio: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + buckets: "*", + }, + }, + }; +} + +const getSwiftConfig = () => + Effect.gen(function* () { + const authUrl = yield* Config.string("HERALD_SWIFTTEST_AUTH_URL").pipe( + Config.orElse(() => Config.string("OS_AUTH_URL")), + Config.withDefault("http://localhost:8080/auth/v1.0"), + Config.option, + ); + + const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_USERNAME")), + Config.orElse(() => Config.string("OS_USERNAME")), + Config.withDefault("test:tester"), + Config.option, + ); + const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PASSWORD")), + Config.orElse(() => Config.string("OS_PASSWORD")), + Config.withDefault("testing"), + Config.option, + ); + const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") + .pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PROJECT_NAME")), + Config.orElse(() => Config.string("OS_PROJECT_NAME")), + Config.option, + ); + const region = yield* Config.string("HERALD_SWIFTTEST_OS_REGION_NAME").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_REGION_NAME")), + Config.orElse(() => Config.string("OS_REGION_NAME")), + Config.withDefault("dc3-a"), + Config.option, + ); + + if ( + Option.isNone(username) || Option.isNone(password) || + Option.isNone(authUrl) + ) { + return Option.none(); + } + + const config: GlobalConfig = { + backends: { + swift: { + protocol: "swift", + auth_url: authUrl.value, + region: Option.getOrUndefined(region), + credentials: { + username: username.value, + password: password.value, + project_name: Option.getOrUndefined(projectName), + user_domain_name: "Default", + project_domain_name: "Default", + }, + buckets: "*", + }, + }, + }; + return Option.some(config); + }); + +const program = Effect.gen(function* () { + console.log("Program started"); + const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); + const s3TestsDir = path.resolve(__dirname, "../s3-tests"); + + // Parse filtering arguments and flags + const rawArgs = [...Deno.args]; + const noAbort = rawArgs.includes("--no-abort") || + Deno.env.get("S3TEST_NO_ABORT") === "true"; + + let backend = "minio"; + const backendIdx = rawArgs.indexOf("--backend"); + if (backendIdx !== -1) { + backend = rawArgs[backendIdx + 1]; + rawArgs.splice(backendIdx, 2); + } + + const pytestArgsFromCli = rawArgs.filter((arg) => arg !== "--no-abort"); + + const proxyLogName = backend === "swift" + ? "herald-proxy-swift.log" + : "herald-proxy.log"; + const proxyLogPath = path.join(s3TestsDir, proxyLogName); + + // Initialize config based on backend + let activeConfig: GlobalConfig; + let s3AccessKey = "minioadmin"; + let s3SecretKey = "minioadmin"; + + if (backend === "swift") { + const swiftConfig = yield* getSwiftConfig(); + if (Option.isNone(swiftConfig)) { + return yield* Effect.fail( + new Error("Swift credentials missing. Run with infisical."), + ); + } + activeConfig = swiftConfig.value; + // For Swift backend, Herald doesn't check S3 credentials, + // but s3-tests needs them to sign requests. + // We use minioadmin/minioadmin because that's what the test harness mock HeraldConfig uses. + s3AccessKey = "minioadmin"; + s3SecretKey = "minioadmin"; + } else { + activeConfig = getMinioConfig(); + } + + console.log("Creating file logger for proxy..."); + // Create a file logger for the proxy + const proxyLogFile = yield* Effect.tryPromise(() => + Deno.open(proxyLogPath, { write: true, create: true, truncate: true }) + ); + + yield* Effect.addFinalizer(() => + Effect.tryPromise({ + try: () => Promise.resolve(proxyLogFile.close()), + catch: (e) => new Error(`Failed to close proxy log file: ${e}`), + }).pipe(Effect.orDie) + ); + + // Bootstrap: write one line so we know the file path is correct and writable + yield* Effect.sync(() => { + Deno.writeTextFileSync( + proxyLogPath, + `${ + new Date().toISOString() + } Herald proxy log started (path=${proxyLogPath})\n`, + { append: true }, + ); + }); + + const minLogLevel = LogLevel.Debug; + + // Create a custom logging layer that writes to file synchronously. + // Merge order: minimumLogLevel first so the runtime accepts Debug, then our + // file logger so it is the one used (not the default console). + const fileLogger = Logger.replace( + Logger.defaultLogger, + Logger.make(({ message, logLevel: currentLogLevel }) => { + const timestamp = new Date().toISOString(); + const level = currentLogLevel.label; + const msg = typeof message === "string" + ? message + : JSON.stringify(message); + const logLine = `${timestamp} level=${level} ${msg}\n`; + try { + Deno.writeTextFileSync(proxyLogPath, logLine, { append: true }); + } catch (e) { + console.error(`Failed to write to proxy log: ${e}`); + } + }), + ); + const FileLoggingLive = Layer.mergeAll( + Logger.minimumLogLevel(minLogLevel), + fileLogger, + ) as Layer.Layer; + + // Provide the file logger to the test harness (the proxy) + const h = yield* makeTestHarness(activeConfig, FileLoggingLive); + + const port = new URL(h.proxyUrl).port; + + // Prove the file logger works in this process (writes to herald-proxy.log) + yield* Effect.logDebug(`Herald proxy listening on port ${port}`).pipe( + Effect.provide(FileLoggingLive), + ); + + // Parse remaining filtering arguments + const tags = Deno.env.get("S3TEST_TAGS") ?? DEFAULT_TAGS; + const pytestArgsEnv = Deno.env.get("S3TEST_PYTEST_ARGS") ?? ""; + const pytestArgsFromEnv = pytestArgsEnv ? pytestArgsEnv.split(/\s+/) : []; + + const pytestArgs = [...pytestArgsFromEnv, ...pytestArgsFromCli]; + + return yield* (Effect.gen(function* () { + // We use console.log for harness output to avoid them going to the proxy log file + console.log( + `Starting Herald (${colors.cyan(backend)} backend) on port ${ + colors.cyan(port) + }`, + ); + console.log(`Proxy logs: ${colors.gray(proxyLogPath)}`); + + const confContent = `[DEFAULT] +host = 127.0.0.1 +port = ${port} +is_secure = no + +[fixtures] +bucket prefix = herald-${backend}-{random}- + +[s3 main] +user_id = main +display_name = main +email = main@example.com +access_key = minioadmin +secret_key = minioadmin + +[s3 alt] +user_id = alt +display_name = alt +email = alt@example.com +access_key = alt +secret_key = alt + +[s3 tenant] +user_id = tenant +display_name = tenant +email = tenant@example.com +access_key = tenant +secret_key = tenant +tenant = testx + +[iam] +email = iam@example.com +user_id = iam +access_key = iam +secret_key = iam +display_name = iam + +[iam root] +access_key = iam_root +secret_key = iam_root +user_id = iam_root +email = iam_root@example.com + +[iam alt root] +access_key = iam_alt_root +secret_key = iam_alt_root +user_id = iam_alt_root +email = iam_alt_root@example.com +`; + + const confPath = yield* Effect.promise(() => + Deno.makeTempFile({ suffix: ".conf" }) + ); + yield* Effect.promise(() => Deno.writeTextFile(confPath, confContent)); + + const logName = backend === "swift" ? "s3-tests-swift.log" : "s3-tests.log"; + const logPath = path.join(s3TestsDir, logName); + + console.log(`s3-tests directory: ${colors.gray(s3TestsDir)}`); + console.log(`Log file: ${colors.gray(logPath)}`); + + // Ensure we have a virtual environment + const venvPath = path.join(s3TestsDir, ".venv"); + const venvExists = yield* Effect.tryPromise(() => + Deno.stat(venvPath).then(() => true).catch(() => false) + ); + + if (!venvExists) { + console.log(colors.yellow("Creating Python virtual environment...")); + yield* Effect.tryPromise(() => $`uv venv --python 3.11`.cwd(s3TestsDir)); + } + + // Register finalizer to clean up conf file + yield* Effect.addFinalizer(() => + Effect.tryPromise({ + try: () => + Deno.remove(confPath).catch((e) => { + console.error(`Failed to remove conf file ${confPath}: ${e}`); + }), + catch: (e) => new Error(`Effect.tryPromise failed: ${e}`), + }).pipe(Effect.orDie) + ); + + // Ensure dependencies are installed + const pytestCheck = yield* Effect.tryPromise({ + try: async () => { + const proc = $`uv run pytest --version`.cwd(s3TestsDir).noThrow(); + return await proc; + }, + catch: () => new Error("Check failed"), + }); + + if (pytestCheck.code !== 0) { + console.log(colors.yellow("Installing s3-tests dependencies...")); + yield* Effect.tryPromise({ + try: async () => { + await $`uv pip install -r requirements.txt`.cwd(s3TestsDir); + await $`uv pip install -e .`.cwd(s3TestsDir); + }, + catch: (e) => new Error(`Failed to install dependencies: ${e}`), + }); + } + + console.log( + `Running s3-tests against Herald on port ${colors.cyan(port)}...`, + ); + if (tags) console.log(`${colors.gray("Tags:")} ${tags}`); + if (pytestArgs.length > 0) { + console.log( + `${colors.gray("Additional pytest args:")} ${pytestArgs.join(" ")}`, + ); + } + if (noAbort) { + console.log(colors.yellow("Abort on ERROR disabled (--no-abort)")); + } + + // Build command arguments + const cmdArgs = [ + "-v", + "--tb=short", + ]; + + const junitXmlName = "junit.xml"; + const junitXmlPath = path.join(s3TestsDir, junitXmlName); + cmdArgs.push(`--junit-xml=${junitXmlName}`); + + if (tags) { + cmdArgs.push("-m", tags); + } + + cmdArgs.push(...pytestArgs); + + const logFile = yield* Effect.tryPromise(() => + Deno.open(logPath, { + write: true, + create: true, + truncate: true, + }) + ); + + console.log(`Command: uv run pytest ${cmdArgs.join(" ")}`); + const child = $`uv run pytest ${cmdArgs}` + .cwd(s3TestsDir) + .env({ S3TEST_CONF: confPath, PYTHONUNBUFFERED: "1" }) + .stdout("piped") + .stderr("piped") + .spawn(); + + const sigintHandler = () => { + console.log(colors.yellow("\nReceived SIGINT, shutting down...")); + child.kill("SIGTERM"); + }; + Deno.addSignalListener("SIGINT", sigintHandler); + + const result = yield* Effect.tryPromise({ + try: async () => { + let collectedInfo = ""; + let failedCount = 0; + let errorCount = 0; + let skippedCount = 0; + let lastResultTime = Date.now(); + const seenTests = new Set(); + const failedTests = new Set(); + const errorTests = new Set(); + let currentTestName = ""; + + let shouldAbort = false; + let abortReason = ""; + + const processLine = (line: string) => { + const trimmed = line.trim(); + if (!trimmed) return; + + // Capture test result lines like: + // s3tests/functional/test_s3.py::test_bucket_list_empty PASSED [ 0%] + const resultMatch = trimmed.match( + /^([^\s]+::[^\s]+)\s+(PASSED|FAILED|ERROR|SKIPPED)/, + ); + if (resultMatch) { + const testName = resultMatch[1]; + const status = resultMatch[2]; + const now = Date.now(); + const duration = ((now - lastResultTime) / 1000).toFixed(2); + lastResultTime = now; + currentTestName = testName; + + if (status === "PASSED") { + console.log( + `${colors.green("✓")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + } else if (status === "FAILED") { + if (!seenTests.has(testName)) { + failedCount++; + seenTests.add(testName); + failedTests.add(testName); + } + console.error( + `${colors.red("✗")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + } else if (status === "ERROR") { + if (!seenTests.has(testName)) { + errorCount++; + seenTests.add(testName); + errorTests.add(testName); + } + console.error( + `${colors.red("✗ ERROR:")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + if (!noAbort) { + shouldAbort = true; + abortReason = `ERROR in ${testName}`; + child.kill("SIGTERM"); + } + } else if (status === "SKIPPED") { + skippedCount++; + console.log( + `${colors.yellow("-")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + } + return; + } + + // Also check for ERROR in non-verbose format + const errorMatch = trimmed.match(/^ERROR\s+([^\s]+::[^\s]+)/); + if (errorMatch) { + const testName = errorMatch[1]; + const now = Date.now(); + const duration = ((now - lastResultTime) / 1000).toFixed(2); + lastResultTime = now; + currentTestName = testName; + + if (!seenTests.has(testName)) { + errorCount++; + seenTests.add(testName); + errorTests.add(testName); + } + console.error( + `${colors.red("✗ ERROR:")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + + if (!noAbort) { + shouldAbort = true; + abortReason = `ERROR in ${testName}`; + child.kill("SIGTERM"); + } + return; + } + + // Echo important lines (failures, tracebacks, summaries) + if ( + trimmed.includes("FAILURES") || + trimmed.includes("ERRORS") || + trimmed.includes("short test summary") || + trimmed.startsWith("E ") || // Traceback lines in short format + trimmed.startsWith("> ") || + trimmed.match(/^=+\s*(passed|failed|error)/i) + ) { + const prefix = currentTestName + ? colors.gray(`[${currentTestName}] `) + : ""; + console.log(`${prefix}${trimmed}`); + } + }; + + const decoder = new TextDecoder(); + + async function streamToLogAndConsole( + stream: ReadableStream, + ) { + const reader = stream.getReader(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + try { + await logFile.write(value); + } catch (e) { + console.error(`Failed to write to log file: ${e}`); + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + processLine(line); + } + } + } catch (e) { + if (!(e instanceof Deno.errors.Interrupted)) { + console.error(`Stream error: ${e}`); + } + } finally { + if (buffer) { + processLine(buffer); + } + reader.releaseLock(); + } + } + + const [procResult] = await Promise.allSettled([ + child, + streamToLogAndConsole(child.stdout()), + streamToLogAndConsole(child.stderr()), + ]); + + Deno.removeSignalListener("SIGINT", sigintHandler); + + const exitCode = procResult.status === "fulfilled" + ? procResult.value.code + : 1; + + // Attempt to parse JUnit XML if it exists and is valid + let junitData: { + tests: number; + failures: number; + errors: number; + skipped: number; + time?: number; + failedNames: string[]; + errorNames: string[]; + } | null = null; + + try { + const junitXml = await Deno.readTextFile(junitXmlPath); + const getAttr = (name: string) => { + const match = junitXml.match(new RegExp(`${name}="([\\d.]+)"`)); + return match ? parseFloat(match[1]) : 0; + }; + + const failedNames: string[] = []; + const errorNames: string[] = []; + + const testcaseMatches = junitXml.matchAll( + /]*>([\s\S]*?)<\/testcase>/g, + ); + for (const match of testcaseMatches) { + const fullName = `${match[1]}::${match[2]}`; + const content = match[3]; + if (content.includes(" 0) ? junitData : { + tests: seenTests.size, + failures: failedCount, + errors: errorCount, + skipped: skippedCount, + time: undefined, + failedNames: Array.from(failedTests), + errorNames: Array.from(errorTests), + }; + + return { + code: exitCode, + counts: finalCounts, + collectedInfo, + shouldAbort, + abortReason, + }; + }, + catch: (e) => new Error(`Failed to run pytest: ${e}`), + }); + + if (result.collectedInfo) { + console.log(colors.gray(result.collectedInfo)); + } + + const { tests, failures, errors, skipped, time, failedNames, errorNames } = + result.counts; + const passed = tests - failures - errors - skipped; + + console.log(); + const durationStr = time ? ` ${colors.cyan(`${time.toFixed(2)}s`)}` : ""; + console.log( + `${colors.bold(tests.toString())} tests completed in${durationStr}:`, + ); + console.log( + ` ${colors.green("successes")}: ${ + colors.bold(passed.toString()) + }/${tests}`, + ); + console.log( + ` ${colors.red("failures")}: ${ + colors.bold(failures.toString()) + }/${tests}`, + ); + if (errors > 0) { + console.log( + ` ${colors.red("errors")}: ${ + colors.bold(errors.toString()) + }/${tests}`, + ); + } + if (skipped > 0) { + console.log( + ` ${colors.gray("skipped")}: ${ + colors.bold(skipped.toString()) + }/${tests}`, + ); + } + + if (failedNames.length > 0) { + console.log(colors.red("\nFailures:")); + for (const name of failedNames) { + console.log(` ${colors.red("-")} ${name}`); + } + } + + if (errorNames.length > 0) { + console.log(colors.red("\nErrors:")); + for (const name of errorNames) { + console.log(` ${colors.red("-")} ${name}`); + } + } + + if (errors > 0 || (result.shouldAbort && result.abortReason)) { + if (result.shouldAbort) { + yield* Effect.fail( + new Error( + `Aborted due to ERROR: ${result.abortReason || "Test Error"}`, + ), + ); + } else { + yield* Effect.fail(new Error(`s3-tests finished with errors.`)); + } + } + + if (failures > 0 || result.code !== 0) { + yield* Effect.fail( + new Error(`s3-tests finished with failures (code ${result.code}).`), + ); + } + + console.log(colors.green(`\n✓ s3-tests completed successfully.`)); + }).pipe( + Effect.provide(Logger.minimumLogLevel(minLogLevel)), + )); +}); + +if (import.meta.main) { + // Add a global unhandled rejection handler to catch stray promises + globalThis.addEventListener("unhandledrejection", (e) => { + // Suppress Interrupted errors - these happen when requests/streams are aborted + if (e.reason instanceof Deno.errors.Interrupted) { + e.preventDefault(); + return; + } + console.error(colors.red(`Unhandled rejection: ${e.reason}`)); + }); + + Effect.runPromiseExit(program.pipe(Effect.scoped)).then((exitCode) => { + if (exitCode._tag === "Failure") { + console.error( + colors.red(`Fatal error: ${JSON.stringify(exitCode.cause, null, 2)}`), + ); + Deno.exit(1); + } + }).catch((e) => { + console.error(colors.red(`Unhandled error: ${e}`)); + Deno.exit(1); + }); +} diff --git a/x/snapdiff.ts b/x/snapdiff.ts new file mode 100755 index 0000000..0c87cb3 --- /dev/null +++ b/x/snapdiff.ts @@ -0,0 +1,110 @@ +#!/usr/bin/env -S deno run --allow-all + +import * as path from "@std/path"; +import { colors } from "cliffy/ansi/colors.ts"; +import { diff } from "jest-diff"; + +async function main() { + const snapFiles: string[] = []; + + async function walk(dir: string) { + for await (const entry of Deno.readDir(dir)) { + if (entry.isDirectory) { + await walk(path.join(dir, entry.name)); + } else if (entry.name.endsWith(".snap")) { + snapFiles.push(path.join(dir, entry.name)); + } + } + } + + try { + await walk("tests"); + } catch { + console.log(colors.red("No snapshots found. Run tests first.")); + Deno.exit(1); + } + + if (snapFiles.length === 0) { + console.log(colors.yellow("No test snapshots found.")); + return; + } + + console.log( + colors.bold( + `\nComparing Baseline (MinIO) vs Proxy (Herald) snapshots...\n`, + ), + ); + + let diffCount = 0; + + for (const snapFile of snapFiles) { + // Import the snap file as a module + const module = await import("file://" + path.resolve(snapFile)); + const snapshots = module.snapshot; + + const testNames = new Set(); + for (const key of Object.keys(snapshots)) { + const match = key.match(/^(Baseline|Proxy)\/(.+) (metadata|body) \d+$/); + if (match) { + testNames.add(match[2]); + } + } + + const sortedTestNames = Array.from(testNames).sort(); + + for (const testName of sortedTestNames) { + let testHasDiff = false; + + for (const component of ["metadata", "body"]) { + const baselineKey = `Baseline/${testName} ${component} 1`; + const proxyKey = `Proxy/${testName} ${component} 1`; + + const baselineVal = snapshots[baselineKey]; + const proxyVal = snapshots[proxyKey]; + + if (baselineVal === undefined || proxyVal === undefined) { + continue; + } + + const d = diff(baselineVal, proxyVal, { + expand: true, + aAnnotation: `Baseline ${component}`, + bAnnotation: `Proxy ${component}`, + }); + + if ( + d !== null && + !d.includes("Compared values have no visual difference.") + ) { + console.log(colors.red(`[DIFF] ${testName} (${component})`)); + console.log(d); + testHasDiff = true; + } + } + + if (testHasDiff) { + console.log("\n" + "=".repeat(80) + "\n"); + diffCount++; + } else { + console.log(colors.green(`[MATCH] ${testName}`)); + } + } + } + + if (diffCount > 0) { + console.log( + colors.red( + `\nFound ${diffCount} tests with differences between Baseline and Proxy.`, + ), + ); + } else { + console.log(colors.green("\nAll Baseline and Proxy snapshots match!")); + } +} + +if (import.meta.main) { + main().catch((e) => { + console.error(colors.red(`Error: ${e}`)); + Deno.exit(1); + }); +} diff --git a/x/utils.ts b/x/utils.ts new file mode 100644 index 0000000..56f54bd --- /dev/null +++ b/x/utils.ts @@ -0,0 +1,23 @@ +import { $ as old$, CommandBuilder } from "@david/dax"; + +/** + * This assumes that the script is run from the x/ directory or via deno run + */ +export const $ = Object.assign( + old$.build$({ + commandBuilder: new CommandBuilder() + .cwd(old$.path(import.meta.resolve("../")).dirname()) + .printCommand(true), + extras: { + relativeDir(path: string) { + return $.path(import.meta.resolve(path)).dirname(); + }, + }, + }), + { + argv: Deno.args, + env: Deno.env.toObject(), + }, +); + +export const DOCKER_CMD = Deno.env.get("DOCKER_CMD") ?? "docker";