diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..775bcd1 --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +HERALD_SWIFTTEST_OS_AUTH=$TF_VAR_OS_PASSWORD +HERALD_SWIFTTEST_OS_PASSWORD=$TF_VAR_OS_PASSWORD +HERALD_SWIFTTEST_OS_PROJECT_NAME=$TF_VAR_OS_PROJECT_NAME +HERALD_SWIFTTEST_OS_USERNAME=$TF_VAR_OS_USERNAME +HEARLD_SWIFTTEST_AUTH_URL=https://api.pub1.infomaniak.cloud/identity/v3 +HEARLD_SWIFTTEST_OS_REGION_NAME=dc3-a diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 1d15299..c5c85ee 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -5,16 +5,16 @@ on: branches: - main paths: - - 'src/**' - - 'tools/**' - - '.github/workflows/build-image.yml' + - "src/**" + - "tools/**" + - ".github/workflows/build-image.yml" pull_request: branches: - main paths: - - 'src/**' - - 'tools/**' - - '.github/workflows/build-image.yml' + - "src/**" + - "tools/**" + - ".github/workflows/build-image.yml" workflow_dispatch: env: diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..13c559e --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,94 @@ +name: checks + +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + +env: + DOCKER_CMD: docker + UV_CACHE_DIR: /tmp/.uv-cache + +jobs: + checks: + runs-on: ubuntu-latest + env: + HERALD_SWIFTTEST_OS_USERNAME: ${{ secrets.OPENSTACK_USERNAME }} + HERALD_SWIFTTEST_OS_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD }} + HERALD_SWIFTTEST_OS_PROJECT_NAME: ${{ secrets.OPENSTACK_PROJECT }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v16 + + - name: Set up Nix cache + uses: DeterminateSystems/magic-nix-cache-action@v9 + + - name: Run pre-commit hooks via prek + run: nix develop --command prek run --all-files + + - name: Cache Deno + uses: actions/cache@v4 + with: + path: ~/.cache/deno + key: ${{ runner.os }}-deno-${{ hashFiles('deno.lock') }} + restore-keys: | + ${{ runner.os }}-deno- + + - name: Restore uv cache + uses: actions/cache@v5 + with: + path: /tmp/.uv-cache + key: uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }} + restore-keys: | + uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }} + uv-${{ runner.os }} + + - name: Start services + run: nix develop --command deno run --allow-all x/compose-up.ts s3 db + + - name: Wait for MinIO + run: | + for i in {1..30}; do + if curl -f http://localhost:9000/minio/health/live; then + echo "MinIO is ready" + exit 0 + fi + echo "Waiting for MinIO..." + sleep 2 + done + echo "MinIO failed to start" + exit 1 + + - name: Integration tests + run: nix develop --command deno task test + + - name: S3 Compatibility (MinIO) + run: nix develop --command deno run --allow-all x/s3-tests.ts --backend minio + + - name: S3 Compatibility (Swift) + if: env.HERALD_SWIFTTEST_OS_USERNAME != '' + env: + HERALD_SWIFTTEST_OS_REGION_NAME: dc3-a + HERALD_SWIFTTEST_AUTH_URL: https://api.pub1.infomaniak.cloud/identity/v3 + run: nix develop --command deno run --allow-all x/s3-tests.ts --backend swift + + - name: Minimize uv cache + run: nix develop --command uv cache prune --ci + + - name: Dump logs on failure + if: failure() + run: | + echo "--- s3-tests/s3-tests.log ---" + cat s3-tests/s3-tests.log || true + echo "--- s3-tests/herald-proxy.log ---" + cat s3-tests/herald-proxy.log || true + echo "--- s3-tests/herald-proxy-swift.log ---" + cat s3-tests/herald-proxy-swift.log || true diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index c878fb7..0000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: pre-commit - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - workflow_dispatch: - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install Nix - uses: DeterminateSystems/nix-installer-action@main - - - name: Set up Nix cache - uses: DeterminateSystems/magic-nix-cache-action@main - - - name: Run pre-commit hooks via prek - run: nix develop --command prek run --all-files diff --git a/.github/workflows/release-request.yml b/.github/workflows/release-request.yml deleted file mode 100644 index e11d30b..0000000 --- a/.github/workflows/release-request.yml +++ /dev/null @@ -1,159 +0,0 @@ -name: Prepare Release - -on: - workflow_dispatch: - push: - branches: - - main - -jobs: - check-version: - name: Check Commitizen Version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.version.outputs.version }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Configure Git - run: | - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - - - name: Get current version (without bumping or pushing) - id: version - uses: commitizen-tools/commitizen-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - push: false - dry_run: true - changelog: false - - prepare-release-pr: - name: Create Release Branch and PR - needs: check-version - if: ${{ needs.check-version.outputs.version != '' && github.ref == 'refs/heads/main' }} - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: main - - - name: Bump version using Commitizen - id: cz - uses: commitizen-tools/commitizen-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - git_name: ${{ github.actor }} - git_email: ${{ github.actor }}@users.noreply.github.com - push: false - changelog: true - dry_run: false - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v8 - with: - title: "Release ${{ steps.cz.outputs.version }}" - body: "Automated PR for version bump to ${{ steps.cz.outputs.version }}" - branch: "release-v${{ steps.cz.outputs.version }}" - delete-branch: true - - check-release: - runs-on: ubuntu-latest - # if: github.ref == 'refs/heads/main' && github.event_name == 'push' - outputs: - release: ${{ steps.check.outputs.release }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: main - - - name: Configure Git - run: | - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - - - name: Get current version - id: version - run: | - VERSION=$(yq '.commitizen.version' .cz.yaml) - echo "version=$VERSION" >> $GITHUB_OUTPUT - - - name: Check if GitHub release already exists - id: check - run: | - VERSION=${{ steps.version.outputs.version }} - echo "Detected version: $VERSION" - - RELEASE_EXISTS=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - https://api.github.com/repos/${{ github.repository }}/releases/tags/v$VERSION \ - | jq -r '.tag_name // empty') - - if [[ "$RELEASE_EXISTS" == "v$VERSION" ]]; then - echo "Release v$VERSION already exists." - echo "release=" >> $GITHUB_OUTPUT - else - echo "Release v$VERSION does not exist yet." - echo "release=$VERSION" >> $GITHUB_OUTPUT - fi - finalize-release: - name: Finalize Release - needs: check-release - if: ${{ needs.check-release.outputs.release != '' }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Tag and Push - run: | - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - git tag -a "v${{ needs.check-release.outputs.release }}" -m "Release v${{ needs.check-release.outputs.release }}" - git push origin "v${{ needs.check-release.outputs.release }}" - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - tag_name: "v${{ needs.check-release.outputs.release }}" - name: "Release v${{ needs.check-release.outputs.release }}" - body_path: "CHANGELOG.md" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - build-docker: - name: Build and Push Docker - needs: check-release - if: ${{ needs.check-release.outputs.release != '' }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and Push Docker - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ghcr.io/${{ github.repository_owner }}/herald:v${{ needs.check-release.outputs.release }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index f09fed5..0000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: test suite -run-name: test suite for ${{ github.event.pull_request.title || github.ref }} -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - types: - - opened - - reopened - - synchronize - - ready_for_review - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - DENO_V: 2.3.5 - GHJK_VERSION: "v0.3.2" - GHJK_ENV: "ci" - -jobs: - changes: - runs-on: ubuntu-latest - permissions: - pull-requests: read - steps: - - uses: actions/checkout@v6 - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - full: - - '.github/workflows/tests.yml' - - 'src/**' - - 'tests/**' - - 'examples/**' - outputs: - full: ${{ steps.filter.outputs.full }} - - pre-commit: - needs: changes - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - uses: denoland/setup-deno@v2 - with: - deno-version: ${{ env.DENO_V }} - - name: Install tofu - run: | - curl --proto '=https' --tlsv1.2 -fsSL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh - chmod +x install-opentofu.sh - ./install-opentofu.sh --install-method deb - rm -f install-opentofu.sh - - - shell: bash - run: | - python -m pip install --upgrade pip - pip install pre-commit - pre-commit install - deno --version - pre-commit run --all-files - - test-full: - needs: [changes] - if: ${{ needs.changes.outputs.full == 'true' && github.event.pull_request.draft == false }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: denoland/setup-deno@v2 - with: - deno-version: ${{ env.DENO_V }} - - name: Download Install Script - run: curl -fsSL "https://raw.github.com/metatypedev/ghjk/$GHJK_VERSION/install.sh" -o install.sh - - name: Execute Install Script - run: yes | bash install.sh - - run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - run: echo "BASH_ENV=$HOME/.local/share/ghjk/env.sh" >> "$GITHUB_ENV" - - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - name: Install tofu - run: | - curl --proto '=https' --tlsv1.2 -fsSL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh - chmod +x install-opentofu.sh - ./install-opentofu.sh --install-method deb - rm -f install-opentofu.sh - - uses: actions/setup-node@v6 - with: - node-version: 18 - - name: setup start-server-and-test - run: npm install -g start-server-and-test - - shell: bash - env: - AUTH_TYPE: "default" - LOG_LEVEL: "DEBUG" - ENV: "DEV" - S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} - S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} - OPENSTACK_USERNAME: ${{ secrets.OPENSTACK_USERNAME }} - OPENSTACK_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD }} - OPENSTACK_PROJECT: ${{ secrets.OPENSTACK_PROJECT }} - AWS_ACCESS_KEY_ID: ${{ secrets.OPENSTACK_USERNAME }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.OPENSTACK_PASSWORD }} - run: | - # run all tests - deno --version - ghjk x dev-compose s3 - sleep 20 - - deno install - - # ghjk x setup-auth - npx start-server-and-test 'deno serve -A --unstable-kv src/main.ts' http://0.0.0.0:8000/ 'deno test -A' diff --git a/.gitignore b/.gitignore index c8be0ce..7f896dd 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,4 @@ token *.db-shm *.db-wal .vscode +symlinks diff --git a/.infisical.json b/.infisical.json new file mode 100644 index 0000000..a6af04a --- /dev/null +++ b/.infisical.json @@ -0,0 +1,5 @@ +{ + "workspaceId": "39bbe4e4-20c2-42fa-8a6c-1bcafcc74faf", + "defaultEnvironment": "", + "gitBranchToEnvironmentMapping": null +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b600640..96b71d2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v6.0.0 hooks: - id: check-added-large-files exclude: tests/res @@ -39,9 +39,14 @@ repos: types: - ts - repo: https://github.com/tofuutils/pre-commit-opentofu - rev: v1.0.3 + rev: v2.2.2 hooks: - id: tofu_fmt + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.36.0 + hooks: + - id: check-dependabot + - id: check-github-workflows # - repo: https://github.com/shellcheck-py/shellcheck-py # rev: v0.10.0.1 # hooks: diff --git a/AGENTS.md b/AGENTS.md index b3db2d5..4410833 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,15 @@ - We're using the effects library https://effect.website/llms.txt - Their HTTP implementation is described in ./HTTP_PLATFORM.md + - **ALWAYS** use `@effect/platform/HttpClient` instead of native `fetch` for + all HTTP requests. - Prefer generators over effect piping. - Use methods on `Effect.Option` like `Option.isNone` instead of looking at _tag. + - **NEVER** use standard `try/catch` or `try/finally` blocks around `yield*` + in Effect generators. Use `Effect.addFinalizer`, `Effect.try`, + `Effect.catchAll`, or `Effect.orElse`. + - **ALWAYS** use the `Config` module from Effect for environment variable + access instead of `Deno.env.get`. - **NEVER** assume default values using `??` or ternary operators for critical configuration or external input (e.g., `bucket.region ?? "us-east-1"`, `request.headers.host ?? "localhost"`). Always fail explicitly with a diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96c5f88..219781a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,8 +5,8 @@ - `src/Domain`: Core logic and data models. Contains Effect Schemas for global configuration and logic for bucket matching. -- `src/Config`: Application configuration loading. Defines the AppConfig service - layer. +- `src/Config`: Application configuration loading. Defines the HeraldConfig + service layer. - `src/Services`: Shared service abstractions and implementations. diff --git a/README.md b/README.md index 12531c4..cf44782 100644 --- a/README.md +++ b/README.md @@ -18,307 +18,65 @@


-## Table of Contents +Herald is an S3 proxy that supports: -- [ Overview](#Overview) -- [ Features](#Features) -- [ Project Structure](#Project-Structure) -- [ Getting Started](#Getting-Started) - - [ Prerequisites](#Prerequisites) - - [ Development](#Development) - - [ herald.yaml config file](#herald.yaml-config-file) - - [ Environment Variables](#Environment-Variables) - - [ Usage](#Usage) - - [ Testing](#Testing) -- [ Project Roadmap](#Project-Roadmap) -- [ Contributing](#Contributing) -- [ Acknowledgments](#Acknowledgments) +- Protocol translation (S3 to S3, S3 to Swift). +- Backend routing based on bucket names. +- Flexible bucket mapping with glob support. ---- +## Config -## Overview - -Herald is an S3 proxy that allows communication to multiple storage services with different communication protocols using the S3 protocol. For instance, you can use herald to connect to an OpenStack swift storage service as you would to S3 storage services like AWS S3 and MinIO. While OpenStack has its own middleware to accept requests in S3 protocol, herald addresses some issues you will face using that S3 middleware. Currently, herald supports two types of backends(storage providers): S3 and OpenStack Swift. A comprehensive list of herald's features and capabilities are listed below. - -## Features - -- Multi Backend Compatibility: interacting with multiple storage backends using a single protocol. You can use the S3 protocol to communicate with storage services that don't necessarily support an S3 protocol natively. Herald supports S3 and OpenStack Swift Storage backends as of right now. -- Kubernetes Native Authentication: herald supports authentication using a service account provisioned by the Kubernetes server. You can register services in your cluster that can access herald and the resources managed by herald to allow a robust authentication pipeline easy to manage. -- Mirroring: a multi-backend commit is another handy feature in herald that allows you to mirror any operations you make to your storage services through herald. You can have a primary storage configured with replicas. The primary and replicas will be in sync and during times of unavailability, herald will use the replicas to fetch data. For write operations, after the operation has been completed, it will be mirrored to a replica storage. The mirroring is done using transactional tasks, stored in a message/task queue, which ensures the replicas are in sync. For read operations, the operation will be forwarded to replica storages if the primary is unavailable. The mirroring operations are done using web workers to avoid the impact on performance. -- IaC Support: Herald supports IaC tech stacks that use the S3 protocol to provision resources such as Terraform and OpenTofu. - -## Project Structure - -```sh -└── herald/ - ├── .github - │ ├── dependabot.yml - │ ├── pull_request_template.md - │ └── workflows - ├── Dockerfile - ├── LICENSE.md - ├── README.md - ├── benchmarks - │ ├── bench_saver.ts - │ ├── result.json - │ └── sdk - ├── deno.jsonc - ├── deno.lock - ├── docker-compose.yml - ├── examples - │ └── simple-bucket-test - ├── ghjk.ts - ├── herald-compose.yaml - ├── herald.yaml - ├── import_map.json - ├── src - │ ├── auth - │ ├── backends - │ ├── buckets - │ ├── config - │ ├── constants - │ ├── main.ts - │ ├── types - │ ├── utils - │ └── workers - ├── tests - │ ├── iac - │ ├── mirror - │ ├── s3 - │ ├── swift - │ └── utils - ├── tools - │ ├── compose - │ ├── deps.ts - │ └── s3-comparison - └── utils - ├── file.ts - └── s3.ts -```` - ---- - -## Getting Started - -### Prerequisites - -Before getting started with herald, ensure your runtime environment meets the following requirements: - -- **Programming Language:** TypeScript -- **Container Runtime:** Docker - -### Development - -Install herald using one of the following methods: - -**Build from source:** - -1. Clone the herald repository: - -```sh -❯ git clone https://github.com/expnt/herald -``` - -2. Navigate to the project directory: - -```sh -❯ cd herald -``` - -3. Install ghjk - -[ghjk](https://github.com/metatypedev/ghjk) is a developer environment management tool used to install dependencies required to run herald. - -4. Install dependencies - -```sh -❯ ghjk p resolve -``` - -5. Run services needed for herald. - -We just spin a minio s3 server and a swift object storage container in docker. - -```sh -❯ ghjk dev-compose up all -``` - -6. Configure herald.yaml - -Configuration for the cloud services that herald connects with are defined here. Other configs such as the port it runs on, temporary dir for tests is also defined here. The object storage for a task store is also defined here. A serialized task store is saved in the specified storage service where durable tasks for mirroring tasks are stored. Service account names are also configured for jwk based authentication. This is a sample configuration file. +Herald is configured via a YAML file (typically `herald.yaml`). The +configuration defines backends and how incoming requests are routed to them. ```yaml -port: 8000 -temp_dir: "./tmp" backends: - minio_s3: - protocol: s3 - openstack_swift: - protocol: swift - exoscale_s3: + # Unique identifier for the backend + minio: + # Backend protocol: "s3" or "swift" protocol: s3 -task_store_backend: - endpoint: "http://localhost:9000" - region: local - forcePathStyle: true - bucket: task-store - credentials: - accessKeyId: minio - secretAccessKey: password - -service_accounts: - - name: "system:serviceaccount:dev-s3-herald:default" - buckets: - - s3-test - - s3-mirror-test - - swift-mirror-test - - iac-s3 - - name: "system:serviceaccount:stg-datacycle:datacycle-app-backend-sa" - buckets: - - s3-test - - s3-mirror-test - - iac-s3 - -buckets: - s3-test: - backend: minio_s3 - config: - endpoint: "http://localhost:9000" - region: local - forcePathStyle: true - bucket: s3-test - credentials: - accessKeyId: minio - secretAccessKey: password - -replicas: - - name: replica-0 - backend: minio_s3 - config: - endpoint: "http://localhost:9090" - region: local - forcePathStyle: true - bucket: s3-test - credentials: - accessKeyId: minio - secretAccessKey: password -``` - - -7. Run herald - -```sh -❯ deno run src/main.ts -``` - -### herald.yaml config file + # Base URL of the backend service + endpoint: http://127.0.0.1:9000 -This configuration YAML file is used to set up and manage the Herald service, which interacts with various storage backends and service accounts. Below is a detailed description of the key sections in the file: + # Default region for this backend + region: us-east-1 -- **port**: Specifies the port number (8000) on which herald will run. -- **temp_dir**: Defines the temporary directory (`./tmp`) used by the service. -- **backends**: Lists the supported storage backends, including `minio_s3`, `openstack_swift`, and `exoscale_s3`, each with its respective protocol. -- **task_store_backend**: Configures the backend for storing tasks, including the endpoint, region, path style, bucket name, and credentials (access key ID and secret access key). -- **service_accounts**: Defines the service accounts with access to specific buckets. Each service account has a name and a list of accessible buckets. -- **buckets**: Specifies the configuration for individual buckets, including the backend type, endpoint, region, path style, bucket name, and credentials. -- **replicas**: Configures replicas for redundancy and load balancing. Each replica has a name, backend type, and configuration similar to the buckets section. + # Authentication credentials for the backend + credentials: + accessKeyId: minioadmin + secretAccessKey: minioadmin -This configuration file allows for flexible and secure management of storage resources and service accounts, ensuring that the Herald service can interact with multiple storage backends and maintain high availability through replicas. - -### Environment Variables -| **Name** | **Default** | **Description** | -|----------------------------|------------------------------|-------------------------------| -| **debug** | — | Boolean string that enables or disables debug mode. | -| **log_level** | (optional) | Logging level; possible values: `NOTSET`, `DEBUG`, `INFO`, `WARN`, `ERROR`, `CRITICAL`. | -| **env** | `DEV` | Environment in which the application runs (`DEV` or `PROD`). | -| **k8s_api** | `https://kubernetes.default.svc` | URL for the Kubernetes API. | -| **cert_path** | `/var/run/secrets/kubernetes.io/serviceaccount/ca.crt` | File path to the Kubernetes service account CA certificate. | -| **config_file_path** | herald.yaml | Path to the Herald configuration file. | -| **service_account_token_path** | `/var/run/secrets/kubernetes.io/serviceaccount/token` | File path to the Kubernetes service account token. | -| **version** | `0.1` | Application version. | -| **sentry_dsn** | (optional) | DSN for Sentry, used for error tracking. | -| **sentry_sample_rate** | `1` | Sampling rate for Sentry events (numeric, 0 to 1). | -| **sentry_traces_sample_rate** | `1` | Sampling rate for Sentry traces (numeric, 0 to 1). | - -### Run using docker - -Pull the image first - -```sh -❯ docker pull ghcr.io/expnt/herald:latest -``` - -Run herald using the following command: -**Using `docker`**   [](https://www.docker.com/) - -```sh -❯ docker run -it expnt/herald:latest -``` - -### Testing - -To run full tests, - -```sh -❯ deno test -A tests + # Bucket routing rules. + # Can be: + # 1. "*" to match all buckets not claimed by other backends + # 2. A glob pattern like "logs-*" + # 3. A map of bucket definitions for granular control + buckets: + # Simple bucket mapping (inherits backend settings) + my-bucket: {} + + # Mapping with overrides + external-data: + # Map proxy bucket "external-data" to backend bucket "data-v1" + bucket_name: data-v1 + # Override endpoint for this specific bucket + endpoint: http://special-endpoint:9000 + # Override region + region: us-west-2 + + # Glob pattern support within the map + "test-*": + region: us-east-1 ``` ---- - -## Project Roadmap - -- [x] **`Task 1`**: Mirroring -- [ ] **`Task 2`**: Event Notification. -- [ ] **`Task 3`**: Advanced Cache Policy - ---- - -## Contributing - -- **💬 [Join the Discussions](https://github.com/expnt/herald/discussions)**: Share your insights, provide feedback, or ask questions. -- **🐛 [Report Issues](https://github.com/expnt/herald/issues)**: Submit bugs found or log feature requests for the `herald` project. -- **💡 [Submit Pull Requests](https://github.com/expnt/herald/blob/main/CONTRIBUTING.md)**: Review open PRs, and submit your own PRs. - -
-Contributing Guidelines - -1. **Fork the Repository**: Start by forking the project repository to your github account. -2. **Clone Locally**: Clone the forked repository to your local machine using a git client. - ```sh - git clone https://github.com/expnt/herald - ``` -3. **Create a New Branch**: Always work on a new branch, giving it a descriptive name. - ```sh - git checkout -b new-feature-x - ``` -4. **Make Your Changes**: Develop and test your changes locally. -5. **Commit Your Changes**: Commit with a clear message describing your updates. - ```sh - git commit -m 'Implemented new feature x.' - ``` -6. **Push to github**: Push the changes to your forked repository. - ```sh - git push origin new-feature-x - ``` -7. **Submit a Pull Request**: Create a PR against the original project repository. Clearly describe the changes and their motivations. -8. **Review**: Once your PR is reviewed and approved, it will be merged into the main branch. Congratulations on your contribution! -
- -
-Contributor Graph -
-

- - - -

-
- ---- - -## Acknowledgments +### Routing Logic -- List any resources, contributors, inspiration, etc. here. +When a request comes in for a bucket (e.g., `GET /my-bucket/file.txt`), Herald +resolves the backend using the following priority: ---- +1. **Direct match**: Looks for `my-bucket` in all backends' `buckets` maps. +2. **Glob match (map)**: Looks for glob patterns (like `test-*`) in all + backends' `buckets` maps. +3. **Glob match (string)**: If a backend has `buckets: "..."`, it checks if the + bucket name matches that pattern. diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..e436adf --- /dev/null +++ b/TODO.md @@ -0,0 +1,133 @@ +# Missing Functionality in Herald3 + +This list represents the S3 functionality that is currently missing in Herald3, +based on a comparison with the `s3-tests` suite and a review of the existing +implementation. + +## 1. Bucket Operations + +- [ ] **Bucket Policies**: Implementation of `GET/PUT/DELETE /?policy`. _(Focus + tests: `test_get_bucket_policy_status`, + `test_post_object_missing_policy_condition`)_ +- [ ] **CORS (Cross-Origin Resource Sharing)**: Implementation of + `GET/PUT/DELETE /?cors` and handling of `OPTIONS` preflight requests. + _(Focus tests: `test_set_cors`, `test_cors_origin_response`, + `test_cors_header_option`)_ +- [ ] **Lifecycle Management**: Implementation of `GET/PUT/DELETE /?lifecycle`. + _(Focus tests: `test_lifecycle_expiration`, `test_lifecycle_transition`)_ +- [ ] **Tagging**: Implementation of `GET/PUT/DELETE /?tagging` for buckets. + _(Focus tests: `test_bucket_tagging_create`, `test_bucket_tagging_get`)_ +- [ ] **Versioning Configuration**: Implementation of `GET/PUT /?versioning`. + (Basic `listVersions` is partially implemented). _(Focus tests: + `test_bucket_list_return_data_versioning`, + `test_versioning_concurrent_multi_object_delete`)_ +- [ ] **ACLs (Access Control Lists)**: Implementation of `GET/PUT /?acl` for + buckets. _(Focus tests: `test_bucket_acl_default`, + `test_put_bucket_acl_grant_group_read`, `test_bucket_header_acl_grants`)_ +- [ ] **Website Configuration**: Implementation of `GET/PUT/DELETE /?website`. + _(Focus tests: `test_website_configuration`, + `test_website_error_document`)_ +- [ ] **Public Access Block**: Implementation of + `GET/PUT/DELETE /?publicAccessBlock`. _(Focus tests: + `test_bucket_public_access_block`)_ +- [ ] **Replication Configuration**: Implementation of + `GET/PUT/DELETE /?replication`. +- [ ] **Notification Configuration (SNS)**: Implementation of + `GET/PUT /?notification`. +- [ ] **Logging Configuration**: Implementation of `GET/PUT /?logging`. _(Focus + tests: `test_bucket_logging_config`)_ +- [ ] **Inventory Configuration**: Implementation of + `GET/PUT/DELETE /?inventory`. +- [ ] **Metrics Configuration**: Implementation of `GET/PUT/DELETE /?metrics`. +- [ ] **Intelligent-Tiering Configuration**: Implementation of + `GET/PUT/DELETE /?intelligent-tiering`. +- [ ] **Ownership Controls**: Implementation of + `GET/PUT/DELETE /?ownershipControls`. + +## 2. Object Operations + +- [ ] **Multi-Object Delete**: Implementation of `POST /?delete`. _(Focus tests: + `test_multi_object_delete`, `test_multi_object_delete_key_limit`)_ +- [ ] **Multipart Upload**: Support for `InitiateMultipartUpload`, `UploadPart`, + `CompleteMultipartUpload`, `AbortMultipartUpload`, and `ListParts`. + _(Focus tests: `test_multipart_upload`, `test_multipart_upload_empty`, + `test_abort_multipart_upload`)_ +- [ ] **GetObject Attributes**: Implementation of `GET /bucket/key?attributes`. + _(Focus tests: `test_get_object_attributes`)_ +- [ ] **HeadObject Consistency**: Fix `404 Not Found` errors on existing objects + during certain test sequences. _(Focus tests: + `test_object_head_zero_bytes`)_ +- [ ] **Unicode Metadata**: Fix support for non-ASCII characters in object + metadata. _(Focus tests: `test_object_set_get_unicode_metadata`)_ +- [ ] **Copy Object**: Support for `PUT` with `x-amz-copy-source` header. + _(Focus tests: `test_object_copy`)_ +- [ ] **Tagging**: Implementation of `GET/PUT/DELETE /?tagging` for objects. + _(Focus tests: `test_object_tagging`)_ +- [ ] **ACLs (Access Control Lists)**: Implementation of `GET/PUT /?acl` for + objects. _(Focus tests: `test_object_acl_default`, `test_object_acl_read`, + `test_object_put_acl_mtime`)_ +- [ ] **Legal Hold & Retention**: Implementation of `GET/PUT /?legal-hold` and + `GET/PUT /?retention` (Object Lock). +- [ ] **Object Lock Configuration**: Implementation of `GET/PUT /?object-lock` + on objects. +- [ ] **S3 Select**: Implementation of `POST /?select&select-type=2`. +- [ ] **Checksums**: Support for `x-amz-checksum-sha1`, `x-amz-checksum-sha256`, + `x-amz-checksum-crc32`, and `x-amz-checksum-crc32c`. +- [ ] **Server-Side Encryption (SSE)**: Handling of + `x-amz-server-side-encryption`, + `x-amz-server-side-encryption-customer-algorithm`, etc. +- [ ] **Restore Object**: Support for `POST /?restore`. + +## 3. Authentication & IAM + +- [ ] **IAM Integration**: Full implementation of IAM policy evaluation for all + requests. +- [ ] **User Policies**: Support for user-specific IAM policies. +- [ ] **Security Token Service (STS)**: Implementation of `GetSessionToken`, + `AssumeRole`, etc. +- [ ] **Web Identity Federation**: Implementation of + `AssumeRoleWithWebIdentity`. + +## 4. Validation, Errors & Protocol + +- [ ] **Bucket Naming Validation**: Implement strict S3 naming rules (no IP + addresses, no double dots, length 3-63, etc.). Currently many naming tests + fail or hang. _(Focus tests: `test_bucket_create_naming_bad_ip`, + `test_bucket_create_naming_dns_dot_dot`, + `test_bucket_create_naming_bad_starts_nonalpha`)_ +- [ ] **Correct Error Codes**: Ensure accurate HTTP status codes for S3 errors + (e.g., return `400 Bad Request` or `403 Forbidden` instead of + `409 Conflict` or `500 Internal Server Error`). _(Focus tests: + `test_bucket_create_exists`, `test_bucket_create_exists_nonowner`, + `test_object_read_not_exist`)_ +- [ ] **Method POST Support**: Fix "Method POST for key [] not implemented" + errors at the bucket root level. _(Focus tests: + `test_multi_object_delete`, `test_post_object_authenticated_request`)_ +- [ ] **Multipart Reliability**: Address `502 Bad Gateway` errors occurring + during `CreateMultipartUpload` and other multipart operations. _(Focus + tests: `test_multipart_upload`)_ +- [ ] **Conditional Requests**: Fix `If-Match`, `If-None-Match`, + `If-Modified-Since`, and `If-Unmodified-Since` behavior. _(Focus tests: + `test_get_object_ifmatch_failed`, `test_get_object_ifnonematch_failed`, + `test_get_object_ifmodifiedsince_failed`)_ +- [ ] **Response Field Completeness**: Ensure expected XML/JSON fields like + `ChecksumSHA256`, `Rules`, `Errors`, and `x-amz-delete-marker` are present + in responses. +- [ ] **Metadata Handling**: Fix incorrect `BucketAlreadyOwnedByYou` errors + being returned on non-create operations (e.g., during `PutBucketPolicy`). + _(Focus tests: `test_bucket_list_return_data`)_ + +## 5. General Compatibility & Compliance + +- [ ] **Strict RFC 2616 Compliance**: Address tests tagged with + `fails_strict_rfc2616`. +- [ ] **S3Proxy Compatibility**: Address tests tagged with `fails_on_s3proxy` to + ensure broader compatibility. +- [ ] **Advanced Header Support**: Comprehensive support for headers like + `Cache-Control`, `Content-Disposition`, `Content-Encoding`, + `Content-Language`, and `Expires`. + +## 5. Non-Standard / Protocol Specific + +- [ ] **Append Object**: Implementation of `appendobject` (often found in + Ceph/RGW). diff --git a/chart/values.yaml b/chart/values.yaml index e55ba44..1691974 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1,4 +1,3 @@ - name: herald namespace: herald @@ -87,9 +86,9 @@ ingress: - path: / pathType: ImplementationSpecific tls: [] - # - secretName: web-tls - # hosts: - # - chart-example.local +# - secretName: web-tls +# hosts: +# - chart-example.local volumeMounts: - name: herald diff --git a/deno.jsonc b/deno.jsonc index c600c50..646ee69 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -30,6 +30,11 @@ "cliffy/ansi/": "https://deno.land/x/cliffy@v1.0.0-rc.3/ansi/" }, "compilerOptions": {}, + "fmt": { + "exclude": [ + "./chart/" + ] + }, "lint": { "exclude": [ "x", diff --git a/deno.lock b/deno.lock index 30451ee..766c865 100644 --- a/deno.lock +++ b/deno.lock @@ -9,6 +9,7 @@ "jsr:@std/assert@^1.0.15": "1.0.16", "jsr:@std/bytes@^1.0.5": "1.0.6", "jsr:@std/fmt@1": "1.0.8", + "jsr:@std/fmt@^1.0.3": "1.0.8", "jsr:@std/fs@1": "1.0.21", "jsr:@std/fs@^1.0.19": "1.0.21", "jsr:@std/fs@^1.0.20": "1.0.21", @@ -25,12 +26,14 @@ "npm:@aws-sdk/client-s3@3": "3.937.0", "npm:@effect/opentelemetry@~0.56.2": "0.56.6_@effect+platform@0.90.10__effect@3.19.14_@opentelemetry+sdk-trace-base@2.3.0__@opentelemetry+api@1.9.0_@opentelemetry+sdk-trace-node@2.3.0__@opentelemetry+api@1.9.0_@opentelemetry+semantic-conventions@1.38.0_effect@3.19.14", "npm:@effect/platform-node@0.96": "0.96.1_@effect+cluster@0.48.16__@effect+platform@0.90.10___effect@3.19.14__@effect+rpc@0.69.5___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+sql@0.44.2___@effect+experimental@0.54.6____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+workflow@0.9.6___@effect+platform@0.90.10____effect@3.19.14___@effect+rpc@0.69.5____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___effect@3.19.14__effect@3.19.14_@effect+platform@0.90.10__effect@3.19.14_@effect+rpc@0.69.5__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_@effect+sql@0.44.2__@effect+experimental@0.54.6___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_effect@3.19.14", + "npm:@effect/platform@*": "0.90.10_effect@3.19.14", "npm:@effect/platform@~0.90.3": "0.90.10_effect@3.19.14", "npm:@opentelemetry/exporter-trace-otlp-http@0.203": "0.203.0_@opentelemetry+api@1.9.0", "npm:@opentelemetry/sdk-trace-base@^2.0.1": "2.3.0_@opentelemetry+api@1.9.0", "npm:@opentelemetry/sdk-trace-node@^2.0.1": "2.3.0_@opentelemetry+api@1.9.0", "npm:@smithy/signature-v4@^4.2.0": "4.2.4", "npm:@smithy/types@^3.7.0": "3.7.2", + "npm:effect@*": "3.19.14", "npm:effect@^3.17.7": "3.19.14", "npm:jest-diff@*": "29.7.0", "npm:jest-diff@^29.7.0": "29.7.0", @@ -47,7 +50,7 @@ "jsr:@david/console-static-text", "jsr:@david/path", "jsr:@david/which", - "jsr:@std/fmt", + "jsr:@std/fmt@1", "jsr:@std/fs@^1.0.20", "jsr:@std/io", "jsr:@std/path@1" diff --git a/flake.nix b/flake.nix index ce8939b..a1e07ab 100644 --- a/flake.nix +++ b/flake.nix @@ -32,7 +32,8 @@ # For systems that do not ship with Python by default (required by `node-gyp`) # python3 - # infisical + infisical + openstack-rs # # opentofu # terragrunt @@ -40,7 +41,9 @@ ]; shellHook = '' export PATH=$PATH:$PWD/x/ - exec $(getent passwd $USER | cut -d: -f7) + if [[ -t 0 ]]; then + exec $(getent passwd $USER | cut -d: -f7) + fi ''; }; diff --git a/ghjk b/ghjk deleted file mode 120000 index 4855491..0000000 --- a/ghjk +++ /dev/null @@ -1 +0,0 @@ -../../rust/ghjk/ \ No newline at end of file diff --git a/herald b/herald deleted file mode 120000 index bfe949e..0000000 --- a/herald +++ /dev/null @@ -1 +0,0 @@ -../herald \ No newline at end of file diff --git a/s3proxy b/s3proxy deleted file mode 120000 index 27ea8bf..0000000 --- a/s3proxy +++ /dev/null @@ -1 +0,0 @@ -../../java/s3proxy/ \ No newline at end of file diff --git a/sample-http b/sample-http deleted file mode 120000 index dfd9d33..0000000 --- a/sample-http +++ /dev/null @@ -1 +0,0 @@ -../sample-http/ \ No newline at end of file diff --git a/sample-rust b/sample-rust deleted file mode 120000 index b7cd242..0000000 --- a/sample-rust +++ /dev/null @@ -1 +0,0 @@ -../../rust/Yohe-Am-backend-1/ \ No newline at end of file diff --git a/src/Api.ts b/src/Api.ts index 2b3afe2..b7a62ec 100644 --- a/src/Api.ts +++ b/src/Api.ts @@ -1,8 +1,11 @@ import { HttpApi, OpenApi } from "@effect/platform"; -import { HealthApi } from "./Frontend/Health/Api.ts"; -import { S3Api } from "./Frontend/Api.ts"; +import { HealthHttpApi } from "./Frontend/Health/Api.ts"; +import { HttpS3Api } from "./Frontend/Api.ts"; -export class Api extends HttpApi.make("api") - .add(HealthApi) - .add(S3Api) +// the http interface is declared first and separately +// and the impl is to adhere to it +// used for openAPI +export class HttpHeraldApi extends HttpApi.make("HeraldHttpApi") + .add(HealthHttpApi) + .add(HttpS3Api) .annotate(OpenApi.Title, "Herald API") {} diff --git a/src/Backends/S3/Backend.ts b/src/Backends/S3/Backend.ts index 9578f27..fc930c7 100644 --- a/src/Backends/S3/Backend.ts +++ b/src/Backends/S3/Backend.ts @@ -1,655 +1,24 @@ -import { Chunk, Effect, Stream } from "effect"; -import { - CreateBucketCommand, - DeleteBucketCommand, - DeleteObjectCommand, - DeleteObjectsCommand, - GetObjectCommand, - HeadBucketCommand, - HeadObjectCommand, - ListBucketsCommand, - type ListBucketsCommandOutput, - ListObjectsCommand, - type ListObjectsCommandOutput, - ListObjectsV2Command, - type ListObjectsV2CommandOutput, - ListObjectVersionsCommand, - PutObjectCommand, -} from "@aws-sdk/client-s3"; +import { Effect } from "effect"; import type { MaterializedBucket } from "../../Domain/Config.ts"; -import { AppConfig } from "../../Config/Layer.ts"; -import { - AccessDenied, - type BackendError, - type BackendService, - BucketAlreadyExists, - BucketAlreadyOwnedByYou, - type BucketInfo, - BucketNotEmpty, - type CommonPrefix, - type DeleteObjectsResult, - InternalError, - type ListObjectsResult, - NoSuchBucket, - NoSuchKey, - type ObjectInfo, -} from "../../Services/Backend.ts"; -import { S3Client } from "./Client.ts"; - -/** - * Strips MinIO metadata suffixes like [minio_cache:v2,return:] from strings. - */ -function stripMinioMetadata(s: string): string { - return s.replace(/\[minio_cache:[^\]]+\]/g, ""); -} - -/** - * Maps S3 SDK exceptions to internal BackendError types. - */ -function mapS3Error(e: unknown, bucketName?: string): BackendError { - const err = e as { - name?: string; - Code?: string; - Message?: string; - message?: string; - $metadata?: { httpStatusCode?: number }; - }; - const name = err?.name || err?.Code || - (e instanceof Error ? e.name : "UnknownError"); - const message = err?.message || err?.Message || - "An unknown S3 error occurred"; - const bucket = bucketName ?? "unknown-bucket"; - - switch (name) { - case "NoSuchBucket": - case "NotFound": - return new NoSuchBucket({ bucketName: bucket, message }); - case "NoSuchKey": - return new NoSuchKey({ - bucketName: bucket, - key: "unknown", - message: message, - }); - case "BucketAlreadyExists": - return new BucketAlreadyExists({ bucketName: bucket, message }); - case "BucketAlreadyOwnedByYou": - return new BucketAlreadyOwnedByYou({ bucketName: bucket, message }); - case "AccessDenied": - case "Forbidden": - return new AccessDenied({ message }); - case "BucketNotEmpty": - case "Conflict": - return new BucketNotEmpty({ bucketName: bucket, message }); - } - - // Handle case where it might be a raw 404 from HEAD request - if (err?.$metadata?.httpStatusCode === 404) { - return new NoSuchKey({ - bucketName: bucket, - key: "unknown", - message: "Not Found", - }); - } - - return new InternalError({ - message: e instanceof Error ? `${e.name}: ${e.message}` : String(e), - }); -} +import type { BackendError, BackendService } from "../../Services/Backend.ts"; +import { makeBucketOps } from "./Buckets.ts"; +import { makeObjectOps } from "./Objects.ts"; +import { getTarget } from "./Utils.ts"; +import type { S3Client } from "./Client.ts"; +import type { HeraldConfig } from "../../Config/Layer.ts"; /** * Creates an S3-specific Backend implementation for a given configuration context. + * Composes bucket and object operations modularly. + * Resolves the target once per backend creation (request-scoped). */ export const makeS3Backend = ( bucket: MaterializedBucket | { backend_id: string }, -): Effect.Effect => - Effect.all({ - s3Service: S3Client, - config: AppConfig, - }).pipe( - Effect.map(({ s3Service, config }) => { - const getTargetBucket = (): MaterializedBucket => { - if ("bucket_name" in bucket) return bucket as MaterializedBucket; - - const backendConfig = config.raw.backends[bucket.backend_id]; - return { - name: "", - backend_id: bucket.backend_id, - protocol: "s3" as const, - endpoint: backendConfig.endpoint, - region: backendConfig.region, - bucket_name: "", - credentials: backendConfig.credentials, - }; - }; - - const targetBucket = getTargetBucket(); - - const service: BackendService = { - listBuckets: () => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send(new ListBucketsCommand({})) as Promise< - ListBucketsCommandOutput - >, - catch: (e) => mapS3Error(e, targetBucket.name), - }) - ), - Effect.flatMap((result) => { - const buckets: BucketInfo[] = []; - for (const b of (result.Buckets ?? [])) { - if (b.Name === undefined) { - return Effect.fail( - new InternalError({ - message: "S3 returned bucket without Name", - }), - ); - } - buckets.push({ - name: b.Name, - creationDate: b.CreationDate, - }); - } - - return Effect.succeed({ - buckets, - owner: { - id: result.Owner?.ID ?? "unknown-owner-id", - displayName: result.Owner?.DisplayName ?? - "unknown-owner-name", - }, - }); - }), - ), - - createBucket: () => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new CreateBucketCommand({ - Bucket: targetBucket.bucket_name, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map(() => undefined), - ), - - deleteBucket: () => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new DeleteBucketCommand({ - Bucket: targetBucket.bucket_name, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map(() => undefined), - ), - - headBucket: () => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new HeadBucketCommand({ Bucket: targetBucket.bucket_name }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map(() => undefined), - ), - - listObjects: (args) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => { - if (args.listType === 2) { - return Effect.tryPromise({ - try: () => - client.send( - new ListObjectsV2Command({ - Bucket: targetBucket.bucket_name, - Prefix: args.prefix, - Delimiter: args.delimiter, - MaxKeys: args.maxKeys, - ContinuationToken: args.continuationToken, - StartAfter: args.startAfter, - }), - ) as Promise, - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }).pipe( - Effect.map((result): ListObjectsResult => ({ - name: result.Name ?? targetBucket.bucket_name, - prefix: result.Prefix, - maxKeys: result.MaxKeys ?? 1000, - delimiter: result.Delimiter, - isTruncated: result.IsTruncated ?? false, - encodingType: args.encodingType, - continuationToken: result.ContinuationToken, - nextContinuationToken: result.NextContinuationToken, - keyCount: result.KeyCount, - listType: 2, - contents: (result.Contents ?? []).map((c): ObjectInfo => ({ - key: stripMinioMetadata(c.Key ?? ""), - lastModified: c.LastModified ?? new Date(), - etag: c.ETag ?? "", - size: c.Size ?? 0, - storageClass: c.StorageClass, - owner: c.Owner - ? { - id: c.Owner.ID ?? "unknown", - displayName: c.Owner.DisplayName ?? "unknown", - } - : undefined, - })), - commonPrefixes: (result.CommonPrefixes ?? []).map(( - cp, - ): CommonPrefix => ({ - prefix: stripMinioMetadata(cp.Prefix ?? ""), - })), - })), - ); - } else { - return Effect.tryPromise({ - try: () => - client.send( - new ListObjectsCommand({ - Bucket: targetBucket.bucket_name, - Prefix: args.prefix, - Delimiter: args.delimiter, - Marker: args.marker, - MaxKeys: args.maxKeys, - }), - ) as Promise, - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }).pipe( - Effect.map((result): ListObjectsResult => ({ - name: result.Name ?? targetBucket.bucket_name, - prefix: result.Prefix, - marker: result.Marker, - nextMarker: result.NextMarker, - maxKeys: result.MaxKeys ?? 1000, - delimiter: result.Delimiter, - isTruncated: result.IsTruncated ?? false, - encodingType: args.encodingType, - listType: 1, - contents: (result.Contents ?? []).map((c): ObjectInfo => ({ - key: stripMinioMetadata(c.Key ?? ""), - lastModified: c.LastModified ?? new Date(), - etag: c.ETag ?? "", - size: c.Size ?? 0, - storageClass: c.StorageClass, - owner: c.Owner - ? { - id: c.Owner.ID ?? "unknown", - displayName: c.Owner.DisplayName ?? "unknown", - } - : undefined, - })), - commonPrefixes: (result.CommonPrefixes ?? []).map(( - cp, - ): CommonPrefix => ({ - prefix: stripMinioMetadata(cp.Prefix ?? ""), - })), - })), - ); - } - }), - ), - - listVersions: (args) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new ListObjectVersionsCommand({ - Bucket: targetBucket.bucket_name, - Prefix: args.prefix, - Delimiter: args.delimiter, - KeyMarker: args.keyMarker, - VersionIdMarker: args.versionIdMarker, - MaxKeys: args.maxKeys, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map((result): ListObjectsResult => ({ - name: result.Name ?? targetBucket.bucket_name, - prefix: result.Prefix, - marker: result.KeyMarker, - nextMarker: result.NextKeyMarker, - maxKeys: result.MaxKeys ?? 1000, - delimiter: result.Delimiter, - isTruncated: result.IsTruncated ?? false, - encodingType: args.encodingType, - listType: 1, // listVersions is similar to V1 - contents: [ - ...(result.Versions ?? []).map((v): ObjectInfo => ({ - key: stripMinioMetadata(v.Key ?? ""), - lastModified: v.LastModified ?? new Date(), - etag: v.ETag ?? "", - size: v.Size ?? 0, - storageClass: v.StorageClass, - versionId: v.VersionId, - isDeleteMarker: false, - isLatest: v.IsLatest, - owner: v.Owner - ? { - id: v.Owner.ID ?? "unknown", - displayName: v.Owner.DisplayName ?? "unknown", - } - : undefined, - })), - ...(result.DeleteMarkers ?? []).map((dm): ObjectInfo => ({ - key: stripMinioMetadata(dm.Key ?? ""), - lastModified: dm.LastModified ?? new Date(), - etag: "", - size: 0, - versionId: dm.VersionId, - isDeleteMarker: true, - isLatest: dm.IsLatest, - owner: dm.Owner - ? { - id: dm.Owner.ID ?? "unknown", - displayName: dm.Owner.DisplayName ?? "unknown", - } - : undefined, - })), - ], - commonPrefixes: (result.CommonPrefixes ?? []).map(( - cp, - ): CommonPrefix => ({ - prefix: stripMinioMetadata(cp.Prefix ?? ""), - })), - })), - ), - - getObject: (key) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new GetObjectCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.flatMap((result) => { - const body = result.Body; - if (!body) { - return Effect.fail( - new InternalError({ - message: "S3 returned empty body for GetObject", - }), - ); - } - - // AWS SDK Body can be many things. In Deno/Browser it has transformToWebStream() - // Use a type-safe check to avoid 'any' - const getWebStream = (): ReadableStream => { - if ( - body && typeof body === "object" && - "transformToWebStream" in body - ) { - const b = body as { transformToWebStream: unknown }; - if (typeof b.transformToWebStream === "function") { - return b.transformToWebStream() as ReadableStream< - Uint8Array - >; - } - } - return body as ReadableStream; - }; - - const stream = Stream.fromReadableStream( - getWebStream, - (e) => new Error(String(e)), - ); - - const metadata: Record = {}; - if (result.Metadata) { - for (const [k, v] of Object.entries(result.Metadata)) { - try { - metadata[k] = decodeURIComponent(v ?? ""); - } catch { - metadata[k] = v ?? ""; - } - } - } - - const headers: Record = {}; - if (result.ContentType) { - headers["content-type"] = result.ContentType; - } - if (result.ETag) headers["etag"] = result.ETag; - if (result.LastModified) { - headers["last-modified"] = result.LastModified.toUTCString(); - } - - for (const [k, v] of Object.entries(metadata)) { - headers[`x-amz-meta-${k}`] = v; - } - - // Buffer the entire stream to ensure it's fully read and connection is closed - // This also addresses issues where the SDK's Body might not be a standard ReadableStream - return Stream.runCollect(stream).pipe( - Effect.mapError((e) => - new InternalError({ message: String(e) }) - ), - Effect.map((chunks) => { - const totalLength = Chunk.reduce( - chunks, - 0, - (acc, chunk) => acc + chunk.length, - ); - const all = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of chunks) { - all.set(chunk, offset); - offset += chunk.length; - } - return { - stream: Stream.succeed(all), - contentType: result.ContentType, - contentLength: all.length, - etag: result.ETag, - lastModified: result.LastModified, - metadata, - headers, - }; - }), - ); - }), - ), - - headObject: (key) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new HeadObjectCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map((result) => { - const metadata: Record = {}; - if (result.Metadata) { - for (const [k, v] of Object.entries(result.Metadata)) { - try { - metadata[k] = decodeURIComponent(v ?? ""); - } catch { - metadata[k] = v ?? ""; - } - } - } - - const headers: Record = {}; - if (result.ContentType) { - headers["content-type"] = result.ContentType; - } - if (result.ContentLength !== undefined) { - headers["content-length"] = String(result.ContentLength); - } - if (result.ETag) headers["etag"] = result.ETag; - if (result.LastModified) { - headers["last-modified"] = result - .LastModified.toUTCString(); - } - - for (const [k, v] of Object.entries(metadata)) { - headers[`x-amz-meta-${k}`] = v; - } - - return { - contentType: result.ContentType, - contentLength: result.ContentLength, - etag: result.ETag, - lastModified: result.LastModified, - metadata, - headers, - }; - }), - ), - - putObject: (key, bodyStream, headers) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Stream.runCollect(bodyStream).pipe( - Effect.mapError((e) => - new InternalError({ message: String(e) }) - ), - Effect.flatMap((chunks) => { - const totalLength = Chunk.reduce( - chunks, - 0, - (acc, chunk) => acc + chunk.length, - ); - const body = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of chunks) { - body.set(chunk, offset); - offset += chunk.length; - } - - const metadata: Record = {}; - for (const [k, v] of Object.entries(headers)) { - if (k.toLowerCase().startsWith("x-amz-meta-")) { - const metaKey = k.substring("x-amz-meta-".length); - const value = String(v); - metadata[metaKey] = /[^\x20-\x7E]/.test(value) - ? encodeURIComponent(value) - : value; - } - } - - const contentType = headers["content-type"]; - - return Effect.tryPromise({ - try: () => - client.send( - new PutObjectCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - Body: body, - ContentType: contentType - ? String(contentType) - : undefined, - Metadata: metadata, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }); - }), - ) - ), - Effect.map((result) => ({ - etag: result.ETag, - versionId: result.VersionId, - })), - ), - - deleteObject: (key) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new DeleteObjectCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map(() => undefined), - ), - - deleteObjects: ( - objects, - ): Effect.Effect => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new DeleteObjectsCommand({ - Bucket: targetBucket.bucket_name, - Delete: { - Objects: objects.map((o) => ({ - Key: o.key, - VersionId: o.versionId === "null" - ? undefined - : o.versionId, - })), - }, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map((result) => ({ - deleted: (result.Deleted ?? []).map((d) => d.Key ?? ""), - errors: (result.Errors ?? []).map((e) => ({ - key: e.Key ?? "unknown", - code: e.Code ?? "InternalError", - message: e.Message ?? "Unknown error", - })), - })), - ), - }; - - return service; - }), - ); +): Effect.Effect => + Effect.gen(function* () { + const target = yield* getTarget(bucket); + return { + ...makeBucketOps(target), + ...makeObjectOps(target), + } satisfies BackendService; + }); diff --git a/src/Backends/S3/Buckets.ts b/src/Backends/S3/Buckets.ts new file mode 100644 index 0000000..b0ff891 --- /dev/null +++ b/src/Backends/S3/Buckets.ts @@ -0,0 +1,74 @@ +import { Effect } from "effect"; +import { + CreateBucketCommand, + DeleteBucketCommand, + HeadBucketCommand, + ListBucketsCommand, + type ListBucketsCommandOutput, +} from "@aws-sdk/client-s3"; +import { type BucketInfo, InternalError } from "../../Services/Backend.ts"; +import { mapS3Error, type S3Target } from "./Utils.ts"; + +export const makeBucketOps = (target: S3Target) => ({ + listBuckets: () => + Effect.gen(function* () { + const { client, name } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send(new ListBucketsCommand({})) as Promise< + ListBucketsCommandOutput + >, + catch: (e) => mapS3Error(e, name), + }); + + const buckets: BucketInfo[] = []; + for (const b of (result.Buckets ?? [])) { + if (b.Name === undefined) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned bucket without Name", + }), + ); + } + buckets.push({ + name: b.Name, + creationDate: b.CreationDate, + }); + } + + return { + buckets, + owner: { + id: result.Owner?.ID ?? "unknown-owner-id", + displayName: result.Owner?.DisplayName ?? "unknown-owner-name", + }, + }; + }), + + createBucket: () => + Effect.gen(function* () { + const { client, bucketName, name } = target; + yield* Effect.tryPromise({ + try: () => client.send(new CreateBucketCommand({ Bucket: bucketName })), + catch: (e) => mapS3Error(e, bucketName || name), + }); + }), + + deleteBucket: () => + Effect.gen(function* () { + const { client, bucketName, name } = target; + yield* Effect.tryPromise({ + try: () => client.send(new DeleteBucketCommand({ Bucket: bucketName })), + catch: (e) => mapS3Error(e, bucketName || name), + }); + }), + + headBucket: () => + Effect.gen(function* () { + const { client, bucketName, name } = target; + yield* Effect.tryPromise({ + try: () => client.send(new HeadBucketCommand({ Bucket: bucketName })), + catch: (e) => mapS3Error(e, bucketName || name), + }); + }), +}); diff --git a/src/Backends/S3/Client.ts b/src/Backends/S3/Client.ts index dc98754..376de09 100644 --- a/src/Backends/S3/Client.ts +++ b/src/Backends/S3/Client.ts @@ -1,7 +1,7 @@ -import { Context, Effect, Layer } from "effect"; +import { Cache, Context, Effect, Layer } from "effect"; import { S3Client as S3ClientSDK } from "@aws-sdk/client-s3"; import type { MaterializedBucket } from "../../Domain/Config.ts"; -import { AppConfig } from "../../Config/Layer.ts"; +import { HeraldConfig } from "../../Config/Layer.ts"; export class S3Client extends Context.Tag("S3Client")< S3Client, @@ -14,94 +14,100 @@ export class S3Client extends Context.Tag("S3Client")< export const S3ClientLive = Layer.effect( S3Client, - AppConfig.pipe( - Effect.flatMap((appConfig) => { - // A simple cache for SDK clients - const clients = new Map(); + Effect.gen(function* () { + const appConfig = yield* HeraldConfig; - return Effect.succeed( - S3Client.of({ - getClient: (bucket: MaterializedBucket | { backend_id: string }) => { - // Resolve full bucket if only backend_id provided - let resolved: MaterializedBucket; - if ("bucket_name" in bucket) { - resolved = bucket; - } else { - const backendConfig = appConfig.raw.backends[bucket.backend_id]; - resolved = { - name: "", - backend_id: bucket.backend_id, - protocol: "s3" as const, - endpoint: backendConfig.endpoint, - region: backendConfig.region, - bucket_name: "", - credentials: backendConfig.credentials, - }; - } + const cache = yield* Cache.make({ + capacity: 100, + timeToLive: "24 hours", // S3 clients can live a long time + lookup: (resolved: MaterializedBucket) => + Effect.gen(function* () { + if (resolved.endpoint === undefined) { + return yield* Effect.fail( + new Error( + `Missing endpoint for backend ${resolved.backend_id}`, + ), + ); + } + + if (resolved.region === undefined) { + return yield* Effect.fail( + new Error(`Missing region for backend ${resolved.backend_id}`), + ); + } + + let accessKeyId: string | undefined; + let secretAccessKey: string | undefined; - const key = - `${resolved.backend_id}:${resolved.endpoint}:${resolved.region}`; - const existing = clients.get(key); - if (existing) { - return Effect.succeed(existing); + if (resolved.credentials) { + const creds = resolved.credentials; + if ("accessKeyId" in creds) { + accessKeyId = creds.accessKeyId; + secretAccessKey = creds.secretAccessKey; + } else if ("username" in creds) { + accessKeyId = creds.username; + secretAccessKey = creds.password; } - if (resolved.endpoint === undefined) { - return Effect.fail( + if (accessKeyId === undefined) { + return yield* Effect.fail( new Error( - `Missing endpoint for backend ${resolved.backend_id}`, + `Missing accessKeyId/username for backend ${resolved.backend_id}`, ), ); } - - if (resolved.region === undefined) { - return Effect.fail( - new Error(`Missing region for backend ${resolved.backend_id}`), + if (secretAccessKey === undefined) { + return yield* Effect.fail( + new Error( + `Missing secretAccessKey/password for backend ${resolved.backend_id}`, + ), ); } + } - if (resolved.credentials) { - if ( - resolved.credentials.accessKeyId === undefined && - resolved.credentials.username === undefined - ) { - return Effect.fail( - new Error( - `Missing accessKeyId/username for backend ${resolved.backend_id}`, - ), - ); - } - if ( - resolved.credentials.secretAccessKey === undefined && - resolved.credentials.password === undefined - ) { - return Effect.fail( - new Error( - `Missing secretAccessKey/password for backend ${resolved.backend_id}`, - ), - ); + return new S3ClientSDK({ + endpoint: resolved.endpoint, + region: resolved.region, + credentials: accessKeyId && secretAccessKey + ? { + accessKeyId, + secretAccessKey, } - } + : undefined, + forcePathStyle: true, + }); + }), + }); - const sdkClient = new S3ClientSDK({ - endpoint: resolved.endpoint, - region: resolved.region, - credentials: resolved.credentials - ? { - accessKeyId: (resolved.credentials.accessKeyId ?? - resolved.credentials.username)!, - secretAccessKey: (resolved.credentials.secretAccessKey ?? - resolved.credentials.password)!, - } - : undefined, - forcePathStyle: true, - }); + return S3Client.of({ + getClient: (bucket: MaterializedBucket | { backend_id: string }) => { + // Resolve full bucket if only backend_id provided + let resolved: MaterializedBucket; + if ("bucket_name" in bucket) { + resolved = bucket; + } else { + const backendConfig = appConfig.raw.backends[bucket.backend_id]; + if (backendConfig && backendConfig.protocol === "s3") { + resolved = { + name: "", + backend_id: bucket.backend_id, + protocol: "s3" as const, + endpoint: backendConfig.endpoint, + region: backendConfig.region, + bucket_name: "", + credentials: backendConfig.credentials, + }; + } else { + return Effect.fail( + new Error( + `Backend ${bucket.backend_id} is not an S3 backend or not found`, + ), + ); + } + } - clients.set(key, sdkClient); - return Effect.succeed(sdkClient); - }, - }), - ); - }), - ), + return cache.get(resolved); + }, + }); + }), ); diff --git a/src/Backends/S3/Objects.ts b/src/Backends/S3/Objects.ts new file mode 100644 index 0000000..58a1f06 --- /dev/null +++ b/src/Backends/S3/Objects.ts @@ -0,0 +1,757 @@ +import { Chunk, Effect, Option, Stream } from "effect"; +import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CreateMultipartUploadCommand, + DeleteObjectCommand, + DeleteObjectsCommand, + GetObjectCommand, + HeadObjectCommand, + ListMultipartUploadsCommand, + ListObjectsCommand, + type ListObjectsCommandOutput, + ListObjectsV2Command, + type ListObjectsV2CommandOutput, + ListObjectVersionsCommand, + ListPartsCommand, + PutObjectCommand, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import { + type CommonPrefix, + InternalError, + type ListObjectsResult, + type ObjectInfo, + type ObjectResponse, +} from "../../Services/Backend.ts"; +import { mapS3Error, type S3Target, stripMinioMetadata } from "./Utils.ts"; + +export const makeObjectOps = (target: S3Target) => ({ + listObjects: (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + encodingType?: string; + continuationToken?: string; + startAfter?: string; + listType?: 1 | 2; + }) => + Effect.gen(function* () { + const { client, bucketName } = target; + if (args.listType === 2) { + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + MaxKeys: args.maxKeys, + ContinuationToken: args.continuationToken, + StartAfter: args.startAfter, + }), + ) as Promise, + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + name: result.Name ?? bucketName, + prefix: result.Prefix, + maxKeys: result.MaxKeys ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: args.encodingType, + continuationToken: result.ContinuationToken, + nextContinuationToken: result.NextContinuationToken, + keyCount: result.KeyCount, + listType: 2, + contents: (result.Contents ?? []).map((c): ObjectInfo => ({ + key: stripMinioMetadata(c.Key ?? ""), + lastModified: c.LastModified ?? new Date(), + etag: c.ETag ?? "", + size: c.Size ?? 0, + storageClass: c.StorageClass, + owner: c.Owner + ? { + id: c.Owner.ID ?? "unknown", + displayName: c.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + commonPrefixes: (result.CommonPrefixes ?? []).map(( + cp, + ): CommonPrefix => ({ + prefix: stripMinioMetadata(cp.Prefix ?? ""), + })), + } satisfies ListObjectsResult; + } else { + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListObjectsCommand({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + Marker: args.marker, + MaxKeys: args.maxKeys, + }), + ) as Promise, + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + name: result.Name ?? bucketName, + prefix: result.Prefix, + marker: result.Marker, + nextMarker: result.NextMarker, + maxKeys: result.MaxKeys ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: args.encodingType, + listType: 1, + contents: (result.Contents ?? []).map((c): ObjectInfo => ({ + key: stripMinioMetadata(c.Key ?? ""), + lastModified: c.LastModified ?? new Date(), + etag: c.ETag ?? "", + size: c.Size ?? 0, + storageClass: c.StorageClass, + owner: c.Owner + ? { + id: c.Owner.ID ?? "unknown", + displayName: c.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + commonPrefixes: (result.CommonPrefixes ?? []).map(( + cp, + ): CommonPrefix => ({ + prefix: stripMinioMetadata(cp.Prefix ?? ""), + })), + } satisfies ListObjectsResult; + } + }), + + listVersions: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + versionIdMarker?: string; + maxKeys?: number; + encodingType?: string; + }) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListObjectVersionsCommand({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + KeyMarker: args.keyMarker, + VersionIdMarker: args.versionIdMarker, + MaxKeys: args.maxKeys, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + name: result.Name ?? bucketName, + prefix: result.Prefix, + marker: result.KeyMarker, + nextMarker: result.NextKeyMarker, + maxKeys: result.MaxKeys ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: args.encodingType, + listType: 1, + contents: [ + ...(result.Versions ?? []).map((v): ObjectInfo => ({ + key: stripMinioMetadata(v.Key ?? ""), + lastModified: v.LastModified ?? new Date(), + etag: v.ETag ?? "", + size: v.Size ?? 0, + storageClass: v.StorageClass, + versionId: v.VersionId, + isDeleteMarker: false, + isLatest: v.IsLatest, + owner: v.Owner + ? { + id: v.Owner.ID ?? "unknown", + displayName: v.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + ...(result.DeleteMarkers ?? []).map((dm): ObjectInfo => ({ + key: stripMinioMetadata(dm.Key ?? ""), + lastModified: dm.LastModified ?? new Date(), + etag: "", + size: 0, + versionId: dm.VersionId, + isDeleteMarker: true, + isLatest: dm.IsLatest, + owner: dm.Owner + ? { + id: dm.Owner.ID ?? "unknown", + displayName: dm.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + ], + commonPrefixes: (result.CommonPrefixes ?? []).map(( + cp, + ): CommonPrefix => ({ + prefix: stripMinioMetadata(cp.Prefix ?? ""), + })), + } satisfies ListObjectsResult; + }), + + getObject: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: key, + Range: (headers["range"] || headers["Range"]) as string, + PartNumber: (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) + ? parseInt( + (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) as string, + ) + : undefined, + IfMatch: (headers["if-match"] || headers["If-Match"]) as string, + IfNoneMatch: (headers["if-none-match"] || + headers["If-None-Match"]) as string, + IfModifiedSince: (headers["if-modified-since"] || + headers["If-Modified-Since"]) + ? new Date( + (headers["if-modified-since"] || + headers["If-Modified-Since"]) as string, + ) + : undefined, + IfUnmodifiedSince: (headers["if-unmodified-since"] || + headers["If-Unmodified-Since"]) + ? new Date( + (headers["if-unmodified-since"] || + headers["If-Unmodified-Since"]) as string, + ) + : undefined, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + const body = result.Body; + if (!body) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned empty body for GetObject", + }), + ); + } + + const getWebStream = (): ReadableStream => { + if ( + body && typeof body === "object" && + "transformToWebStream" in body + ) { + const b = body as { transformToWebStream: unknown }; + if (typeof b.transformToWebStream === "function") { + return b.transformToWebStream() as ReadableStream< + Uint8Array + >; + } + } + return body as ReadableStream; + }; + + const stream = Stream.fromReadableStream( + getWebStream, + (e) => new Error(String(e)), + ); + + const metadata: Record = {}; + if (result.Metadata) { + for (const [k, v] of Object.entries(result.Metadata)) { + metadata[k] = Option.liftThrowable(decodeURIComponent)( + v ?? "", + ).pipe( + Option.getOrElse(() => v ?? ""), + ); + } + } + + const s3Headers: Record = {}; + if (result.ContentType) { + s3Headers["content-type"] = result.ContentType; + } + if (result.ContentLength !== undefined) { + s3Headers["content-length"] = String(result.ContentLength); + } + if (result.ETag) s3Headers["etag"] = result.ETag; + if (result.PartsCount !== undefined) { + s3Headers["x-amz-mp-parts-count"] = String(result.PartsCount); + } + if (result.VersionId) { + s3Headers["x-amz-version-id"] = result.VersionId; + } + if (result.LastModified) { + s3Headers["last-modified"] = result.LastModified.toUTCString(); + } + + for (const [k, v] of Object.entries(metadata)) { + s3Headers[`x-amz-meta-${k}`] = v; + } + + return yield* Stream.runCollect(stream).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + Effect.map((chunks) => { + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const all = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + all.set(chunk, offset); + offset += chunk.length; + } + return { + stream: Stream.succeed(all), + contentType: result.ContentType, + contentLength: all.length, + etag: result.ETag, + lastModified: result.LastModified, + metadata, + headers: s3Headers, + } satisfies ObjectResponse; + }), + ); + }), + + headObject: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const commandInput = { + Bucket: bucketName, + Key: key, + PartNumber: (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) + ? parseInt( + (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) as string, + ) + : undefined, + }; + const result = yield* Effect.tryPromise({ + try: () => client.send(new HeadObjectCommand(commandInput)), + catch: (e) => mapS3Error(e, bucketName), + }); + + const metadata: Record = {}; + if (result.Metadata) { + for (const [k, v] of Object.entries(result.Metadata)) { + metadata[k] = Option.liftThrowable(decodeURIComponent)( + v ?? "", + ).pipe( + Option.getOrElse(() => v ?? ""), + ); + } + } + + const s3Headers: Record = {}; + if (result.ContentType) { + s3Headers["content-type"] = result.ContentType; + } + if (result.ContentLength !== undefined) { + s3Headers["content-length"] = String(result.ContentLength); + } + if (result.ETag) s3Headers["etag"] = result.ETag; + if (result.PartsCount !== undefined) { + s3Headers["x-amz-mp-parts-count"] = String(result.PartsCount); + } + if (result.VersionId) { + s3Headers["x-amz-version-id"] = result.VersionId; + } + if (result.LastModified) { + s3Headers["last-modified"] = result + .LastModified.toUTCString(); + } + + for (const [k, v] of Object.entries(metadata)) { + s3Headers[`x-amz-meta-${k}`] = v; + } + + return { + contentType: result.ContentType, + contentLength: result.ContentLength, + etag: result.ETag, + lastModified: result.LastModified, + metadata, + headers: s3Headers, + }; + }), + + putObject: ( + key: string, + bodyStream: Stream.Stream, + headers: Record, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const chunks = yield* Stream.runCollect(bodyStream).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + ); + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const body = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.length; + } + + const metadata: Record = {}; + for (const [k, v] of Object.entries(headers)) { + if (k.toLowerCase().startsWith("x-amz-meta-")) { + const metaKey = k.substring("x-amz-meta-".length); + const value = String(v); + metadata[metaKey] = /[^\x20-\x7E]/.test(value) + ? encodeURIComponent(value) + : value; + } + } + + const contentType = headers["content-type"]; + + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: key, + Body: body, + ContentType: contentType ? String(contentType) : undefined, + Metadata: metadata, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + etag: result.ETag, + versionId: result.VersionId, + }; + }), + + deleteObject: (key: string) => + Effect.gen(function* () { + const { client, bucketName } = target; + yield* Effect.tryPromise({ + try: () => + client.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: key, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + }), + + deleteObjects: (objects: readonly { key: string; versionId?: string }[]) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects.map((o) => ({ + Key: o.key, + VersionId: o.versionId === "null" ? undefined : o.versionId, + })), + }, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + deleted: (result.Deleted ?? []).map((d) => d.Key ?? ""), + errors: (result.Errors ?? []).map((e) => ({ + key: e.Key ?? "unknown", + code: e.Code ?? "InternalError", + message: e.Message ?? "Unknown error", + })), + }; + }), + + createMultipartUpload: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const metadata: Record = {}; + for (const [k, v] of Object.entries(headers)) { + if (k.toLowerCase().startsWith("x-amz-meta-")) { + const metaKey = k.substring("x-amz-meta-".length); + metadata[metaKey] = String(v); + } + } + const contentType = headers["content-type"]; + + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + Metadata: metadata, + ContentType: contentType ? String(contentType) : undefined, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + if (!result.UploadId) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned empty UploadId", + }), + ); + } + return { uploadId: result.UploadId }; + }), + + uploadPart: ( + key: string, + uploadId: string, + partNumber: number, + bodyStream: Stream.Stream, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const chunks = yield* Stream.runCollect(bodyStream).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + ); + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const body = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.length; + } + + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + PartNumber: partNumber, + Body: body, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + if (!result.ETag) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned empty ETag for UploadPart", + }), + ); + } + return { etag: result.ETag }; + }), + + completeMultipartUpload: ( + key: string, + uploadId: string, + parts: readonly { etag: string; partNumber: number }[], + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: parts.map((p) => ({ + ETag: p.etag, + PartNumber: p.partNumber, + })), + }, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + if ( + !result.Location || !result.Bucket || !result.Key || + !result.ETag + ) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned incomplete CompleteMultipartUploadResult", + }), + ); + } + return { + location: result.Location, + bucket: result.Bucket, + key: result.Key, + etag: result.ETag, + versionId: result.VersionId, + }; + }), + + abortMultipartUpload: (key: string, uploadId: string) => + Effect.gen(function* () { + const { client, bucketName } = target; + yield* Effect.tryPromise({ + try: () => + client.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + }), + + listMultipartUploads: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + uploadIdMarker?: string; + maxUploads?: number; + encodingType?: string; + }) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListMultipartUploadsCommand({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + KeyMarker: args.keyMarker, + UploadIdMarker: args.uploadIdMarker, + MaxUploads: args.maxUploads, + EncodingType: args.encodingType as "url" | undefined, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + bucket: result.Bucket ?? bucketName, + prefix: result.Prefix, + keyMarker: result.KeyMarker, + uploadIdMarker: result.UploadIdMarker, + nextKeyMarker: result.NextKeyMarker, + nextUploadIdMarker: result.NextUploadIdMarker, + maxUploads: result.MaxUploads ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: result.EncodingType as string, + uploads: (result.Uploads ?? []).map((u) => ({ + key: u.Key ?? "", + uploadId: u.UploadId ?? "", + owner: { + id: u.Owner?.ID ?? "", + displayName: u.Owner?.DisplayName ?? "", + }, + initiator: { + id: u.Initiator?.ID ?? "", + displayName: u.Initiator?.DisplayName ?? "", + }, + storageClass: u.StorageClass ?? "STANDARD", + initiated: u.Initiated ?? new Date(), + })), + commonPrefixes: (result.CommonPrefixes ?? []).map((cp) => ({ + prefix: cp.Prefix ?? "", + })), + }; + }), + + listParts: (key: string, uploadId: string) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListPartsCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + bucket: result.Bucket ?? bucketName, + key: result.Key ?? key, + uploadId: result.UploadId ?? uploadId, + owner: { + id: result.Owner?.ID ?? "", + displayName: result.Owner?.DisplayName ?? "", + }, + initiator: { + id: result.Initiator?.ID ?? "", + displayName: result.Initiator?.DisplayName ?? "", + }, + storageClass: result.StorageClass ?? "STANDARD", + partNumberMarker: result.PartNumberMarker + ? parseInt(String(result.PartNumberMarker)) + : 0, + nextPartNumberMarker: result.NextPartNumberMarker + ? parseInt(String(result.NextPartNumberMarker)) + : 0, + maxParts: result.MaxParts ?? 1000, + isTruncated: result.IsTruncated ?? false, + parts: (result.Parts ?? []).map((p) => ({ + partNumber: p.PartNumber ?? 0, + lastModified: p.LastModified ?? new Date(), + etag: p.ETag ?? "", + size: p.Size ?? 0, + })), + }; + }), +}); diff --git a/src/Backends/S3/Signer.ts b/src/Backends/S3/Signer.ts index 6bf6b59..c0d467d 100644 --- a/src/Backends/S3/Signer.ts +++ b/src/Backends/S3/Signer.ts @@ -22,10 +22,21 @@ function getV4Signer(config: BackendConfig) { ); } - const accessKeyId = config.credentials.accessKeyId ?? - config.credentials.username; - const secretAccessKey = config.credentials.secretAccessKey ?? - config.credentials.password; + const creds = config.credentials; + let accessKeyId: string | undefined; + let secretAccessKey: string | undefined; + + if ("accessKeyId" in creds) { + accessKeyId = creds.accessKeyId; + } else if ("username" in creds) { + accessKeyId = creds.username; + } + + if ("secretAccessKey" in creds) { + secretAccessKey = creds.secretAccessKey; + } else if ("password" in creds) { + secretAccessKey = creds.password; + } if (!accessKeyId || !secretAccessKey) { return yield* Effect.fail( diff --git a/src/Backends/S3/Utils.ts b/src/Backends/S3/Utils.ts new file mode 100644 index 0000000..f11d486 --- /dev/null +++ b/src/Backends/S3/Utils.ts @@ -0,0 +1,147 @@ +import { Effect } from "effect"; +import type { S3Client as S3ClientSDK } from "@aws-sdk/client-s3"; +import type { MaterializedBucket } from "../../Domain/Config.ts"; +import { HeraldConfig } from "../../Config/Layer.ts"; +import { + AccessDenied, + type BackendError, + BucketAlreadyExists, + BucketAlreadyOwnedByYou, + BucketNotEmpty, + EntityTooSmall, + InternalError, + InvalidPart, + InvalidPartOrder, + InvalidRequest, + MalformedXML, + NoSuchBucket, + NoSuchKey, + NoSuchUpload, +} from "../../Services/Backend.ts"; +import { S3Client } from "./Client.ts"; + +export interface S3Target { + readonly client: S3ClientSDK; + readonly bucketName: string; + readonly name: string; +} + +/** + * Strips MinIO metadata suffixes like [minio_cache:v2,return:] from strings. + */ +export function stripMinioMetadata(s: string): string { + return s.replace(/\[minio_cache:[^\]]+\]/g, ""); +} + +/** + * Maps S3 SDK exceptions to internal BackendError types. + */ +export function mapS3Error(e: unknown, bucketName?: string): BackendError { + const err = e as { + name?: string; + Code?: string; + Message?: string; + message?: string; + $metadata?: { httpStatusCode?: number }; + }; + const name = err?.name || err?.Code || + (e instanceof Error ? e.name : "UnknownError"); + const message = err?.message || err?.Message || + "An unknown S3 error occurred"; + const bucket = bucketName ?? "unknown-bucket"; + + switch (name) { + case "NoSuchBucket": + case "NotFound": + return new NoSuchBucket({ bucketName: bucket, message }); + case "NoSuchKey": + return new NoSuchKey({ + bucketName: bucket, + key: "unknown", + message: message, + }); + case "NoSuchUpload": + return new NoSuchUpload({ + uploadId: "unknown", + message: message, + }); + case "InvalidPart": + case "InvalidPartNumber": + return new InvalidPart({ message }); + case "InvalidPartOrder": + return new InvalidPartOrder({ message }); + case "EntityTooSmall": + return new EntityTooSmall({ message }); + case "InvalidRequest": + if (message.includes("at least one part")) { + return new MalformedXML({ message }); + } + return new InvalidRequest({ message }); + case "MalformedXML": + return new MalformedXML({ message }); + case "BucketAlreadyExists": + return new BucketAlreadyExists({ bucketName: bucket, message }); + case "BucketAlreadyOwnedByYou": + return new BucketAlreadyOwnedByYou({ bucketName: bucket, message }); + case "AccessDenied": + case "Forbidden": + return new AccessDenied({ message }); + case "BucketNotEmpty": + case "Conflict": + return new BucketNotEmpty({ bucketName: bucket, message }); + } + + // Handle case where it might be a raw 404 from HEAD request + if (err?.$metadata?.httpStatusCode === 404) { + return new NoSuchKey({ + bucketName: bucket, + key: "unknown", + message: "Not Found", + }); + } + + return new InternalError({ + message: e instanceof Error ? `${e.name}: ${e.message}` : String(e), + }); +} + +/** + * Resolves the target bucket configuration and acquires the S3 client. + * This ensures the backend remains a stateless proxy that picks up request-local configuration and clients. + */ +export const getTarget = ( + bucket: MaterializedBucket | { backend_id: string }, +): Effect.Effect => + Effect.gen(function* () { + const s3Service = yield* S3Client; + const config = yield* HeraldConfig; + + const resolveTargetBucket = (): MaterializedBucket => { + if ("bucket_name" in bucket) return bucket as MaterializedBucket; + + const backendConfig = config.raw.backends[bucket.backend_id]; + if (backendConfig && backendConfig.protocol === "s3") { + return { + name: "", + backend_id: bucket.backend_id, + protocol: "s3" as const, + endpoint: backendConfig.endpoint, + region: backendConfig.region, + bucket_name: "", + credentials: backendConfig.credentials, + }; + } + throw new Error(`Backend ${bucket.backend_id} is not an S3 backend`); + }; + + const targetBucket = resolveTargetBucket(); + const client = yield* s3Service.getClient(targetBucket).pipe( + Effect.mapError((e) => mapS3Error(e, targetBucket.name)), + ); + + return { + client, + bucketName: targetBucket.bucket_name, + name: targetBucket.name, + }; + }); diff --git a/src/Backends/Swift/Backend.ts b/src/Backends/Swift/Backend.ts new file mode 100644 index 0000000..1c50070 --- /dev/null +++ b/src/Backends/Swift/Backend.ts @@ -0,0 +1,29 @@ +import { Effect } from "effect"; +import { HttpClient } from "@effect/platform"; +import type { BackendError, BackendService } from "../../Services/Backend.ts"; +import type { MaterializedBucket } from "../../Domain/Config.ts"; +import { makeBucketOps } from "./Buckets.ts"; +import { makeObjectOps } from "./Objects.ts"; +import { getTarget } from "./Utils.ts"; +import type { SwiftClient } from "./Client.ts"; + +/** + * Creates a Swift-specific Backend implementation for a given configuration context. + * Composes bucket and object operations modularly. + * Resolves the target and client once per backend creation (request-scoped). + */ +export const makeSwiftBackend = ( + bucket: MaterializedBucket | { backend_id: string }, +): Effect.Effect< + BackendService, + BackendError, + SwiftClient | HttpClient.HttpClient +> => + Effect.gen(function* () { + const target = yield* getTarget(bucket); + const client = yield* HttpClient.HttpClient; + return { + ...makeBucketOps(target, client), + ...makeObjectOps(target, client), + } satisfies BackendService; + }); diff --git a/src/Backends/Swift/Buckets.ts b/src/Backends/Swift/Buckets.ts new file mode 100644 index 0000000..c6371fd --- /dev/null +++ b/src/Backends/Swift/Buckets.ts @@ -0,0 +1,143 @@ +import { Effect } from "effect"; +import { type HttpClient, HttpClientRequest } from "@effect/platform"; +import { + BucketAlreadyOwnedByYou, + type BucketInfo, + type OwnerInfo, +} from "../../Services/Backend.ts"; +import { mapError, type SwiftTarget } from "./Utils.ts"; + +export interface SwiftContainer { + readonly name: string; + readonly last_modified?: string; +} + +export const makeBucketOps = ( + target: SwiftTarget, + client: HttpClient.HttpClient, +) => ({ + listBuckets: () => + Effect.gen(function* () { + const { storageUrl, token } = target; + const response = yield* client.execute( + HttpClientRequest.get(`${storageUrl}?format=json`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), "")), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", "", "GET"), + ); + } + + const containers = (yield* response.json.pipe( + Effect.mapError((e) => + mapError(500, `Failed to parse Swift response: ${e}`, "") + ), + )) as readonly SwiftContainer[]; + + const bucketInfos: BucketInfo[] = containers.map((b) => ({ + name: b.name, + creationDate: b.last_modified ? new Date(b.last_modified) : undefined, + })); + + const owner: OwnerInfo = { id: "swift", displayName: "Swift User" }; + + return { buckets: bucketInfos, owner }; + }), + + createBucket: () => + Effect.gen(function* () { + const { url, token, container } = target; + const response = yield* client.execute( + HttpClientRequest.put(url).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status === 201) { + return; + } + + if (response.status === 202) { + return yield* Effect.fail( + new BucketAlreadyOwnedByYou({ + bucketName: container, + message: "Bucket already exists", + }), + ); + } + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", container, "PUT"), + ); + } + }), + + deleteBucket: () => + Effect.gen(function* () { + const { url, token, container } = target; + const response = yield* client.execute( + HttpClientRequest.del(url).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + yield* Effect.logDebug( + `Swift deleteBucket container=[${container}] status=${response.status}`, + ); + + if (response.status === 204) { + return; + } + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "DELETE", + ), + ); + } + }), + + headBucket: () => + Effect.gen(function* () { + const { url, token, container } = target; + const response = yield* client.execute( + HttpClientRequest.head(url).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", container, "HEAD"), + ); + } + }), +}); diff --git a/src/Backends/Swift/Client.ts b/src/Backends/Swift/Client.ts new file mode 100644 index 0000000..0feda08 --- /dev/null +++ b/src/Backends/Swift/Client.ts @@ -0,0 +1,187 @@ +import { Cache, Context, Effect, Layer, type Schema } from "effect"; +import { HttpClient, HttpClientRequest } from "@effect/platform"; +import type { MaterializedBucket, SwiftConfig } from "../../Domain/Config.ts"; +import { HeraldConfig } from "../../Config/Layer.ts"; + +export interface SwiftAuthMeta { + readonly token: string; + readonly storageUrl: string; +} + +export class SwiftClient extends Context.Tag("SwiftClient")< + SwiftClient, + { + readonly getAuthMeta: ( + bucket: MaterializedBucket | { backend_id: string }, + ) => Effect.Effect; + } +>() {} + +interface SwiftEndpoint { + readonly region: string; + readonly interface: "public" | "internal" | "admin"; + readonly url: string; +} + +interface SwiftService { + readonly type: string; + readonly endpoints: readonly SwiftEndpoint[]; +} + +interface SwiftTokenResponse { + readonly token: { + readonly catalog: readonly SwiftService[]; + }; +} + +export const SwiftClientLive = Layer.effect( + SwiftClient, + Effect.gen(function* () { + const appConfig = yield* HeraldConfig; + const client = yield* HttpClient.HttpClient; + + const fetchAuthMeta = ( + config: Schema.Schema.Type, + ): Effect.Effect => { + const { auth_url, credentials, region } = config; + + if (!credentials || !("username" in credentials)) { + return Effect.fail( + new Error( + "Swift credentials (username, password, etc.) are required", + ), + ); + } + + const { + username, + password, + project_name, + user_domain_name = "Default", + project_domain_name = "Default", + } = credentials; + + const requestBody = { + auth: { + identity: { + methods: ["password"], + password: { + user: { + name: username, + domain: { name: user_domain_name }, + password: password, + }, + }, + }, + scope: { + project: { + domain: { name: project_domain_name }, + name: project_name, + }, + }, + }, + }; + + return Effect.gen(function* () { + const request = yield* HttpClientRequest.post(`${auth_url}/auth/tokens`) + .pipe( + HttpClientRequest.bodyJson(requestBody), + Effect.mapError((e) => new Error(String(e))), + ); + const response = yield* client.execute(request).pipe( + Effect.mapError((e) => new Error(String(e))), + ); + + if (response.status < 200 || response.status >= 300) { + const msg = yield* response.text.pipe( + Effect.orElseSucceed(() => "Unknown error"), + ); + return yield* Effect.fail( + new Error(`Failed to authenticate with Swift: ${msg}`), + ); + } + + const token = response.headers["x-subject-token"]; + const tokenStr = Array.isArray(token) ? token[0] : token; + + if (!tokenStr) { + return yield* Effect.fail( + new Error( + "X-Subject-Token header missing from Swift response", + ), + ); + } + + const body = (yield* response.json.pipe( + Effect.mapError((e) => new Error(String(e))), + )) as SwiftTokenResponse; + + const catalog = body.token.catalog; + const storageService = catalog.find((s) => s.type === "object-store"); + + if (!storageService) { + return yield* Effect.fail( + new Error( + "Object Store service not found in Swift catalog", + ), + ); + } + + const endpoint = storageService.endpoints.find( + (e) => + (region ? e.region === region : true) && + e.interface === "public", + ); + + if (!endpoint) { + return yield* Effect.fail( + new Error( + `Public Swift endpoint not found (region: ${region ?? "any"})`, + ), + ); + } + + return { + token: tokenStr, + storageUrl: endpoint.url, + }; + }); + }; + + const cache = yield* Cache.make({ + capacity: 100, + timeToLive: "50 minutes", // Swift tokens usually last 1h + lookup: (config: Schema.Schema.Type) => + fetchAuthMeta(config), + }); + + return SwiftClient.of({ + getAuthMeta: ( + bucket: MaterializedBucket | { backend_id: string }, + ) => { + let backend_id: string; + let config: Schema.Schema.Type; + + if ("protocol" in bucket) { + backend_id = bucket.backend_id; + config = appConfig.raw.backends[backend_id] as Schema.Schema.Type< + typeof SwiftConfig + >; + } else { + backend_id = bucket.backend_id; + config = appConfig.raw.backends[backend_id] as Schema.Schema.Type< + typeof SwiftConfig + >; + } + + if (!config || config.protocol !== "swift") { + return Effect.fail( + new Error(`Backend ${backend_id} is not a Swift backend`), + ); + } + + return cache.get(config); + }, + }); + }), +); diff --git a/src/Backends/Swift/Objects.ts b/src/Backends/Swift/Objects.ts new file mode 100644 index 0000000..431394b --- /dev/null +++ b/src/Backends/Swift/Objects.ts @@ -0,0 +1,529 @@ +import { Effect, Option, type Stream } from "effect"; +import { type HttpClient, HttpClientRequest } from "@effect/platform"; +import { + type CommonPrefix, + type DeleteObjectsResult, + InternalError, + type ListObjectsResult, + type ObjectInfo, + type ObjectResponse, + type PutObjectResult, +} from "../../Services/Backend.ts"; +import { mapError, type SwiftTarget } from "./Utils.ts"; +import { fixHeaderEncoding } from "../../Frontend/Utils.ts"; + +export interface SwiftObject { + readonly name?: string; + readonly hash?: string; + readonly bytes?: number; + readonly content_type?: string; + readonly last_modified?: string; + readonly subdir?: string; +} + +export const makeObjectOps = ( + target: SwiftTarget, + client: HttpClient.HttpClient, +) => { + const listObjects = (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + encodingType?: string; + continuationToken?: string; + startAfter?: string; + listType?: 1 | 2; + }) => + Effect.gen(function* () { + const { url, token, container } = target; + const limit = args.maxKeys ?? 1000; + const query = new URLSearchParams({ format: "json" }); + if (args.prefix) query.set("prefix", args.prefix); + if (args.delimiter) query.set("delimiter", args.delimiter); + if (args.marker) query.set("marker", args.marker); + query.set("limit", String(limit + 1)); + if (args.continuationToken) query.set("marker", args.continuationToken); + if (args.startAfter) query.set("marker", args.startAfter); + + const response = yield* client.execute( + HttpClientRequest.get(`${url}?${query.toString()}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + yield* Effect.logDebug( + `Swift listObjects query=[${query.toString()}] status=${response.status}`, + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", container, "GET"), + ); + } + + const rawObjects = (yield* response.json.pipe( + Effect.mapError((e) => + mapError(500, `Failed to parse Swift response: ${e}`, container) + ), + )) as readonly SwiftObject[]; + + const isTruncated = rawObjects.length > limit; + const objects = isTruncated ? rawObjects.slice(0, limit) : rawObjects; + + const contents: ObjectInfo[] = []; + const commonPrefixes: CommonPrefix[] = []; + + for (const obj of objects) { + if (obj.subdir) { + commonPrefixes.push({ prefix: obj.subdir }); + } else if (obj.name) { + contents.push({ + key: obj.name, + lastModified: obj.last_modified + ? new Date(obj.last_modified) + : new Date(), + etag: obj.hash ? `"${obj.hash}"` : "", + size: obj.bytes ?? 0, + storageClass: "STANDARD", + owner: { id: "swift", displayName: "Swift User" }, + }); + } + } + + const nextMarker = isTruncated && objects.length > 0 + ? objects[objects.length - 1].name || + objects[objects.length - 1].subdir + : undefined; + + return { + name: container, + prefix: args.prefix, + maxKeys: limit, + delimiter: args.delimiter, + isTruncated, + marker: args.marker, + nextMarker, + contents, + commonPrefixes, + encodingType: args.encodingType, + listType: args.listType ?? 1, + nextContinuationToken: args.listType === 2 ? nextMarker : undefined, + keyCount: contents.length + commonPrefixes.length, + } satisfies ListObjectsResult; + }); + + return { + listObjects: (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + encodingType?: string; + continuationToken?: string; + startAfter?: string; + listType?: 1 | 2; + }) => listObjects(args), + + listVersions: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + versionIdMarker?: string; + maxKeys?: number; + encodingType?: string; + }) => + Effect.gen(function* () { + const result = yield* listObjects({ + prefix: args.prefix, + delimiter: args.delimiter, + marker: args.keyMarker, + maxKeys: args.maxKeys, + }); + return { + ...result, + contents: result.contents.map((c) => ({ + ...c, + versionId: "null", + isLatest: true, + })), + }; + }), + + getObject: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const swiftHeaders: Record = { + "X-Auth-Token": token, + }; + if (headers["range"] || headers["Range"]) { + swiftHeaders["Range"] = String( + headers["range"] || headers["Range"], + ); + } + if (headers["if-match"] || headers["If-Match"]) { + swiftHeaders["If-Match"] = String( + headers["if-match"] || + headers["If-Match"], + ); + } + if (headers["if-none-match"] || headers["If-None-Match"]) { + swiftHeaders["If-None-Match"] = String( + headers["if-none-match"] || + headers["If-None-Match"], + ); + } + if (headers["if-modified-since"] || headers["If-Modified-Since"]) { + swiftHeaders["If-Modified-Since"] = String( + headers["if-modified-since"] || + headers["If-Modified-Since"], + ); + } + if ( + headers["if-unmodified-since"] || headers["If-Unmodified-Since"] + ) { + swiftHeaders["If-Unmodified-Since"] = String( + headers["if-unmodified-since"] || + headers["If-Unmodified-Since"], + ); + } + + const response = yield* client.execute( + HttpClientRequest.get(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders(swiftHeaders), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "GET", + key, + ), + ); + } + + const metadata: Record = {}; + const s3Headers: Record = {}; + + for (const [k, v] of Object.entries(response.headers)) { + const lowK = k.toLowerCase(); + const value = Array.isArray(v) ? v.join(", ") : v; + if (lowK.startsWith("x-object-meta-")) { + const metaKey = lowK.substring("x-object-meta-".length); + const decodedValue = (value.includes("%")) + ? Option.liftThrowable(decodeURIComponent)(value).pipe( + Option.getOrElse(() => value), + ) + : value; + metadata[metaKey] = decodedValue; + s3Headers[`x-amz-meta-${metaKey}`] = decodedValue; + } else if (lowK === "content-type") { + s3Headers["Content-Type"] = value; + } else if (lowK === "content-length") { + s3Headers["Content-Length"] = value; + } else if (lowK === "etag") { + s3Headers["ETag"] = value; + } else if (lowK === "last-modified") { + s3Headers["Last-Modified"] = value; + } + } + + const contentLengthHeader = response.headers["content-length"]; + const contentLength = Array.isArray(contentLengthHeader) + ? parseInt(contentLengthHeader[0] || "0") + : parseInt(contentLengthHeader || "0"); + + const etagHeader = response.headers["etag"]; + const etag = Array.isArray(etagHeader) ? etagHeader[0] : etagHeader; + + const lastModifiedHeader = response.headers["last-modified"]; + const lastModified = Array.isArray(lastModifiedHeader) + ? lastModifiedHeader[0] + : lastModifiedHeader; + + return { + stream: response.stream, + contentType: (Array.isArray(response.headers["content-type"]) + ? response.headers["content-type"][0] + : response.headers["content-type"]) || undefined, + contentLength, + etag: etag || undefined, + lastModified: lastModified ? new Date(lastModified) : undefined, + metadata, + headers: s3Headers, + } satisfies ObjectResponse; + }), + + headObject: ( + key: string, + _headers: Record, + ) => + Effect.gen(function* () { + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const swiftHeaders: Record = { + "X-Auth-Token": token, + }; + // ... handle headers if needed + const response = yield* client.execute( + HttpClientRequest.head(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders(swiftHeaders), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "HEAD", + key, + ), + ); + } + + const metadata: Record = {}; + const s3Headers: Record = {}; + + for (const [k, v] of Object.entries(response.headers)) { + const lowK = k.toLowerCase(); + const value = Array.isArray(v) ? v.join(", ") : v; + if (lowK.startsWith("x-object-meta-")) { + const metaKey = lowK.substring("x-object-meta-".length); + const decodedValue = (value.includes("%")) + ? Option.liftThrowable(decodeURIComponent)(value).pipe( + Option.getOrElse(() => value), + ) + : value; + metadata[metaKey] = decodedValue; + s3Headers[`x-amz-meta-${metaKey}`] = decodedValue; + } else if (lowK === "content-type") { + s3Headers["Content-Type"] = value; + } else if (lowK === "content-length") { + s3Headers["Content-Length"] = value; + } else if (lowK === "etag") { + s3Headers["ETag"] = value; + } else if (lowK === "last-modified") { + s3Headers["Last-Modified"] = value; + } + } + + const contentLengthHeader = response.headers["content-length"]; + const contentLength = Array.isArray(contentLengthHeader) + ? parseInt(contentLengthHeader[0] || "0") + : parseInt(contentLengthHeader || "0"); + + const etagHeader = response.headers["etag"]; + const etag = Array.isArray(etagHeader) ? etagHeader[0] : etagHeader; + + const lastModifiedHeader = response.headers["last-modified"]; + const lastModified = Array.isArray(lastModifiedHeader) + ? lastModifiedHeader[0] + : lastModifiedHeader; + + return { + contentType: (Array.isArray(response.headers["content-type"]) + ? response.headers["content-type"][0] + : response.headers["content-type"]) || undefined, + contentLength, + etag: etag || undefined, + lastModified: lastModified ? new Date(lastModified) : undefined, + metadata, + headers: s3Headers, + }; + }), + + putObject: ( + key: string, + stream: Stream.Stream, + headers: Record, + ) => + Effect.gen(function* () { + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const contentLength = headers["content-length"] || + headers["Content-Length"]; + + const swiftHeaders: Record = { + "X-Auth-Token": token, + "Content-Type": (headers["content-type"] || headers["Content-Type"] || + "application/octet-stream") as string, + ...(contentLength ? { "Content-Length": String(contentLength) } : {}), + }; + + for (const [k, v] of Object.entries(headers)) { + const lowK = k.toLowerCase(); + if (lowK.startsWith("x-amz-meta-")) { + const metaKey = lowK.substring("x-amz-meta-".length); + const value = fixHeaderEncoding(String(v)); + swiftHeaders[`X-Object-Meta-${metaKey}`] = + /[^\x20-\x7E]/.test(value) + ? encodeURIComponent(value) + : value; + } + } + + const request = HttpClientRequest.put(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders(swiftHeaders), + HttpClientRequest.bodyStream(stream), + ); + + const response = yield* client.execute(request).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + yield* Effect.logDebug( + `Swift putObject key=[${key}] status=${response.status}`, + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "PUT", + key, + ), + ); + } + + const etagHeader = response.headers["etag"]; + const etagValue = Array.isArray(etagHeader) + ? etagHeader[0] + : etagHeader; + + return { + etag: etagValue || undefined, + } satisfies PutObjectResult; + }), + + deleteObject: (key: string) => + Effect.gen(function* () { + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const response = yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status < 200 || response.status >= 300) { + if (response.status === 404) { + return; + } + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "DELETE", + key, + ), + ); + } + }), + + deleteObjects: (objects: readonly { key: string; versionId?: string }[]) => + Effect.gen(function* () { + const { url, token, container } = target; + const deleted: string[] = []; + const errors: { key: string; code: string; message: string }[] = []; + + for (const obj of objects) { + const encodedKey = obj.key.split("/").map(encodeURIComponent).join( + "/", + ); + const response = yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + yield* Effect.logDebug( + `Swift deleteObject key=[${obj.key}] status=${response.status}`, + ); + + if ( + (response.status >= 200 && response.status < 300) || + response.status === 204 || response.status === 404 + ) { + deleted.push(obj.key); + } else { + const errorBody = yield* response.text.pipe( + Effect.orElseSucceed(() => "Unknown error"), + ); + errors.push({ + key: obj.key, + code: String(response.status), + message: errorBody, + }); + } + } + + return { deleted, errors } satisfies DeleteObjectsResult; + }), + + createMultipartUpload: ( + _key: string, + _headers: Record, + ) => Effect.fail(new InternalError({ message: "Not implemented" })), + uploadPart: ( + _key: string, + _uploadId: string, + _partNumber: number, + _body: Stream.Stream, + ) => Effect.fail(new InternalError({ message: "Not implemented" })), + completeMultipartUpload: ( + _key: string, + _uploadId: string, + _parts: readonly { etag: string; partNumber: number }[], + ) => Effect.fail(new InternalError({ message: "Not implemented" })), + abortMultipartUpload: (_key: string, _uploadId: string) => + Effect.fail(new InternalError({ message: "Not implemented" })), + listMultipartUploads: (_args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + uploadIdMarker?: string; + maxUploads?: number; + encodingType?: string; + }) => Effect.fail(new InternalError({ message: "Not implemented" })), + listParts: (_key: string, _uploadId: string) => + Effect.fail(new InternalError({ message: "Not implemented" })), + }; +}; diff --git a/src/Backends/Swift/Utils.ts b/src/Backends/Swift/Utils.ts new file mode 100644 index 0000000..62295b0 --- /dev/null +++ b/src/Backends/Swift/Utils.ts @@ -0,0 +1,74 @@ +import { Effect } from "effect"; +import { + type BackendError, + BucketAlreadyExists, + BucketAlreadyOwnedByYou, + BucketNotEmpty, + InternalError, + NoSuchBucket, + NoSuchKey, +} from "../../Services/Backend.ts"; +import type { MaterializedBucket } from "../../Domain/Config.ts"; +import { SwiftClient } from "./Client.ts"; + +export interface SwiftTarget { + readonly storageUrl: string; + readonly token: string; + readonly container: string; + readonly url: string; +} + +export const mapError = ( + status: number, + message: string, + bucketName: string, + method?: string, + key?: string, +): BackendError => { + switch (status) { + case 404: + if (key) { + return new NoSuchKey({ bucketName, key, message }); + } + return new NoSuchBucket({ bucketName, message }); + case 409: + if (method === "DELETE") { + return new BucketNotEmpty({ bucketName, message }); + } + return new BucketAlreadyExists({ bucketName, message }); + case 202: + if (method === "PUT") { + return new BucketAlreadyOwnedByYou({ bucketName, message }); + } + return new InternalError({ + message: `Swift error (${status}): ${message}`, + }); + default: + return new InternalError({ + message: `Swift error (${status}): ${message}`, + }); + } +}; + +/** + * Resolves the target container and acquires the Swift token dynamically. + */ +export const getTarget = ( + bucket: MaterializedBucket | { backend_id: string }, +): Effect.Effect => + Effect.gen(function* () { + const swiftClient = yield* SwiftClient; + const auth = yield* swiftClient.getAuthMeta(bucket).pipe( + Effect.mapError((e) => new InternalError({ message: e.message })), + ); + const container = "bucket_name" in bucket ? bucket.bucket_name : ""; + const encodedContainer = container ? encodeURIComponent(container) : ""; + return { + storageUrl: auth.storageUrl, + token: auth.token, + container, + url: encodedContainer + ? `${auth.storageUrl}/${encodedContainer}` + : auth.storageUrl, + }; + }); diff --git a/src/Config/Layer.ts b/src/Config/Layer.ts index 0470bb0..8efd2d0 100644 --- a/src/Config/Layer.ts +++ b/src/Config/Layer.ts @@ -1,39 +1,140 @@ -import { Context, Effect, Layer, type Option } from "effect"; +import { Config, Context, Effect, Layer, type Option, Schema } from "effect"; import { parse } from "@std/yaml"; import { GlobalConfig, lookupBucket, type MaterializedBucket, } from "../Domain/Config.ts"; -import { Schema } from "effect"; -export class AppConfig extends Context.Tag("AppConfig")< - AppConfig, +export class HeraldConfig extends Context.Tag("HeraldConfig")< + HeraldConfig, { readonly raw: GlobalConfig; readonly lookupBucket: (name: string) => Option.Option; } >() {} -export const AppConfigLive = Layer.effect( - AppConfig, +function toConfigKey(str: string): string { + const mapping: Record = { + "AUTH_URL": "auth_url", + "PROJECT_NAME": "project_name", + "USER_DOMAIN_NAME": "user_domain_name", + "PROJECT_DOMAIN_NAME": "project_domain_name", + "ACCESS_KEY_ID": "accessKeyId", + "SECRET_ACCESS_KEY": "secretAccessKey", + }; + return mapping[str] || str.toLowerCase(); +} + +export function parseConfig( + yamlConfig: unknown, + env: Record, +): GlobalConfig { + const yamlBackends = + (yamlConfig && typeof yamlConfig === "object" && "backends" in yamlConfig) + ? (yamlConfig as { backends: Record> }) + .backends + : {}; + + const backends: Record> = {}; + for (const [k, v] of Object.entries(yamlBackends)) { + backends[k] = { ...v }; + } + + const commonKeys = [ + "PROTOCOL", + "ENDPOINT", + "REGION", + "BUCKETS", + "ACCESS_KEY_ID", + "SECRET_ACCESS_KEY", + "AUTH_URL", + "CONTAINER", + "USERNAME", + "PASSWORD", + "PROJECT_NAME", + "USER_DOMAIN_NAME", + "PROJECT_DOMAIN_NAME", + ]; + + for (const [key, value] of Object.entries(env)) { + if (!key.startsWith("HERALD_")) continue; + if (key === "HERALD_CONFIG_PATH") continue; + if (key === "HERALD_LOG_LEVEL") continue; + + const parts = key.substring(7).split("_"); + let backendName: string; + let configParts: string[]; + + if (parts.length === 1 || commonKeys.includes(parts[0])) { + backendName = "default"; + configParts = parts; + } else { + backendName = parts[0].toLowerCase(); + configParts = parts.slice(1); + } + + const configKey = toConfigKey(configParts.join("_")); + if (!backends[backendName]) backends[backendName] = {}; + const backend = backends[backendName]; + + const credentialKeys = [ + "accessKeyId", + "secretAccessKey", + "username", + "password", + "project_name", + "user_domain_name", + "project_domain_name", + ]; + + if (credentialKeys.includes(configKey)) { + if (!backend.credentials) { + backend.credentials = {} as Record; + } + (backend.credentials as Record)[configKey] = value; + } else { + backend[configKey] = value; + } + } + + // Default backend fallback if no backends defined at all + if (Object.keys(backends).length === 0) { + backends["default"] = { + protocol: "s3", + buckets: "*", + }; + } + + return Schema.decodeUnknownSync(GlobalConfig)({ backends }); +} + +export const HeraldConfigLive = Layer.effect( + HeraldConfig, Effect.gen(function* () { - const configPath = yield* Effect.succeed( - Deno.env.get("CONFIG_PATH") ?? "herald.yaml", + const configPath = yield* Config.string("HERALD_CONFIG_PATH").pipe( + Config.orElse(() => Config.string("CONFIG_PATH")), + Config.withDefault("herald.yaml"), ); - const content = yield* Effect.tryPromise({ + const yamlConfig = yield* Effect.tryPromise({ try: () => Deno.readTextFile(configPath), - catch: (e) => - new Error(`Failed to read config file at ${configPath}: ${e}`), - }); + catch: () => new Error("Config file missing"), + }).pipe( + Effect.flatMap((content) => + Effect.try({ + try: () => parse(content), + catch: (e) => new Error(`YAML parse error: ${e}`), + }) + ), + Effect.orElseSucceed(() => ({ backends: {} })), + ); - const yaml = yield* Effect.try({ - try: () => parse(content) as unknown, - catch: (e) => new Error(`Failed to parse YAML: ${e}`), - }); + // Discovery needs the full environment. In Deno we use Deno.env.toObject(). + // We can wrap this in an Effect to be more idiomatic. + const env = yield* Effect.sync(() => Deno.env.toObject()); - const raw = yield* Schema.decodeUnknown(GlobalConfig)(yaml); + const raw = parseConfig(yamlConfig, env); return { raw, diff --git a/src/Domain/Config.ts b/src/Domain/Config.ts index b0239e1..77f32a0 100644 --- a/src/Domain/Config.ts +++ b/src/Domain/Config.ts @@ -1,12 +1,20 @@ import { Option, Schema } from "effect"; -export const Credentials = Schema.Struct({ - username: Schema.optional(Schema.String), - password: Schema.optional(Schema.String), +export const S3Credentials = Schema.Struct({ accessKeyId: Schema.optional(Schema.String), secretAccessKey: Schema.optional(Schema.String), }); +export const SwiftCredentials = Schema.Struct({ + username: Schema.optional(Schema.String), + password: Schema.optional(Schema.String), + project_name: Schema.optional(Schema.String), + user_domain_name: Schema.optional(Schema.String), + project_domain_name: Schema.optional(Schema.String), +}); + +export const Credentials = Schema.Union(S3Credentials, SwiftCredentials); + export const BucketOverride = Schema.Struct({ endpoint: Schema.optional(Schema.String), bucket_name: Schema.optional(Schema.String), @@ -15,20 +23,33 @@ export const BucketOverride = Schema.Struct({ export type BucketOverride = Schema.Schema.Type; -export const BackendConfig = Schema.Struct({ - protocol: Schema.Literal("s3", "swift"), +export const BucketsConfig = Schema.optionalWith( + Schema.Union( + Schema.Record({ key: Schema.String, value: BucketOverride }), + Schema.String, + ), + { default: () => "*" }, +); + +export const S3Config = Schema.Struct({ + protocol: Schema.Literal("s3"), endpoint: Schema.optional(Schema.String), region: Schema.optional(Schema.String), - credentials: Schema.optional(Credentials), - buckets: Schema.optionalWith( - Schema.Union( - Schema.Record({ key: Schema.String, value: BucketOverride }), - Schema.String, - ), - { default: () => "*" }, - ), + credentials: Schema.optional(S3Credentials), + buckets: BucketsConfig, }); +export const SwiftConfig = Schema.Struct({ + protocol: Schema.Literal("swift"), + auth_url: Schema.String, + region: Schema.optional(Schema.String), + container: Schema.optional(Schema.String), + credentials: Schema.optional(SwiftCredentials), + buckets: BucketsConfig, +}); + +export const BackendConfig = Schema.Union(S3Config, SwiftConfig); + export type BackendConfig = Schema.Schema.Type; export const GlobalConfig = Schema.Struct({ @@ -45,6 +66,9 @@ export const MaterializedBucket = Schema.Struct({ region: Schema.optional(Schema.String), bucket_name: Schema.String, credentials: Schema.optional(Credentials), + // Swift specific + auth_url: Schema.optional(Schema.String), + container: Schema.optional(Schema.String), }); export type MaterializedBucket = Schema.Schema.Type; @@ -68,17 +92,20 @@ export const lookupBucket = ( const buckets = backend.buckets; if (buckets && typeof buckets !== "string" && buckets[bucketName]) { const override = buckets[bucketName]; - return Option.some( - { - name: bucketName, - backend_id, - protocol: backend.protocol, - endpoint: override.endpoint ?? backend.endpoint, - region: override.region ?? backend.region, - bucket_name: override.bucket_name ?? bucketName, - credentials: backend.credentials, - } as const, - ); + const base: MaterializedBucket = { + name: bucketName, + backend_id, + protocol: backend.protocol, + endpoint: override.endpoint ?? + (backend.protocol === "s3" ? backend.endpoint : undefined), + region: override.region ?? backend.region, + bucket_name: override.bucket_name ?? bucketName, + credentials: backend.credentials, + auth_url: backend.protocol === "swift" ? backend.auth_url : undefined, + container: backend.protocol === "swift" ? backend.container : undefined, + }; + + return Option.some(base); } } @@ -88,19 +115,25 @@ export const lookupBucket = ( if (buckets && typeof buckets !== "string") { for (const [key, override] of Object.entries(buckets)) { if (globToRegex(key).test(bucketName)) { - return Option.some( - { - name: bucketName, - backend_id, - protocol: backend.protocol, - endpoint: (override as BucketOverride).endpoint ?? - backend.endpoint, - region: (override as BucketOverride).region ?? backend.region, - bucket_name: (override as BucketOverride).bucket_name ?? - bucketName, - credentials: backend.credentials, - } as const, - ); + const base: MaterializedBucket = { + name: bucketName, + backend_id, + protocol: backend.protocol, + endpoint: (override as BucketOverride).endpoint ?? + (backend.protocol === "s3" ? backend.endpoint : undefined), + region: (override as BucketOverride).region ?? backend.region, + bucket_name: (override as BucketOverride).bucket_name ?? + bucketName, + credentials: backend.credentials, + auth_url: backend.protocol === "swift" + ? backend.auth_url + : undefined, + container: backend.protocol === "swift" + ? backend.container + : undefined, + }; + + return Option.some(base); } } } @@ -111,17 +144,21 @@ export const lookupBucket = ( const buckets = backend.buckets; if (buckets && typeof buckets === "string") { if (globToRegex(buckets).test(bucketName)) { - return Option.some( - { - name: bucketName, - backend_id, - protocol: backend.protocol, - endpoint: backend.endpoint, - region: backend.region, - bucket_name: bucketName, - credentials: backend.credentials, - } as const, - ); + const base: MaterializedBucket = { + name: bucketName, + backend_id, + protocol: backend.protocol, + endpoint: backend.protocol === "s3" ? backend.endpoint : undefined, + region: backend.region, + bucket_name: bucketName, + credentials: backend.credentials, + auth_url: backend.protocol === "swift" ? backend.auth_url : undefined, + container: backend.protocol === "swift" + ? backend.container + : undefined, + }; + + return Option.some(base); } } } diff --git a/src/Frontend/Api.ts b/src/Frontend/Api.ts index 0c8c502..f4a7fdc 100644 --- a/src/Frontend/Api.ts +++ b/src/Frontend/Api.ts @@ -5,7 +5,11 @@ export class BadGateway extends Schema.TaggedError()("BadGateway", { message: Schema.String, }) {} -export const S3Api = HttpApiGroup.make("s3") +export const HttpS3Api = HttpApiGroup.make("s3") + .add( + HttpApiEndpoint.post("postRoot", "/") + .addError(BadGateway, { status: 502 }), + ) .add( HttpApiEndpoint.get("listBuckets", "/") .addError(BadGateway, { status: 502 }), diff --git a/src/Frontend/Buckets/Create.ts b/src/Frontend/Buckets/Create.ts index f40475c..68c32a9 100644 --- a/src/Frontend/Buckets/Create.ts +++ b/src/Frontend/Buckets/Create.ts @@ -1,12 +1,37 @@ import { Effect } from "effect"; import { HttpServerResponse } from "@effect/platform"; -import { resolveBucket } from "../Utils.ts"; - -export const createBucket = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - yield* backend.createBucket(); +import { RequestContext } from "../Utils.ts"; + +export const createBucket = () => + Effect.gen(function* () { + const { backend, bucket, params, request } = yield* RequestContext; + + yield* Effect.logDebug( + `createBucket bucket=[${bucket}] url=[${request.url}]`, + ); + + if (params.acl !== undefined) { + // PutBucketAcl + // Check for canned ACL validity if present + const cannedAcl = request.headers["x-amz-acl"]; + const validCannedAcls = [ + "private", + "public-read", + "public-read-write", + "authenticated-read", + ]; + if (cannedAcl && !validCannedAcls.includes(cannedAcl)) { + return HttpServerResponse.text( + `InvalidArgumentArgument x-amz-acl is invalid.`, + { status: 400, headers: { "Content-Type": "application/xml" } }, + ); + } + + // For now, we just return 200 OK if the bucket exists + yield* backend.headBucket(); return HttpServerResponse.text("", { status: 200 }); - })); + } + + yield* backend.createBucket(); + return HttpServerResponse.text("", { status: 200 }); + }); diff --git a/src/Frontend/Buckets/Delete.ts b/src/Frontend/Buckets/Delete.ts index de3a301..6c7fbe1 100644 --- a/src/Frontend/Buckets/Delete.ts +++ b/src/Frontend/Buckets/Delete.ts @@ -1,12 +1,10 @@ import { Effect } from "effect"; import { HttpServerResponse } from "@effect/platform"; -import { resolveBucket } from "../Utils.ts"; +import { RequestContext } from "../Utils.ts"; -export const deleteBucket = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - yield* backend.deleteBucket(); - return HttpServerResponse.empty({ status: 204 }); - })); +export const deleteBucket = () => + Effect.gen(function* () { + const { backend } = yield* RequestContext; + yield* backend.deleteBucket(); + return HttpServerResponse.empty({ status: 204 }); + }); diff --git a/src/Frontend/Buckets/Head.ts b/src/Frontend/Buckets/Head.ts index fb371d8..a076d71 100644 --- a/src/Frontend/Buckets/Head.ts +++ b/src/Frontend/Buckets/Head.ts @@ -1,12 +1,10 @@ import { Effect } from "effect"; import { HttpServerResponse } from "@effect/platform"; -import { resolveBucket } from "../Utils.ts"; +import { RequestContext } from "../Utils.ts"; -export const headBucket = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - yield* backend.headBucket(); - return HttpServerResponse.empty({ status: 200 }); - })); +export const headBucket = () => + Effect.gen(function* () { + const { backend } = yield* RequestContext; + yield* backend.headBucket(); + return HttpServerResponse.empty({ status: 200 }); + }); diff --git a/src/Frontend/Buckets/List.ts b/src/Frontend/Buckets/List.ts index b11f411..4bb13f5 100644 --- a/src/Frontend/Buckets/List.ts +++ b/src/Frontend/Buckets/List.ts @@ -1,23 +1,24 @@ import { Effect } from "effect"; -import { AppConfig } from "../../Config/Layer.ts"; +import { HeraldConfig } from "../../Config/Layer.ts"; import { S3Xml } from "../../Services/S3Xml.ts"; import { resolveBackend } from "../Utils.ts"; export const listBuckets = () => Effect.gen(function* () { - const config = yield* AppConfig; + const config = yield* HeraldConfig; // For ListBuckets, we need to decide which backend to proxy to. - const s3BackendId = Object.keys(config.raw.backends).find((id) => + // We prefer an S3 backend if available, otherwise we take the first one. + const backendId = Object.keys(config.raw.backends).find((id) => config.raw.backends[id].protocol === "s3" - ); + ) ?? Object.keys(config.raw.backends)[0]; - if (!s3BackendId) { + if (!backendId) { const s3Xml = yield* S3Xml; - return s3Xml.formatError("No S3 backend configured"); + return s3Xml.formatError("No backend configured"); } - return yield* resolveBackend(s3BackendId, (backend) => + return yield* resolveBackend(backendId, (backend) => Effect.gen(function* () { const result = yield* backend.listBuckets(); const s3xml = yield* S3Xml; diff --git a/src/Frontend/Health/Api.ts b/src/Frontend/Health/Api.ts index 70032e5..829ae5f 100644 --- a/src/Frontend/Health/Api.ts +++ b/src/Frontend/Health/Api.ts @@ -1,7 +1,7 @@ import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "@effect/platform"; import { Schema } from "effect"; -export class HealthApi extends HttpApiGroup.make("health") +export class HealthHttpApi extends HttpApiGroup.make("health") .add( HttpApiEndpoint.get("getStatus", "/health") .addSuccess(Schema.Struct({ status: Schema.Literal("ok") })), diff --git a/src/Frontend/Health/Http.ts b/src/Frontend/Health/Http.ts index 52e64f2..0f186df 100644 --- a/src/Frontend/Health/Http.ts +++ b/src/Frontend/Health/Http.ts @@ -1,9 +1,9 @@ import { HttpApiBuilder } from "@effect/platform"; import { Effect } from "effect"; -import { Api } from "../../Api.ts"; +import { HttpHeraldApi } from "../../Api.ts"; export const HttpHealthLive = HttpApiBuilder.group( - Api, + HttpHeraldApi, "health", (handlers) => handlers.handle( diff --git a/src/Frontend/Http.ts b/src/Frontend/Http.ts index c1b4b12..90b4541 100644 --- a/src/Frontend/Http.ts +++ b/src/Frontend/Http.ts @@ -1,6 +1,6 @@ -import { HttpApiBuilder } from "@effect/platform"; -import { Layer } from "effect"; -import { Api } from "../Api.ts"; +import { HttpApiBuilder, HttpServerResponse } from "@effect/platform"; +import { Effect, Layer } from "effect"; +import { HttpHeraldApi } from "../Api.ts"; import { listBuckets } from "./Buckets/List.ts"; import { createBucket } from "./Buckets/Create.ts"; import { deleteBucket } from "./Buckets/Delete.ts"; @@ -12,27 +12,40 @@ import { deleteObject } from "./Objects/Delete.ts"; import { headObject } from "./Objects/Head.ts"; import { postObject } from "./Objects/Post.ts"; import { S3ClientLive } from "../Backends/S3/Client.ts"; +import { SwiftClientLive } from "../Backends/Swift/Client.ts"; import { S3XmlLive } from "../Services/S3Xml.ts"; import { BackendResolverLive } from "../Services/BackendResolver.ts"; +import { provideRequestContext } from "./Utils.ts"; export const HttpS3Live = HttpApiBuilder.group( - Api, + HttpHeraldApi, "s3", (handlers) => handlers + // handleRaw is preferred througout since + // we want to return XML directly + // after setting our own + .handleRaw("postRoot", (_handlers) => + Effect.gen(function* () { + yield* Effect.logDebug("POST / received"); + // FIXME: what's the purose of this handler? + // 200 diverges from 502 as defiend in the openapi + return HttpServerResponse.text("", { status: 200 }); + })) .handleRaw("listBuckets", listBuckets) - .handleRaw("createBucket", createBucket) - .handleRaw("deleteBucket", deleteBucket) - .handleRaw("headBucket", headBucket) - .handleRaw("listObjects", listObjects) - .handleRaw("postBucket", postObject) - .handleRaw("getObject", getObject) - .handleRaw("putObject", putObject) - .handleRaw("postObject", postObject) - .handleRaw("deleteObject", deleteObject) - .handleRaw("headObject", headObject), + .handleRaw("createBucket", provideRequestContext(createBucket)) + .handleRaw("deleteBucket", provideRequestContext(deleteBucket)) + .handleRaw("headBucket", provideRequestContext(headBucket)) + .handleRaw("listObjects", provideRequestContext(listObjects)) + .handleRaw("postBucket", provideRequestContext(postObject)) + .handleRaw("getObject", provideRequestContext(getObject)) + .handleRaw("putObject", provideRequestContext(putObject)) + .handleRaw("postObject", provideRequestContext(postObject)) + .handleRaw("deleteObject", provideRequestContext(deleteObject)) + .handleRaw("headObject", provideRequestContext(headObject)), ).pipe( Layer.provide(BackendResolverLive), Layer.provide(S3ClientLive), + Layer.provide(SwiftClientLive), Layer.provide(S3XmlLive), ); diff --git a/src/Frontend/Objects/Delete.ts b/src/Frontend/Objects/Delete.ts index f2c9270..3e1856b 100644 --- a/src/Frontend/Objects/Delete.ts +++ b/src/Frontend/Objects/Delete.ts @@ -1,18 +1,20 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; /** * Handler for DeleteObject (DELETE /:bucket/*) */ -export const deleteObject = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const key = extractKey(request.url, bucket); +export const deleteObject = () => + Effect.gen(function* () { + const { backend, key, params } = yield* RequestContext; - yield* backend.deleteObject(key); + if (params.uploadId) { + // Abort Multipart Upload + yield* backend.abortMultipartUpload(key, params.uploadId); return HttpServerResponse.empty({ status: 204 }); - })); + } + + yield* backend.deleteObject(key); + return HttpServerResponse.empty({ status: 204 }); + }); diff --git a/src/Frontend/Objects/Get.ts b/src/Frontend/Objects/Get.ts index fff8504..c223fec 100644 --- a/src/Frontend/Objects/Get.ts +++ b/src/Frontend/Objects/Get.ts @@ -1,20 +1,35 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; /** * Handler for GetObject (GET /:bucket/*) + * Also handles ListParts (?uploadId=...). */ -export const getObject = ({ path: { bucket } }: { path: { bucket: string } }) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const key = extractKey(request.url, bucket); +export const getObject = () => + Effect.gen(function* () { + const { backend, key, params, request } = yield* RequestContext; + const s3Xml = yield* S3Xml; - const result = yield* backend.getObject(key); - return HttpServerResponse.stream(result.stream, { - status: 200, - headers: result.headers, - contentType: result.contentType, - }); - })); + if (params.uploadId) { + // List Parts + const result = yield* backend.listParts(key, params.uploadId); + return s3Xml.formatListParts(result); + } + + const combinedHeaders = { ...request.headers }; + if (params.partNumber) { + combinedHeaders["x-amz-part-number"] = String(params.partNumber); + } + + const result = yield* backend.getObject(key, combinedHeaders); + const status = (request.headers["range"] || request.headers["Range"]) + ? 206 + : 200; + return HttpServerResponse.stream(result.stream, { + status, + headers: result.headers, + contentType: result.contentType, + }); + }); diff --git a/src/Frontend/Objects/Head.ts b/src/Frontend/Objects/Head.ts index b91a0a2..b3daa57 100644 --- a/src/Frontend/Objects/Head.ts +++ b/src/Frontend/Objects/Head.ts @@ -1,21 +1,22 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; /** * Handler for HeadObject (HEAD /:bucket/*) */ -export const headObject = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const key = extractKey(request.url, bucket); +export const headObject = () => + Effect.gen(function* () { + const { backend, key, params, request } = yield* RequestContext; - const result = yield* backend.headObject(key); - return HttpServerResponse.empty({ - status: 200, - headers: result.headers, - }); - })); + const combinedHeaders = { ...request.headers }; + if (params.partNumber) { + combinedHeaders["x-amz-part-number"] = String(params.partNumber); + } + + const result = yield* backend.headObject(key, combinedHeaders); + return HttpServerResponse.empty({ + status: 200, + headers: result.headers, + }); + }); diff --git a/src/Frontend/Objects/List.ts b/src/Frontend/Objects/List.ts index 3552e2e..883f247 100644 --- a/src/Frontend/Objects/List.ts +++ b/src/Frontend/Objects/List.ts @@ -1,47 +1,49 @@ import { Effect } from "effect"; -import { HttpServerRequest } from "@effect/platform"; -import { resolveBucket } from "../Utils.ts"; +import { RequestContext } from "../Utils.ts"; import { S3Xml } from "../../Services/S3Xml.ts"; /** * Handler for ListObjects (GET /:bucket) */ -export const listObjects = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const s3Xml = yield* S3Xml; - const url = new URL(request.url, "http://localhost"); - const searchParams = url.searchParams; +export const listObjects = () => + Effect.gen(function* () { + const { backend, params } = yield* RequestContext; + const s3Xml = yield* S3Xml; - if (searchParams.has("versions")) { - const result = yield* backend.listVersions({ - prefix: searchParams.get("prefix") ?? undefined, - delimiter: searchParams.get("delimiter") ?? undefined, - keyMarker: searchParams.get("key-marker") ?? undefined, - versionIdMarker: searchParams.get("version-id-marker") ?? undefined, - maxKeys: searchParams.has("max-keys") - ? parseInt(searchParams.get("max-keys")!) - : undefined, - encodingType: searchParams.get("encoding-type") ?? undefined, - }); - return s3Xml.formatListVersions(result); - } + if (params.versions !== undefined) { + const result = yield* backend.listVersions({ + prefix: params.prefix, + delimiter: params.delimiter, + keyMarker: params["key-marker"], + versionIdMarker: params["version-id-marker"], + maxKeys: params["max-keys"], + encodingType: params["encoding-type"], + }); + return s3Xml.formatListVersions(result); + } - const result = yield* backend.listObjects({ - prefix: searchParams.get("prefix") ?? undefined, - delimiter: searchParams.get("delimiter") ?? undefined, - marker: searchParams.get("marker") ?? undefined, - maxKeys: searchParams.has("max-keys") - ? parseInt(searchParams.get("max-keys")!) - : undefined, - encodingType: searchParams.get("encoding-type") ?? undefined, - continuationToken: searchParams.get("continuation-token") ?? undefined, - startAfter: searchParams.get("start-after") ?? undefined, - listType: searchParams.get("list-type") === "2" ? 2 : 1, + if (params.uploads !== undefined) { + const result = yield* backend.listMultipartUploads({ + prefix: params.prefix, + delimiter: params.delimiter, + keyMarker: params["key-marker"], + uploadIdMarker: params["upload-id-marker"], + maxUploads: params["max-uploads"], + encodingType: params["encoding-type"], }); + return s3Xml.formatListMultipartUploads(result); + } + + const result = yield* backend.listObjects({ + prefix: params.prefix, + delimiter: params.delimiter, + marker: params.marker, + maxKeys: params["max-keys"], + encodingType: params["encoding-type"], + continuationToken: params["continuation-token"], + startAfter: params["start-after"], + listType: params["list-type"] === "2" ? 2 : 1, + }); - return s3Xml.formatListObjects(result); - })); + return s3Xml.formatListObjects(result); + }); diff --git a/src/Frontend/Objects/Post.ts b/src/Frontend/Objects/Post.ts index 84f7039..dffade7 100644 --- a/src/Frontend/Objects/Post.ts +++ b/src/Frontend/Objects/Post.ts @@ -1,81 +1,151 @@ -import { Effect, Stream } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { Effect, Option, Stream } from "effect"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; /** * Handler for POST requests on buckets or objects. * Primarily used for Multi-Object Delete (POST /:bucket?delete). + * Also handles InitiateMultipartUpload (?uploads) and CompleteMultipartUpload (?uploadId=...). */ -export const postObject = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const url = new URL(request.url, "http://localhost"); - const searchParams = url.searchParams; - const key = extractKey(request.url, bucket); +export const postObject = () => + Effect.gen(function* () { + const { backend, bucket, key, params, request } = yield* RequestContext; + const s3Xml = yield* S3Xml; - if (searchParams.has("delete")) { - // Multi-Object Delete - const bodyChunks = yield* Stream.runCollect(request.stream); - let totalLength = 0; - for (const chunk of Array.from(bodyChunks)) { - totalLength += chunk.length; - } - const bodyBytes = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of Array.from(bodyChunks)) { - bodyBytes.set(chunk, offset); - offset += chunk.length; - } - const bodyText = new TextDecoder().decode(bodyBytes); - - const objects: { key: string; versionId?: string }[] = []; - // Simple XML parsing for Multi-Object Delete - const objectMatches = Array.from( - bodyText.matchAll(/(.*?)<\/Object>/gs), - ); - for (const match of objectMatches) { - const content = match[1]; - const keyMatch = content.match(/(.*?)<\/Key>/); - const versionIdMatch = content.match(/(.*?)<\/VersionId>/); - if (keyMatch) { - try { - objects.push({ - key: decodeURIComponent(keyMatch[1]), - versionId: versionIdMatch ? versionIdMatch[1] : undefined, - }); - } catch { - objects.push({ - key: keyMatch[1], - versionId: versionIdMatch ? versionIdMatch[1] : undefined, - }); - } - } - } + if (params.delete !== undefined) { + // Multi-Object Delete + const bodyChunks = yield* Stream.runCollect(request.stream); + let totalLength = 0; + for (const chunk of Array.from(bodyChunks)) { + totalLength += chunk.length; + } + const bodyBytes = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of Array.from(bodyChunks)) { + bodyBytes.set(chunk, offset); + offset += chunk.length; + } + const bodyText = new TextDecoder().decode(bodyBytes); - if (objects.length > 0) { - const deleteResult = yield* backend.deleteObjects(objects); - const xml = - `${ - deleteResult.deleted.map((k) => - `${k}` - ).join("") - }`; - return HttpServerResponse.text(xml, { - headers: { "Content-Type": "application/xml" }, + const objects: { key: string; versionId?: string }[] = []; + // Simple XML parsing for Multi-Object Delete + const objectMatches = Array.from( + bodyText.matchAll(/(.*?)<\/Object>/gs), + ); + for (const match of objectMatches) { + const content = match[1]; + const keyMatch = content.match(/(.*?)<\/Key>/); + const versionIdMatch = content.match(/(.*?)<\/VersionId>/); + if (keyMatch) { + const rawKey = keyMatch[1]; + const key = Option.liftThrowable(decodeURIComponent)(rawKey).pipe( + Option.getOrElse(() => rawKey), + ); + yield* Effect.logDebug(`DeleteObjects extracted key=[${key}]`); + objects.push({ + key, + versionId: versionIdMatch ? versionIdMatch[1] : undefined, }); } - // If no keys, still return empty result + } + + if (objects.length > 0) { + const deleteResult = yield* backend.deleteObjects(objects); + const deletedXml = deleteResult.deleted.map((k) => + `${k}` + ).join(""); + const errorsXml = deleteResult.errors.map((e) => + `${e.key}${e.code}${e.message}` + ).join(""); + const xml = - ``; + `${deletedXml}${errorsXml}`; return HttpServerResponse.text(xml, { headers: { "Content-Type": "application/xml" }, }); } + // If no keys, still return empty result + const xml = + ``; + return HttpServerResponse.text(xml, { + headers: { "Content-Type": "application/xml" }, + }); + } + + if (params.uploads !== undefined) { + // Initiate Multipart Upload + const result = yield* backend.createMultipartUpload( + key, + request.headers, + ); + return s3Xml.formatInitiateMultipartUpload( + bucket, + key, + result.uploadId, + ); + } + + if (params.uploadId) { + // Complete Multipart Upload + const bodyChunks = yield* Stream.runCollect(request.stream); + let totalLength = 0; + for (const chunk of Array.from(bodyChunks)) { + totalLength += chunk.length; + } + const bodyBytes = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of Array.from(bodyChunks)) { + bodyBytes.set(chunk, offset); + offset += chunk.length; + } + const bodyText = new TextDecoder().decode(bodyBytes); - return yield* Effect.fail( - new Error(`Method POST for key [${key}] not implemented`), + const parts: { etag: string; partNumber: number }[] = []; + const partMatches = Array.from( + bodyText.matchAll(/(.*?)<\/Part>/gs), ); - })); + for (const match of partMatches) { + const content = match[1]; + const partNumberMatch = content.match( + /(.*?)<\/PartNumber>/, + ); + const etagMatch = content.match(/(.*?)<\/ETag>/); + if (partNumberMatch && etagMatch) { + parts.push({ + partNumber: parseInt(partNumberMatch[1]), + etag: etagMatch[1].replace(/"/g, '"'), + }); + } + } + + const result = yield* backend.completeMultipartUpload( + key, + params.uploadId, + parts, + ).pipe( + Effect.catchTag("NoSuchUpload", (e) => + Effect.gen(function* () { + // Idempotency: check if object already exists + const head = yield* backend.headObject(key, {}).pipe( + Effect.orElseFail(() => e), + ); + if (head.etag) { + return { + location: `http://localhost/${bucket}/${key}`, // Approximate + bucket, + key, + etag: head.etag, + versionId: head.headers["x-amz-version-id"], + }; + } + return yield* Effect.fail(e); + })), + ); + return s3Xml.formatCompleteMultipartUpload(result); + } + + return yield* Effect.fail( + new Error(`Method POST for key [${key}] not implemented`), + ); + }); diff --git a/src/Frontend/Objects/Put.ts b/src/Frontend/Objects/Put.ts index 62894cc..08421cd 100644 --- a/src/Frontend/Objects/Put.ts +++ b/src/Frontend/Objects/Put.ts @@ -1,27 +1,39 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; /** * Handler for PutObject (PUT /:bucket/*) */ -export const putObject = ({ path: { bucket } }: { path: { bucket: string } }) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const key = extractKey(request.url, bucket); +export const putObject = () => + Effect.gen(function* () { + const { backend, key, params, request } = yield* RequestContext; - const result = yield* backend.putObject( + if (params.partNumber && params.uploadId) { + // Upload Part + const result = yield* backend.uploadPart( key, + params.uploadId, + params.partNumber, request.stream, - request.headers, ); - const headers: Record = {}; - if (result.etag) headers["etag"] = result.etag; - if (result.versionId) headers["x-amz-version-id"] = result.versionId; - return HttpServerResponse.empty({ status: 200, - headers, + headers: { ETag: result.etag }, }); - })); + } + + const result = yield* backend.putObject( + key, + request.stream, + request.headers, + ); + const headers: Record = {}; + if (result.etag) headers["ETag"] = result.etag; + if (result.versionId) headers["x-amz-version-id"] = result.versionId; + + return HttpServerResponse.empty({ + status: 200, + headers, + }); + }); diff --git a/src/Frontend/Utils.ts b/src/Frontend/Utils.ts index dec42b1..972743c 100644 --- a/src/Frontend/Utils.ts +++ b/src/Frontend/Utils.ts @@ -1,4 +1,4 @@ -import { Effect, Option } from "effect"; +import { Context, Effect, Either, Option, Schema } from "effect"; import { BackendResolver } from "../Services/BackendResolver.ts"; import { S3Xml } from "../Services/S3Xml.ts"; import { @@ -8,22 +8,51 @@ import { BucketAlreadyOwnedByYou, BucketNotEmpty, DeleteObjectsError, + EntityTooSmall, InternalError, + InvalidPart, + InvalidPartOrder, + InvalidRequest, + MalformedXML, NoSuchBucket, NoSuchKey, + NoSuchUpload, } from "../Services/Backend.ts"; -import { HttpServerRequest, type HttpServerResponse } from "@effect/platform"; -import type { AppConfig } from "../Config/Layer.ts"; +import { + HttpServerRequest, + type HttpServerResponse, + Url, +} from "@effect/platform"; +import type { HeraldConfig } from "../Config/Layer.ts"; import type { S3Client } from "../Backends/S3/Client.ts"; +import type { SwiftClient } from "../Backends/Swift/Client.ts"; import { BadGateway } from "./Api.ts"; +/** + * Fixes header values that might have been incorrectly decoded as Latin-1 + * instead of UTF-8 by the HTTP server. + */ +export function fixHeaderEncoding(value: string): string { + // deno-lint-ignore no-control-regex + if (!/[^\x00-\x7F]/.test(value)) { + return value; + } + return Option.liftThrowable(() => { + const bytes = Uint8Array.from(value, (c) => c.charCodeAt(0)); + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + })().pipe( + Option.getOrElse(() => value), + ); +} + /** * Extracts the object key from the request URL, given the bucket name. */ export function extractKey(requestUrl: string, bucket: string): string { - const pathname = requestUrl.startsWith("/") - ? requestUrl - : new URL(requestUrl).pathname; + const urlResult = Url.fromString(requestUrl, "http://localhost"); + const pathname = Either.isRight(urlResult) + ? urlResult.right.pathname + : requestUrl; const [pathOnly] = pathname.split("?"); const bucketPrefixWithSlash = `/${bucket}/`; @@ -37,6 +66,112 @@ export function extractKey(requestUrl: string, bucket: string): string { return ""; } +/** + * Context for S3 operations (bucket or object). + */ +export class RequestContext extends Context.Tag("RequestContext")< + RequestContext, + { + readonly backend: typeof Backend.Service; + readonly bucket: string; + readonly key: string; + readonly params: S3QueryParams; + readonly request: HttpServerRequest.HttpServerRequest; + } +>() {} + +/** + * Higher-order function to handle S3 context. + */ +export function provideRequestContext< + A extends HttpServerResponse.HttpServerResponse, + E, + R, +>( + fn: () => Effect.Effect, +): ( + args: { path: { bucket: string } }, +) => Effect.Effect< + HttpServerResponse.HttpServerResponse, + BadGateway, + | Exclude + | BackendResolver + | S3Xml + | HeraldConfig + | S3Client + | SwiftClient + | HttpServerRequest.HttpServerRequest +> { + return ({ path: { bucket } }) => + resolveBucket(bucket, (backend) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const urlResult = Url.fromString(request.url, "http://localhost"); + if (Either.isLeft(urlResult)) { + return yield* Effect.fail( + new InternalError({ message: String(urlResult.left) }), + ); + } + const url = urlResult.right; + const key = extractKey(request.url, bucket); + const params = yield* parseQueryParams(url.searchParams, S3QueryParams); + const ctx = { + backend, + bucket, + key, + params, + request, + }; + return yield* fn().pipe(Effect.provideService(RequestContext, ctx)); + }) as unknown as Effect.Effect< + HttpServerResponse.HttpServerResponse, + BadGateway, + Exclude + >); +} + +/** + * Common S3 Query Parameters Schema + */ +export const S3QueryParams = Schema.Struct({ + uploadId: Schema.optional(Schema.String), + partNumber: Schema.optional(Schema.NumberFromString), + prefix: Schema.optional(Schema.String), + delimiter: Schema.optional(Schema.String), + marker: Schema.optional(Schema.String), + "max-keys": Schema.optional(Schema.NumberFromString), + "max-uploads": Schema.optional(Schema.NumberFromString), + "encoding-type": Schema.optional(Schema.String), + "continuation-token": Schema.optional(Schema.String), + "start-after": Schema.optional(Schema.String), + "list-type": Schema.optional(Schema.String), + "version-id-marker": Schema.optional(Schema.String), + "key-marker": Schema.optional(Schema.String), + "upload-id-marker": Schema.optional(Schema.String), + versions: Schema.optional(Schema.String), + uploads: Schema.optional(Schema.String), + delete: Schema.optional(Schema.String), + acl: Schema.optional(Schema.String), +}); + +export type S3QueryParams = Schema.Schema.Type; + +/** + * Utility to parse search params using a Schema. + */ +export function parseQueryParams( + searchParams: URLSearchParams, + schema: Schema.Schema, +): Effect.Effect { + const paramsRecord: Record = {}; + searchParams.forEach((value, key) => { + paramsRecord[key] = value; + }); + return Schema.decodeUnknown(schema)(paramsRecord).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + ); +} + /** * Resolves a bucket by name and runs the provided effect with the resolved backend. * Centralizes error handling via S3Xml.formatError. @@ -54,8 +189,9 @@ export function resolveBucket< | R | BackendResolver | S3Xml - | AppConfig + | HeraldConfig | S3Client + | SwiftClient | HttpServerRequest.HttpServerRequest > { return Effect.gen(function* () { @@ -68,6 +204,26 @@ export function resolveBucket< ? request.value.method === "HEAD" : false; + if (Option.isSome(request)) { + const auth = request.value.headers["authorization"]; + yield* Effect.logDebug( + `${request.value.method} ${request.value.url} auth: [${auth}]`, + ); + if ( + !auth || auth.trim() === "" || + (auth.startsWith("AWS ") && auth.split(":").length < 2 && + !auth.includes("Signature=")) || + (auth.startsWith("AWS4-") && !auth.includes("Signature=")) + ) { + return s3Xml.formatError( + new AccessDenied({ + message: "Access Denied", + }), + isHead, + ); + } + } + const program = Effect.gen(function* () { const backend = yield* Backend; return yield* fn(backend); @@ -83,6 +239,12 @@ export function resolveBucket< e instanceof InternalError || e instanceof AccessDenied || e instanceof BucketNotEmpty || + e instanceof NoSuchUpload || + e instanceof InvalidPart || + e instanceof InvalidPartOrder || + e instanceof EntityTooSmall || + e instanceof InvalidRequest || + e instanceof MalformedXML || e instanceof DeleteObjectsError ) { return Effect.succeed(s3Xml.formatError(e, isHead)); @@ -120,8 +282,9 @@ export function resolveBackend< | R | BackendResolver | S3Xml - | AppConfig + | HeraldConfig | S3Client + | SwiftClient | HttpServerRequest.HttpServerRequest > { return Effect.gen(function* () { @@ -149,6 +312,12 @@ export function resolveBackend< e instanceof InternalError || e instanceof AccessDenied || e instanceof BucketNotEmpty || + e instanceof NoSuchUpload || + e instanceof InvalidPart || + e instanceof InvalidPartOrder || + e instanceof EntityTooSmall || + e instanceof InvalidRequest || + e instanceof MalformedXML || e instanceof DeleteObjectsError ) { return Effect.succeed(s3Xml.formatError(e, isHead)); diff --git a/src/Http.ts b/src/Http.ts index e77671f..3c3c693 100644 --- a/src/Http.ts +++ b/src/Http.ts @@ -9,33 +9,38 @@ import { Config, Effect, Layer } from "effect"; // deno-lint-ignore no-external-import import { createServer } from "node:http"; -export { Api } from "./Api.ts"; +export { HttpHeraldApi as HeraldHttpApi } from "./Api.ts"; export { HttpHealthLive } from "./Frontend/Health/Http.ts"; export { HttpS3Live } from "./Frontend/Http.ts"; -import { AppConfigLive } from "./Config/Layer.ts"; +import { HeraldConfigLive } from "./Config/Layer.ts"; import { HttpHealthLive } from "./Frontend/Health/Http.ts"; import { HttpS3Live } from "./Frontend/Http.ts"; -import { Api } from "./Api.ts"; +import { HttpHeraldApi } from "./Api.ts"; -export const ApiLive = HttpApiBuilder.api(Api).pipe( +export const HttpHeraldLive = HttpApiBuilder.api(HttpHeraldApi).pipe( Layer.provide(HttpHealthLive), Layer.provide(HttpS3Live), ); -export const HttpLive = Layer.unwrapEffect( +export const HttpServerHeraldLive = Layer.unwrapEffect( Effect.gen(function* () { const port = yield* Config.withDefault( Config.integer("PORT"), 3000, ); return HttpApiBuilder.serve(HttpMiddleware.logger).pipe( + // provides swagger ui for http api Layer.provide(HttpApiSwagger.layer()), + // provides openapi.json endpoint Layer.provide(HttpApiBuilder.middlewareOpenApi()), + // adds cors support + // FIXME: config support Layer.provide(HttpApiBuilder.middlewareCors()), - Layer.provide(ApiLive), + Layer.provide(HttpHeraldLive), + // log address at startup HttpServer.withLogAddress, Layer.provide(NodeHttpServer.layer(createServer, { port })), - Layer.provide(AppConfigLive), + Layer.provide(HeraldConfigLive), ); }), ); diff --git a/src/Logging/Layer.ts b/src/Logging/Layer.ts index c2d5ada..1c2c233 100644 --- a/src/Logging/Layer.ts +++ b/src/Logging/Layer.ts @@ -1,8 +1,39 @@ -import { Effect, Layer, Logger, LogLevel } from "effect"; +import { Config, Effect, Layer, Logger, LogLevel, Option } from "effect"; export const LoggingLive = Layer.mergeAll( - Logger.minimumLogLevel(LogLevel.Info), - // You can add more logger configuration here, like changing the format to JSON for production + Layer.unwrapEffect( + Effect.gen(function* () { + const logLevelStr = yield* Config.option( + Config.string("HERALD_LOG_LEVEL"), + ); + + if (Option.isNone(logLevelStr)) { + return Logger.minimumLogLevel(LogLevel.Info); + } + + const level = logLevelStr.value.toUpperCase(); + switch (level) { + case "ALL": + return Logger.minimumLogLevel(LogLevel.All); + case "TRACE": + return Logger.minimumLogLevel(LogLevel.Trace); + case "DEBUG": + return Logger.minimumLogLevel(LogLevel.Debug); + case "INFO": + return Logger.minimumLogLevel(LogLevel.Info); + case "WARN": + return Logger.minimumLogLevel(LogLevel.Warning); + case "ERROR": + return Logger.minimumLogLevel(LogLevel.Error); + case "FATAL": + return Logger.minimumLogLevel(LogLevel.Fatal); + case "NONE": + return Logger.minimumLogLevel(LogLevel.None); + default: + return Logger.minimumLogLevel(LogLevel.Info); + } + }), + ), ); /** diff --git a/src/Services/Backend.ts b/src/Services/Backend.ts index 8da9574..9f3f1f4 100644 --- a/src/Services/Backend.ts +++ b/src/Services/Backend.ts @@ -1,3 +1,7 @@ +/** + * The `Backend` service represents a single impl that herald can proxy to. + */ + import { Context, type Effect, Schema, type Stream } from "effect"; export interface BucketInfo { @@ -68,6 +72,67 @@ export interface PutObjectResult { readonly versionId?: string; } +export interface MultipartUploadResult { + readonly uploadId: string; +} + +export interface UploadPartResult { + readonly etag: string; +} + +export interface CompleteMultipartUploadResult { + readonly location: string; + readonly bucket: string; + readonly key: string; + readonly etag: string; + readonly versionId?: string; +} + +export interface PartInfo { + readonly partNumber: number; + readonly lastModified: Date; + readonly etag: string; + readonly size: number; +} + +export interface ListPartsResult { + readonly bucket: string; + readonly key: string; + readonly uploadId: string; + readonly owner: OwnerInfo; + readonly initiator: OwnerInfo; + readonly storageClass: string; + readonly partNumberMarker: number; + readonly nextPartNumberMarker: number; + readonly maxParts: number; + readonly isTruncated: boolean; + readonly parts: readonly PartInfo[]; +} + +export interface MultipartUploadInfo { + readonly key: string; + readonly uploadId: string; + readonly owner: OwnerInfo; + readonly initiator: OwnerInfo; + readonly storageClass: string; + readonly initiated: Date; +} + +export interface ListMultipartUploadsResult { + readonly bucket: string; + readonly prefix?: string; + readonly keyMarker?: string; + readonly uploadIdMarker?: string; + readonly nextKeyMarker?: string; + readonly nextUploadIdMarker?: string; + readonly maxUploads: number; + readonly delimiter?: string; + readonly isTruncated: boolean; + readonly uploads: readonly MultipartUploadInfo[]; + readonly commonPrefixes: readonly CommonPrefix[]; + readonly encodingType?: string; +} + export class NoSuchBucket extends Schema.TaggedError()("NoSuchBucket", { bucketName: Schema.String, @@ -111,6 +176,37 @@ export class BucketNotEmpty message: Schema.String, }) {} +export class NoSuchUpload + extends Schema.TaggedError()("NoSuchUpload", { + uploadId: Schema.String, + message: Schema.String, + }) {} + +export class InvalidPart + extends Schema.TaggedError()("InvalidPart", { + message: Schema.String, + }) {} + +export class InvalidPartOrder + extends Schema.TaggedError()("InvalidPartOrder", { + message: Schema.String, + }) {} + +export class EntityTooSmall + extends Schema.TaggedError()("EntityTooSmall", { + message: Schema.String, + }) {} + +export class InvalidRequest + extends Schema.TaggedError()("InvalidRequest", { + message: Schema.String, + }) {} + +export class MalformedXML + extends Schema.TaggedError()("MalformedXML", { + message: Schema.String, + }) {} + export interface DeleteError { readonly key: string; readonly code: string; @@ -141,7 +237,13 @@ export type BackendError = | AccessDenied | NoSuchKey | BucketNotEmpty - | DeleteObjectsError; + | DeleteObjectsError + | NoSuchUpload + | InvalidPart + | InvalidPartOrder + | EntityTooSmall + | InvalidRequest + | MalformedXML; export interface BackendService { readonly listBuckets: () => Effect.Effect< @@ -171,9 +273,11 @@ export interface BackendService { }) => Effect.Effect; readonly getObject: ( key: string, + headers: Record, ) => Effect.Effect; readonly headObject: ( key: string, + headers: Record, ) => Effect.Effect; readonly putObject: ( key: string, @@ -184,6 +288,39 @@ export interface BackendService { readonly deleteObjects: ( objects: readonly { key: string; versionId?: string }[], ) => Effect.Effect; + + // Multipart Upload + readonly createMultipartUpload: ( + key: string, + headers: Record, + ) => Effect.Effect; + readonly uploadPart: ( + key: string, + uploadId: string, + partNumber: number, + body: Stream.Stream, + ) => Effect.Effect; + readonly completeMultipartUpload: ( + key: string, + uploadId: string, + parts: readonly { etag: string; partNumber: number }[], + ) => Effect.Effect; + readonly abortMultipartUpload: ( + key: string, + uploadId: string, + ) => Effect.Effect; + readonly listMultipartUploads: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + uploadIdMarker?: string; + maxUploads?: number; + encodingType?: string; + }) => Effect.Effect; + readonly listParts: ( + key: string, + uploadId: string, + ) => Effect.Effect; } /** diff --git a/src/Services/BackendResolver.ts b/src/Services/BackendResolver.ts index b04ed8e..4ef9801 100644 --- a/src/Services/BackendResolver.ts +++ b/src/Services/BackendResolver.ts @@ -1,8 +1,11 @@ -import { Context, Effect, Layer, Option } from "effect"; -import { AppConfig } from "../Config/Layer.ts"; -import { Backend, type BackendService } from "./Backend.ts"; +import { Cache, Context, Effect, Layer, Option } from "effect"; +import { HeraldConfig } from "../Config/Layer.ts"; +import { Backend } from "./Backend.ts"; import type { S3Client } from "../Backends/S3/Client.ts"; import { makeS3Backend } from "../Backends/S3/Backend.ts"; +import { makeSwiftBackend } from "../Backends/Swift/Backend.ts"; +import type { SwiftClient } from "../Backends/Swift/Client.ts"; +import type { MaterializedBucket } from "../Domain/Config.ts"; /** * BackendResolver handles dynamic resolution and provisioning of Backend implementations @@ -17,7 +20,7 @@ export class BackendResolver extends Context.Tag("BackendResolver")< ) => Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client + Exclude | HeraldConfig | S3Client | SwiftClient >; readonly provideForBackendId: ( @@ -26,7 +29,7 @@ export class BackendResolver extends Context.Tag("BackendResolver")< ) => Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client + Exclude | HeraldConfig | S3Client | SwiftClient >; } >() {} @@ -34,50 +37,78 @@ export class BackendResolver extends Context.Tag("BackendResolver")< export const BackendResolverLive = Layer.effect( BackendResolver, Effect.gen(function* () { - const config = yield* AppConfig; + const config = yield* HeraldConfig; - // Dynamic provision logic with memoization. - const bucketCache = new Map(); - const backendCache = new Map(); + const makeBackend = ( + bucketConfig: MaterializedBucket | { backend_id: string }, + ) => + Effect.gen(function* () { + const protocol = "protocol" in bucketConfig + ? bucketConfig.protocol + : config.raw.backends[bucketConfig.backend_id]?.protocol; - return { - provideForBucket: ( - bucketName: string, - effect: Effect.Effect, - ) => - Effect.gen(function* () { - if (bucketCache.has(bucketName)) { - return yield* Effect.provideService( - effect, - Backend, - bucketCache.get(bucketName)!, - ); - } + if (protocol === "s3") { + return yield* makeS3Backend(bucketConfig); + } else if (protocol === "swift") { + return yield* makeSwiftBackend(bucketConfig); + } else { + return yield* Effect.fail( + new Error(`Unsupported protocol: ${protocol}`), + ); + } + }); + + // We cache by the string identifier (bucket name or backend ID). + // The BackendService itself is request-scoped because makeBackend yields requirements + // that are resolved from the current context when the cache is lookep up. + // Wait, Cache.get(key) will execute the lookup if not present. + // If we want the BackendService to be truly request-scoped but cached, + // we have a conflict if the requirements (like HeraldConfig) change per request. + // However, in Herald, HeraldConfig is usually a singleton for the app. + // If it's a singleton, then caching the BackendService is fine. + const bucketCache = yield* Cache.make({ + capacity: 100, + timeToLive: "24 hours", + lookup: (bucketName: string) => + Effect.gen(function* () { const matched = config.lookupBucket(bucketName); if (Option.isNone(matched)) { return yield* Effect.fail( new Error(`No configuration found for bucket: ${bucketName}`), ); } + return yield* makeBackend(matched.value); + }), + }); - const bucketConfig = matched.value; - let backendImpl: BackendService; - - if (bucketConfig.protocol === "s3") { - backendImpl = yield* makeS3Backend(bucketConfig); - } else { + const backendCache = yield* Cache.make({ + capacity: 100, + timeToLive: "24 hours", + lookup: (backendId: string) => + Effect.gen(function* () { + const backendConfig = config.raw.backends[backendId]; + if (!backendConfig) { return yield* Effect.fail( - new Error(`Unsupported protocol: ${bucketConfig.protocol}`), + new Error(`No configuration found for backend: ${backendId}`), ); } + return yield* makeBackend({ backend_id: backendId }); + }), + }); - bucketCache.set(bucketName, backendImpl); + return { + provideForBucket: ( + bucketName: string, + effect: Effect.Effect, + ) => + Effect.gen(function* () { + const backendImpl = yield* bucketCache.get(bucketName); return yield* Effect.provideService(effect, Backend, backendImpl); }) as Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client + Exclude | HeraldConfig | S3Client | SwiftClient >, provideForBackendId: ( @@ -85,37 +116,12 @@ export const BackendResolverLive = Layer.effect( effect: Effect.Effect, ) => Effect.gen(function* () { - if (backendCache.has(backendId)) { - return yield* Effect.provideService( - effect, - Backend, - backendCache.get(backendId)!, - ); - } - - const backendConfig = config.raw.backends[backendId]; - if (!backendConfig) { - return yield* Effect.fail( - new Error(`No configuration found for backend: ${backendId}`), - ); - } - - let backendImpl: BackendService; - - if (backendConfig.protocol === "s3") { - backendImpl = yield* makeS3Backend({ backend_id: backendId }); - } else { - return yield* Effect.fail( - new Error(`Unsupported protocol: ${backendConfig.protocol}`), - ); - } - - backendCache.set(backendId, backendImpl); + const backendImpl = yield* backendCache.get(backendId); return yield* Effect.provideService(effect, Backend, backendImpl); }) as Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client + Exclude | HeraldConfig | S3Client | SwiftClient >, }; }), diff --git a/src/Services/S3Xml.ts b/src/Services/S3Xml.ts index 987ea54..06815fb 100644 --- a/src/Services/S3Xml.ts +++ b/src/Services/S3Xml.ts @@ -6,13 +6,24 @@ import { BucketAlreadyOwnedByYou, type BucketInfo, BucketNotEmpty, + EntityTooSmall, InternalError, + InvalidPart, + InvalidPartOrder, + InvalidRequest, + type ListMultipartUploadsResult, type ListObjectsResult, + type ListPartsResult, + MalformedXML, NoSuchBucket, NoSuchKey, + NoSuchUpload, type OwnerInfo, } from "./Backend.ts"; +/** + * This service centeralizes XML authoring logic. + */ export class S3Xml extends Context.Tag("S3Xml")< S3Xml, { @@ -30,6 +41,25 @@ export class S3Xml extends Context.Tag("S3Xml")< readonly formatListVersions: ( result: ListObjectsResult, ) => HttpServerResponse.HttpServerResponse; + readonly formatListMultipartUploads: ( + result: ListMultipartUploadsResult, + ) => HttpServerResponse.HttpServerResponse; + readonly formatInitiateMultipartUpload: ( + bucket: string, + key: string, + uploadId: string, + ) => HttpServerResponse.HttpServerResponse; + readonly formatCompleteMultipartUpload: ( + result: { + location: string; + bucket: string; + key: string; + etag: string; + }, + ) => HttpServerResponse.HttpServerResponse; + readonly formatListParts: ( + result: ListPartsResult, + ) => HttpServerResponse.HttpServerResponse; } >() {} @@ -66,6 +96,30 @@ export const S3XmlLive = Layer.succeed( code = "BucketNotEmpty"; message = e.message; status = 409; + } else if (e instanceof NoSuchUpload) { + code = "NoSuchUpload"; + message = e.message; + status = 404; + } else if (e instanceof InvalidPart) { + code = "InvalidPart"; + message = e.message; + status = 400; + } else if (e instanceof InvalidPartOrder) { + code = "InvalidPartOrder"; + message = e.message; + status = 400; + } else if (e instanceof EntityTooSmall) { + code = "EntityTooSmall"; + message = e.message; + status = 400; + } else if (e instanceof InvalidRequest) { + code = "InvalidRequest"; + message = e.message; + status = 400; + } else if (e instanceof MalformedXML) { + code = "MalformedXML"; + message = e.message; + status = 400; } else if (e instanceof InternalError) { code = "InternalError"; message = e.message; @@ -108,99 +162,82 @@ export const S3XmlLive = Layer.succeed( formatListObjects: (result) => { const encode = (s: string) => - result.encodingType === "url" + result.encodingType?.toLowerCase() === "url" ? encodeURIComponent(s).replace(/%2F/g, "/") : s; - const contentsXml = result.contents.map((c) => ` - - ${encode(c.key)} - ${c.lastModified.toISOString()} - ${c.etag} - ${c.size} - ${c.storageClass ?? "STANDARD"} - ${ - c.owner - ? `${c.owner.id}${c.owner.displayName}` - : "" - } - - `).join(""); + const contentsXml = result.contents.map((c) => + `${ + encode(c.key) + }${c.lastModified.toISOString()}${c.etag}${c.size}${ + c.storageClass ?? + "STANDARD" + }${ + c.owner + ? `${c.owner.id}${c.owner.displayName}` + : "" + }` + ).join(""); - const commonPrefixesXml = result.commonPrefixes.map((cp) => ` - - ${encode(cp.prefix)} - - `).join(""); + const commonPrefixesXml = result.commonPrefixes.map((cp) => + `${encode(cp.prefix)}` + ).join(""); let xml: string; if (result.listType === 2) { // ListObjectsV2 - xml = ` - - ${result.name} - ${encode(result.prefix ?? "")} - ${ - result.keyCount ?? - (result.contents.length + result.commonPrefixes.length) - } - ${result.maxKeys} - ${encode(result.delimiter ?? "")} - ${result.isTruncated} - ${ - result.continuationToken - ? `${result.continuationToken}` - : "" - } - ${ - result.nextContinuationToken - ? `${result.nextContinuationToken}` - : "" - } - ${ - result.startAfter - ? `${encode(result.startAfter)}` - : "" - } - ${ - result.encodingType - ? `${result.encodingType}` - : "" - } - ${contentsXml} - ${commonPrefixesXml} - - `; + xml = + `${result.name}${ + encode( + result.prefix ?? "", + ) + }${ + result.keyCount ?? + (result.contents.length + result.commonPrefixes.length) + }${result.maxKeys}${ + encode( + result.delimiter ?? "", + ) + }${result.isTruncated}${ + result.continuationToken + ? `${result.continuationToken}` + : "" + }${ + result.nextContinuationToken + ? `${result.nextContinuationToken}` + : "" + }${ + result.startAfter + ? `${encode(result.startAfter)}` + : "" + }${ + result.encodingType + ? `${result.encodingType}` + : "" + }${contentsXml}${commonPrefixesXml}`; } else { // ListObjectsV1 - xml = ` - - ${result.name} - ${encode(result.prefix ?? "")} - ${encode(result.marker ?? "")} - ${ - result.nextMarker - ? `${encode(result.nextMarker)}` - : "" - } - ${result.maxKeys} - ${encode(result.delimiter ?? "")} - ${result.isTruncated} - ${ - result.encodingType - ? `${result.encodingType}` - : "" - } - ${contentsXml} - ${commonPrefixesXml} - - `; + xml = + `${result.name}${ + encode( + result.prefix ?? "", + ) + }${encode(result.marker ?? "")}${ + result.nextMarker + ? `${encode(result.nextMarker)}` + : "" + }${result.maxKeys}${ + encode( + result.delimiter ?? "", + ) + }${result.isTruncated}${ + result.encodingType + ? `${result.encodingType}` + : "" + }${contentsXml}${commonPrefixesXml}`; } - // Clean up whitespace between tags - const cleanXml = xml.replace(/>\s+<").trim(); - - return HttpServerResponse.text(cleanXml, { + return HttpServerResponse.text(xml, { headers: { "Content-Type": "application/xml", }, @@ -209,68 +246,131 @@ export const S3XmlLive = Layer.succeed( formatListVersions: (result) => { const encode = (s: string) => - result.encodingType === "url" + result.encodingType?.toLowerCase() === "url" ? encodeURIComponent(s).replace(/%2F/g, "/") : s; const versionsXml = result.contents.filter((c) => !c.isDeleteMarker).map( - (v) => ` - - ${encode(v.key)} - ${v.versionId ?? "null"} - ${v.isLatest ?? true} - ${v.lastModified.toISOString()} - ${v.etag} - ${v.size} - ${v.storageClass ?? "STANDARD"} - ${ - v.owner - ? `${v.owner.id}${v.owner.displayName}` - : "" - } - - `, + (v) => + `${encode(v.key)}${ + v.versionId ?? + "null" + }${ + v.isLatest ?? + true + }${v.lastModified.toISOString()}${v.etag}${v.size}${ + v.storageClass ?? + "STANDARD" + }${ + v.owner + ? `${v.owner.id}${v.owner.displayName}` + : "" + }`, ).join(""); const deleteMarkersXml = result.contents.filter((c) => c.isDeleteMarker) - .map((dm) => ` - - ${encode(dm.key)} - ${dm.versionId ?? "null"} - ${dm.isLatest ?? true} - ${dm.lastModified.toISOString()} - ${ - dm.owner - ? `${dm.owner.id}${dm.owner.displayName}` + .map((dm) => + `${encode(dm.key)}${ + dm.versionId ?? + "null" + }${ + dm.isLatest ?? + true + }${dm.lastModified.toISOString()}${ + dm.owner + ? `${dm.owner.id}${dm.owner.displayName}` + : "" + }` + ).join(""); + + const commonPrefixesXml = result.commonPrefixes.map((cp) => + `${encode(cp.prefix)}` + ).join(""); + + const xml = + `${result.name}${ + encode( + result.prefix ?? "", + ) + }${ + encode( + result.marker ?? "", + ) + }${result.maxKeys}${ + encode( + result.delimiter ?? "", + ) + }${result.isTruncated}${ + result.nextMarker + ? `${ + encode(result.nextMarker) + }null` : "" - } - - `).join(""); + }${versionsXml}${deleteMarkersXml}${commonPrefixesXml}`; - const commonPrefixesXml = result.commonPrefixes.map((cp) => ` - - ${encode(cp.prefix)} - - `).join(""); + return HttpServerResponse.text(xml, { + headers: { + "Content-Type": "application/xml", + }, + }); + }, - const xml = ` - - ${result.name} - ${encode(result.prefix ?? "")} - ${encode(result.marker ?? "")} - - ${result.maxKeys} - ${encode(result.delimiter ?? "")} - ${result.isTruncated} - ${versionsXml} - ${deleteMarkersXml} - ${commonPrefixesXml} - - `; + formatListMultipartUploads: (result) => { + const uploadsXml = result.uploads.map((u) => + `${u.key}${u.uploadId}${u.initiator.id}${u.initiator.displayName}${u.owner.id}${u.owner.displayName}${u.storageClass}${u.initiated.toISOString()}` + ).join(""); - const cleanXml = xml.replace(/>\s+<").trim(); + const commonPrefixesXml = result.commonPrefixes.map((cp) => + `${cp.prefix}` + ).join(""); - return HttpServerResponse.text(cleanXml, { + const xml = + `${result.bucket}${ + result.keyMarker ?? "" + }${ + result.uploadIdMarker ?? "" + }${ + result.nextKeyMarker ?? "" + }${ + result.nextUploadIdMarker ?? "" + }${result.maxUploads}${result.isTruncated}${uploadsXml}${commonPrefixesXml}`; + + return HttpServerResponse.text(xml, { + headers: { "Content-Type": "application/xml" }, + }); + }, + + formatInitiateMultipartUpload: (bucket, key, uploadId) => { + const xml = + `${bucket}${key}${uploadId}`; + + return HttpServerResponse.text(xml, { + headers: { + "Content-Type": "application/xml", + }, + }); + }, + + formatCompleteMultipartUpload: (result) => { + const xml = + `${result.location}${result.bucket}${result.key}${result.etag}`; + + return HttpServerResponse.text(xml, { + headers: { + "Content-Type": "application/xml", + }, + }); + }, + + formatListParts: (result) => { + const partsXml = result.parts.map((p) => + `${p.partNumber}${p.lastModified.toISOString()}${p.etag}${p.size}` + ).join(""); + + const xml = + `${result.bucket}${result.key}${result.uploadId}${result.initiator.id}${result.initiator.displayName}${result.owner.id}${result.owner.displayName}${result.storageClass}${result.partNumberMarker}${result.nextPartNumberMarker}${result.maxParts}${result.isTruncated}${partsXml}`; + + return HttpServerResponse.text(xml, { headers: { "Content-Type": "application/xml", }, diff --git a/src/main.ts b/src/main.ts index cff9dea..56ccc3b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,13 +2,19 @@ import { FetchHttpClient } from "@effect/platform"; import { NodeRuntime } from "@effect/platform-node"; import { Layer } from "effect"; // our http server impl layer -import { HttpLive } from "./Http.ts"; +import { HttpServerHeraldLive } from "./Http.ts"; // otel tracing layer import { TracingLive } from "./Tracing.ts"; -HttpLive.pipe( +HttpServerHeraldLive.pipe( Layer.provide(TracingLive), + // provider an HttpClient impl based on `fetch` + // used to talk the the swift impl Layer.provide(FetchHttpClient.layer), + // run layer until interrupted Layer.launch, + // add support for Cli goodies like + // signal mgmt, teardown, exit codes and stdio impl + // for Logger NodeRuntime.runMain, ); diff --git a/tests/config.test.ts b/tests/config.test.ts index f9b82ef..eb5981a 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -6,8 +6,9 @@ import { BackendResolver, BackendResolverLive, } from "../src/Services/BackendResolver.ts"; -import { AppConfig } from "../src/Config/Layer.ts"; +import { HeraldConfig, parseConfig } from "../src/Config/Layer.ts"; import { S3Client } from "../src/Backends/S3/Client.ts"; +import { SwiftClient } from "../src/Backends/Swift/Client.ts"; import type { S3Client as S3ClientSDK } from "@aws-sdk/client-s3"; import { Backend } from "../src/Services/Backend.ts"; @@ -204,6 +205,50 @@ const cases: TestCase[] = [ "data-customer-internal": { bucket_name: "infix-match" }, }, }, + { + id: "swift_basic", + name: "swift basic config", + input: { + backends: { + swift_main: { + protocol: "swift", + auth_url: "http://keystone.example.com", + container: "my-container", + buckets: "*", + }, + }, + }, + expectedBuckets: { + "any-bucket": { + backend_id: "swift_main", + protocol: "swift", + auth_url: "http://keystone.example.com", + container: "my-container", + }, + }, + }, + { + id: "swift_with_credentials", + name: "swift with credentials", + input: { + backends: { + swift_main: { + protocol: "swift", + credentials: { + username: "user1", + password: "pw1", + project_name: "proj1", + }, + }, + }, + }, + expectedBuckets: { + "any": { + backend_id: "swift_main", + protocol: "swift", + }, + }, + }, ]; for (const tc of cases) { @@ -243,13 +288,52 @@ for (const tc of cases) { })); } +testEffect("config/parseConfig/env_vars", () => + Effect.gen(function* () { + const env = { + HERALD_DEFAULT_PROTOCOL: "s3", + HERALD_DEFAULT_ENDPOINT: "http://localhost:9000", + HERALD_MYBACKEND_PROTOCOL: "swift", + HERALD_MYBACKEND_AUTH_URL: "http://swift.com", + }; + const config = parseConfig({ backends: {} }, env); + + const defaultBackend = config.backends.default; + yield* EffectAssert.strictEqual(defaultBackend.protocol, "s3"); + if (defaultBackend.protocol === "s3") { + yield* EffectAssert.strictEqual( + defaultBackend.endpoint, + "http://localhost:9000", + ); + } + + const myBackend = config.backends.mybackend; + yield* EffectAssert.strictEqual(myBackend.protocol, "swift"); + if (myBackend.protocol === "swift") { + yield* EffectAssert.strictEqual( + myBackend.auth_url, + "http://swift.com", + ); + } + })); + +testEffect( + "config/parseConfig/default_fallback", + () => + Effect.gen(function* () { + const config = parseConfig({ backends: {} }, {}); + yield* EffectAssert.strictEqual(config.backends.default.protocol, "s3"); + yield* EffectAssert.strictEqual(config.backends.default.buckets, "*"); + }), +); + interface ResolverTestCase { id: string; name: string; config: GlobalConfig; op: ( resolver: Context.Tag.Service, - ) => Effect.Effect; + ) => Effect.Effect; expectedError?: string; } @@ -320,7 +404,7 @@ const resolverCases: ResolverTestCase[] = [ for (const tc of resolverCases) { testEffect(`resolver/${tc.id}`, () => Effect.gen(function* () { - const AppConfigLive = Layer.succeed(AppConfig, { + const HeraldConfigLive = Layer.succeed(HeraldConfig, { raw: tc.config, lookupBucket: (name: string) => lookupBucket(tc.config, name), }); @@ -330,13 +414,20 @@ for (const tc of resolverCases) { getClient: () => Effect.succeed({} as S3ClientSDK), }); + // Mock SwiftClient + const SwiftClientLive = Layer.succeed(SwiftClient, { + getAuthMeta: () => + Effect.succeed({ token: "test", storageUrl: "http://test" }), + }); + const program = Effect.gen(function* () { const resolver = yield* BackendResolver; return yield* tc.op(resolver); }).pipe( Effect.provide(BackendResolverLive), - Effect.provide(AppConfigLive), + Effect.provide(HeraldConfigLive), Effect.provide(S3ClientLive), + Effect.provide(SwiftClientLive), Effect.either, ); diff --git a/tests/health.test.ts b/tests/health.test.ts index 8ce8e58..296d6c5 100644 --- a/tests/health.test.ts +++ b/tests/health.test.ts @@ -5,27 +5,29 @@ import { HttpApiClient, HttpServer, } from "@effect/platform"; -import { Api, HttpHealthLive, HttpS3Live } from "../src/Http.ts"; -import { AppConfig } from "../src/Config/Layer.ts"; +import { HeraldHttpApi, HttpHealthLive, HttpS3Live } from "../src/Http.ts"; +import { HeraldConfig } from "../src/Config/Layer.ts"; import { S3ClientLive } from "../src/Backends/S3/Client.ts"; +import { SwiftClientLive } from "../src/Backends/Swift/Client.ts"; import { S3XmlLive } from "../src/Services/S3Xml.ts"; import { BackendResolverLive } from "../src/Services/BackendResolver.ts"; import { EffectAssert, testEffect } from "./utils.ts"; testEffect("health/getStatus", () => Effect.gen(function* () { - const AppConfigLive = Layer.succeed(AppConfig, { + const HeraldConfigLive = Layer.succeed(HeraldConfig, { raw: { backends: {} }, lookupBucket: () => Option.none(), }); - const ApiWithRequirements = HttpApiBuilder.api(Api).pipe( + const ApiWithRequirements = HttpApiBuilder.api(HeraldHttpApi).pipe( Layer.provide(HttpHealthLive), Layer.provide(HttpS3Live), - Layer.provide(S3ClientLive), Layer.provide(BackendResolverLive), + Layer.provide(S3ClientLive), + Layer.provide(SwiftClientLive), Layer.provide(S3XmlLive), - Layer.provide(AppConfigLive), + Layer.provide(HeraldConfigLive), Layer.provide(FetchHttpClient.layer), Layer.provideMerge(HttpServer.layerContext), ); @@ -34,7 +36,7 @@ testEffect("health/getStatus", () => const webHandler = HttpApiBuilder.toWebHandler(ApiWithRequirements); const clientProgram = Effect.gen(function* () { - const client = yield* HttpApiClient.make(Api, { + const client = yield* HttpApiClient.make(HeraldHttpApi, { baseUrl: "http://localhost", }); return yield* client.health.getStatus(); diff --git a/tests/integration/__snapshots__/buckets.test.ts.snap b/tests/integration/__snapshots__/buckets.test.ts.snap index eb093fc..76add56 100644 --- a/tests/integration/__snapshots__/buckets.test.ts.snap +++ b/tests/integration/__snapshots__/buckets.test.ts.snap @@ -20,6 +20,13 @@ snapshot[`Proxy/buckets/create/new metadata 1`] = ` } `; +snapshot[`Swift/buckets/create/new metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/buckets/create/existing metadata 1`] = ` { headers: { @@ -40,6 +47,13 @@ snapshot[`Proxy/buckets/create/existing metadata 1`] = ` } `; +snapshot[`Swift/buckets/create/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/buckets/delete/existing metadata 1`] = ` { headers: { @@ -60,6 +74,13 @@ snapshot[`Proxy/buckets/delete/existing metadata 1`] = ` } `; +snapshot[`Swift/buckets/delete/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/buckets/delete/non-existent metadata 1`] = ` { headers: { @@ -92,6 +113,18 @@ snapshot[`Proxy/buckets/delete/non-existent metadata 1`] = ` snapshot[`Proxy/buckets/delete/non-existent body 1`] = `'NoSuchBucketThe specified bucket does not exist'`; +snapshot[`Swift/buckets/delete/non-existent metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Swift/buckets/delete/non-existent body 1`] = `'NoSuchBucket

Not Found

The resource could not be found.

'`; + snapshot[`Baseline/buckets/head/existing metadata 1`] = ` { headers: { @@ -112,6 +145,13 @@ snapshot[`Proxy/buckets/head/existing metadata 1`] = ` } `; +snapshot[`Swift/buckets/head/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/buckets/head/non-existent metadata 1`] = ` { headers: { @@ -135,11 +175,16 @@ snapshot[`Proxy/buckets/head/non-existent metadata 1`] = ` } `; +snapshot[`Swift/buckets/head/non-existent metadata 1`] = ` +{ + headers: {}, + status: 404, +} +`; + snapshot[`Baseline/buckets/list metadata 1`] = ` { headers: { - "accept-ranges": "bytes", - "content-length": "275", "content-type": "application/xml", "strict-transport-security": "max-age=31536000; includeSubDomains", "x-content-type-options": "nosniff", @@ -152,7 +197,7 @@ snapshot[`Baseline/buckets/list metadata 1`] = ` snapshot[`Baseline/buckets/list body 1`] = ` ' -02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4minio' +02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4minioherald-25gnaqqph3oof5kljqtdeu-72026-01-15T00:00:00.000Zherald-74khf7szf4qtrzrth2weoi-2192026-01-15T00:00:00.000Zherald-88ztrfgehycvaw5bh5t625-12026-01-15T00:00:00.000Zherald-almi4r3xt6pj4vmf25mpkc-362026-01-15T00:00:00.000Zherald-b3y7kg3dn3u5q0awin9aj8-72026-01-15T00:00:00.000Zherald-ferrwumx0p2j3tdhrfle4o-642026-01-15T00:00:00.000Zherald-iqm95px2zlxt75mcsx3dms-12026-01-15T00:00:00.000Zherald-l84igcd8jggs3wioh4msk8-12026-01-15T00:00:00.000Zherald-quy3o0n429jznm43dcga5l-312026-01-15T00:00:00.000Zherald-unz0kp56250vjw6umbd0va-12026-01-15T00:00:00.000Zherald-zrs5hcqpud1tn54vd9u700-152026-01-15T00:00:00.000Zherald-zunthialhf5qffc4p9xthk-742026-01-15T00:00:00.000Z' `; snapshot[`Proxy/buckets/list metadata 1`] = ` @@ -165,4 +210,16 @@ snapshot[`Proxy/buckets/list metadata 1`] = ` } `; -snapshot[`Proxy/buckets/list body 1`] = `'02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4minio'`; +snapshot[`Proxy/buckets/list body 1`] = `'02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4minioherald-25gnaqqph3oof5kljqtdeu-72026-01-15T00:00:00.000Zherald-74khf7szf4qtrzrth2weoi-2192026-01-15T00:00:00.000Zherald-88ztrfgehycvaw5bh5t625-12026-01-15T00:00:00.000Zherald-almi4r3xt6pj4vmf25mpkc-362026-01-15T00:00:00.000Zherald-b3y7kg3dn3u5q0awin9aj8-72026-01-15T00:00:00.000Zherald-ferrwumx0p2j3tdhrfle4o-642026-01-15T00:00:00.000Zherald-iqm95px2zlxt75mcsx3dms-12026-01-15T00:00:00.000Zherald-l84igcd8jggs3wioh4msk8-12026-01-15T00:00:00.000Zherald-quy3o0n429jznm43dcga5l-312026-01-15T00:00:00.000Zherald-unz0kp56250vjw6umbd0va-12026-01-15T00:00:00.000Zherald-zrs5hcqpud1tn54vd9u700-152026-01-15T00:00:00.000Zherald-zunthialhf5qffc4p9xthk-742026-01-15T00:00:00.000Z'`; + +snapshot[`Swift/buckets/list metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 200, +} +`; + +snapshot[`Swift/buckets/list body 1`] = `'swiftSwift User192.168.5.1232026-01-15T00:00:00.000Za2026-01-15T00:00:00.000Zaa2026-01-15T00:00:00.000Zbuilds2026-01-15T00:00:00.000Zfoo-2026-01-15T00:00:00.000Zfoo-.bar2026-01-15T00:00:00.000Zfoo.-bar2026-01-15T00:00:00.000Zfoo..bar2026-01-15T00:00:00.000Zfoo_bar2026-01-15T00:00:00.000Zherald-swift-2w97l75ompcxiypo-12026-01-15T00:00:00.000Zherald-swift-5dfyoor543wddfpb-12026-01-15T00:00:00.000Zherald-swift-5m8pru9nzpno98zp-1572026-01-15T00:00:00.000Zherald-swift-5txup4vs19i8tr6s-292026-01-15T00:00:00.000Zherald-swift-cze1vw7y05q33782-1462026-01-15T00:00:00.000Zherald-swift-fd0oi5radob46p39-12026-01-15T00:00:00.000Zherald-swift-m55m3lqytoaxuxro-132026-01-15T00:00:00.000Zherald-swift-oda2k1hu2ds0wir6-22026-01-15T00:00:00.000Zherald-swift-sy6d1ftl2i7g78jj-12026-01-15T00:00:00.000Zherald-swift-yx0xlhaeebv1g9c7-12026-01-15T00:00:00.000Zherald-task-store-mr-120-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-127-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-130-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-131-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-132-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-137-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-139-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-143-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-144-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-145-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-146-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-147-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-149-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-150-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-151-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-154-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-155-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-157-vivavox2026-01-15T00:00:00.000Zherald-task-store-prd-vivavox2026-01-15T00:00:00.000Zherald-task-store-stg-vivavox2026-01-15T00:00:00.000Ziac-swift2026-01-15T00:00:00.000Zmr-101-vivavox2026-01-15T00:00:00.000Zmr-109-vivavox2026-01-15T00:00:00.000Zmr-111-vivavox2026-01-15T00:00:00.000Zmr-115-vivavox2026-01-15T00:00:00.000Zmr-116-vivavox2026-01-15T00:00:00.000Zmr-120-vivavox2026-01-15T00:00:00.000Zmr-121-vivavox2026-01-15T00:00:00.000Zmr-122-vivavox2026-01-15T00:00:00.000Zmr-124-vivavox2026-01-15T00:00:00.000Zmr-126-vivavox2026-01-15T00:00:00.000Zmr-127-vivavox2026-01-15T00:00:00.000Zmr-130-vivavox2026-01-15T00:00:00.000Zmr-131-vivavox2026-01-15T00:00:00.000Zmr-132-vivavox2026-01-15T00:00:00.000Zmr-137-vivavox2026-01-15T00:00:00.000Zmr-139-vivavox2026-01-15T00:00:00.000Zmr-143-vivavox2026-01-15T00:00:00.000Zmr-144-vivavox2026-01-15T00:00:00.000Zmr-145-vivavox2026-01-15T00:00:00.000Zmr-146-vivavox2026-01-15T00:00:00.000Zmr-147-vivavox2026-01-15T00:00:00.000Zmr-149-vivavox2026-01-15T00:00:00.000Zmr-150-vivavox2026-01-15T00:00:00.000Zmr-151-vivavox2026-01-15T00:00:00.000Zmr-154-vivavox2026-01-15T00:00:00.000Zmr-155-vivavox2026-01-15T00:00:00.000Zmr-157-vivavox2026-01-15T00:00:00.000Zprd-vivavox2026-01-15T00:00:00.000Zstg-vivavox2026-01-15T00:00:00.000Zstg-vivavox+segments2026-01-15T00:00:00.000Ztest-objects-bucket2026-01-15T00:00:00.000Z'`; diff --git a/tests/integration/__snapshots__/objects.test.ts.snap b/tests/integration/__snapshots__/objects.test.ts.snap index c490bdf..63b70f8 100644 --- a/tests/integration/__snapshots__/objects.test.ts.snap +++ b/tests/integration/__snapshots__/objects.test.ts.snap @@ -20,6 +20,13 @@ snapshot[`Proxy/objects/put metadata 1`] = ` } `; +snapshot[`Swift/objects/put metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/objects/get/existing metadata 1`] = ` { headers: { @@ -40,6 +47,13 @@ snapshot[`Proxy/objects/get/existing metadata 1`] = ` } `; +snapshot[`Swift/objects/get/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/objects/get/non-existent metadata 1`] = ` { headers: { @@ -72,6 +86,18 @@ snapshot[`Proxy/objects/get/non-existent metadata 1`] = ` snapshot[`Proxy/objects/get/non-existent body 1`] = `'NoSuchKeyThe specified key does not exist.'`; +snapshot[`Swift/objects/get/non-existent metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Swift/objects/get/non-existent body 1`] = `'NoSuchKey

Not Found

The resource could not be found.

'`; + snapshot[`Baseline/objects/head/existing metadata 1`] = ` { headers: { @@ -92,6 +118,13 @@ snapshot[`Proxy/objects/head/existing metadata 1`] = ` } `; +snapshot[`Swift/objects/head/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/objects/head/non-existent metadata 1`] = ` { headers: { @@ -115,6 +148,13 @@ snapshot[`Proxy/objects/head/non-existent metadata 1`] = ` } `; +snapshot[`Swift/objects/head/non-existent metadata 1`] = ` +{ + headers: {}, + status: 404, +} +`; + snapshot[`Baseline/objects/delete/existing metadata 1`] = ` { headers: { @@ -134,3 +174,102 @@ snapshot[`Proxy/objects/delete/existing metadata 1`] = ` status: 204, } `; + +snapshot[`Swift/objects/delete/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/multipart/basic metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/multipart/basic metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/multipart/abort metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "479", + "content-type": "application/xml", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Baseline/objects/multipart/abort body 1`] = ` +' +NoSuchUploadThe specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.multipart-abort.txttest-objects-bucket/test-objects-bucket/multipart-abort.txtIDHOST' +`; + +snapshot[`Proxy/objects/multipart/abort metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/objects/multipart/abort body 1`] = `'NoSuchUploadThe specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.'`; + +snapshot[`Baseline/objects/multipart/list-parts metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/multipart/list-parts metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/multipart/empty metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/multipart/empty metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; diff --git a/tests/integration/objects.test.ts b/tests/integration/objects.test.ts index e2799f1..a4f2de9 100644 --- a/tests/integration/objects.test.ts +++ b/tests/integration/objects.test.ts @@ -1,12 +1,17 @@ import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, CreateBucketCommand, + CreateMultipartUploadCommand, DeleteBucketCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, + ListPartsCommand, PutObjectCommand, type S3Client, S3ServiceException, + UploadPartCommand, } from "@aws-sdk/client-s3"; import { harness, type ProxyTestCase } from "../utils.ts"; import type { GlobalConfig } from "../../src/Domain/Config.ts"; @@ -121,6 +126,180 @@ const specs: ObjectTestSpec[] = [ ); }, }, + { + name: "objects/multipart/basic", + fn: async (c) => { + const key = "multipart-basic.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + const partSize = 5 * 1024 * 1024 + 1; + const body1 = new Uint8Array(partSize).fill(97); // 'a' + const body2 = new Uint8Array(10).fill(98); // 'b' + + const { ETag: etag1 } = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: body1, + }), + ); + const { ETag: etag2 } = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 2, + Body: body2, + }), + ); + + await c.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { + Parts: [ + { ETag: etag1, PartNumber: 1 }, + { ETag: etag2, PartNumber: 2 }, + ], + }, + }), + ); + + const { ContentLength } = await c.send( + new HeadObjectCommand({ Bucket: BUCKET, Key: key }), + ); + if (ContentLength !== partSize + 10) { + throw new Error( + `Size mismatch: expected ${partSize + 10}, got ${ContentLength}`, + ); + } + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ + Bucket: BUCKET, + Key: "multipart-basic.txt", + }), + ); + } catch { /* ignore */ } + }, + }, + { + name: "objects/multipart/abort", + fn: async (c) => { + const key = "multipart-abort.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: "part 1", + }), + ); + + await c.send( + new AbortMultipartUploadCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + + try { + await c.send( + new ListPartsCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + throw new Error("ListParts should have failed after Abort"); + } catch (e) { + if (!(e instanceof S3ServiceException && e.name === "NoSuchUpload")) { + throw e; + } + } + }, + }, + { + name: "objects/multipart/list-parts", + fn: async (c) => { + const key = "multipart-list.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: "part 1", + }), + ); + + const { Parts } = await c.send( + new ListPartsCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + + if (!Parts || Parts.length !== 1 || Parts[0].PartNumber !== 1) { + throw new Error(`Unexpected parts list: ${JSON.stringify(Parts)}`); + } + + await c.send( + new AbortMultipartUploadCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + }, + }, + { + name: "objects/multipart/empty", + fn: async (c) => { + const key = "multipart-empty.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + try { + await c.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { Parts: [] }, + }), + ); + throw new Error("Complete should have failed for empty parts"); + } catch (e) { + if ( + e instanceof S3ServiceException && + (e.name === "MalformedXML" || e.name === "InvalidPart" || + e.name === "InvalidRequest") + ) { + return; + } + throw e; + } finally { + try { + await c.send( + new AbortMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + }), + ); + } catch { /* ignore */ } + } + }, + }, ]; async function runObjectTest(tc: ObjectTestSpec, client: S3Client) { diff --git a/tests/utils.ts b/tests/utils.ts index d2d778f..e5446b5 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -1,10 +1,11 @@ import { S3Client } from "@aws-sdk/client-s3"; -import { Effect, Layer } from "effect"; -import { ApiLive } from "../src/Http.ts"; -import { AppConfig } from "../src/Config/Layer.ts"; +import { Config, Effect, Layer, Logger, LogLevel, Option } from "effect"; +import { HttpHeraldLive } from "../src/Http.ts"; +import { HeraldConfig } from "../src/Config/Layer.ts"; import { lookupBucket } from "../src/Domain/Config.ts"; import { BackendResolverLive } from "../src/Services/BackendResolver.ts"; import { S3ClientLive } from "../src/Backends/S3/Client.ts"; +import { SwiftClientLive } from "../src/Backends/Swift/Client.ts"; import { S3XmlLive } from "../src/Services/S3Xml.ts"; import { HttpApiBuilder, HttpServer } from "@effect/platform"; import { FetchHttpClient } from "@effect/platform"; @@ -31,20 +32,27 @@ export type Snapshot = { body: string; }; -export const makeTestHarness = (config: GlobalConfig) => +export const makeTestHarness = ( + config: GlobalConfig, + loggingLayer: Layer.Layer = Logger.minimumLogLevel( + LogLevel.Info, + ), +) => Effect.gen(function* () { - const AppConfigLive = Layer.succeed(AppConfig, { + const HeraldConfigLive = Layer.succeed(HeraldConfig, { raw: config, lookupBucket: (name: string) => lookupBucket(config, name), }); - const ApiWithRequirements = ApiLive.pipe( + const ApiWithRequirements = HttpHeraldLive.pipe( Layer.provide(BackendResolverLive), Layer.provide(S3ClientLive), + Layer.provide(SwiftClientLive), Layer.provide(S3XmlLive), - Layer.provide(AppConfigLive), + Layer.provide(HeraldConfigLive), Layer.provide(FetchHttpClient.layer), Layer.provideMerge(HttpServer.layerContext), + Layer.provideMerge(loggingLayer), ); // In @effect/platform 0.90.x, toWebHandler returns the object directly, not an Effect. @@ -53,7 +61,9 @@ export const makeTestHarness = (config: GlobalConfig) => // Start Deno.serve on a random port const server = Deno.serve( { port: 0, onListen: () => {} }, - (req) => webHandler.handler(req), + (req) => { + return webHandler.handler(req); + }, ); // Ensure cleanup @@ -388,6 +398,137 @@ function proxyRunner(tc: ProxyTestCase, t: Deno.TestContext) { ); } +const getSwiftConfig = () => + Effect.gen(function* () { + const authUrl = yield* Config.string("HEARLD_SWIFTTEST_AUTH_URL").pipe( + Config.orElse(() => Config.string("HERALD_SWIFTTEST_AUTH_URL")), + Config.orElse(() => Config.string("OS_AUTH_URL")), + Config.withDefault("https://api.pub1.infomaniak.cloud/identity/v3"), + Config.option, + ); + + const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_USERNAME")), + Config.orElse(() => Config.string("OS_USERNAME")), + Config.option, + ); + const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PASSWORD")), + Config.orElse(() => Config.string("OS_PASSWORD")), + Config.option, + ); + const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") + .pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PROJECT_NAME")), + Config.orElse(() => Config.string("OS_PROJECT_NAME")), + Config.option, + ); + const region = yield* Config.string("HEARLD_SWIFTTEST_OS_REGION_NAME").pipe( + Config.orElse(() => Config.string("HERALD_SWIFTTEST_OS_REGION_NAME")), + Config.orElse(() => Config.string("TF_VAR_OS_REGION_NAME")), + Config.orElse(() => Config.string("OS_REGION_NAME")), + Config.withDefault("dc3-a"), + Config.option, + ); + + if ( + Option.isNone(username) || Option.isNone(password) || + Option.isNone(projectName) || Option.isNone(authUrl) + ) { + return Option.none(); + } + + const config: GlobalConfig = { + backends: { + swift: { + protocol: "swift", + auth_url: authUrl.value, + region: Option.getOrUndefined(region), + credentials: { + username: username.value, + password: password.value, + project_name: projectName.value, + user_domain_name: "Default", + project_domain_name: "Default", + }, + buckets: "*", + }, + }, + }; + return Option.some(config); + }); + +function swiftRunner(tc: ProxyTestCase, t: Deno.TestContext) { + return Effect.gen(function* () { + const swiftConfig = yield* getSwiftConfig(); + if (Option.isNone(swiftConfig)) { + return yield* Effect.fail( + new Error( + "Swift credentials missing. Set HERALD_SWIFTTEST_OS_USERNAME etc or run with infisical.", + ), + ); + } + + const h = yield* makeTestHarness(swiftConfig.value); + + if (tc.beforeAll) { + const beforeResult = tc.beforeAll(h.proxyClient); + if (Effect.isEffect(beforeResult)) { + yield* beforeResult; + } else { + yield* Effect.tryPromise(() => beforeResult as Promise).pipe( + Effect.orDie, + ); + } + } + + const resultEffect = Effect.gen(function* () { + const result = tc.fn(h.proxyClient); + if (Effect.isEffect(result)) { + yield* result; + } else { + yield* Effect.tryPromise({ + try: () => result as Promise, + catch: (e) => new Error(`Test function failed for ${tc.name}: ${e}`), + }); + } + }); + + yield* resultEffect; + + const lastResponse = h.getLastResponse(); + if (lastResponse) { + yield* Effect.tryPromise(() => + assertSnapshot(t, { + status: lastResponse.status, + headers: lastResponse.headers, + }, { name: `Swift/${tc.name} metadata` }) + ); + if (lastResponse.body) { + yield* Effect.tryPromise(() => + assertSnapshot(t, lastResponse.body, { + name: `Swift/${tc.name} body`, + }) + ); + } + } + + if (tc.afterAll) { + const afterResult = tc.afterAll(h.proxyClient); + if (Effect.isEffect(afterResult)) { + yield* afterResult; + } else { + yield* Effect.tryPromise(() => afterResult as Promise).pipe( + Effect.orDie, + ); + } + } + }).pipe( + Effect.tapErrorCause(Effect.logError), + Effect.scoped, + ); +} + export function harness(cases: ProxyTestCase[]) { const namePrefix = ""; for (const tc of cases) { @@ -403,5 +544,9 @@ export function harness(cases: ProxyTestCase[]) { ignore: tc.ignore, only: tc.only, }); + testEffect(`${namePrefix}Swift/${tc.name}`, (t) => swiftRunner(tc, t), { + ignore: tc.ignore, + only: tc.only, + }); } } diff --git a/tools/compose.yml b/tools/compose.yml index 0b5de2d..0067818 100644 --- a/tools/compose.yml +++ b/tools/compose.yml @@ -1,6 +1,7 @@ name: herald services: redis: + profiles: ["db"] image: docker.io/library/redis:alpine command: --save 60 1 --loglevel warning healthcheck: @@ -15,6 +16,7 @@ services: - redisdata:/data minio: + profiles: ["s3"] image: docker.io/minio/minio:latest command: server /data --console-address ":9001" ports: diff --git a/x/compose-down.ts b/x/compose-down.ts index 4094166..04f52d5 100755 --- a/x/compose-down.ts +++ b/x/compose-down.ts @@ -2,4 +2,6 @@ import { $, DOCKER_CMD } from "./utils.ts"; -await $.raw`${DOCKER_CMD} compose down`.cwd($.relativeDir("../tools/")); +await $.raw`${DOCKER_CMD} compose -f compose.yml down`.cwd( + $.path(import.meta.resolve("../tools/")), +); diff --git a/x/compose-up.ts b/x/compose-up.ts index 367e9a6..e6cfe3d 100755 --- a/x/compose-up.ts +++ b/x/compose-up.ts @@ -6,6 +6,6 @@ const profiles = $.argv .map((prof) => `--profile ${prof}`) .join(" "); -await $.raw`${DOCKER_CMD} compose ${profiles} up -d`.cwd( - $.relativeDir("../tools/"), +await $.raw`${DOCKER_CMD} compose -f compose.yml ${profiles} up -d`.cwd( + $.path(import.meta.resolve("../tools/")), ); diff --git a/x/purge-minio.ts b/x/purge-minio.ts index 204714a..3bf8ccd 100644 --- a/x/purge-minio.ts +++ b/x/purge-minio.ts @@ -2,7 +2,6 @@ import { DeleteBucketCommand, DeleteObjectsCommand, ListBucketsCommand, - ListObjectsV2Command, ListObjectVersionsCommand, S3Client, } from "npm:@aws-sdk/client-s3"; diff --git a/x/s3-tests.ts b/x/s3-tests.ts index af1c316..d4e958e 100755 --- a/x/s3-tests.ts +++ b/x/s3-tests.ts @@ -1,253 +1,678 @@ #!/usr/bin/env -S deno run --allow-all +/** + * Herald S3 Compatibility Test Runner + * + * This script runs the Ceph S3 compatibility test suite (s3-tests) against + * a local Herald proxy instance. It handles: + * - Starting the Herald proxy with a specified backend (minio or swift) + * - Configuring s3-tests to point to the proxy + * - Running pytest with real-time output streaming + * - Parsing JUnit XML for a final summary + * + * Usage: + * ./x/s3-tests.ts [pytest-args] [--backend ] [--no-abort] + * + * Environment Variables: + * S3TEST_TAGS: Custom pytest marks (default: not fails_on_s3proxy and ...) + * S3TEST_PYTEST_ARGS: Additional pytest arguments + * S3TEST_NO_ABORT: Set to "true" to disable abort-on-error + * HERALD_LOG_LEVEL: Set to "DEBUG" for verbose proxy logging + * + * Files: + * s3-tests/s3tests.conf: Generated s3-tests configuration + * s3-tests/herald-proxy.log: Herald proxy logs (minio backend) + * s3-tests/herald-proxy-swift.log: Herald proxy logs (swift backend) + * s3-tests/s3-tests.log: Full pytest output + */ -import { Effect } from "effect"; -import { LoggingLive } from "../src/Logging/Layer.ts"; -import { makeTestHarness } from "../tests/utils.ts"; -import type { GlobalConfig } from "../src/Domain/Config.ts"; +import { Config, Effect, Logger, LogLevel, Option } from "effect"; import * as path from "@std/path"; -import { $ } from "./utils.ts"; - -// Default tags taken from s3proxy/src/test/resources/run-s3-tests.sh -const DEFAULT_TAGS = [ - "not fails_on_s3proxy", - "and not appendobject", - "and not bucket_policy", - "and not checksum", - "and not copy", - "and not cors", - "and not encryption", - "and not fails_strict_rfc2616", - "and not iam_tenant", - "and not lifecycle", - "and not object_lock", - "and not policy", - "and not policy_status", - "and not s3select", - "and not s3website", - "and not sse_s3", - "and not tagging", - "and not test_of_sts", - "and not user_policy", - "and not versioning", - "and not webidentity_test", -].join(" "); - -const config: GlobalConfig = { - backends: { - minio: { - protocol: "s3", - endpoint: "http://localhost:9000", - region: "us-east-1", - credentials: { - accessKeyId: "minioadmin", - secretAccessKey: "minioadmin", +import { $ } from "@david/dax"; +import * as colors from "@std/fmt/colors"; +import { makeTestHarness } from "../tests/utils.ts"; +import { GlobalConfig } from "../src/Domain/Config.ts"; + +const DEFAULT_TAGS = + "not fails_on_s3proxy and not appendobject and not bucket_policy and not copy and not cors and not encryption and not fails_strict_rfc2616 and not iam_tenant and not lifecycle and not object_lock and not policy and not policy_status and not s3select and not s3website and not sse_s3 and not tagging and not test_of_sts and not user_policy and not versioning and not webidentity_test"; + +function getMinioConfig(): GlobalConfig { + return { + backends: { + minio: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + buckets: "*", }, - buckets: "*", }, - }, -}; + }; +} + +const getSwiftConfig = () => + Effect.gen(function* () { + const authUrl = yield* Config.string("HEARLD_SWIFTTEST_AUTH_URL").pipe( + Config.orElse(() => Config.string("HERALD_SWIFTTEST_AUTH_URL")), + Config.orElse(() => Config.string("OS_AUTH_URL")), + Config.withDefault("https://api.pub1.infomaniak.cloud/identity/v3"), + Config.option, + ); + + const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_USERNAME")), + Config.orElse(() => Config.string("OS_USERNAME")), + Config.option, + ); + const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PASSWORD")), + Config.orElse(() => Config.string("OS_PASSWORD")), + Config.option, + ); + const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") + .pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PROJECT_NAME")), + Config.orElse(() => Config.string("OS_PROJECT_NAME")), + Config.option, + ); + const region = yield* Config.string("HEARLD_SWIFTTEST_OS_REGION_NAME").pipe( + Config.orElse(() => Config.string("HERALD_SWIFTTEST_OS_REGION_NAME")), + Config.orElse(() => Config.string("TF_VAR_OS_REGION_NAME")), + Config.orElse(() => Config.string("OS_REGION_NAME")), + Config.withDefault("dc3-a"), + Config.option, + ); + + if ( + Option.isNone(username) || Option.isNone(password) || + Option.isNone(projectName) || Option.isNone(authUrl) + ) { + return Option.none(); + } + + const config: GlobalConfig = { + backends: { + swift: { + protocol: "swift", + auth_url: authUrl.value, + region: Option.getOrUndefined(region), + credentials: { + username: username.value, + password: password.value, + project_name: projectName.value, + user_domain_name: "Default", + project_domain_name: "Default", + }, + buckets: "*", + }, + }, + }; + return Option.some(config); + }); -const program = makeTestHarness(config).pipe( - Effect.flatMap((h) => { - const port = new URL(h.proxyUrl).port; +const program = Effect.gen(function* () { + console.log("Program started"); + const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); + const s3TestsDir = path.resolve(__dirname, "../s3-tests"); - // Parse filtering arguments - const tags = $.env.S3TEST_TAGS ?? DEFAULT_TAGS; - const pytestArgsEnv = $.env.S3TEST_PYTEST_ARGS ?? ""; - const pytestArgsFromEnv = pytestArgsEnv ? pytestArgsEnv.split(/\s+/) : []; - const pytestArgsFromCli = $.argv; - const pytestArgs = [...pytestArgsFromEnv, ...pytestArgsFromCli]; + // Parse filtering arguments and flags + const rawArgs = [...Deno.args]; + const noAbort = rawArgs.includes("--no-abort") || + Deno.env.get("S3TEST_NO_ABORT") === "true"; + + let backend = "minio"; + const backendIdx = rawArgs.indexOf("--backend"); + if (backendIdx !== -1) { + backend = rawArgs[backendIdx + 1]; + rawArgs.splice(backendIdx, 2); + } + + const pytestArgsFromCli = rawArgs.filter((arg) => arg !== "--no-abort"); + + const proxyLogName = backend === "swift" + ? "herald-proxy-swift.log" + : "herald-proxy.log"; + const proxyLogPath = path.join(s3TestsDir, proxyLogName); + + // Initialize config based on backend + let activeConfig: GlobalConfig; + let s3AccessKey = "minioadmin"; + let s3SecretKey = "minioadmin"; + + if (backend === "swift") { + const swiftConfig = yield* getSwiftConfig(); + if (Option.isNone(swiftConfig)) { + return yield* Effect.fail( + new Error("Swift credentials missing. Run with infisical."), + ); + } + activeConfig = swiftConfig.value; + // For Swift backend, Herald doesn't check S3 credentials, + // but s3-tests needs them to sign requests. + s3AccessKey = "dummy"; + s3SecretKey = "dummy"; + } else { + activeConfig = getMinioConfig(); + } + + console.log("Creating file logger for proxy..."); + // Create a file logger for the proxy + const proxyLogFile = yield* Effect.tryPromise(() => + Deno.open(proxyLogPath, { write: true, create: true, truncate: true }) + ); + + yield* Effect.addFinalizer(() => + Effect.tryPromise({ + try: () => Promise.resolve(proxyLogFile.close()), + catch: (e) => new Error(`Failed to close proxy log file: ${e}`), + }).pipe(Effect.orDie) + ); + + const logLevel = yield* Config.string("HERALD_LOG_LEVEL").pipe( + Config.withDefault("INFO"), + ); + const minLogLevel = LogLevel.Debug; + + // Create a custom logging layer that writes to file synchronously + const FileLoggingLive = Logger.replace( + Logger.defaultLogger, + Logger.make(({ message, logLevel: currentLogLevel }) => { + const timestamp = new Date().toISOString(); + const level = currentLogLevel.label; + const msg = typeof message === "string" ? message : String(message); + const logLine = `${timestamp} level=${level} ${msg}\n`; + try { + proxyLogFile.writeSync(new TextEncoder().encode(logLine)); + } catch (e) { + console.error(`Failed to write to proxy log: ${e}`); + } + }), + ); - return Effect.gen(function* () { - yield* Effect.logInfo(`Starting Herald proxy on port ${port}`); + // Provide the file logger to the test harness (the proxy) + const h = yield* makeTestHarness(activeConfig, FileLoggingLive); - const confContent = `[DEFAULT] + const port = new URL(h.proxyUrl).port; + + // Parse remaining filtering arguments + const tags = Deno.env.get("S3TEST_TAGS") ?? DEFAULT_TAGS; + const pytestArgsEnv = Deno.env.get("S3TEST_PYTEST_ARGS") ?? ""; + const pytestArgsFromEnv = pytestArgsEnv ? pytestArgsEnv.split(/\s+/) : []; + + const pytestArgs = [...pytestArgsFromEnv, ...pytestArgsFromCli]; + + return yield* (Effect.gen(function* () { + // We use console.log for harness output to avoid them going to the proxy log file + console.log( + `Starting Herald (${colors.cyan(backend)} backend) on port ${ + colors.cyan(port) + }`, + ); + console.log(`Proxy logs: ${colors.gray(proxyLogPath)}`); + + const confContent = `[DEFAULT] host = 127.0.0.1 port = ${port} is_secure = no [fixtures] -bucket prefix = herald-{random}- +bucket prefix = herald-${backend}-{random}- [s3 main] user_id = main display_name = main email = main@example.com -access_key = minioadmin -secret_key = minioadmin +access_key = ${s3AccessKey} +secret_key = ${s3SecretKey} [s3 alt] user_id = alt display_name = alt email = alt@example.com -access_key = minioadmin -secret_key = minioadmin +access_key = ${s3AccessKey} +secret_key = ${s3SecretKey} [s3 tenant] user_id = tenant display_name = tenant email = tenant@example.com -access_key = minioadmin -secret_key = minioadmin +access_key = ${s3AccessKey} +secret_key = ${s3SecretKey} tenant = testx [iam] email = iam@example.com user_id = iam -access_key = minioadmin -secret_key = minioadmin +access_key = ${s3AccessKey} +secret_key = ${s3SecretKey} display_name = iam [iam root] -access_key = minioadmin -secret_key = minioadmin +access_key = ${s3AccessKey} +secret_key = ${s3SecretKey} user_id = iam_root email = iam_root@example.com [iam alt root] -access_key = minioadmin -secret_key = minioadmin +access_key = ${s3AccessKey} +secret_key = ${s3SecretKey} user_id = iam_alt_root email = iam_alt_root@example.com `; - const confPath = yield* Effect.promise(() => - Deno.makeTempFile({ suffix: ".conf" }) - ); - yield* Effect.promise(() => Deno.writeTextFile(confPath, confContent)); + const confPath = yield* Effect.promise(() => + Deno.makeTempFile({ suffix: ".conf" }) + ); + yield* Effect.promise(() => Deno.writeTextFile(confPath, confContent)); - const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); - const s3TestsDir = path.resolve(__dirname, "../s3-tests"); - const logPath = path.join(s3TestsDir, "s3-tests.log"); + const logPath = path.join(s3TestsDir, "s3-tests.log"); - yield* Effect.logInfo(`s3-tests directory: ${s3TestsDir}`); - yield* Effect.logInfo(`Log file: ${logPath}`); + console.log(`s3-tests directory: ${colors.gray(s3TestsDir)}`); + console.log(`Log file: ${colors.gray(logPath)}`); - // Ensure we have a virtual environment - const venvPath = path.join(s3TestsDir, ".venv"); - const venvExists = yield* Effect.tryPromise(() => - Deno.stat(venvPath).then(() => true).catch(() => false) - ); + // Ensure we have a virtual environment + const venvPath = path.join(s3TestsDir, ".venv"); + const venvExists = yield* Effect.tryPromise(() => + Deno.stat(venvPath).then(() => true).catch(() => false) + ); - if (!venvExists) { - yield* Effect.logInfo("Creating Python virtual environment..."); - yield* Effect.tryPromise(() => - $`uv venv --python 3.11`.cwd(s3TestsDir) - ); - } + if (!venvExists) { + console.log(colors.yellow("Creating Python virtual environment...")); + yield* Effect.tryPromise(() => $`uv venv --python 3.11`.cwd(s3TestsDir)); + } + + // Register finalizer to clean up conf file + yield* Effect.addFinalizer(() => + Effect.tryPromise({ + try: () => + Deno.remove(confPath).catch((e) => { + console.error(`Failed to remove conf file ${confPath}: ${e}`); + }), + catch: (e) => new Error(`Effect.tryPromise failed: ${e}`), + }).pipe(Effect.orDie) + ); + + // Ensure dependencies are installed + const pytestCheck = yield* Effect.tryPromise({ + try: async () => { + const proc = $`uv run pytest --version`.cwd(s3TestsDir).noThrow(); + return await proc; + }, + catch: () => new Error("Check failed"), + }); - yield* Effect.logInfo( - `Running s3-tests against Herald on port ${port}...`, + if (pytestCheck.code !== 0) { + console.log(colors.yellow("Installing s3-tests dependencies...")); + yield* Effect.tryPromise({ + try: async () => { + await $`uv pip install -r requirements.txt`.cwd(s3TestsDir); + await $`uv pip install -e .`.cwd(s3TestsDir); + }, + catch: (e) => new Error(`Failed to install dependencies: ${e}`), + }); + } + + console.log( + `Running s3-tests against Herald on port ${colors.cyan(port)}...`, + ); + if (tags) console.log(`${colors.gray("Tags:")} ${tags}`); + if (pytestArgs.length > 0) { + console.log( + `${colors.gray("Additional pytest args:")} ${pytestArgs.join(" ")}`, ); - yield* Effect.logInfo(`Tags: ${tags}`); - yield* Effect.logInfo(`Additional pytest args: ${pytestArgs.join(" ")}`); + } + if (noAbort) { + console.log(colors.yellow("Abort on ERROR disabled (--no-abort)")); + } - // Run pytest with timeout - const timeoutId = setTimeout(() => {}, 300000); // 5 minutes + // Build command arguments + const cmdArgs = [ + "-v", + "--tb=short", + ]; - try { - // Build command arguments - const cmdArgs: string[] = ["run", "pytest", "-v", "--tb=long"]; + const junitXmlName = "junit.xml"; + const junitXmlPath = path.join(s3TestsDir, junitXmlName); + cmdArgs.push(`--junit-xml=${junitXmlName}`); - if (tags) { - cmdArgs.push("-m", tags); - } + if (tags) { + cmdArgs.push("-m", tags); + } - // Add user-provided pytest arguments - cmdArgs.push(...pytestArgs); + cmdArgs.push(...pytestArgs); - // Add test path if not already specified - const hasTestPath = pytestArgs.some((arg) => - arg.includes("s3tests/") || arg.includes("test_") - ); - if (!hasTestPath) { - cmdArgs.push("s3tests/functional/test_s3.py"); - } + const logFile = yield* Effect.tryPromise(() => + Deno.open(logPath, { + write: true, + create: true, + truncate: true, + }) + ); - const result = yield* Effect.tryPromise({ - try: async () => { - const proc = $`uv ${cmdArgs}` - .cwd(s3TestsDir) - .env({ - S3TEST_CONF: confPath, - UV_PYTHON: "3.11", - }) - .noThrow() - .stdout("piped") - .stderr("piped"); - return await proc; - }, - catch: (e) => new Error(`Failed to run pytest: ${e}`), - }); - - // Write output to log file - const stdoutBytes = yield* Effect.sync(() => { - const stdout = result.stdout as unknown; - if (stdout instanceof Uint8Array) { - return stdout; + console.log(`Command: uv run pytest ${cmdArgs.join(" ")}`); + const child = $`uv run pytest ${cmdArgs}` + .cwd(s3TestsDir) + .env({ S3TEST_CONF: confPath, PYTHONUNBUFFERED: "1" }) + .stdout("piped") + .stderr("piped") + .spawn(); + + const sigintHandler = () => { + child.kill(); + Deno.exit(0); + }; + Deno.addSignalListener("SIGINT", sigintHandler); + + const result = yield* Effect.tryPromise({ + try: async () => { + let collectedInfo = ""; + let failedCount = 0; + let errorCount = 0; + let skippedCount = 0; + let lastResultTime = Date.now(); + const seenTests = new Set(); + const failedTests = new Set(); + const errorTests = new Set(); + let currentTestName = ""; + + let shouldAbort = false; + let abortReason = ""; + + const processLine = (line: string) => { + const trimmed = line.trim(); + if (!trimmed) return; + + // Capture test result lines like: + // s3tests/functional/test_s3.py::test_bucket_list_empty PASSED [ 0%] + const resultMatch = trimmed.match( + /^([^\s]+::[^\s]+)\s+(PASSED|FAILED|ERROR|SKIPPED)/, + ); + if (resultMatch) { + const testName = resultMatch[1]; + const status = resultMatch[2]; + const now = Date.now(); + const duration = ((now - lastResultTime) / 1000).toFixed(2); + lastResultTime = now; + currentTestName = testName; + + if (status === "PASSED") { + console.log( + `${colors.green("✓")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + } else if (status === "FAILED") { + if (!seenTests.has(testName)) { + failedCount++; + seenTests.add(testName); + failedTests.add(testName); + } + console.error( + `${colors.red("✗")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + } else if (status === "ERROR") { + if (!seenTests.has(testName)) { + errorCount++; + seenTests.add(testName); + errorTests.add(testName); + } + console.error( + `${colors.red("✗ ERROR:")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + if (!noAbort) { + shouldAbort = true; + abortReason = `ERROR in ${testName}`; + child.kill(); + } + } else if (status === "SKIPPED") { + skippedCount++; + console.log( + `${colors.yellow("-")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + } + return; } - return new TextEncoder().encode(String(stdout)); - }); - const stderrBytes = yield* Effect.sync(() => { - const stderr = result.stderr as unknown; - if (stderr instanceof Uint8Array) { - return stderr; + + // Also check for ERROR in non-verbose format + const errorMatch = trimmed.match(/^ERROR\s+([^\s]+::[^\s]+)/); + if (errorMatch) { + const testName = errorMatch[1]; + const now = Date.now(); + const duration = ((now - lastResultTime) / 1000).toFixed(2); + lastResultTime = now; + currentTestName = testName; + + if (!seenTests.has(testName)) { + errorCount++; + seenTests.add(testName); + errorTests.add(testName); + } + console.error( + `${colors.red("✗ ERROR:")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + + if (!noAbort) { + shouldAbort = true; + abortReason = `ERROR in ${testName}`; + child.kill(); + } + return; } - return new TextEncoder().encode(String(stderr)); - }); - const combined = new Uint8Array( - stdoutBytes.length + stderrBytes.length, - ); - combined.set(stdoutBytes); - combined.set(stderrBytes, stdoutBytes.length); - yield* Effect.tryPromise(() => Deno.writeFile(logPath, combined)); - if (result.code !== 0) { - yield* Effect.logError( - `s3-tests finished with exit code ${result.code}`, - ); + // Echo important lines (failures, tracebacks, summaries) + if ( + trimmed.includes("FAILURES") || + trimmed.includes("ERRORS") || + trimmed.includes("short test summary") || + trimmed.startsWith("E ") || // Traceback lines in short format + trimmed.startsWith("> ") || + trimmed.match(/^=+\s*(passed|failed|error)/i) + ) { + const prefix = currentTestName + ? colors.gray(`[${currentTestName}] `) + : ""; + console.log(`${prefix}${trimmed}`); + } + }; - // Show last 20 lines of log - const tailResult = yield* Effect.tryPromise({ - try: async () => { - const proc = $`tail -n 20 ${logPath}`.stdout("piped"); - return await proc; - }, - catch: (e) => new Error(`Failed to tail log file: ${e}`), - }); - yield* Effect.logError("Last 20 lines of log:"); - const tailOutput = yield* Effect.sync(() => { - const stdout = tailResult.stdout as unknown; - if (stdout instanceof Uint8Array) { - return new TextDecoder().decode(stdout); + const decoder = new TextDecoder(); + + async function streamToLogAndConsole( + stream: ReadableStream, + ) { + const reader = stream.getReader(); + let buffer = ""; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + try { + await logFile.write(value); + } catch (e) { + console.error(`Failed to write to log file: ${e}`); + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + processLine(line); } - return String(stdout); - }); - yield* Effect.logError(tailOutput); + } + if (buffer) { + processLine(buffer); + } + reader.releaseLock(); + } + + const [procResult] = await Promise.all([ + child, + streamToLogAndConsole(child.stdout()), + streamToLogAndConsole(child.stderr()), + ]); + + Deno.removeSignalListener("SIGINT", sigintHandler); - yield* Effect.fail( - new Error(`s3-tests failed with exit code ${result.code}`), + // Attempt to parse JUnit XML if it exists and is valid + let junitData: { + tests: number; + failures: number; + errors: number; + skipped: number; + time?: number; + failedNames: string[]; + errorNames: string[]; + } | null = null; + + try { + const junitXml = await Deno.readTextFile(junitXmlPath); + const getAttr = (name: string) => { + const match = junitXml.match(new RegExp(`${name}="([\\d.]+)"`)); + return match ? parseFloat(match[1]) : 0; + }; + + const failedNames: string[] = []; + const errorNames: string[] = []; + + const testcaseMatches = junitXml.matchAll( + /]*>([\s\S]*?)<\/testcase>/g, ); - } else { - yield* Effect.logInfo("s3-tests passed!"); + for (const match of testcaseMatches) { + const fullName = `${match[1]}::${match[2]}`; + const content = match[3]; + if (content.includes(" Deno.remove(confPath).catch(() => {})); - } + + // Use streaming counts if JUnit failed or reported 0 + const finalCounts = (junitData && junitData.tests > 0) ? junitData : { + tests: seenTests.size, + failures: failedCount, + errors: errorCount, + skipped: skippedCount, + time: undefined, + failedNames: Array.from(failedTests), + errorNames: Array.from(errorTests), + }; + + return { + code: procResult.code, + counts: finalCounts, + collectedInfo, + shouldAbort, + abortReason, + }; + }, + catch: (e) => new Error(`Failed to run pytest: ${e}`), }); - }), - Effect.scoped, - Effect.provide(LoggingLive), -); + + if (result.collectedInfo) { + console.log(colors.gray(result.collectedInfo)); + } + + const { tests, failures, errors, skipped, time, failedNames, errorNames } = + result.counts; + const passed = tests - failures - errors - skipped; + + console.log(); + const durationStr = time ? ` ${colors.cyan(`${time.toFixed(2)}s`)}` : ""; + console.log( + `${colors.bold(tests.toString())} tests completed in${durationStr}:`, + ); + console.log( + ` ${colors.green("successes")}: ${ + colors.bold(passed.toString()) + }/${tests}`, + ); + console.log( + ` ${colors.red("failures")}: ${ + colors.bold(failures.toString()) + }/${tests}`, + ); + if (errors > 0) { + console.log( + ` ${colors.red("errors")}: ${ + colors.bold(errors.toString()) + }/${tests}`, + ); + } + if (skipped > 0) { + console.log( + ` ${colors.gray("skipped")}: ${ + colors.bold(skipped.toString()) + }/${tests}`, + ); + } + + if (failedNames.length > 0) { + console.log(colors.red("\nFailures:")); + for (const name of failedNames) { + console.log(` ${colors.red("-")} ${name}`); + } + } + + if (errorNames.length > 0) { + console.log(colors.red("\nErrors:")); + for (const name of errorNames) { + console.log(` ${colors.red("-")} ${name}`); + } + } + + if (errors > 0 || (result.shouldAbort && result.abortReason)) { + if (result.shouldAbort) { + yield* Effect.fail( + new Error( + `Aborted due to ERROR: ${result.abortReason || "Test Error"}`, + ), + ); + } else { + yield* Effect.fail(new Error(`s3-tests finished with errors.`)); + } + } + + if (failures > 0 || result.code !== 0) { + yield* Effect.fail( + new Error(`s3-tests finished with failures (code ${result.code}).`), + ); + } + + console.log(colors.green(`\n✓ s3-tests completed successfully.`)); + }).pipe( + Effect.provide(Logger.minimumLogLevel(minLogLevel)), + )); +}); if (import.meta.main) { - Effect.runPromiseExit(program).then((exitCode) => { + Effect.runPromiseExit(program.pipe(Effect.scoped)).then((exitCode) => { if (exitCode._tag === "Failure") { + console.error( + colors.red(`Fatal error: ${JSON.stringify(exitCode.cause, null, 2)}`), + ); Deno.exit(1); } }).catch((e) => { - console.error(`Error: ${e}`); + console.error(colors.red(`Unhandled error: ${e}`)); Deno.exit(1); }); } diff --git a/x/swift-debug.ts b/x/swift-debug.ts new file mode 100644 index 0000000..76e888c --- /dev/null +++ b/x/swift-debug.ts @@ -0,0 +1,42 @@ +#!/usr/bin/env -S deno run --allow-all +import { Effect, Logger, LogLevel } from "effect"; +import { SwiftClient, SwiftClientLive } from "../src/Backends/Swift/Client.ts"; +import { HeraldConfigLive } from "../src/Config/Layer.ts"; +import { makeSwiftBackend } from "../src/Backends/Swift/Backend.ts"; +import { FetchHttpClient } from "@effect/platform"; + +const program = Effect.gen(function* () { + console.log("Checking Swift connection..."); + + // We'll use the 'default' backend which should be configured via HERALD_ env vars + const backendId = "default"; + + const swiftClient = yield* SwiftClient; + const auth = yield* swiftClient.getAuthMeta({ backend_id: backendId }); + + console.log("Auth successful!"); + console.log(`Storage URL: ${auth.storageUrl}`); + console.log(`Token: ${auth.token.substring(0, 10)}...`); + + const backend = yield* makeSwiftBackend({ backend_id: backendId }); + const { buckets } = yield* backend.listBuckets(); + + console.log(`Found ${buckets.length} buckets:`); + for (const b of buckets) { + console.log(` - ${b.name} (created: ${b.creationDate})`); + } +}).pipe( + Effect.provide(SwiftClientLive), + Effect.provide(HeraldConfigLive), + Effect.provide(FetchHttpClient.layer), + Effect.provide(Logger.minimumLogLevel(LogLevel.Debug)), +); + +if (import.meta.main) { + Effect.runPromiseExit(program).then((exit) => { + if (exit._tag === "Failure") { + console.error("Program failed:", exit.cause); + Deno.exit(1); + } + }); +} diff --git a/x/swift-s3-tests.ts b/x/swift-s3-tests.ts new file mode 100644 index 0000000..1ca12ec --- /dev/null +++ b/x/swift-s3-tests.ts @@ -0,0 +1,207 @@ +#!/usr/bin/env -S deno run --allow-all + +/** + * Herald Swift Compatibility Test Runner + * + * This script runs the Ceph s3-tests suite against a Herald proxy instance + * configured with an OpenStack Swift backend. + */ + +import { Config, Effect, Layer, Logger, LogLevel } from "effect"; +import { makeTestHarness } from "../tests/utils.ts"; +import type { GlobalConfig } from "../src/Domain/Config.ts"; +import * as path from "@std/path"; +import { $ } from "./utils.ts"; +import * as colors from "@std/fmt/colors"; + +const program = Effect.gen(function* () { + const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); + const s3TestsDir = path.resolve(__dirname, "../s3-tests"); + const proxyLogPath = path.join(s3TestsDir, "herald-proxy-swift.log"); + + // Read Swift config from environment + const authUrl = yield* Config.string("HERALD_SWIFTTEST_AUTH_URL").pipe( + Config.orElse(() => Config.string("HEARLD_SWIFTTEST_AUTH_URL")), + Config.withDefault(""), + ); + const region = yield* Config.string("HERALD_SWIFTTEST_OS_REGION_NAME").pipe( + Config.orElse(() => Config.string("HEARLD_SWIFTTEST_OS_REGION_NAME")), + Config.withDefault(""), + ); + const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( + Config.withDefault(""), + ); + const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( + Config.withDefault(""), + ); + const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") + .pipe(Config.withDefault("")); + + if (!authUrl || !username || !password || !projectName) { + return yield* Effect.fail( + new Error( + "Swift environment variables (HERALD_SWIFTTEST_...) are missing. Run with infisical.", + ), + ); + } + + const swiftConfig: GlobalConfig = { + backends: { + swift: { + protocol: "swift", + auth_url: authUrl, + region: region || undefined, + credentials: { + username, + password, + project_name: projectName, + user_domain_name: "Default", + project_domain_name: "Default", + }, + buckets: "*", + }, + }, + }; + + // Create a file logger for the proxy + const proxyLogFile = yield* Effect.tryPromise(() => + Deno.open(proxyLogPath, { write: true, create: true, truncate: true }) + ); + + yield* Effect.addFinalizer(() => + Effect.tryPromise({ + try: () => Promise.resolve(proxyLogFile.close()), + catch: (e) => new Error(`Failed to close proxy log file: ${e}`), + }).pipe(Effect.orDie) + ); + + // Provide the test harness + const h = yield* makeTestHarness(swiftConfig); + const port = new URL(h.proxyUrl).port; + + console.log(`Starting Herald (Swift backend) on port ${colors.cyan(port)}`); + console.log(`Proxy logs: ${colors.gray(proxyLogPath)}`); + + const confContent = `[DEFAULT] +host = 127.0.0.1 +port = ${port} +is_secure = no + +[fixtures] +bucket prefix = herald-swift-{random}- + +[s3 main] +user_id = main +display_name = main +email = main@example.com +access_key = dummy +secret_key = dummy + +[s3 alt] +user_id = alt +display_name = alt +email = alt@example.com +access_key = dummy +secret_key = dummy +`; + + const confPath = yield* Effect.promise(() => + Deno.makeTempFile({ suffix: ".conf" }) + ); + yield* Effect.promise(() => Deno.writeTextFile(confPath, confContent)); + yield* Effect.addFinalizer(() => + Effect.promise(() => + Deno.remove(confPath).catch((e) => { + console.error(`Failed to remove conf file ${confPath}: ${e}`); + }) + ) + ); + + const logPath = path.join(s3TestsDir, "s3-tests-swift.log"); + const junitXmlPath = path.join(s3TestsDir, "junit-swift.xml"); + + const rawArgs = $.argv; + const noAbort = rawArgs.includes("--no-abort"); + const pytestArgsFromCli = rawArgs.filter((arg) => arg !== "--no-abort"); + + const cmdArgs: string[] = [ + "run", + "pytest", + "-v", + "--tb=short", + `--junit-xml=${junitXmlPath}`, + ...pytestArgsFromCli, + ]; + + // If no specific test path, default to test_s3.py + if ( + !pytestArgsFromCli.some((arg) => + arg.includes("s3tests/") || arg.endsWith(".py") + ) + ) { + cmdArgs.push("s3tests/functional/test_s3.py"); + } + + console.log(`Running s3-tests against Herald (Swift)...`); + + const logFile = yield* Effect.tryPromise(() => + Deno.open(logPath, { write: true, create: true, truncate: true }) + ); + yield* Effect.addFinalizer(() => + Effect.promise(() => Promise.resolve(logFile.close())) + ); + + const result = yield* Effect.tryPromise({ + try: async () => { + const child = $`uv ${cmdArgs}` + .cwd(s3TestsDir) + .env({ + S3TEST_CONF: confPath, + UV_PYTHON: "3.11", + PYTHONUNBUFFERED: "1", + }) + .noThrow() + .stdout("piped") + .stderr("piped") + .spawn(); + + const decoder = new TextDecoder(); + async function streamToLog(stream: ReadableStream) { + const reader = stream.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) break; + await logFile.write(value); + Deno.stdout.writeSync(value); // Echo to console for now + } + } + + const [procResult] = await Promise.all([ + child, + streamToLog(child.stdout()), + streamToLog(child.stderr()), + ]); + + return procResult; + }, + catch: (e) => new Error(`Failed to run pytest: ${e}`), + }); + + if (result.code !== 0) { + yield* Effect.fail(new Error(`s3-tests failed with code ${result.code}`)); + } + + console.log(colors.green(`\n✓ s3-tests completed successfully.`)); +}).pipe( + Effect.scoped, + Effect.provide(Logger.minimumLogLevel(LogLevel.Info)), +); + +if (import.meta.main) { + Effect.runPromiseExit(program).then((exit) => { + if (exit._tag === "Failure") { + console.error(colors.red(`Error: ${exit.cause}`)); + Deno.exit(1); + } + }); +}