From 9e5152a8f3ec310557a7dc5a23c37f427846038a Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Tue, 20 Jan 2026 03:05:36 +0300 Subject: [PATCH 1/3] wip: swift object CRUD --- .env.example | 6 + .github/workflows/build-image.yml | 12 +- .github/workflows/pre-commit.yml | 4 +- .gitignore | 90 ++ .infisical.json | 5 + AGENTS.md | 5 + README.md | 61 +- TODO.md | 133 +++ deno.lock | 3 +- flake.nix | 3 +- src/Backends/S3/Backend.ts | 43 +- src/Backends/S3/Client.ts | 56 +- src/Backends/S3/Signer.ts | 19 +- src/Backends/Swift/Backend.ts | 592 +++++++++++++ src/Backends/Swift/Client.ts | 194 +++++ src/Config/Layer.ts | 127 ++- src/Domain/Config.ts | 133 +-- src/Frontend/Api.ts | 4 + src/Frontend/Buckets/Create.ts | 30 +- src/Frontend/Buckets/List.ts | 11 +- src/Frontend/Http.ts | 11 +- src/Frontend/Objects/Post.ts | 35 +- src/Frontend/Utils.ts | 47 +- src/Logging/Layer.ts | 37 +- src/Services/BackendResolver.ts | 17 +- src/Services/S3Xml.ts | 250 +++--- tests/config.test.ts | 95 ++- .../__snapshots__/buckets.test.ts.snap | 65 +- .../__snapshots__/objects.test.ts.snap | 47 ++ tests/utils.ts | 151 +++- x/s3-tests.ts | 784 ++++++++++++++---- x/swift-debug.ts | 41 + x/swift-s3-tests.ts | 207 +++++ 33 files changed, 2842 insertions(+), 476 deletions(-) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 .infisical.json create mode 100644 TODO.md create mode 100644 src/Backends/Swift/Backend.ts create mode 100644 src/Backends/Swift/Client.ts create mode 100644 x/swift-debug.ts create mode 100644 x/swift-s3-tests.ts diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..775bcd1 --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +HERALD_SWIFTTEST_OS_AUTH=$TF_VAR_OS_PASSWORD +HERALD_SWIFTTEST_OS_PASSWORD=$TF_VAR_OS_PASSWORD +HERALD_SWIFTTEST_OS_PROJECT_NAME=$TF_VAR_OS_PROJECT_NAME +HERALD_SWIFTTEST_OS_USERNAME=$TF_VAR_OS_USERNAME +HEARLD_SWIFTTEST_AUTH_URL=https://api.pub1.infomaniak.cloud/identity/v3 +HEARLD_SWIFTTEST_OS_REGION_NAME=dc3-a diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 1d15299..c5c85ee 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -5,16 +5,16 @@ on: branches: - main paths: - - 'src/**' - - 'tools/**' - - '.github/workflows/build-image.yml' + - "src/**" + - "tools/**" + - ".github/workflows/build-image.yml" pull_request: branches: - main paths: - - 'src/**' - - 'tools/**' - - '.github/workflows/build-image.yml' + - "src/**" + - "tools/**" + - ".github/workflows/build-image.yml" workflow_dispatch: env: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index c878fb7..630ff40 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -2,9 +2,9 @@ name: pre-commit on: push: - branches: [ main ] + branches: [main] pull_request: - branches: [ main ] + branches: [main] workflow_dispatch: jobs: diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c8be0ce --- /dev/null +++ b/.gitignore @@ -0,0 +1,90 @@ +logs +*.log + +# MacOS specific +.DS_Store + +# Local environment variables +.env +.env.local +.env.*.local + +# build +/build +test.http + +############################ +# OS X +############################ + +.DS_Store +.AppleDouble +.LSOverride +Icon +.Spotlight-V100 +.Trashes +._* + +############################ +# Linux +############################ + +*~ + +############################ +# Windows +############################ + +Thumbs.db +ehthumbs.db +Desktop.ini +$RECYCLE.BIN/ +*.cab +*.msi +*.msm +*.msp + +############################ +# Logs and databases +############################ + +.tmp +*.log +*.sql +*.sqlite +*.sqlite3 + +############################ +# Misc. +############################ + +*# +ssl +.idea +nbproject +public/uploads/* +!public/uploads/.gitkeep + +**/.terraform/* +*.tfstate +*.tfstate.* +crash.log +*.tfvars +.terragrunt-cache + +############################ +# Others +############################ +*tshscript.sh +tests/**/.terraform.lock.hcl +examples/**/.terraform.lock.hcl +*.crt +token +venv/ +bucket_differences.json +*.crt +token +*.db +*.db-shm +*.db-wal +.vscode diff --git a/.infisical.json b/.infisical.json new file mode 100644 index 0000000..a6af04a --- /dev/null +++ b/.infisical.json @@ -0,0 +1,5 @@ +{ + "workspaceId": "39bbe4e4-20c2-42fa-8a6c-1bcafcc74faf", + "defaultEnvironment": "", + "gitBranchToEnvironmentMapping": null +} diff --git a/AGENTS.md b/AGENTS.md index b3db2d5..fbc11d0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,6 +3,11 @@ - Prefer generators over effect piping. - Use methods on `Effect.Option` like `Option.isNone` instead of looking at _tag. + - **NEVER** use standard `try/catch` or `try/finally` blocks around `yield*` + in Effect generators. Use `Effect.addFinalizer`, `Effect.try`, + `Effect.catchAll`, or `Effect.orElse`. + - **ALWAYS** use the `Config` module from Effect for environment variable + access instead of `Deno.env.get`. - **NEVER** assume default values using `??` or ternary operators for critical configuration or external input (e.g., `bucket.region ?? "us-east-1"`, `request.headers.host ?? "localhost"`). Always fail explicitly with a diff --git a/README.md b/README.md index 21701f7..05e3049 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,63 @@ Herald is an S3 proxy that supports: -- TODO +- Protocol translation (S3 to S3, S3 to Swift). +- Backend routing based on bucket names. +- Flexible bucket mapping with glob support. + +## Config + +Herald is configured via a YAML file (typically `herald.yaml`). The +configuration defines backends and how incoming requests are routed to them. + +```yaml +backends: + # Unique identifier for the backend + minio: + # Backend protocol: "s3" or "swift" + protocol: s3 + + # Base URL of the backend service + endpoint: http://127.0.0.1:9000 + + # Default region for this backend + region: us-east-1 + + # Authentication credentials for the backend + credentials: + accessKeyId: minioadmin + secretAccessKey: minioadmin + + # Bucket routing rules. + # Can be: + # 1. "*" to match all buckets not claimed by other backends + # 2. A glob pattern like "logs-*" + # 3. A map of bucket definitions for granular control + buckets: + # Simple bucket mapping (inherits backend settings) + my-bucket: {} + + # Mapping with overrides + external-data: + # Map proxy bucket "external-data" to backend bucket "data-v1" + bucket_name: data-v1 + # Override endpoint for this specific bucket + endpoint: http://special-endpoint:9000 + # Override region + region: us-west-2 + + # Glob pattern support within the map + "test-*": + region: us-east-1 +``` + +### Routing Logic + +When a request comes in for a bucket (e.g., `GET /my-bucket/file.txt`), Herald +resolves the backend using the following priority: + +1. **Direct match**: Looks for `my-bucket` in all backends' `buckets` maps. +2. **Glob match (map)**: Looks for glob patterns (like `test-*`) in all + backends' `buckets` maps. +3. **Glob match (string)**: If a backend has `buckets: "..."`, it checks if the + bucket name matches that pattern. diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..e436adf --- /dev/null +++ b/TODO.md @@ -0,0 +1,133 @@ +# Missing Functionality in Herald3 + +This list represents the S3 functionality that is currently missing in Herald3, +based on a comparison with the `s3-tests` suite and a review of the existing +implementation. + +## 1. Bucket Operations + +- [ ] **Bucket Policies**: Implementation of `GET/PUT/DELETE /?policy`. _(Focus + tests: `test_get_bucket_policy_status`, + `test_post_object_missing_policy_condition`)_ +- [ ] **CORS (Cross-Origin Resource Sharing)**: Implementation of + `GET/PUT/DELETE /?cors` and handling of `OPTIONS` preflight requests. + _(Focus tests: `test_set_cors`, `test_cors_origin_response`, + `test_cors_header_option`)_ +- [ ] **Lifecycle Management**: Implementation of `GET/PUT/DELETE /?lifecycle`. + _(Focus tests: `test_lifecycle_expiration`, `test_lifecycle_transition`)_ +- [ ] **Tagging**: Implementation of `GET/PUT/DELETE /?tagging` for buckets. + _(Focus tests: `test_bucket_tagging_create`, `test_bucket_tagging_get`)_ +- [ ] **Versioning Configuration**: Implementation of `GET/PUT /?versioning`. + (Basic `listVersions` is partially implemented). _(Focus tests: + `test_bucket_list_return_data_versioning`, + `test_versioning_concurrent_multi_object_delete`)_ +- [ ] **ACLs (Access Control Lists)**: Implementation of `GET/PUT /?acl` for + buckets. _(Focus tests: `test_bucket_acl_default`, + `test_put_bucket_acl_grant_group_read`, `test_bucket_header_acl_grants`)_ +- [ ] **Website Configuration**: Implementation of `GET/PUT/DELETE /?website`. + _(Focus tests: `test_website_configuration`, + `test_website_error_document`)_ +- [ ] **Public Access Block**: Implementation of + `GET/PUT/DELETE /?publicAccessBlock`. _(Focus tests: + `test_bucket_public_access_block`)_ +- [ ] **Replication Configuration**: Implementation of + `GET/PUT/DELETE /?replication`. +- [ ] **Notification Configuration (SNS)**: Implementation of + `GET/PUT /?notification`. +- [ ] **Logging Configuration**: Implementation of `GET/PUT /?logging`. _(Focus + tests: `test_bucket_logging_config`)_ +- [ ] **Inventory Configuration**: Implementation of + `GET/PUT/DELETE /?inventory`. +- [ ] **Metrics Configuration**: Implementation of `GET/PUT/DELETE /?metrics`. +- [ ] **Intelligent-Tiering Configuration**: Implementation of + `GET/PUT/DELETE /?intelligent-tiering`. +- [ ] **Ownership Controls**: Implementation of + `GET/PUT/DELETE /?ownershipControls`. + +## 2. Object Operations + +- [ ] **Multi-Object Delete**: Implementation of `POST /?delete`. _(Focus tests: + `test_multi_object_delete`, `test_multi_object_delete_key_limit`)_ +- [ ] **Multipart Upload**: Support for `InitiateMultipartUpload`, `UploadPart`, + `CompleteMultipartUpload`, `AbortMultipartUpload`, and `ListParts`. + _(Focus tests: `test_multipart_upload`, `test_multipart_upload_empty`, + `test_abort_multipart_upload`)_ +- [ ] **GetObject Attributes**: Implementation of `GET /bucket/key?attributes`. + _(Focus tests: `test_get_object_attributes`)_ +- [ ] **HeadObject Consistency**: Fix `404 Not Found` errors on existing objects + during certain test sequences. _(Focus tests: + `test_object_head_zero_bytes`)_ +- [ ] **Unicode Metadata**: Fix support for non-ASCII characters in object + metadata. _(Focus tests: `test_object_set_get_unicode_metadata`)_ +- [ ] **Copy Object**: Support for `PUT` with `x-amz-copy-source` header. + _(Focus tests: `test_object_copy`)_ +- [ ] **Tagging**: Implementation of `GET/PUT/DELETE /?tagging` for objects. + _(Focus tests: `test_object_tagging`)_ +- [ ] **ACLs (Access Control Lists)**: Implementation of `GET/PUT /?acl` for + objects. _(Focus tests: `test_object_acl_default`, `test_object_acl_read`, + `test_object_put_acl_mtime`)_ +- [ ] **Legal Hold & Retention**: Implementation of `GET/PUT /?legal-hold` and + `GET/PUT /?retention` (Object Lock). +- [ ] **Object Lock Configuration**: Implementation of `GET/PUT /?object-lock` + on objects. +- [ ] **S3 Select**: Implementation of `POST /?select&select-type=2`. +- [ ] **Checksums**: Support for `x-amz-checksum-sha1`, `x-amz-checksum-sha256`, + `x-amz-checksum-crc32`, and `x-amz-checksum-crc32c`. +- [ ] **Server-Side Encryption (SSE)**: Handling of + `x-amz-server-side-encryption`, + `x-amz-server-side-encryption-customer-algorithm`, etc. +- [ ] **Restore Object**: Support for `POST /?restore`. + +## 3. Authentication & IAM + +- [ ] **IAM Integration**: Full implementation of IAM policy evaluation for all + requests. +- [ ] **User Policies**: Support for user-specific IAM policies. +- [ ] **Security Token Service (STS)**: Implementation of `GetSessionToken`, + `AssumeRole`, etc. +- [ ] **Web Identity Federation**: Implementation of + `AssumeRoleWithWebIdentity`. + +## 4. Validation, Errors & Protocol + +- [ ] **Bucket Naming Validation**: Implement strict S3 naming rules (no IP + addresses, no double dots, length 3-63, etc.). Currently many naming tests + fail or hang. _(Focus tests: `test_bucket_create_naming_bad_ip`, + `test_bucket_create_naming_dns_dot_dot`, + `test_bucket_create_naming_bad_starts_nonalpha`)_ +- [ ] **Correct Error Codes**: Ensure accurate HTTP status codes for S3 errors + (e.g., return `400 Bad Request` or `403 Forbidden` instead of + `409 Conflict` or `500 Internal Server Error`). _(Focus tests: + `test_bucket_create_exists`, `test_bucket_create_exists_nonowner`, + `test_object_read_not_exist`)_ +- [ ] **Method POST Support**: Fix "Method POST for key [] not implemented" + errors at the bucket root level. _(Focus tests: + `test_multi_object_delete`, `test_post_object_authenticated_request`)_ +- [ ] **Multipart Reliability**: Address `502 Bad Gateway` errors occurring + during `CreateMultipartUpload` and other multipart operations. _(Focus + tests: `test_multipart_upload`)_ +- [ ] **Conditional Requests**: Fix `If-Match`, `If-None-Match`, + `If-Modified-Since`, and `If-Unmodified-Since` behavior. _(Focus tests: + `test_get_object_ifmatch_failed`, `test_get_object_ifnonematch_failed`, + `test_get_object_ifmodifiedsince_failed`)_ +- [ ] **Response Field Completeness**: Ensure expected XML/JSON fields like + `ChecksumSHA256`, `Rules`, `Errors`, and `x-amz-delete-marker` are present + in responses. +- [ ] **Metadata Handling**: Fix incorrect `BucketAlreadyOwnedByYou` errors + being returned on non-create operations (e.g., during `PutBucketPolicy`). + _(Focus tests: `test_bucket_list_return_data`)_ + +## 5. General Compatibility & Compliance + +- [ ] **Strict RFC 2616 Compliance**: Address tests tagged with + `fails_strict_rfc2616`. +- [ ] **S3Proxy Compatibility**: Address tests tagged with `fails_on_s3proxy` to + ensure broader compatibility. +- [ ] **Advanced Header Support**: Comprehensive support for headers like + `Cache-Control`, `Content-Disposition`, `Content-Encoding`, + `Content-Language`, and `Expires`. + +## 5. Non-Standard / Protocol Specific + +- [ ] **Append Object**: Implementation of `appendobject` (often found in + Ceph/RGW). diff --git a/deno.lock b/deno.lock index 30451ee..da816a9 100644 --- a/deno.lock +++ b/deno.lock @@ -9,6 +9,7 @@ "jsr:@std/assert@^1.0.15": "1.0.16", "jsr:@std/bytes@^1.0.5": "1.0.6", "jsr:@std/fmt@1": "1.0.8", + "jsr:@std/fmt@^1.0.3": "1.0.8", "jsr:@std/fs@1": "1.0.21", "jsr:@std/fs@^1.0.19": "1.0.21", "jsr:@std/fs@^1.0.20": "1.0.21", @@ -47,7 +48,7 @@ "jsr:@david/console-static-text", "jsr:@david/path", "jsr:@david/which", - "jsr:@std/fmt", + "jsr:@std/fmt@1", "jsr:@std/fs@^1.0.20", "jsr:@std/io", "jsr:@std/path@1" diff --git a/flake.nix b/flake.nix index ce8939b..1e0d33f 100644 --- a/flake.nix +++ b/flake.nix @@ -32,7 +32,8 @@ # For systems that do not ship with Python by default (required by `node-gyp`) # python3 - # infisical + infisical + openstack-rs # # opentofu # terragrunt diff --git a/src/Backends/S3/Backend.ts b/src/Backends/S3/Backend.ts index 9578f27..e888527 100644 --- a/src/Backends/S3/Backend.ts +++ b/src/Backends/S3/Backend.ts @@ -1,4 +1,4 @@ -import { Chunk, Effect, Stream } from "effect"; +import { Chunk, Effect, Option, Stream } from "effect"; import { CreateBucketCommand, DeleteBucketCommand, @@ -111,15 +111,18 @@ export const makeS3Backend = ( if ("bucket_name" in bucket) return bucket as MaterializedBucket; const backendConfig = config.raw.backends[bucket.backend_id]; - return { - name: "", - backend_id: bucket.backend_id, - protocol: "s3" as const, - endpoint: backendConfig.endpoint, - region: backendConfig.region, - bucket_name: "", - credentials: backendConfig.credentials, - }; + if (backendConfig && backendConfig.protocol === "s3") { + return { + name: "", + backend_id: bucket.backend_id, + protocol: "s3" as const, + endpoint: backendConfig.endpoint, + region: backendConfig.region, + bucket_name: "", + credentials: backendConfig.credentials, + }; + } + throw new Error(`Backend ${bucket.backend_id} is not an S3 backend`); }; const targetBucket = getTargetBucket(); @@ -431,11 +434,11 @@ export const makeS3Backend = ( const metadata: Record = {}; if (result.Metadata) { for (const [k, v] of Object.entries(result.Metadata)) { - try { - metadata[k] = decodeURIComponent(v ?? ""); - } catch { - metadata[k] = v ?? ""; - } + metadata[k] = Option.liftThrowable(decodeURIComponent)( + v ?? "", + ).pipe( + Option.getOrElse(() => v ?? ""), + ); } } @@ -503,11 +506,11 @@ export const makeS3Backend = ( const metadata: Record = {}; if (result.Metadata) { for (const [k, v] of Object.entries(result.Metadata)) { - try { - metadata[k] = decodeURIComponent(v ?? ""); - } catch { - metadata[k] = v ?? ""; - } + metadata[k] = Option.liftThrowable(decodeURIComponent)( + v ?? "", + ).pipe( + Option.getOrElse(() => v ?? ""), + ); } } diff --git a/src/Backends/S3/Client.ts b/src/Backends/S3/Client.ts index dc98754..bf79c56 100644 --- a/src/Backends/S3/Client.ts +++ b/src/Backends/S3/Client.ts @@ -28,15 +28,23 @@ export const S3ClientLive = Layer.effect( resolved = bucket; } else { const backendConfig = appConfig.raw.backends[bucket.backend_id]; - resolved = { - name: "", - backend_id: bucket.backend_id, - protocol: "s3" as const, - endpoint: backendConfig.endpoint, - region: backendConfig.region, - bucket_name: "", - credentials: backendConfig.credentials, - }; + if (backendConfig && backendConfig.protocol === "s3") { + resolved = { + name: "", + backend_id: bucket.backend_id, + protocol: "s3" as const, + endpoint: backendConfig.endpoint, + region: backendConfig.region, + bucket_name: "", + credentials: backendConfig.credentials, + }; + } else { + return Effect.fail( + new Error( + `Backend ${bucket.backend_id} is not an S3 backend or not found`, + ), + ); + } } const key = @@ -60,21 +68,27 @@ export const S3ClientLive = Layer.effect( ); } + let accessKeyId: string | undefined; + let secretAccessKey: string | undefined; + if (resolved.credentials) { - if ( - resolved.credentials.accessKeyId === undefined && - resolved.credentials.username === undefined - ) { + const creds = resolved.credentials; + if ("accessKeyId" in creds) { + accessKeyId = creds.accessKeyId; + secretAccessKey = creds.secretAccessKey; + } else if ("username" in creds) { + accessKeyId = creds.username; + secretAccessKey = creds.password; + } + + if (accessKeyId === undefined) { return Effect.fail( new Error( `Missing accessKeyId/username for backend ${resolved.backend_id}`, ), ); } - if ( - resolved.credentials.secretAccessKey === undefined && - resolved.credentials.password === undefined - ) { + if (secretAccessKey === undefined) { return Effect.fail( new Error( `Missing secretAccessKey/password for backend ${resolved.backend_id}`, @@ -86,12 +100,10 @@ export const S3ClientLive = Layer.effect( const sdkClient = new S3ClientSDK({ endpoint: resolved.endpoint, region: resolved.region, - credentials: resolved.credentials + credentials: accessKeyId && secretAccessKey ? { - accessKeyId: (resolved.credentials.accessKeyId ?? - resolved.credentials.username)!, - secretAccessKey: (resolved.credentials.secretAccessKey ?? - resolved.credentials.password)!, + accessKeyId, + secretAccessKey, } : undefined, forcePathStyle: true, diff --git a/src/Backends/S3/Signer.ts b/src/Backends/S3/Signer.ts index 6bf6b59..c0d467d 100644 --- a/src/Backends/S3/Signer.ts +++ b/src/Backends/S3/Signer.ts @@ -22,10 +22,21 @@ function getV4Signer(config: BackendConfig) { ); } - const accessKeyId = config.credentials.accessKeyId ?? - config.credentials.username; - const secretAccessKey = config.credentials.secretAccessKey ?? - config.credentials.password; + const creds = config.credentials; + let accessKeyId: string | undefined; + let secretAccessKey: string | undefined; + + if ("accessKeyId" in creds) { + accessKeyId = creds.accessKeyId; + } else if ("username" in creds) { + accessKeyId = creds.username; + } + + if ("secretAccessKey" in creds) { + secretAccessKey = creds.secretAccessKey; + } else if ("password" in creds) { + secretAccessKey = creds.password; + } if (!accessKeyId || !secretAccessKey) { return yield* Effect.fail( diff --git a/src/Backends/Swift/Backend.ts b/src/Backends/Swift/Backend.ts new file mode 100644 index 0000000..32079da --- /dev/null +++ b/src/Backends/Swift/Backend.ts @@ -0,0 +1,592 @@ +import { Effect, Option, Stream } from "effect"; +import { + type BackendError, + type BackendService, + BucketAlreadyExists, + BucketAlreadyOwnedByYou, + type BucketInfo, + BucketNotEmpty, + type CommonPrefix, + type DeleteObjectsResult, + type HeadObjectResult, + InternalError, + type ListObjectsResult, + NoSuchBucket, + NoSuchKey, + type ObjectInfo, + type ObjectResponse, + type OwnerInfo, + type PutObjectResult, +} from "../../Services/Backend.ts"; +import type { MaterializedBucket } from "../../Domain/Config.ts"; +import { SwiftClient } from "./Client.ts"; +import { fixHeaderEncoding } from "../../Frontend/Utils.ts"; + +interface SwiftContainer { + readonly name: string; + readonly last_modified?: string; +} + +interface SwiftObject { + readonly name?: string; + readonly hash?: string; + readonly bytes?: number; + readonly content_type?: string; + readonly last_modified?: string; + readonly subdir?: string; +} + +export const makeSwiftBackend = ( + bucket: MaterializedBucket | { backend_id: string }, +): Effect.Effect => + Effect.gen(function* () { + const swiftClient = yield* SwiftClient; + + const getTarget = () => + Effect.gen(function* () { + const auth = yield* swiftClient.getAuthMeta(bucket).pipe( + Effect.mapError((e) => new InternalError({ message: e.message })), + ); + const container = "bucket_name" in bucket ? bucket.bucket_name : ""; + const encodedContainer = container ? encodeURIComponent(container) : ""; + return { + storageUrl: auth.storageUrl, + token: auth.token, + container, + url: encodedContainer + ? `${auth.storageUrl}/${encodedContainer}` + : auth.storageUrl, + }; + }); + + const mapError = ( + status: number, + message: string, + bucketName: string, + method?: string, + key?: string, + ): BackendError => { + switch (status) { + case 404: + if (key) { + return new NoSuchKey({ bucketName, key, message }); + } + return new NoSuchBucket({ bucketName, message }); + case 409: + if (method === "DELETE") { + return new BucketNotEmpty({ bucketName, message }); + } + return new BucketAlreadyExists({ bucketName, message }); + case 202: + if (method === "PUT") { + return new BucketAlreadyOwnedByYou({ bucketName, message }); + } + return new InternalError({ + message: `Swift error (${status}): ${message}`, + }); + default: + return new InternalError({ + message: `Swift error (${status}): ${message}`, + }); + } + }; + + const listObjects = (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + encodingType?: string; + continuationToken?: string; + startAfter?: string; + listType?: 1 | 2; + }) => + Effect.gen(function* () { + const { url, token, container } = yield* getTarget(); + const limit = args.maxKeys ?? 1000; + const query = new URLSearchParams({ format: "json" }); + if (args.prefix) query.set("prefix", args.prefix); + if (args.delimiter) query.set("delimiter", args.delimiter); + if (args.marker) query.set("marker", args.marker); + query.set("limit", String(limit + 1)); + if (args.continuationToken) query.set("marker", args.continuationToken); + if (args.startAfter) query.set("marker", args.startAfter); + + const response = yield* Effect.tryPromise({ + try: () => + fetch(`${url}?${query.toString()}`, { + headers: { "X-Auth-Token": token }, + }), + catch: (e) => new InternalError({ message: String(e) }), + }); + + yield* Effect.logDebug( + `Swift listObjects query=[${query.toString()}] status=${response.status}`, + ); + + if (!response.ok) { + return yield* Effect.fail( + mapError(response.status, response.statusText, container, "GET"), + ); + } + + const rawObjects = (yield* Effect.tryPromise({ + try: () => response.json(), + catch: (e) => + new InternalError({ + message: `Failed to parse Swift response: ${e}`, + }), + })) as readonly SwiftObject[]; + + const isTruncated = rawObjects.length > limit; + const objects = isTruncated ? rawObjects.slice(0, limit) : rawObjects; + + const contents: ObjectInfo[] = []; + const commonPrefixes: CommonPrefix[] = []; + + for (const obj of objects) { + if (obj.subdir) { + commonPrefixes.push({ prefix: obj.subdir }); + } else if (obj.name) { + contents.push({ + key: obj.name, + lastModified: obj.last_modified + ? new Date(obj.last_modified) + : new Date(), + etag: obj.hash ? `"${obj.hash}"` : "", + size: obj.bytes ?? 0, + storageClass: "STANDARD", + owner: { id: "swift", displayName: "Swift User" }, + }); + } + } + + const nextMarker = isTruncated && objects.length > 0 + ? objects[objects.length - 1].name || + objects[objects.length - 1].subdir + : undefined; + + return { + name: container, + prefix: args.prefix, + maxKeys: limit, + delimiter: args.delimiter, + isTruncated, + marker: args.marker, + nextMarker, + contents, + commonPrefixes, + encodingType: args.encodingType, + listType: args.listType ?? 1, + nextContinuationToken: args.listType === 2 ? nextMarker : undefined, + keyCount: contents.length + commonPrefixes.length, + } satisfies ListObjectsResult; + }); + + return { + listBuckets: () => + Effect.gen(function* () { + const { storageUrl, token } = yield* getTarget(); + const response = yield* Effect.tryPromise({ + try: () => + fetch(`${storageUrl}?format=json`, { + headers: { "X-Auth-Token": token }, + }), + catch: (e) => new InternalError({ message: String(e) }), + }); + + if (!response.ok) { + return yield* Effect.fail( + mapError(response.status, response.statusText, "", "GET"), + ); + } + + const buckets = (yield* Effect.tryPromise({ + try: () => response.json(), + catch: (e) => + new InternalError({ + message: `Failed to parse Swift response: ${e}`, + }), + })) as readonly SwiftContainer[]; + + const bucketInfos: BucketInfo[] = buckets.map((b) => ({ + name: b.name, + creationDate: b.last_modified + ? new Date(b.last_modified) + : undefined, + })); + + const owner: OwnerInfo = { id: "swift", displayName: "Swift User" }; + + return { buckets: bucketInfos, owner }; + }), + + createBucket: () => + Effect.gen(function* () { + const { url, token, container } = yield* getTarget(); + const response = yield* Effect.tryPromise({ + try: () => + fetch(url, { + method: "PUT", + headers: { "X-Auth-Token": token }, + }), + catch: (e) => new InternalError({ message: String(e) }), + }); + + if (response.status === 201) { + return yield* Effect.void; + } + + if (response.status === 202) { + return yield* Effect.fail( + new BucketAlreadyOwnedByYou({ + bucketName: container, + message: "Bucket already exists", + }), + ); + } + + if (!response.ok) { + return yield* Effect.fail( + mapError(response.status, response.statusText, container, "PUT"), + ); + } + + return yield* Effect.void; + }), + + deleteBucket: () => + Effect.gen(function* () { + const { url, token, container } = yield* getTarget(); + const response = yield* Effect.tryPromise({ + try: () => + fetch(url, { + method: "DELETE", + headers: { "X-Auth-Token": token }, + }), + catch: (e) => new InternalError({ message: String(e) }), + }); + + yield* Effect.logDebug( + `Swift deleteBucket container=[${container}] status=${response.status}`, + ); + + if (response.status === 204) { + return yield* Effect.void; + } + + if (!response.ok) { + return yield* Effect.fail( + mapError( + response.status, + response.statusText, + container, + "DELETE", + ), + ); + } + + return yield* Effect.void; + }), + + headBucket: () => + Effect.gen(function* () { + const { url, token, container } = yield* getTarget(); + const response = yield* Effect.tryPromise({ + try: () => + fetch(url, { + method: "HEAD", + headers: { "X-Auth-Token": token }, + }), + catch: (e) => new InternalError({ message: String(e) }), + }); + + if (!response.ok) { + return yield* Effect.fail( + mapError(response.status, response.statusText, container, "HEAD"), + ); + } + + return yield* Effect.void; + }), + + listObjects, + + listVersions: (args) => + Effect.gen(function* () { + const result = yield* listObjects({ + prefix: args.prefix, + delimiter: args.delimiter, + marker: args.keyMarker, + maxKeys: args.maxKeys, + }); + return { + ...result, + contents: result.contents.map((c) => ({ + ...c, + versionId: "null", + isLatest: true, + })), + }; + }), + + getObject: (key: string) => + Effect.gen(function* () { + const { url, token, container } = yield* getTarget(); + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const response = yield* Effect.tryPromise({ + try: () => + fetch(`${url}/${encodedKey}`, { + headers: { "X-Auth-Token": token }, + }), + catch: (e) => new InternalError({ message: String(e) }), + }); + + if (!response.ok) { + return yield* Effect.fail( + mapError( + response.status, + response.statusText, + container, + "GET", + key, + ), + ); + } + + const metadata: Record = {}; + const s3Headers: Record = {}; + response.headers.forEach((v, k) => { + const lowK = k.toLowerCase(); + if (lowK.startsWith("x-object-meta-")) { + const metaKey = lowK.substring("x-object-meta-".length); + const value = (v.includes("%")) + ? Option.liftThrowable(decodeURIComponent)(v).pipe( + Option.getOrElse(() => v), + ) + : v; + metadata[metaKey] = value; + s3Headers[`x-amz-meta-${metaKey}`] = value; + } else if (lowK === "content-type") { + s3Headers["Content-Type"] = v; + } else if (lowK === "content-length") { + s3Headers["Content-Length"] = v; + } else if (lowK === "etag") { + s3Headers["ETag"] = v; + } else if (lowK === "last-modified") { + s3Headers["Last-Modified"] = v; + } + }); + + return { + stream: Stream.fromReadableStream( + () => response.body!, + (e) => new InternalError({ message: String(e) }), + ), + contentType: response.headers.get("Content-Type") || undefined, + contentLength: parseInt( + response.headers.get("Content-Length") || "0", + ), + etag: response.headers.get("ETag") || undefined, + lastModified: response.headers.get("Last-Modified") + ? new Date(response.headers.get("Last-Modified")!) + : undefined, + metadata, + headers: s3Headers, + } satisfies ObjectResponse; + }), + + headObject: (key: string) => + Effect.gen(function* () { + const { url, token, container } = yield* getTarget(); + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const response = yield* Effect.tryPromise({ + try: () => + fetch(`${url}/${encodedKey}`, { + method: "HEAD", + headers: { "X-Auth-Token": token }, + }), + catch: (e) => new InternalError({ message: String(e) }), + }); + + if (!response.ok) { + return yield* Effect.fail( + mapError( + response.status, + response.statusText, + container, + "HEAD", + key, + ), + ); + } + + const metadata: Record = {}; + const s3Headers: Record = {}; + response.headers.forEach((v, k) => { + const lowK = k.toLowerCase(); + if (lowK.startsWith("x-object-meta-")) { + const metaKey = lowK.substring("x-object-meta-".length); + const value = (v.includes("%")) + ? Option.liftThrowable(decodeURIComponent)(v).pipe( + Option.getOrElse(() => v), + ) + : v; + metadata[metaKey] = value; + s3Headers[`x-amz-meta-${metaKey}`] = value; + } else if (lowK === "content-type") { + s3Headers["Content-Type"] = v; + } else if (lowK === "content-length") { + s3Headers["Content-Length"] = v; + } else if (lowK === "etag") { + s3Headers["ETag"] = v; + } else if (lowK === "last-modified") { + s3Headers["Last-Modified"] = v; + } + }); + + return { + contentType: response.headers.get("Content-Type") || undefined, + contentLength: parseInt( + response.headers.get("Content-Length") || "0", + ), + etag: response.headers.get("ETag") || undefined, + lastModified: response.headers.get("Last-Modified") + ? new Date(response.headers.get("Last-Modified")!) + : undefined, + metadata, + headers: s3Headers, + } satisfies HeadObjectResult; + }), + + putObject: (key, stream, headers) => + Effect.gen(function* () { + const { url, token, container } = yield* getTarget(); + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const contentLength = headers["content-length"] || + headers["Content-Length"]; + + const swiftHeaders: Record = { + "X-Auth-Token": token, + "Content-Type": + (headers["content-type"] || headers["Content-Type"] || + "application/octet-stream") as string, + ...(contentLength + ? { "Content-Length": String(contentLength) } + : {}), + }; + + for (const [k, v] of Object.entries(headers)) { + const lowK = k.toLowerCase(); + if (lowK.startsWith("x-amz-meta-")) { + const metaKey = lowK.substring("x-amz-meta-".length); + const value = fixHeaderEncoding(String(v)); + swiftHeaders[`X-Object-Meta-${metaKey}`] = + /[^\x20-\x7E]/.test(value) ? encodeURIComponent(value) : value; + } + } + + const response = yield* Effect.tryPromise({ + try: () => + fetch(`${url}/${encodedKey}`, { + method: "PUT", + headers: swiftHeaders, + body: Stream.toReadableStream(stream), + // @ts-ignore: duplex is required for streaming body in fetch + duplex: "half", + }), + catch: (e) => new InternalError({ message: String(e) }), + }); + yield* Effect.logDebug( + `Swift putObject key=[${key}] status=${response.status}`, + ); + + if (!response.ok) { + return yield* Effect.fail( + mapError( + response.status, + response.statusText, + container, + "PUT", + key, + ), + ); + } + + return { + etag: response.headers.get("ETag") || undefined, + } satisfies PutObjectResult; + }), + + deleteObject: (key: string) => + Effect.gen(function* () { + const { url, token, container } = yield* getTarget(); + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const response = yield* Effect.tryPromise({ + try: () => + fetch(`${url}/${encodedKey}`, { + method: "DELETE", + headers: { "X-Auth-Token": token }, + }), + catch: (e) => new InternalError({ message: String(e) }), + }); + + if (!response.ok && response.status !== 204) { + return yield* Effect.fail( + mapError( + response.status, + response.statusText, + container, + "DELETE", + key, + ), + ); + } + + return yield* Effect.void; + }), + + deleteObjects: (objects) => + Effect.gen(function* () { + const { url, token, container: _container } = yield* getTarget(); + const deleted: string[] = []; + const errors: { key: string; code: string; message: string }[] = []; + + for (const obj of objects) { + const encodedKey = obj.key.split("/").map(encodeURIComponent).join( + "/", + ); + const response = yield* Effect.tryPromise({ + try: () => + fetch(`${url}/${encodedKey}`, { + method: "DELETE", + headers: { "X-Auth-Token": token }, + }), + catch: (e) => new InternalError({ message: String(e) }), + }); + + yield* Effect.logDebug( + `Swift deleteObject key=[${obj.key}] status=${response.status}`, + ); + + if ( + response.ok || response.status === 204 || response.status === 404 + ) { + deleted.push(obj.key); + } else { + const errorBody = yield* Effect.tryPromise(() => response.text()) + .pipe( + Effect.orElseSucceed(() => "Unknown error"), + ); + errors.push({ + key: obj.key, + code: String(response.status), + message: errorBody, + }); + } + } + + return { deleted, errors } satisfies DeleteObjectsResult; + }), + }; + }); diff --git a/src/Backends/Swift/Client.ts b/src/Backends/Swift/Client.ts new file mode 100644 index 0000000..7be8123 --- /dev/null +++ b/src/Backends/Swift/Client.ts @@ -0,0 +1,194 @@ +import { Context, Effect, Layer, type Schema } from "effect"; +import type { MaterializedBucket, SwiftConfig } from "../../Domain/Config.ts"; +import { AppConfig } from "../../Config/Layer.ts"; + +export interface SwiftAuthMeta { + readonly token: string; + readonly storageUrl: string; +} + +export class SwiftClient extends Context.Tag("SwiftClient")< + SwiftClient, + { + readonly getAuthMeta: ( + bucket: MaterializedBucket | { backend_id: string }, + ) => Effect.Effect; + } +>() {} + +interface SwiftEndpoint { + readonly region: string; + readonly interface: "public" | "internal" | "admin"; + readonly url: string; +} + +interface SwiftService { + readonly type: string; + readonly endpoints: readonly SwiftEndpoint[]; +} + +interface SwiftTokenResponse { + readonly token: { + readonly catalog: readonly SwiftService[]; + }; +} + +export const SwiftClientLive = Layer.effect( + SwiftClient, + AppConfig.pipe( + Effect.flatMap((appConfig) => { + const cache = new Map(); + + const fetchAuthMeta = ( + config: Schema.Schema.Type, + ): Effect.Effect => { + const { auth_url, credentials, region } = config; + + if (!auth_url) { + return Effect.fail( + new Error("auth_url is required for Swift backend"), + ); + } + if (!credentials || !("username" in credentials)) { + return Effect.fail( + new Error( + "Swift credentials (username, password, etc.) are required", + ), + ); + } + + const { + username, + password, + project_name, + user_domain_name = "Default", + project_domain_name = "Default", + } = credentials; + + const requestBody = JSON.stringify({ + auth: { + identity: { + methods: ["password"], + password: { + user: { + name: username, + domain: { name: user_domain_name }, + password: password, + }, + }, + }, + scope: { + project: { + domain: { name: project_domain_name }, + name: project_name, + }, + }, + }, + }); + + return Effect.tryPromise({ + try: async () => { + const response = await fetch(`${auth_url}/auth/tokens`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: requestBody, + }); + + if (!response.ok) { + const msg = await response.text(); + throw new Error(`Failed to authenticate with Swift: ${msg}`); + } + + const token = response.headers.get("X-Subject-Token"); + if (!token) { + throw new Error( + "X-Subject-Token header missing from Swift response", + ); + } + + const body = (await response.json()) as SwiftTokenResponse; + const catalog = body.token.catalog; + const storageService = catalog.find((s) => + s.type === "object-store" + ); + + if (!storageService) { + throw new Error( + "Object Store service not found in Swift catalog", + ); + } + + const endpoint = storageService.endpoints.find( + (e) => + (region ? e.region === region : true) && + e.interface === "public", + ); + + if (!endpoint) { + throw new Error( + `Public Swift endpoint not found (region: ${region ?? "any"})`, + ); + } + + return { + token, + storageUrl: endpoint.url, + }; + }, + catch: (e) => e as Error, + }); + }; + + return Effect.succeed( + SwiftClient.of({ + getAuthMeta: ( + bucket: MaterializedBucket | { backend_id: string }, + ) => { + let backend_id: string; + let config: Schema.Schema.Type; + + if ("protocol" in bucket) { + backend_id = bucket.backend_id; + config = appConfig.raw.backends[backend_id] as Schema.Schema.Type< + typeof SwiftConfig + >; + } else { + backend_id = bucket.backend_id; + config = appConfig.raw.backends[backend_id] as Schema.Schema.Type< + typeof SwiftConfig + >; + } + + if (!config || config.protocol !== "swift") { + return Effect.fail( + new Error(`Backend ${backend_id} is not a Swift backend`), + ); + } + + const cacheKey = + `${backend_id}:${config.auth_url}:${config.region}`; + const cached = cache.get(cacheKey); + const now = Date.now(); + + if (cached && cached.expires > now) { + return Effect.succeed({ + token: cached.token, + storageUrl: cached.storageUrl, + }); + } + + return fetchAuthMeta(config).pipe( + Effect.tap((meta) => { + // Cache for 50 minutes (Swift tokens usually last 1h) + cache.set(cacheKey, { + ...meta, + expires: now + 50 * 60 * 1000, + }); + }), + ); + }, + }), + ); + }), + ), +); diff --git a/src/Config/Layer.ts b/src/Config/Layer.ts index 0470bb0..4acdfeb 100644 --- a/src/Config/Layer.ts +++ b/src/Config/Layer.ts @@ -1,11 +1,10 @@ -import { Context, Effect, Layer, type Option } from "effect"; +import { Config, Context, Effect, Layer, type Option, Schema } from "effect"; import { parse } from "@std/yaml"; import { GlobalConfig, lookupBucket, type MaterializedBucket, } from "../Domain/Config.ts"; -import { Schema } from "effect"; export class AppConfig extends Context.Tag("AppConfig")< AppConfig, @@ -15,25 +14,127 @@ export class AppConfig extends Context.Tag("AppConfig")< } >() {} +function toConfigKey(str: string): string { + const mapping: Record = { + "AUTH_URL": "auth_url", + "PROJECT_NAME": "project_name", + "USER_DOMAIN_NAME": "user_domain_name", + "PROJECT_DOMAIN_NAME": "project_domain_name", + "ACCESS_KEY_ID": "accessKeyId", + "SECRET_ACCESS_KEY": "secretAccessKey", + }; + return mapping[str] || str.toLowerCase(); +} + +export function parseConfig( + yamlConfig: unknown, + env: Record, +): GlobalConfig { + const yamlBackends = + (yamlConfig && typeof yamlConfig === "object" && "backends" in yamlConfig) + ? (yamlConfig as { backends: Record> }) + .backends + : {}; + + const backends: Record> = {}; + for (const [k, v] of Object.entries(yamlBackends)) { + backends[k] = { ...v }; + } + + const commonKeys = [ + "PROTOCOL", + "ENDPOINT", + "REGION", + "BUCKETS", + "ACCESS_KEY_ID", + "SECRET_ACCESS_KEY", + "AUTH_URL", + "CONTAINER", + "USERNAME", + "PASSWORD", + "PROJECT_NAME", + "USER_DOMAIN_NAME", + "PROJECT_DOMAIN_NAME", + ]; + + for (const [key, value] of Object.entries(env)) { + if (!key.startsWith("HERALD_")) continue; + if (key === "HERALD_CONFIG_PATH") continue; + if (key === "HERALD_LOG_LEVEL") continue; + + const parts = key.substring(7).split("_"); + let backendName: string; + let configParts: string[]; + + if (parts.length === 1 || commonKeys.includes(parts[0])) { + backendName = "default"; + configParts = parts; + } else { + backendName = parts[0].toLowerCase(); + configParts = parts.slice(1); + } + + const configKey = toConfigKey(configParts.join("_")); + if (!backends[backendName]) backends[backendName] = {}; + const backend = backends[backendName]; + + const credentialKeys = [ + "accessKeyId", + "secretAccessKey", + "username", + "password", + "project_name", + "user_domain_name", + "project_domain_name", + ]; + + if (credentialKeys.includes(configKey)) { + if (!backend.credentials) { + backend.credentials = {} as Record; + } + (backend.credentials as Record)[configKey] = value; + } else { + backend[configKey] = value; + } + } + + // Default backend fallback if no backends defined at all + if (Object.keys(backends).length === 0) { + backends["default"] = { + protocol: "s3", + buckets: "*", + }; + } + + return Schema.decodeUnknownSync(GlobalConfig)({ backends }); +} + export const AppConfigLive = Layer.effect( AppConfig, Effect.gen(function* () { - const configPath = yield* Effect.succeed( - Deno.env.get("CONFIG_PATH") ?? "herald.yaml", + const configPath = yield* Config.string("HERALD_CONFIG_PATH").pipe( + Config.orElse(() => Config.string("CONFIG_PATH")), + Config.withDefault("herald.yaml"), ); - const content = yield* Effect.tryPromise({ + const yamlConfig = yield* Effect.tryPromise({ try: () => Deno.readTextFile(configPath), - catch: (e) => - new Error(`Failed to read config file at ${configPath}: ${e}`), - }); + catch: () => new Error("Config file missing"), + }).pipe( + Effect.flatMap((content) => + Effect.try({ + try: () => parse(content), + catch: (e) => new Error(`YAML parse error: ${e}`), + }) + ), + Effect.orElseSucceed(() => ({ backends: {} })), + ); - const yaml = yield* Effect.try({ - try: () => parse(content) as unknown, - catch: (e) => new Error(`Failed to parse YAML: ${e}`), - }); + // Discovery needs the full environment. In Deno we use Deno.env.toObject(). + // We can wrap this in an Effect to be more idiomatic. + const env = yield* Effect.sync(() => Deno.env.toObject()); - const raw = yield* Schema.decodeUnknown(GlobalConfig)(yaml); + const raw = parseConfig(yamlConfig, env); return { raw, diff --git a/src/Domain/Config.ts b/src/Domain/Config.ts index b0239e1..337f560 100644 --- a/src/Domain/Config.ts +++ b/src/Domain/Config.ts @@ -1,12 +1,20 @@ import { Option, Schema } from "effect"; -export const Credentials = Schema.Struct({ - username: Schema.optional(Schema.String), - password: Schema.optional(Schema.String), +export const S3Credentials = Schema.Struct({ accessKeyId: Schema.optional(Schema.String), secretAccessKey: Schema.optional(Schema.String), }); +export const SwiftCredentials = Schema.Struct({ + username: Schema.optional(Schema.String), + password: Schema.optional(Schema.String), + project_name: Schema.optional(Schema.String), + user_domain_name: Schema.optional(Schema.String), + project_domain_name: Schema.optional(Schema.String), +}); + +export const Credentials = Schema.Union(S3Credentials, SwiftCredentials); + export const BucketOverride = Schema.Struct({ endpoint: Schema.optional(Schema.String), bucket_name: Schema.optional(Schema.String), @@ -15,20 +23,33 @@ export const BucketOverride = Schema.Struct({ export type BucketOverride = Schema.Schema.Type; -export const BackendConfig = Schema.Struct({ - protocol: Schema.Literal("s3", "swift"), +export const BucketsConfig = Schema.optionalWith( + Schema.Union( + Schema.Record({ key: Schema.String, value: BucketOverride }), + Schema.String, + ), + { default: () => "*" }, +); + +export const S3Config = Schema.Struct({ + protocol: Schema.Literal("s3"), endpoint: Schema.optional(Schema.String), region: Schema.optional(Schema.String), - credentials: Schema.optional(Credentials), - buckets: Schema.optionalWith( - Schema.Union( - Schema.Record({ key: Schema.String, value: BucketOverride }), - Schema.String, - ), - { default: () => "*" }, - ), + credentials: Schema.optional(S3Credentials), + buckets: BucketsConfig, }); +export const SwiftConfig = Schema.Struct({ + protocol: Schema.Literal("swift"), + auth_url: Schema.optional(Schema.String), + region: Schema.optional(Schema.String), + container: Schema.optional(Schema.String), + credentials: Schema.optional(SwiftCredentials), + buckets: BucketsConfig, +}); + +export const BackendConfig = Schema.Union(S3Config, SwiftConfig); + export type BackendConfig = Schema.Schema.Type; export const GlobalConfig = Schema.Struct({ @@ -45,6 +66,9 @@ export const MaterializedBucket = Schema.Struct({ region: Schema.optional(Schema.String), bucket_name: Schema.String, credentials: Schema.optional(Credentials), + // Swift specific + auth_url: Schema.optional(Schema.String), + container: Schema.optional(Schema.String), }); export type MaterializedBucket = Schema.Schema.Type; @@ -68,17 +92,20 @@ export const lookupBucket = ( const buckets = backend.buckets; if (buckets && typeof buckets !== "string" && buckets[bucketName]) { const override = buckets[bucketName]; - return Option.some( - { - name: bucketName, - backend_id, - protocol: backend.protocol, - endpoint: override.endpoint ?? backend.endpoint, - region: override.region ?? backend.region, - bucket_name: override.bucket_name ?? bucketName, - credentials: backend.credentials, - } as const, - ); + const base: MaterializedBucket = { + name: bucketName, + backend_id, + protocol: backend.protocol, + endpoint: override.endpoint ?? + (backend.protocol === "s3" ? backend.endpoint : undefined), + region: override.region ?? backend.region, + bucket_name: override.bucket_name ?? bucketName, + credentials: backend.credentials, + auth_url: backend.protocol === "swift" ? backend.auth_url : undefined, + container: backend.protocol === "swift" ? backend.container : undefined, + }; + + return Option.some(base); } } @@ -88,19 +115,25 @@ export const lookupBucket = ( if (buckets && typeof buckets !== "string") { for (const [key, override] of Object.entries(buckets)) { if (globToRegex(key).test(bucketName)) { - return Option.some( - { - name: bucketName, - backend_id, - protocol: backend.protocol, - endpoint: (override as BucketOverride).endpoint ?? - backend.endpoint, - region: (override as BucketOverride).region ?? backend.region, - bucket_name: (override as BucketOverride).bucket_name ?? - bucketName, - credentials: backend.credentials, - } as const, - ); + const base: MaterializedBucket = { + name: bucketName, + backend_id, + protocol: backend.protocol, + endpoint: (override as BucketOverride).endpoint ?? + (backend.protocol === "s3" ? backend.endpoint : undefined), + region: (override as BucketOverride).region ?? backend.region, + bucket_name: (override as BucketOverride).bucket_name ?? + bucketName, + credentials: backend.credentials, + auth_url: backend.protocol === "swift" + ? backend.auth_url + : undefined, + container: backend.protocol === "swift" + ? backend.container + : undefined, + }; + + return Option.some(base); } } } @@ -111,17 +144,21 @@ export const lookupBucket = ( const buckets = backend.buckets; if (buckets && typeof buckets === "string") { if (globToRegex(buckets).test(bucketName)) { - return Option.some( - { - name: bucketName, - backend_id, - protocol: backend.protocol, - endpoint: backend.endpoint, - region: backend.region, - bucket_name: bucketName, - credentials: backend.credentials, - } as const, - ); + const base: MaterializedBucket = { + name: bucketName, + backend_id, + protocol: backend.protocol, + endpoint: backend.protocol === "s3" ? backend.endpoint : undefined, + region: backend.region, + bucket_name: bucketName, + credentials: backend.credentials, + auth_url: backend.protocol === "swift" ? backend.auth_url : undefined, + container: backend.protocol === "swift" + ? backend.container + : undefined, + }; + + return Option.some(base); } } } diff --git a/src/Frontend/Api.ts b/src/Frontend/Api.ts index 0c8c502..91a78de 100644 --- a/src/Frontend/Api.ts +++ b/src/Frontend/Api.ts @@ -6,6 +6,10 @@ export class BadGateway extends Schema.TaggedError()("BadGateway", { }) {} export const S3Api = HttpApiGroup.make("s3") + .add( + HttpApiEndpoint.post("postRoot", "/") + .addError(BadGateway, { status: 502 }), + ) .add( HttpApiEndpoint.get("listBuckets", "/") .addError(BadGateway, { status: 502 }), diff --git a/src/Frontend/Buckets/Create.ts b/src/Frontend/Buckets/Create.ts index f40475c..7d42506 100644 --- a/src/Frontend/Buckets/Create.ts +++ b/src/Frontend/Buckets/Create.ts @@ -1,5 +1,5 @@ import { Effect } from "effect"; -import { HttpServerResponse } from "@effect/platform"; +import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; import { resolveBucket } from "../Utils.ts"; export const createBucket = ( @@ -7,6 +7,34 @@ export const createBucket = ( ) => resolveBucket(bucket, (backend) => Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const url = new URL(request.url, "http://localhost"); + yield* Effect.logDebug( + `createBucket bucket=[${bucket}] url=[${request.url}]`, + ); + + if (url.searchParams.has("acl")) { + // PutBucketAcl + // Check for canned ACL validity if present + const cannedAcl = request.headers["x-amz-acl"]; + const validCannedAcls = [ + "private", + "public-read", + "public-read-write", + "authenticated-read", + ]; + if (cannedAcl && !validCannedAcls.includes(cannedAcl)) { + return HttpServerResponse.text( + `InvalidArgumentArgument x-amz-acl is invalid.`, + { status: 400, headers: { "Content-Type": "application/xml" } }, + ); + } + + // For now, we just return 200 OK if the bucket exists + yield* backend.headBucket(); + return HttpServerResponse.text("", { status: 200 }); + } + yield* backend.createBucket(); return HttpServerResponse.text("", { status: 200 }); })); diff --git a/src/Frontend/Buckets/List.ts b/src/Frontend/Buckets/List.ts index b11f411..a6759fc 100644 --- a/src/Frontend/Buckets/List.ts +++ b/src/Frontend/Buckets/List.ts @@ -8,16 +8,17 @@ export const listBuckets = () => const config = yield* AppConfig; // For ListBuckets, we need to decide which backend to proxy to. - const s3BackendId = Object.keys(config.raw.backends).find((id) => + // We prefer an S3 backend if available, otherwise we take the first one. + const backendId = Object.keys(config.raw.backends).find((id) => config.raw.backends[id].protocol === "s3" - ); + ) ?? Object.keys(config.raw.backends)[0]; - if (!s3BackendId) { + if (!backendId) { const s3Xml = yield* S3Xml; - return s3Xml.formatError("No S3 backend configured"); + return s3Xml.formatError("No backend configured"); } - return yield* resolveBackend(s3BackendId, (backend) => + return yield* resolveBackend(backendId, (backend) => Effect.gen(function* () { const result = yield* backend.listBuckets(); const s3xml = yield* S3Xml; diff --git a/src/Frontend/Http.ts b/src/Frontend/Http.ts index c1b4b12..28825cc 100644 --- a/src/Frontend/Http.ts +++ b/src/Frontend/Http.ts @@ -1,5 +1,5 @@ -import { HttpApiBuilder } from "@effect/platform"; -import { Layer } from "effect"; +import { HttpApiBuilder, HttpServerResponse } from "@effect/platform"; +import { Effect, Layer } from "effect"; import { Api } from "../Api.ts"; import { listBuckets } from "./Buckets/List.ts"; import { createBucket } from "./Buckets/Create.ts"; @@ -12,6 +12,7 @@ import { deleteObject } from "./Objects/Delete.ts"; import { headObject } from "./Objects/Head.ts"; import { postObject } from "./Objects/Post.ts"; import { S3ClientLive } from "../Backends/S3/Client.ts"; +import { SwiftClientLive } from "../Backends/Swift/Client.ts"; import { S3XmlLive } from "../Services/S3Xml.ts"; import { BackendResolverLive } from "../Services/BackendResolver.ts"; @@ -20,6 +21,11 @@ export const HttpS3Live = HttpApiBuilder.group( "s3", (handlers) => handlers + .handleRaw("postRoot", (_handlers) => + Effect.gen(function* () { + yield* Effect.logDebug("POST / received"); + return HttpServerResponse.text("", { status: 200 }); + })) .handleRaw("listBuckets", listBuckets) .handleRaw("createBucket", createBucket) .handleRaw("deleteBucket", deleteBucket) @@ -34,5 +40,6 @@ export const HttpS3Live = HttpApiBuilder.group( ).pipe( Layer.provide(BackendResolverLive), Layer.provide(S3ClientLive), + Layer.provide(SwiftClientLive), Layer.provide(S3XmlLive), ); diff --git a/src/Frontend/Objects/Post.ts b/src/Frontend/Objects/Post.ts index 84f7039..c61c6ba 100644 --- a/src/Frontend/Objects/Post.ts +++ b/src/Frontend/Objects/Post.ts @@ -1,4 +1,4 @@ -import { Effect, Stream } from "effect"; +import { Effect, Option, Stream } from "effect"; import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; import { extractKey, resolveBucket } from "../Utils.ts"; @@ -41,28 +41,29 @@ export const postObject = ( const keyMatch = content.match(/(.*?)<\/Key>/); const versionIdMatch = content.match(/(.*?)<\/VersionId>/); if (keyMatch) { - try { - objects.push({ - key: decodeURIComponent(keyMatch[1]), - versionId: versionIdMatch ? versionIdMatch[1] : undefined, - }); - } catch { - objects.push({ - key: keyMatch[1], - versionId: versionIdMatch ? versionIdMatch[1] : undefined, - }); - } + const rawKey = keyMatch[1]; + const key = Option.liftThrowable(decodeURIComponent)(rawKey).pipe( + Option.getOrElse(() => rawKey), + ); + yield* Effect.logDebug(`DeleteObjects extracted key=[${key}]`); + objects.push({ + key, + versionId: versionIdMatch ? versionIdMatch[1] : undefined, + }); } } if (objects.length > 0) { const deleteResult = yield* backend.deleteObjects(objects); + const deletedXml = deleteResult.deleted.map((k) => + `${k}` + ).join(""); + const errorsXml = deleteResult.errors.map((e) => + `${e.key}${e.code}${e.message}` + ).join(""); + const xml = - `${ - deleteResult.deleted.map((k) => - `${k}` - ).join("") - }`; + `${deletedXml}${errorsXml}`; return HttpServerResponse.text(xml, { headers: { "Content-Type": "application/xml" }, }); diff --git a/src/Frontend/Utils.ts b/src/Frontend/Utils.ts index dec42b1..9e08b74 100644 --- a/src/Frontend/Utils.ts +++ b/src/Frontend/Utils.ts @@ -7,7 +7,6 @@ import { BucketAlreadyExists, BucketAlreadyOwnedByYou, BucketNotEmpty, - DeleteObjectsError, InternalError, NoSuchBucket, NoSuchKey, @@ -15,8 +14,26 @@ import { import { HttpServerRequest, type HttpServerResponse } from "@effect/platform"; import type { AppConfig } from "../Config/Layer.ts"; import type { S3Client } from "../Backends/S3/Client.ts"; +import type { SwiftClient } from "../Backends/Swift/Client.ts"; import { BadGateway } from "./Api.ts"; +/** + * Fixes header values that might have been incorrectly decoded as Latin-1 + * instead of UTF-8 by the HTTP server. + */ +export function fixHeaderEncoding(value: string): string { + // deno-lint-ignore no-control-regex + if (!/[^\x00-\x7F]/.test(value)) { + return value; + } + return Option.liftThrowable(() => { + const bytes = Uint8Array.from(value, (c) => c.charCodeAt(0)); + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + })().pipe( + Option.getOrElse(() => value), + ); +} + /** * Extracts the object key from the request URL, given the bucket name. */ @@ -56,6 +73,7 @@ export function resolveBucket< | S3Xml | AppConfig | S3Client + | SwiftClient | HttpServerRequest.HttpServerRequest > { return Effect.gen(function* () { @@ -68,6 +86,26 @@ export function resolveBucket< ? request.value.method === "HEAD" : false; + if (Option.isSome(request)) { + const auth = request.value.headers["authorization"]; + yield* Effect.logDebug( + `${request.value.method} ${request.value.url} auth: [${auth}]`, + ); + if ( + !auth || auth.trim() === "" || + (auth.startsWith("AWS ") && auth.split(":").length < 2 && + !auth.includes("Signature=")) || + (auth.startsWith("AWS4-") && !auth.includes("Signature=")) + ) { + return s3Xml.formatError( + new AccessDenied({ + message: "Access Denied", + }), + isHead, + ); + } + } + const program = Effect.gen(function* () { const backend = yield* Backend; return yield* fn(backend); @@ -82,8 +120,7 @@ export function resolveBucket< e instanceof BucketAlreadyOwnedByYou || e instanceof InternalError || e instanceof AccessDenied || - e instanceof BucketNotEmpty || - e instanceof DeleteObjectsError + e instanceof BucketNotEmpty ) { return Effect.succeed(s3Xml.formatError(e, isHead)); } @@ -122,6 +159,7 @@ export function resolveBackend< | S3Xml | AppConfig | S3Client + | SwiftClient | HttpServerRequest.HttpServerRequest > { return Effect.gen(function* () { @@ -148,8 +186,7 @@ export function resolveBackend< e instanceof BucketAlreadyOwnedByYou || e instanceof InternalError || e instanceof AccessDenied || - e instanceof BucketNotEmpty || - e instanceof DeleteObjectsError + e instanceof BucketNotEmpty ) { return Effect.succeed(s3Xml.formatError(e, isHead)); } diff --git a/src/Logging/Layer.ts b/src/Logging/Layer.ts index c2d5ada..ab0c5ed 100644 --- a/src/Logging/Layer.ts +++ b/src/Logging/Layer.ts @@ -1,8 +1,39 @@ -import { Effect, Layer, Logger, LogLevel } from "effect"; +import { Config, Effect, Layer, Logger, LogLevel } from "effect"; export const LoggingLive = Layer.mergeAll( - Logger.minimumLogLevel(LogLevel.Info), - // You can add more logger configuration here, like changing the format to JSON for production + Layer.unwrapEffect( + Effect.gen(function* () { + const logLevelStr = yield* Config.option( + Config.string("HERALD_LOG_LEVEL"), + ); + + if (logLevelStr._tag === "None") { + return Logger.minimumLogLevel(LogLevel.Info); + } + + const level = logLevelStr.value.toUpperCase(); + switch (level) { + case "ALL": + return Logger.minimumLogLevel(LogLevel.All); + case "TRACE": + return Logger.minimumLogLevel(LogLevel.Trace); + case "DEBUG": + return Logger.minimumLogLevel(LogLevel.Debug); + case "INFO": + return Logger.minimumLogLevel(LogLevel.Info); + case "WARN": + return Logger.minimumLogLevel(LogLevel.Warning); + case "ERROR": + return Logger.minimumLogLevel(LogLevel.Error); + case "FATAL": + return Logger.minimumLogLevel(LogLevel.Fatal); + case "NONE": + return Logger.minimumLogLevel(LogLevel.None); + default: + return Logger.minimumLogLevel(LogLevel.Info); + } + }), + ), ); /** diff --git a/src/Services/BackendResolver.ts b/src/Services/BackendResolver.ts index b04ed8e..979866d 100644 --- a/src/Services/BackendResolver.ts +++ b/src/Services/BackendResolver.ts @@ -3,6 +3,8 @@ import { AppConfig } from "../Config/Layer.ts"; import { Backend, type BackendService } from "./Backend.ts"; import type { S3Client } from "../Backends/S3/Client.ts"; import { makeS3Backend } from "../Backends/S3/Backend.ts"; +import { makeSwiftBackend } from "../Backends/Swift/Backend.ts"; +import type { SwiftClient } from "../Backends/Swift/Client.ts"; /** * BackendResolver handles dynamic resolution and provisioning of Backend implementations @@ -17,7 +19,7 @@ export class BackendResolver extends Context.Tag("BackendResolver")< ) => Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client + Exclude | AppConfig | S3Client | SwiftClient >; readonly provideForBackendId: ( @@ -26,7 +28,7 @@ export class BackendResolver extends Context.Tag("BackendResolver")< ) => Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client + Exclude | AppConfig | S3Client | SwiftClient >; } >() {} @@ -66,6 +68,8 @@ export const BackendResolverLive = Layer.effect( if (bucketConfig.protocol === "s3") { backendImpl = yield* makeS3Backend(bucketConfig); + } else if (bucketConfig.protocol === "swift") { + backendImpl = yield* makeSwiftBackend(bucketConfig); } else { return yield* Effect.fail( new Error(`Unsupported protocol: ${bucketConfig.protocol}`), @@ -77,7 +81,7 @@ export const BackendResolverLive = Layer.effect( }) as Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client + Exclude | AppConfig | S3Client | SwiftClient >, provideForBackendId: ( @@ -104,9 +108,12 @@ export const BackendResolverLive = Layer.effect( if (backendConfig.protocol === "s3") { backendImpl = yield* makeS3Backend({ backend_id: backendId }); + } else if (backendConfig.protocol === "swift") { + backendImpl = yield* makeSwiftBackend({ backend_id: backendId }); } else { + const protocol = (backendConfig as { protocol: string }).protocol; return yield* Effect.fail( - new Error(`Unsupported protocol: ${backendConfig.protocol}`), + new Error(`Unsupported protocol: ${protocol}`), ); } @@ -115,7 +122,7 @@ export const BackendResolverLive = Layer.effect( }) as Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client + Exclude | AppConfig | S3Client | SwiftClient >, }; }), diff --git a/src/Services/S3Xml.ts b/src/Services/S3Xml.ts index 987ea54..444cfb5 100644 --- a/src/Services/S3Xml.ts +++ b/src/Services/S3Xml.ts @@ -108,99 +108,82 @@ export const S3XmlLive = Layer.succeed( formatListObjects: (result) => { const encode = (s: string) => - result.encodingType === "url" + result.encodingType?.toLowerCase() === "url" ? encodeURIComponent(s).replace(/%2F/g, "/") : s; - const contentsXml = result.contents.map((c) => ` - - ${encode(c.key)} - ${c.lastModified.toISOString()} - ${c.etag} - ${c.size} - ${c.storageClass ?? "STANDARD"} - ${ - c.owner - ? `${c.owner.id}${c.owner.displayName}` - : "" - } - - `).join(""); + const contentsXml = result.contents.map((c) => + `${ + encode(c.key) + }${c.lastModified.toISOString()}${c.etag}${c.size}${ + c.storageClass ?? + "STANDARD" + }${ + c.owner + ? `${c.owner.id}${c.owner.displayName}` + : "" + }` + ).join(""); - const commonPrefixesXml = result.commonPrefixes.map((cp) => ` - - ${encode(cp.prefix)} - - `).join(""); + const commonPrefixesXml = result.commonPrefixes.map((cp) => + `${encode(cp.prefix)}` + ).join(""); let xml: string; if (result.listType === 2) { // ListObjectsV2 - xml = ` - - ${result.name} - ${encode(result.prefix ?? "")} - ${ - result.keyCount ?? - (result.contents.length + result.commonPrefixes.length) - } - ${result.maxKeys} - ${encode(result.delimiter ?? "")} - ${result.isTruncated} - ${ - result.continuationToken - ? `${result.continuationToken}` - : "" - } - ${ - result.nextContinuationToken - ? `${result.nextContinuationToken}` - : "" - } - ${ - result.startAfter - ? `${encode(result.startAfter)}` - : "" - } - ${ - result.encodingType - ? `${result.encodingType}` - : "" - } - ${contentsXml} - ${commonPrefixesXml} - - `; + xml = + `${result.name}${ + encode( + result.prefix ?? "", + ) + }${ + result.keyCount ?? + (result.contents.length + result.commonPrefixes.length) + }${result.maxKeys}${ + encode( + result.delimiter ?? "", + ) + }${result.isTruncated}${ + result.continuationToken + ? `${result.continuationToken}` + : "" + }${ + result.nextContinuationToken + ? `${result.nextContinuationToken}` + : "" + }${ + result.startAfter + ? `${encode(result.startAfter)}` + : "" + }${ + result.encodingType + ? `${result.encodingType}` + : "" + }${contentsXml}${commonPrefixesXml}`; } else { // ListObjectsV1 - xml = ` - - ${result.name} - ${encode(result.prefix ?? "")} - ${encode(result.marker ?? "")} - ${ - result.nextMarker - ? `${encode(result.nextMarker)}` - : "" - } - ${result.maxKeys} - ${encode(result.delimiter ?? "")} - ${result.isTruncated} - ${ - result.encodingType - ? `${result.encodingType}` - : "" - } - ${contentsXml} - ${commonPrefixesXml} - - `; + xml = + `${result.name}${ + encode( + result.prefix ?? "", + ) + }${encode(result.marker ?? "")}${ + result.nextMarker + ? `${encode(result.nextMarker)}` + : "" + }${result.maxKeys}${ + encode( + result.delimiter ?? "", + ) + }${result.isTruncated}${ + result.encodingType + ? `${result.encodingType}` + : "" + }${contentsXml}${commonPrefixesXml}`; } - // Clean up whitespace between tags - const cleanXml = xml.replace(/>\s+<").trim(); - - return HttpServerResponse.text(cleanXml, { + return HttpServerResponse.text(xml, { headers: { "Content-Type": "application/xml", }, @@ -209,68 +192,69 @@ export const S3XmlLive = Layer.succeed( formatListVersions: (result) => { const encode = (s: string) => - result.encodingType === "url" + result.encodingType?.toLowerCase() === "url" ? encodeURIComponent(s).replace(/%2F/g, "/") : s; const versionsXml = result.contents.filter((c) => !c.isDeleteMarker).map( - (v) => ` - - ${encode(v.key)} - ${v.versionId ?? "null"} - ${v.isLatest ?? true} - ${v.lastModified.toISOString()} - ${v.etag} - ${v.size} - ${v.storageClass ?? "STANDARD"} - ${ - v.owner - ? `${v.owner.id}${v.owner.displayName}` - : "" - } - - `, + (v) => + `${encode(v.key)}${ + v.versionId ?? + "null" + }${ + v.isLatest ?? + true + }${v.lastModified.toISOString()}${v.etag}${v.size}${ + v.storageClass ?? + "STANDARD" + }${ + v.owner + ? `${v.owner.id}${v.owner.displayName}` + : "" + }`, ).join(""); const deleteMarkersXml = result.contents.filter((c) => c.isDeleteMarker) - .map((dm) => ` - - ${encode(dm.key)} - ${dm.versionId ?? "null"} - ${dm.isLatest ?? true} - ${dm.lastModified.toISOString()} - ${ - dm.owner - ? `${dm.owner.id}${dm.owner.displayName}` - : "" - } - - `).join(""); - - const commonPrefixesXml = result.commonPrefixes.map((cp) => ` - - ${encode(cp.prefix)} - - `).join(""); + .map((dm) => + `${encode(dm.key)}${ + dm.versionId ?? + "null" + }${ + dm.isLatest ?? + true + }${dm.lastModified.toISOString()}${ + dm.owner + ? `${dm.owner.id}${dm.owner.displayName}` + : "" + }` + ).join(""); - const xml = ` - - ${result.name} - ${encode(result.prefix ?? "")} - ${encode(result.marker ?? "")} - - ${result.maxKeys} - ${encode(result.delimiter ?? "")} - ${result.isTruncated} - ${versionsXml} - ${deleteMarkersXml} - ${commonPrefixesXml} - - `; + const commonPrefixesXml = result.commonPrefixes.map((cp) => + `${encode(cp.prefix)}` + ).join(""); - const cleanXml = xml.replace(/>\s+<").trim(); + const xml = + `${result.name}${ + encode( + result.prefix ?? "", + ) + }${ + encode( + result.marker ?? "", + ) + }${result.maxKeys}${ + encode( + result.delimiter ?? "", + ) + }${result.isTruncated}${ + result.nextMarker + ? `${ + encode(result.nextMarker) + }null` + : "" + }${versionsXml}${deleteMarkersXml}${commonPrefixesXml}`; - return HttpServerResponse.text(cleanXml, { + return HttpServerResponse.text(xml, { headers: { "Content-Type": "application/xml", }, diff --git a/tests/config.test.ts b/tests/config.test.ts index f9b82ef..2e1f485 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -6,8 +6,9 @@ import { BackendResolver, BackendResolverLive, } from "../src/Services/BackendResolver.ts"; -import { AppConfig } from "../src/Config/Layer.ts"; +import { AppConfig, parseConfig } from "../src/Config/Layer.ts"; import { S3Client } from "../src/Backends/S3/Client.ts"; +import { SwiftClient } from "../src/Backends/Swift/Client.ts"; import type { S3Client as S3ClientSDK } from "@aws-sdk/client-s3"; import { Backend } from "../src/Services/Backend.ts"; @@ -204,6 +205,50 @@ const cases: TestCase[] = [ "data-customer-internal": { bucket_name: "infix-match" }, }, }, + { + id: "swift_basic", + name: "swift basic config", + input: { + backends: { + swift_main: { + protocol: "swift", + auth_url: "http://keystone.example.com", + container: "my-container", + buckets: "*", + }, + }, + }, + expectedBuckets: { + "any-bucket": { + backend_id: "swift_main", + protocol: "swift", + auth_url: "http://keystone.example.com", + container: "my-container", + }, + }, + }, + { + id: "swift_with_credentials", + name: "swift with credentials", + input: { + backends: { + swift_main: { + protocol: "swift", + credentials: { + username: "user1", + password: "pw1", + project_name: "proj1", + }, + }, + }, + }, + expectedBuckets: { + "any": { + backend_id: "swift_main", + protocol: "swift", + }, + }, + }, ]; for (const tc of cases) { @@ -243,13 +288,52 @@ for (const tc of cases) { })); } +testEffect("config/parseConfig/env_vars", () => + Effect.gen(function* () { + const env = { + HERALD_DEFAULT_PROTOCOL: "s3", + HERALD_DEFAULT_ENDPOINT: "http://localhost:9000", + HERALD_MYBACKEND_PROTOCOL: "swift", + HERALD_MYBACKEND_AUTH_URL: "http://swift.com", + }; + const config = parseConfig({ backends: {} }, env); + + const defaultBackend = config.backends.default; + yield* EffectAssert.strictEqual(defaultBackend.protocol, "s3"); + if (defaultBackend.protocol === "s3") { + yield* EffectAssert.strictEqual( + defaultBackend.endpoint, + "http://localhost:9000", + ); + } + + const myBackend = config.backends.mybackend; + yield* EffectAssert.strictEqual(myBackend.protocol, "swift"); + if (myBackend.protocol === "swift") { + yield* EffectAssert.strictEqual( + myBackend.auth_url, + "http://swift.com", + ); + } + })); + +testEffect( + "config/parseConfig/default_fallback", + () => + Effect.gen(function* () { + const config = parseConfig({ backends: {} }, {}); + yield* EffectAssert.strictEqual(config.backends.default.protocol, "s3"); + yield* EffectAssert.strictEqual(config.backends.default.buckets, "*"); + }), +); + interface ResolverTestCase { id: string; name: string; config: GlobalConfig; op: ( resolver: Context.Tag.Service, - ) => Effect.Effect; + ) => Effect.Effect; expectedError?: string; } @@ -330,6 +414,12 @@ for (const tc of resolverCases) { getClient: () => Effect.succeed({} as S3ClientSDK), }); + // Mock SwiftClient + const SwiftClientLive = Layer.succeed(SwiftClient, { + getAuthMeta: () => + Effect.succeed({ token: "test", storageUrl: "http://test" }), + }); + const program = Effect.gen(function* () { const resolver = yield* BackendResolver; return yield* tc.op(resolver); @@ -337,6 +427,7 @@ for (const tc of resolverCases) { Effect.provide(BackendResolverLive), Effect.provide(AppConfigLive), Effect.provide(S3ClientLive), + Effect.provide(SwiftClientLive), Effect.either, ); diff --git a/tests/integration/__snapshots__/buckets.test.ts.snap b/tests/integration/__snapshots__/buckets.test.ts.snap index eb093fc..c982eb5 100644 --- a/tests/integration/__snapshots__/buckets.test.ts.snap +++ b/tests/integration/__snapshots__/buckets.test.ts.snap @@ -20,6 +20,13 @@ snapshot[`Proxy/buckets/create/new metadata 1`] = ` } `; +snapshot[`Swift/buckets/create/new metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/buckets/create/existing metadata 1`] = ` { headers: { @@ -40,6 +47,13 @@ snapshot[`Proxy/buckets/create/existing metadata 1`] = ` } `; +snapshot[`Swift/buckets/create/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/buckets/delete/existing metadata 1`] = ` { headers: { @@ -60,6 +74,13 @@ snapshot[`Proxy/buckets/delete/existing metadata 1`] = ` } `; +snapshot[`Swift/buckets/delete/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/buckets/delete/non-existent metadata 1`] = ` { headers: { @@ -92,6 +113,18 @@ snapshot[`Proxy/buckets/delete/non-existent metadata 1`] = ` snapshot[`Proxy/buckets/delete/non-existent body 1`] = `'NoSuchBucketThe specified bucket does not exist'`; +snapshot[`Swift/buckets/delete/non-existent metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Swift/buckets/delete/non-existent body 1`] = `'NoSuchBucketNot Found'`; + snapshot[`Baseline/buckets/head/existing metadata 1`] = ` { headers: { @@ -112,6 +145,13 @@ snapshot[`Proxy/buckets/head/existing metadata 1`] = ` } `; +snapshot[`Swift/buckets/head/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/buckets/head/non-existent metadata 1`] = ` { headers: { @@ -135,11 +175,16 @@ snapshot[`Proxy/buckets/head/non-existent metadata 1`] = ` } `; +snapshot[`Swift/buckets/head/non-existent metadata 1`] = ` +{ + headers: {}, + status: 404, +} +`; + snapshot[`Baseline/buckets/list metadata 1`] = ` { headers: { - "accept-ranges": "bytes", - "content-length": "275", "content-type": "application/xml", "strict-transport-security": "max-age=31536000; includeSubDomains", "x-content-type-options": "nosniff", @@ -152,7 +197,7 @@ snapshot[`Baseline/buckets/list metadata 1`] = ` snapshot[`Baseline/buckets/list body 1`] = ` ' -02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4minio' +02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4minioherald-25gnaqqph3oof5kljqtdeu-72026-01-15T00:00:00.000Zherald-74khf7szf4qtrzrth2weoi-2192026-01-15T00:00:00.000Zherald-88ztrfgehycvaw5bh5t625-12026-01-15T00:00:00.000Zherald-almi4r3xt6pj4vmf25mpkc-362026-01-15T00:00:00.000Zherald-b3y7kg3dn3u5q0awin9aj8-72026-01-15T00:00:00.000Zherald-ferrwumx0p2j3tdhrfle4o-642026-01-15T00:00:00.000Zherald-iqm95px2zlxt75mcsx3dms-12026-01-15T00:00:00.000Zherald-l84igcd8jggs3wioh4msk8-12026-01-15T00:00:00.000Zherald-quy3o0n429jznm43dcga5l-312026-01-15T00:00:00.000Zherald-unz0kp56250vjw6umbd0va-12026-01-15T00:00:00.000Zherald-zrs5hcqpud1tn54vd9u700-152026-01-15T00:00:00.000Zherald-zunthialhf5qffc4p9xthk-742026-01-15T00:00:00.000Z' `; snapshot[`Proxy/buckets/list metadata 1`] = ` @@ -165,4 +210,16 @@ snapshot[`Proxy/buckets/list metadata 1`] = ` } `; -snapshot[`Proxy/buckets/list body 1`] = `'02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4minio'`; +snapshot[`Proxy/buckets/list body 1`] = `'02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4minioherald-25gnaqqph3oof5kljqtdeu-72026-01-15T00:00:00.000Zherald-74khf7szf4qtrzrth2weoi-2192026-01-15T00:00:00.000Zherald-88ztrfgehycvaw5bh5t625-12026-01-15T00:00:00.000Zherald-almi4r3xt6pj4vmf25mpkc-362026-01-15T00:00:00.000Zherald-b3y7kg3dn3u5q0awin9aj8-72026-01-15T00:00:00.000Zherald-ferrwumx0p2j3tdhrfle4o-642026-01-15T00:00:00.000Zherald-iqm95px2zlxt75mcsx3dms-12026-01-15T00:00:00.000Zherald-l84igcd8jggs3wioh4msk8-12026-01-15T00:00:00.000Zherald-quy3o0n429jznm43dcga5l-312026-01-15T00:00:00.000Zherald-unz0kp56250vjw6umbd0va-12026-01-15T00:00:00.000Zherald-zrs5hcqpud1tn54vd9u700-152026-01-15T00:00:00.000Zherald-zunthialhf5qffc4p9xthk-742026-01-15T00:00:00.000Z'`; + +snapshot[`Swift/buckets/list metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 200, +} +`; + +snapshot[`Swift/buckets/list body 1`] = `'swiftSwift User192.168.5.1232026-01-15T00:00:00.000Za2026-01-15T00:00:00.000Zaa2026-01-15T00:00:00.000Zbuilds2026-01-15T00:00:00.000Zfoo-2026-01-15T00:00:00.000Zfoo-.bar2026-01-15T00:00:00.000Zfoo.-bar2026-01-15T00:00:00.000Zfoo..bar2026-01-15T00:00:00.000Zfoo_bar2026-01-15T00:00:00.000Zherald-swift-2w97l75ompcxiypo-12026-01-15T00:00:00.000Zherald-swift-5dfyoor543wddfpb-12026-01-15T00:00:00.000Zherald-swift-5m8pru9nzpno98zp-1572026-01-15T00:00:00.000Zherald-swift-5txup4vs19i8tr6s-292026-01-15T00:00:00.000Zherald-swift-cze1vw7y05q33782-1462026-01-15T00:00:00.000Zherald-swift-fd0oi5radob46p39-12026-01-15T00:00:00.000Zherald-swift-oda2k1hu2ds0wir6-22026-01-15T00:00:00.000Zherald-swift-sy6d1ftl2i7g78jj-12026-01-15T00:00:00.000Zherald-swift-yx0xlhaeebv1g9c7-12026-01-15T00:00:00.000Zherald-task-store-mr-120-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-127-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-130-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-131-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-132-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-137-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-139-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-143-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-144-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-145-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-146-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-147-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-149-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-150-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-151-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-154-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-155-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-157-vivavox2026-01-15T00:00:00.000Zherald-task-store-prd-vivavox2026-01-15T00:00:00.000Zherald-task-store-stg-vivavox2026-01-15T00:00:00.000Ziac-swift2026-01-15T00:00:00.000Zmr-101-vivavox2026-01-15T00:00:00.000Zmr-109-vivavox2026-01-15T00:00:00.000Zmr-111-vivavox2026-01-15T00:00:00.000Zmr-115-vivavox2026-01-15T00:00:00.000Zmr-116-vivavox2026-01-15T00:00:00.000Zmr-120-vivavox2026-01-15T00:00:00.000Zmr-121-vivavox2026-01-15T00:00:00.000Zmr-122-vivavox2026-01-15T00:00:00.000Zmr-124-vivavox2026-01-15T00:00:00.000Zmr-126-vivavox2026-01-15T00:00:00.000Zmr-127-vivavox2026-01-15T00:00:00.000Zmr-130-vivavox2026-01-15T00:00:00.000Zmr-131-vivavox2026-01-15T00:00:00.000Zmr-132-vivavox2026-01-15T00:00:00.000Zmr-137-vivavox2026-01-15T00:00:00.000Zmr-139-vivavox2026-01-15T00:00:00.000Zmr-143-vivavox2026-01-15T00:00:00.000Zmr-144-vivavox2026-01-15T00:00:00.000Zmr-145-vivavox2026-01-15T00:00:00.000Zmr-146-vivavox2026-01-15T00:00:00.000Zmr-147-vivavox2026-01-15T00:00:00.000Zmr-149-vivavox2026-01-15T00:00:00.000Zmr-150-vivavox2026-01-15T00:00:00.000Zmr-151-vivavox2026-01-15T00:00:00.000Zmr-154-vivavox2026-01-15T00:00:00.000Zmr-155-vivavox2026-01-15T00:00:00.000Zmr-157-vivavox2026-01-15T00:00:00.000Zprd-vivavox2026-01-15T00:00:00.000Zstg-vivavox2026-01-15T00:00:00.000Zstg-vivavox+segments2026-01-15T00:00:00.000Z'`; diff --git a/tests/integration/__snapshots__/objects.test.ts.snap b/tests/integration/__snapshots__/objects.test.ts.snap index c490bdf..ba27552 100644 --- a/tests/integration/__snapshots__/objects.test.ts.snap +++ b/tests/integration/__snapshots__/objects.test.ts.snap @@ -20,6 +20,13 @@ snapshot[`Proxy/objects/put metadata 1`] = ` } `; +snapshot[`Swift/objects/put metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/objects/get/existing metadata 1`] = ` { headers: { @@ -40,6 +47,13 @@ snapshot[`Proxy/objects/get/existing metadata 1`] = ` } `; +snapshot[`Swift/objects/get/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/objects/get/non-existent metadata 1`] = ` { headers: { @@ -72,6 +86,18 @@ snapshot[`Proxy/objects/get/non-existent metadata 1`] = ` snapshot[`Proxy/objects/get/non-existent body 1`] = `'NoSuchKeyThe specified key does not exist.'`; +snapshot[`Swift/objects/get/non-existent metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Swift/objects/get/non-existent body 1`] = `'NoSuchKeyNot Found'`; + snapshot[`Baseline/objects/head/existing metadata 1`] = ` { headers: { @@ -92,6 +118,13 @@ snapshot[`Proxy/objects/head/existing metadata 1`] = ` } `; +snapshot[`Swift/objects/head/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/objects/head/non-existent metadata 1`] = ` { headers: { @@ -115,6 +148,13 @@ snapshot[`Proxy/objects/head/non-existent metadata 1`] = ` } `; +snapshot[`Swift/objects/head/non-existent metadata 1`] = ` +{ + headers: {}, + status: 404, +} +`; + snapshot[`Baseline/objects/delete/existing metadata 1`] = ` { headers: { @@ -134,3 +174,10 @@ snapshot[`Proxy/objects/delete/existing metadata 1`] = ` status: 204, } `; + +snapshot[`Swift/objects/delete/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; diff --git a/tests/utils.ts b/tests/utils.ts index d2d778f..6748cd1 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -1,10 +1,11 @@ import { S3Client } from "@aws-sdk/client-s3"; -import { Effect, Layer } from "effect"; +import { Config, Effect, Layer, Logger, LogLevel, Option } from "effect"; import { ApiLive } from "../src/Http.ts"; import { AppConfig } from "../src/Config/Layer.ts"; import { lookupBucket } from "../src/Domain/Config.ts"; import { BackendResolverLive } from "../src/Services/BackendResolver.ts"; import { S3ClientLive } from "../src/Backends/S3/Client.ts"; +import { SwiftClientLive } from "../src/Backends/Swift/Client.ts"; import { S3XmlLive } from "../src/Services/S3Xml.ts"; import { HttpApiBuilder, HttpServer } from "@effect/platform"; import { FetchHttpClient } from "@effect/platform"; @@ -31,7 +32,12 @@ export type Snapshot = { body: string; }; -export const makeTestHarness = (config: GlobalConfig) => +export const makeTestHarness = ( + config: GlobalConfig, + loggingLayer: Layer.Layer = Logger.minimumLogLevel( + LogLevel.Info, + ), +) => Effect.gen(function* () { const AppConfigLive = Layer.succeed(AppConfig, { raw: config, @@ -41,10 +47,12 @@ export const makeTestHarness = (config: GlobalConfig) => const ApiWithRequirements = ApiLive.pipe( Layer.provide(BackendResolverLive), Layer.provide(S3ClientLive), + Layer.provide(SwiftClientLive), Layer.provide(S3XmlLive), Layer.provide(AppConfigLive), Layer.provide(FetchHttpClient.layer), Layer.provideMerge(HttpServer.layerContext), + Layer.provideMerge(loggingLayer), ); // In @effect/platform 0.90.x, toWebHandler returns the object directly, not an Effect. @@ -53,7 +61,9 @@ export const makeTestHarness = (config: GlobalConfig) => // Start Deno.serve on a random port const server = Deno.serve( { port: 0, onListen: () => {} }, - (req) => webHandler.handler(req), + (req) => { + return webHandler.handler(req); + }, ); // Ensure cleanup @@ -388,6 +398,137 @@ function proxyRunner(tc: ProxyTestCase, t: Deno.TestContext) { ); } +const getSwiftConfig = () => + Effect.gen(function* () { + const authUrl = yield* Config.string("HEARLD_SWIFTTEST_AUTH_URL").pipe( + Config.orElse(() => Config.string("HERALD_SWIFTTEST_AUTH_URL")), + Config.orElse(() => Config.string("OS_AUTH_URL")), + Config.withDefault("https://api.pub1.infomaniak.cloud/identity/v3"), + Config.option, + ); + + const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_USERNAME")), + Config.orElse(() => Config.string("OS_USERNAME")), + Config.option, + ); + const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PASSWORD")), + Config.orElse(() => Config.string("OS_PASSWORD")), + Config.option, + ); + const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") + .pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PROJECT_NAME")), + Config.orElse(() => Config.string("OS_PROJECT_NAME")), + Config.option, + ); + const region = yield* Config.string("HEARLD_SWIFTTEST_OS_REGION_NAME").pipe( + Config.orElse(() => Config.string("HERALD_SWIFTTEST_OS_REGION_NAME")), + Config.orElse(() => Config.string("TF_VAR_OS_REGION_NAME")), + Config.orElse(() => Config.string("OS_REGION_NAME")), + Config.withDefault("dc3-a"), + Config.option, + ); + + if ( + Option.isNone(username) || Option.isNone(password) || + Option.isNone(projectName) || Option.isNone(authUrl) + ) { + return Option.none(); + } + + const config: GlobalConfig = { + backends: { + swift: { + protocol: "swift", + auth_url: authUrl.value, + region: Option.getOrUndefined(region), + credentials: { + username: username.value, + password: password.value, + project_name: projectName.value, + user_domain_name: "Default", + project_domain_name: "Default", + }, + buckets: "*", + }, + }, + }; + return Option.some(config); + }); + +function swiftRunner(tc: ProxyTestCase, t: Deno.TestContext) { + return Effect.gen(function* () { + const swiftConfig = yield* getSwiftConfig(); + if (Option.isNone(swiftConfig)) { + return yield* Effect.fail( + new Error( + "Swift credentials missing. Set HERALD_SWIFTTEST_OS_USERNAME etc or run with infisical.", + ), + ); + } + + const h = yield* makeTestHarness(swiftConfig.value); + + if (tc.beforeAll) { + const beforeResult = tc.beforeAll(h.proxyClient); + if (Effect.isEffect(beforeResult)) { + yield* beforeResult; + } else { + yield* Effect.tryPromise(() => beforeResult as Promise).pipe( + Effect.orDie, + ); + } + } + + const resultEffect = Effect.gen(function* () { + const result = tc.fn(h.proxyClient); + if (Effect.isEffect(result)) { + yield* result; + } else { + yield* Effect.tryPromise({ + try: () => result as Promise, + catch: (e) => new Error(`Test function failed for ${tc.name}: ${e}`), + }); + } + }); + + yield* resultEffect; + + const lastResponse = h.getLastResponse(); + if (lastResponse) { + yield* Effect.tryPromise(() => + assertSnapshot(t, { + status: lastResponse.status, + headers: lastResponse.headers, + }, { name: `Swift/${tc.name} metadata` }) + ); + if (lastResponse.body) { + yield* Effect.tryPromise(() => + assertSnapshot(t, lastResponse.body, { + name: `Swift/${tc.name} body`, + }) + ); + } + } + + if (tc.afterAll) { + const afterResult = tc.afterAll(h.proxyClient); + if (Effect.isEffect(afterResult)) { + yield* afterResult; + } else { + yield* Effect.tryPromise(() => afterResult as Promise).pipe( + Effect.orDie, + ); + } + } + }).pipe( + Effect.tapErrorCause(Effect.logError), + Effect.scoped, + ); +} + export function harness(cases: ProxyTestCase[]) { const namePrefix = ""; for (const tc of cases) { @@ -403,5 +544,9 @@ export function harness(cases: ProxyTestCase[]) { ignore: tc.ignore, only: tc.only, }); + testEffect(`${namePrefix}Swift/${tc.name}`, (t) => swiftRunner(tc, t), { + ignore: tc.ignore, + only: tc.only, + }); } } diff --git a/x/s3-tests.ts b/x/s3-tests.ts index af1c316..9c9bc41 100755 --- a/x/s3-tests.ts +++ b/x/s3-tests.ts @@ -1,253 +1,681 @@ #!/usr/bin/env -S deno run --allow-all +/** + * Herald S3 Compatibility Test Runner + * + * This script runs the Ceph S3 compatibility test suite (s3-tests) against + * a local Herald proxy instance. It handles: + * - Starting the Herald proxy with a specified backend (minio or swift) + * - Configuring s3-tests to point to the proxy + * - Running pytest with real-time output streaming + * - Parsing JUnit XML for a final summary + * + * Usage: + * ./x/s3-tests.ts [pytest-args] [--backend ] [--no-abort] + * + * Environment Variables: + * S3TEST_TAGS: Custom pytest marks (default: not fails_on_s3proxy and ...) + * S3TEST_PYTEST_ARGS: Additional pytest arguments + * S3TEST_NO_ABORT: Set to "true" to disable abort-on-error + * HERALD_LOG_LEVEL: Set to "DEBUG" for verbose proxy logging + * + * Files: + * s3-tests/s3tests.conf: Generated s3-tests configuration + * s3-tests/herald-proxy.log: Herald proxy logs (minio backend) + * s3-tests/herald-proxy-swift.log: Herald proxy logs (swift backend) + * s3-tests/s3-tests.log: Full pytest output + */ -import { Effect } from "effect"; -import { LoggingLive } from "../src/Logging/Layer.ts"; -import { makeTestHarness } from "../tests/utils.ts"; -import type { GlobalConfig } from "../src/Domain/Config.ts"; +import { Config, Effect, Logger, LogLevel, Option } from "effect"; import * as path from "@std/path"; -import { $ } from "./utils.ts"; - -// Default tags taken from s3proxy/src/test/resources/run-s3-tests.sh -const DEFAULT_TAGS = [ - "not fails_on_s3proxy", - "and not appendobject", - "and not bucket_policy", - "and not checksum", - "and not copy", - "and not cors", - "and not encryption", - "and not fails_strict_rfc2616", - "and not iam_tenant", - "and not lifecycle", - "and not object_lock", - "and not policy", - "and not policy_status", - "and not s3select", - "and not s3website", - "and not sse_s3", - "and not tagging", - "and not test_of_sts", - "and not user_policy", - "and not versioning", - "and not webidentity_test", -].join(" "); - -const config: GlobalConfig = { - backends: { - minio: { - protocol: "s3", - endpoint: "http://localhost:9000", - region: "us-east-1", - credentials: { - accessKeyId: "minioadmin", - secretAccessKey: "minioadmin", +import { $ } from "@david/dax"; +import * as colors from "@std/fmt/colors"; +import { makeTestHarness } from "../tests/utils.ts"; +import { GlobalConfig } from "../src/Domain/Config.ts"; + +const DEFAULT_TAGS = + "not fails_on_s3proxy and not appendobject and not bucket_policy and not copy and not cors and not encryption and not fails_strict_rfc2616 and not iam_tenant and not lifecycle and not object_lock and not policy and not policy_status and not s3select and not s3website and not sse_s3 and not tagging and not test_of_sts and not user_policy and not versioning and not webidentity_test"; + +function getMinioConfig(): GlobalConfig { + return { + backends: { + minio: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + buckets: "*", }, - buckets: "*", }, - }, -}; + }; +} + +const getSwiftConfig = () => + Effect.gen(function* () { + const authUrl = yield* Config.string("HEARLD_SWIFTTEST_AUTH_URL").pipe( + Config.orElse(() => Config.string("HERALD_SWIFTTEST_AUTH_URL")), + Config.orElse(() => Config.string("OS_AUTH_URL")), + Config.withDefault("https://api.pub1.infomaniak.cloud/identity/v3"), + Config.option, + ); + + const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_USERNAME")), + Config.orElse(() => Config.string("OS_USERNAME")), + Config.option, + ); + const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PASSWORD")), + Config.orElse(() => Config.string("OS_PASSWORD")), + Config.option, + ); + const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") + .pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PROJECT_NAME")), + Config.orElse(() => Config.string("OS_PROJECT_NAME")), + Config.option, + ); + const region = yield* Config.string("HEARLD_SWIFTTEST_OS_REGION_NAME").pipe( + Config.orElse(() => Config.string("HERALD_SWIFTTEST_OS_REGION_NAME")), + Config.orElse(() => Config.string("TF_VAR_OS_REGION_NAME")), + Config.orElse(() => Config.string("OS_REGION_NAME")), + Config.withDefault("dc3-a"), + Config.option, + ); + + if ( + Option.isNone(username) || Option.isNone(password) || + Option.isNone(projectName) || Option.isNone(authUrl) + ) { + return Option.none(); + } + + const config: GlobalConfig = { + backends: { + swift: { + protocol: "swift", + auth_url: authUrl.value, + region: Option.getOrUndefined(region), + credentials: { + username: username.value, + password: password.value, + project_name: projectName.value, + user_domain_name: "Default", + project_domain_name: "Default", + }, + buckets: "*", + }, + }, + }; + return Option.some(config); + }); -const program = makeTestHarness(config).pipe( - Effect.flatMap((h) => { - const port = new URL(h.proxyUrl).port; +const program = Effect.gen(function* () { + console.log("Program started"); + const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); + const s3TestsDir = path.resolve(__dirname, "../s3-tests"); - // Parse filtering arguments - const tags = $.env.S3TEST_TAGS ?? DEFAULT_TAGS; - const pytestArgsEnv = $.env.S3TEST_PYTEST_ARGS ?? ""; - const pytestArgsFromEnv = pytestArgsEnv ? pytestArgsEnv.split(/\s+/) : []; - const pytestArgsFromCli = $.argv; - const pytestArgs = [...pytestArgsFromEnv, ...pytestArgsFromCli]; + // Parse filtering arguments and flags + const rawArgs = [...Deno.args]; + const noAbort = rawArgs.includes("--no-abort") || + Deno.env.get("S3TEST_NO_ABORT") === "true"; - return Effect.gen(function* () { - yield* Effect.logInfo(`Starting Herald proxy on port ${port}`); + let backend = "minio"; + const backendIdx = rawArgs.indexOf("--backend"); + if (backendIdx !== -1) { + backend = rawArgs[backendIdx + 1]; + rawArgs.splice(backendIdx, 2); + } + + const pytestArgsFromCli = rawArgs.filter((arg) => arg !== "--no-abort"); + + const proxyLogName = backend === "swift" + ? "herald-proxy-swift.log" + : "herald-proxy.log"; + const proxyLogPath = path.join(s3TestsDir, proxyLogName); + + // Initialize config based on backend + let activeConfig: GlobalConfig; + let s3AccessKey = "minioadmin"; + let s3SecretKey = "minioadmin"; + + if (backend === "swift") { + const swiftConfig = yield* getSwiftConfig(); + if (Option.isNone(swiftConfig)) { + return yield* Effect.fail( + new Error("Swift credentials missing. Run with infisical."), + ); + } + activeConfig = swiftConfig.value; + // For Swift backend, Herald doesn't check S3 credentials, + // but s3-tests needs them to sign requests. + s3AccessKey = "dummy"; + s3SecretKey = "dummy"; + } else { + activeConfig = getMinioConfig(); + } + + console.log("Creating file logger for proxy..."); + // Create a file logger for the proxy + const proxyLogFile = yield* Effect.tryPromise(() => + Deno.open(proxyLogPath, { write: true, create: true, truncate: true }) + ); + + yield* Effect.addFinalizer(() => + Effect.tryPromise({ + try: () => Promise.resolve(proxyLogFile.close()), + catch: (e) => new Error(`Failed to close proxy log file: ${e}`), + }).pipe(Effect.orDie) + ); + + const logLevel = yield* Config.string("HERALD_LOG_LEVEL").pipe( + Config.withDefault("INFO"), + ); + const minLogLevel = LogLevel.Debug; + + // Create a custom logging layer that writes to file synchronously + const FileLoggingLive = Logger.replace( + Logger.defaultLogger, + Logger.make(({ message, logLevel: currentLogLevel }) => { + if (currentLogLevel.syslog > minLogLevel.syslog) { + return; + } + const timestamp = new Date().toISOString(); + const level = currentLogLevel.label; + const msg = typeof message === "string" ? message : String(message); + const logLine = `${timestamp} level=${level} ${msg}\n`; + try { + proxyLogFile.writeSync(new TextEncoder().encode(logLine)); + } catch (e) { + console.error(`Failed to write to proxy log: ${e}`); + } + }), + ); - const confContent = `[DEFAULT] + // Provide the file logger to the test harness (the proxy) + const h = yield* makeTestHarness(activeConfig, FileLoggingLive); + + const port = new URL(h.proxyUrl).port; + + // Parse remaining filtering arguments + const tags = Deno.env.get("S3TEST_TAGS") ?? DEFAULT_TAGS; + const pytestArgsEnv = Deno.env.get("S3TEST_PYTEST_ARGS") ?? ""; + const pytestArgsFromEnv = pytestArgsEnv ? pytestArgsEnv.split(/\s+/) : []; + + const pytestArgs = [...pytestArgsFromEnv, ...pytestArgsFromCli]; + + return yield* (Effect.gen(function* () { + // We use console.log for harness output to avoid them going to the proxy log file + console.log( + `Starting Herald (${colors.cyan(backend)} backend) on port ${ + colors.cyan(port) + }`, + ); + console.log(`Proxy logs: ${colors.gray(proxyLogPath)}`); + + const confContent = `[DEFAULT] host = 127.0.0.1 port = ${port} is_secure = no [fixtures] -bucket prefix = herald-{random}- +bucket prefix = herald-${backend}-{random}- [s3 main] user_id = main display_name = main email = main@example.com -access_key = minioadmin -secret_key = minioadmin +access_key = ${s3AccessKey} +secret_key = ${s3SecretKey} [s3 alt] user_id = alt display_name = alt email = alt@example.com -access_key = minioadmin -secret_key = minioadmin +access_key = ${s3AccessKey} +secret_key = ${s3SecretKey} [s3 tenant] user_id = tenant display_name = tenant email = tenant@example.com -access_key = minioadmin -secret_key = minioadmin +access_key = ${s3AccessKey} +secret_key = ${s3SecretKey} tenant = testx [iam] email = iam@example.com user_id = iam -access_key = minioadmin -secret_key = minioadmin +access_key = ${s3AccessKey} +secret_key = ${s3SecretKey} display_name = iam [iam root] -access_key = minioadmin -secret_key = minioadmin +access_key = ${s3AccessKey} +secret_key = ${s3SecretKey} user_id = iam_root email = iam_root@example.com [iam alt root] -access_key = minioadmin -secret_key = minioadmin +access_key = ${s3AccessKey} +secret_key = ${s3SecretKey} user_id = iam_alt_root email = iam_alt_root@example.com `; - const confPath = yield* Effect.promise(() => - Deno.makeTempFile({ suffix: ".conf" }) - ); - yield* Effect.promise(() => Deno.writeTextFile(confPath, confContent)); + const confPath = yield* Effect.promise(() => + Deno.makeTempFile({ suffix: ".conf" }) + ); + yield* Effect.promise(() => Deno.writeTextFile(confPath, confContent)); - const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); - const s3TestsDir = path.resolve(__dirname, "../s3-tests"); - const logPath = path.join(s3TestsDir, "s3-tests.log"); + const logPath = path.join(s3TestsDir, "s3-tests.log"); - yield* Effect.logInfo(`s3-tests directory: ${s3TestsDir}`); - yield* Effect.logInfo(`Log file: ${logPath}`); + console.log(`s3-tests directory: ${colors.gray(s3TestsDir)}`); + console.log(`Log file: ${colors.gray(logPath)}`); - // Ensure we have a virtual environment - const venvPath = path.join(s3TestsDir, ".venv"); - const venvExists = yield* Effect.tryPromise(() => - Deno.stat(venvPath).then(() => true).catch(() => false) - ); + // Ensure we have a virtual environment + const venvPath = path.join(s3TestsDir, ".venv"); + const venvExists = yield* Effect.tryPromise(() => + Deno.stat(venvPath).then(() => true).catch(() => false) + ); - if (!venvExists) { - yield* Effect.logInfo("Creating Python virtual environment..."); - yield* Effect.tryPromise(() => - $`uv venv --python 3.11`.cwd(s3TestsDir) - ); - } + if (!venvExists) { + console.log(colors.yellow("Creating Python virtual environment...")); + yield* Effect.tryPromise(() => $`uv venv --python 3.11`.cwd(s3TestsDir)); + } + + // Register finalizer to clean up conf file + yield* Effect.addFinalizer(() => + Effect.tryPromise({ + try: () => + Deno.remove(confPath).catch((e) => { + console.error(`Failed to remove conf file ${confPath}: ${e}`); + }), + catch: (e) => new Error(`Effect.tryPromise failed: ${e}`), + }).pipe(Effect.orDie) + ); + + // Ensure dependencies are installed + const pytestCheck = yield* Effect.tryPromise({ + try: async () => { + const proc = $`uv run pytest --version`.cwd(s3TestsDir).noThrow(); + return await proc; + }, + catch: () => new Error("Check failed"), + }); + + if (pytestCheck.code !== 0) { + console.log(colors.yellow("Installing s3-tests dependencies...")); + yield* Effect.tryPromise({ + try: async () => { + await $`uv pip install -r requirements.txt`.cwd(s3TestsDir); + await $`uv pip install -e .`.cwd(s3TestsDir); + }, + catch: (e) => new Error(`Failed to install dependencies: ${e}`), + }); + } - yield* Effect.logInfo( - `Running s3-tests against Herald on port ${port}...`, + console.log( + `Running s3-tests against Herald on port ${colors.cyan(port)}...`, + ); + if (tags) console.log(`${colors.gray("Tags:")} ${tags}`); + if (pytestArgs.length > 0) { + console.log( + `${colors.gray("Additional pytest args:")} ${pytestArgs.join(" ")}`, ); - yield* Effect.logInfo(`Tags: ${tags}`); - yield* Effect.logInfo(`Additional pytest args: ${pytestArgs.join(" ")}`); + } + if (noAbort) { + console.log(colors.yellow("Abort on ERROR disabled (--no-abort)")); + } - // Run pytest with timeout - const timeoutId = setTimeout(() => {}, 300000); // 5 minutes + // Build command arguments + const cmdArgs = [ + "-v", + "--tb=short", + ]; - try { - // Build command arguments - const cmdArgs: string[] = ["run", "pytest", "-v", "--tb=long"]; + const junitXmlName = "junit.xml"; + const junitXmlPath = path.join(s3TestsDir, junitXmlName); + cmdArgs.push(`--junit-xml=${junitXmlName}`); - if (tags) { - cmdArgs.push("-m", tags); - } + if (tags) { + cmdArgs.push("-m", tags); + } - // Add user-provided pytest arguments - cmdArgs.push(...pytestArgs); + cmdArgs.push(...pytestArgs); - // Add test path if not already specified - const hasTestPath = pytestArgs.some((arg) => - arg.includes("s3tests/") || arg.includes("test_") - ); - if (!hasTestPath) { - cmdArgs.push("s3tests/functional/test_s3.py"); - } + const logFile = yield* Effect.tryPromise(() => + Deno.open(logPath, { + write: true, + create: true, + truncate: true, + }) + ); - const result = yield* Effect.tryPromise({ - try: async () => { - const proc = $`uv ${cmdArgs}` - .cwd(s3TestsDir) - .env({ - S3TEST_CONF: confPath, - UV_PYTHON: "3.11", - }) - .noThrow() - .stdout("piped") - .stderr("piped"); - return await proc; - }, - catch: (e) => new Error(`Failed to run pytest: ${e}`), - }); - - // Write output to log file - const stdoutBytes = yield* Effect.sync(() => { - const stdout = result.stdout as unknown; - if (stdout instanceof Uint8Array) { - return stdout; + console.log(`Command: uv run pytest ${cmdArgs.join(" ")}`); + const child = $`uv run pytest ${cmdArgs}` + .cwd(s3TestsDir) + .env({ S3TEST_CONF: confPath, PYTHONUNBUFFERED: "1" }) + .stdout("piped") + .stderr("piped") + .spawn(); + + const sigintHandler = () => { + child.kill(); + Deno.exit(0); + }; + Deno.addSignalListener("SIGINT", sigintHandler); + + const result = yield* Effect.tryPromise({ + try: async () => { + let collectedInfo = ""; + let failedCount = 0; + let errorCount = 0; + let skippedCount = 0; + let lastResultTime = Date.now(); + const seenTests = new Set(); + const failedTests = new Set(); + const errorTests = new Set(); + let currentTestName = ""; + + let shouldAbort = false; + let abortReason = ""; + + const processLine = (line: string) => { + const trimmed = line.trim(); + if (!trimmed) return; + + // Capture test result lines like: + // s3tests/functional/test_s3.py::test_bucket_list_empty PASSED [ 0%] + const resultMatch = trimmed.match( + /^([^\s]+::[^\s]+)\s+(PASSED|FAILED|ERROR|SKIPPED)/, + ); + if (resultMatch) { + const testName = resultMatch[1]; + const status = resultMatch[2]; + const now = Date.now(); + const duration = ((now - lastResultTime) / 1000).toFixed(2); + lastResultTime = now; + currentTestName = testName; + + if (status === "PASSED") { + console.log( + `${colors.green("✓")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + } else if (status === "FAILED") { + if (!seenTests.has(testName)) { + failedCount++; + seenTests.add(testName); + failedTests.add(testName); + } + console.error( + `${colors.red("✗")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + } else if (status === "ERROR") { + if (!seenTests.has(testName)) { + errorCount++; + seenTests.add(testName); + errorTests.add(testName); + } + console.error( + `${colors.red("✗ ERROR:")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + if (!noAbort) { + shouldAbort = true; + abortReason = `ERROR in ${testName}`; + child.kill(); + } + } else if (status === "SKIPPED") { + skippedCount++; + console.log( + `${colors.yellow("-")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + } + return; } - return new TextEncoder().encode(String(stdout)); - }); - const stderrBytes = yield* Effect.sync(() => { - const stderr = result.stderr as unknown; - if (stderr instanceof Uint8Array) { - return stderr; + + // Also check for ERROR in non-verbose format + const errorMatch = trimmed.match(/^ERROR\s+([^\s]+::[^\s]+)/); + if (errorMatch) { + const testName = errorMatch[1]; + const now = Date.now(); + const duration = ((now - lastResultTime) / 1000).toFixed(2); + lastResultTime = now; + currentTestName = testName; + + if (!seenTests.has(testName)) { + errorCount++; + seenTests.add(testName); + errorTests.add(testName); + } + console.error( + `${colors.red("✗ ERROR:")} ${testName} ${ + colors.gray(`(${duration}s)`) + }`, + ); + + if (!noAbort) { + shouldAbort = true; + abortReason = `ERROR in ${testName}`; + child.kill(); + } + return; } - return new TextEncoder().encode(String(stderr)); - }); - const combined = new Uint8Array( - stdoutBytes.length + stderrBytes.length, - ); - combined.set(stdoutBytes); - combined.set(stderrBytes, stdoutBytes.length); - yield* Effect.tryPromise(() => Deno.writeFile(logPath, combined)); - if (result.code !== 0) { - yield* Effect.logError( - `s3-tests finished with exit code ${result.code}`, - ); + // Echo important lines (failures, tracebacks, summaries) + if ( + trimmed.includes("FAILURES") || + trimmed.includes("ERRORS") || + trimmed.includes("short test summary") || + trimmed.startsWith("E ") || // Traceback lines in short format + trimmed.startsWith("> ") || + trimmed.match(/^=+\s*(passed|failed|error)/i) + ) { + const prefix = currentTestName + ? colors.gray(`[${currentTestName}] `) + : ""; + console.log(`${prefix}${trimmed}`); + } + }; - // Show last 20 lines of log - const tailResult = yield* Effect.tryPromise({ - try: async () => { - const proc = $`tail -n 20 ${logPath}`.stdout("piped"); - return await proc; - }, - catch: (e) => new Error(`Failed to tail log file: ${e}`), - }); - yield* Effect.logError("Last 20 lines of log:"); - const tailOutput = yield* Effect.sync(() => { - const stdout = tailResult.stdout as unknown; - if (stdout instanceof Uint8Array) { - return new TextDecoder().decode(stdout); + const decoder = new TextDecoder(); + + async function streamToLogAndConsole( + stream: ReadableStream, + ) { + const reader = stream.getReader(); + let buffer = ""; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + try { + await logFile.write(value); + } catch (e) { + console.error(`Failed to write to log file: ${e}`); + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + processLine(line); } - return String(stdout); - }); - yield* Effect.logError(tailOutput); + } + if (buffer) { + processLine(buffer); + } + reader.releaseLock(); + } + + const [procResult] = await Promise.all([ + child, + streamToLogAndConsole(child.stdout()), + streamToLogAndConsole(child.stderr()), + ]); + + Deno.removeSignalListener("SIGINT", sigintHandler); - yield* Effect.fail( - new Error(`s3-tests failed with exit code ${result.code}`), + // Attempt to parse JUnit XML if it exists and is valid + let junitData: { + tests: number; + failures: number; + errors: number; + skipped: number; + time?: number; + failedNames: string[]; + errorNames: string[]; + } | null = null; + + try { + const junitXml = await Deno.readTextFile(junitXmlPath); + const getAttr = (name: string) => { + const match = junitXml.match(new RegExp(`${name}="([\\d.]+)"`)); + return match ? parseFloat(match[1]) : 0; + }; + + const failedNames: string[] = []; + const errorNames: string[] = []; + + const testcaseMatches = junitXml.matchAll( + /]*>([\s\S]*?)<\/testcase>/g, ); - } else { - yield* Effect.logInfo("s3-tests passed!"); + for (const match of testcaseMatches) { + const fullName = `${match[1]}::${match[2]}`; + const content = match[3]; + if (content.includes(" Deno.remove(confPath).catch(() => {})); - } + + // Use streaming counts if JUnit failed or reported 0 + const finalCounts = (junitData && junitData.tests > 0) ? junitData : { + tests: seenTests.size, + failures: failedCount, + errors: errorCount, + skipped: skippedCount, + time: undefined, + failedNames: Array.from(failedTests), + errorNames: Array.from(errorTests), + }; + + return { + code: procResult.code, + counts: finalCounts, + collectedInfo, + shouldAbort, + abortReason, + }; + }, + catch: (e) => new Error(`Failed to run pytest: ${e}`), }); - }), - Effect.scoped, - Effect.provide(LoggingLive), -); + + if (result.collectedInfo) { + console.log(colors.gray(result.collectedInfo)); + } + + const { tests, failures, errors, skipped, time, failedNames, errorNames } = + result.counts; + const passed = tests - failures - errors - skipped; + + console.log(); + const durationStr = time ? ` ${colors.cyan(`${time.toFixed(2)}s`)}` : ""; + console.log( + `${colors.bold(tests.toString())} tests completed in${durationStr}:`, + ); + console.log( + ` ${colors.green("successes")}: ${ + colors.bold(passed.toString()) + }/${tests}`, + ); + console.log( + ` ${colors.red("failures")}: ${ + colors.bold(failures.toString()) + }/${tests}`, + ); + if (errors > 0) { + console.log( + ` ${colors.red("errors")}: ${ + colors.bold(errors.toString()) + }/${tests}`, + ); + } + if (skipped > 0) { + console.log( + ` ${colors.gray("skipped")}: ${ + colors.bold(skipped.toString()) + }/${tests}`, + ); + } + + if (failedNames.length > 0) { + console.log(colors.red("\nFailures:")); + for (const name of failedNames) { + console.log(` ${colors.red("-")} ${name}`); + } + } + + if (errorNames.length > 0) { + console.log(colors.red("\nErrors:")); + for (const name of errorNames) { + console.log(` ${colors.red("-")} ${name}`); + } + } + + if (errors > 0 || (result.shouldAbort && result.abortReason)) { + if (result.shouldAbort) { + yield* Effect.fail( + new Error( + `Aborted due to ERROR: ${result.abortReason || "Test Error"}`, + ), + ); + } else { + yield* Effect.fail(new Error(`s3-tests finished with errors.`)); + } + } + + if (failures > 0 || result.code !== 0) { + yield* Effect.fail( + new Error(`s3-tests finished with failures (code ${result.code}).`), + ); + } + + console.log(colors.green(`\n✓ s3-tests completed successfully.`)); + }).pipe( + Effect.provide(Logger.minimumLogLevel(minLogLevel)), + )); +}); if (import.meta.main) { - Effect.runPromiseExit(program).then((exitCode) => { + Effect.runPromiseExit(program.pipe(Effect.scoped)).then((exitCode) => { if (exitCode._tag === "Failure") { + console.error( + colors.red(`Fatal error: ${JSON.stringify(exitCode.cause, null, 2)}`), + ); Deno.exit(1); } }).catch((e) => { - console.error(`Error: ${e}`); + console.error(colors.red(`Unhandled error: ${e}`)); Deno.exit(1); }); } diff --git a/x/swift-debug.ts b/x/swift-debug.ts new file mode 100644 index 0000000..986d71b --- /dev/null +++ b/x/swift-debug.ts @@ -0,0 +1,41 @@ +#!/usr/bin/env -S deno run --allow-all +import { Effect, Logger, LogLevel } from "effect"; +import { SwiftClient, SwiftClientLive } from "../src/Backends/Swift/Client.ts"; +import { AppConfigLive } from "../src/Config/Layer.ts"; +import { makeSwiftBackend } from "../src/Backends/Swift/Backend.ts"; +import { Backend } from "../src/Services/Backend.ts"; + +const program = Effect.gen(function* () { + console.log("Checking Swift connection..."); + + // We'll use the 'default' backend which should be configured via HERALD_ env vars + const backendId = "default"; + + const swiftClient = yield* SwiftClient; + const auth = yield* swiftClient.getAuthMeta({ backend_id: backendId }); + + console.log("Auth successful!"); + console.log(`Storage URL: ${auth.storageUrl}`); + console.log(`Token: ${auth.token.substring(0, 10)}...`); + + const backend = yield* makeSwiftBackend({ backend_id: backendId }); + const { buckets } = yield* backend.listBuckets(); + + console.log(`Found ${buckets.length} buckets:`); + for (const b of buckets) { + console.log(` - ${b.name} (created: ${b.creationDate})`); + } +}).pipe( + Effect.provide(SwiftClientLive), + Effect.provide(AppConfigLive), + Effect.provide(Logger.minimumLogLevel(LogLevel.Debug)), +); + +if (import.meta.main) { + Effect.runPromiseExit(program).then((exit) => { + if (exit._tag === "Failure") { + console.error("Program failed:", exit.cause); + Deno.exit(1); + } + }); +} diff --git a/x/swift-s3-tests.ts b/x/swift-s3-tests.ts new file mode 100644 index 0000000..1ca12ec --- /dev/null +++ b/x/swift-s3-tests.ts @@ -0,0 +1,207 @@ +#!/usr/bin/env -S deno run --allow-all + +/** + * Herald Swift Compatibility Test Runner + * + * This script runs the Ceph s3-tests suite against a Herald proxy instance + * configured with an OpenStack Swift backend. + */ + +import { Config, Effect, Layer, Logger, LogLevel } from "effect"; +import { makeTestHarness } from "../tests/utils.ts"; +import type { GlobalConfig } from "../src/Domain/Config.ts"; +import * as path from "@std/path"; +import { $ } from "./utils.ts"; +import * as colors from "@std/fmt/colors"; + +const program = Effect.gen(function* () { + const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); + const s3TestsDir = path.resolve(__dirname, "../s3-tests"); + const proxyLogPath = path.join(s3TestsDir, "herald-proxy-swift.log"); + + // Read Swift config from environment + const authUrl = yield* Config.string("HERALD_SWIFTTEST_AUTH_URL").pipe( + Config.orElse(() => Config.string("HEARLD_SWIFTTEST_AUTH_URL")), + Config.withDefault(""), + ); + const region = yield* Config.string("HERALD_SWIFTTEST_OS_REGION_NAME").pipe( + Config.orElse(() => Config.string("HEARLD_SWIFTTEST_OS_REGION_NAME")), + Config.withDefault(""), + ); + const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( + Config.withDefault(""), + ); + const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( + Config.withDefault(""), + ); + const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") + .pipe(Config.withDefault("")); + + if (!authUrl || !username || !password || !projectName) { + return yield* Effect.fail( + new Error( + "Swift environment variables (HERALD_SWIFTTEST_...) are missing. Run with infisical.", + ), + ); + } + + const swiftConfig: GlobalConfig = { + backends: { + swift: { + protocol: "swift", + auth_url: authUrl, + region: region || undefined, + credentials: { + username, + password, + project_name: projectName, + user_domain_name: "Default", + project_domain_name: "Default", + }, + buckets: "*", + }, + }, + }; + + // Create a file logger for the proxy + const proxyLogFile = yield* Effect.tryPromise(() => + Deno.open(proxyLogPath, { write: true, create: true, truncate: true }) + ); + + yield* Effect.addFinalizer(() => + Effect.tryPromise({ + try: () => Promise.resolve(proxyLogFile.close()), + catch: (e) => new Error(`Failed to close proxy log file: ${e}`), + }).pipe(Effect.orDie) + ); + + // Provide the test harness + const h = yield* makeTestHarness(swiftConfig); + const port = new URL(h.proxyUrl).port; + + console.log(`Starting Herald (Swift backend) on port ${colors.cyan(port)}`); + console.log(`Proxy logs: ${colors.gray(proxyLogPath)}`); + + const confContent = `[DEFAULT] +host = 127.0.0.1 +port = ${port} +is_secure = no + +[fixtures] +bucket prefix = herald-swift-{random}- + +[s3 main] +user_id = main +display_name = main +email = main@example.com +access_key = dummy +secret_key = dummy + +[s3 alt] +user_id = alt +display_name = alt +email = alt@example.com +access_key = dummy +secret_key = dummy +`; + + const confPath = yield* Effect.promise(() => + Deno.makeTempFile({ suffix: ".conf" }) + ); + yield* Effect.promise(() => Deno.writeTextFile(confPath, confContent)); + yield* Effect.addFinalizer(() => + Effect.promise(() => + Deno.remove(confPath).catch((e) => { + console.error(`Failed to remove conf file ${confPath}: ${e}`); + }) + ) + ); + + const logPath = path.join(s3TestsDir, "s3-tests-swift.log"); + const junitXmlPath = path.join(s3TestsDir, "junit-swift.xml"); + + const rawArgs = $.argv; + const noAbort = rawArgs.includes("--no-abort"); + const pytestArgsFromCli = rawArgs.filter((arg) => arg !== "--no-abort"); + + const cmdArgs: string[] = [ + "run", + "pytest", + "-v", + "--tb=short", + `--junit-xml=${junitXmlPath}`, + ...pytestArgsFromCli, + ]; + + // If no specific test path, default to test_s3.py + if ( + !pytestArgsFromCli.some((arg) => + arg.includes("s3tests/") || arg.endsWith(".py") + ) + ) { + cmdArgs.push("s3tests/functional/test_s3.py"); + } + + console.log(`Running s3-tests against Herald (Swift)...`); + + const logFile = yield* Effect.tryPromise(() => + Deno.open(logPath, { write: true, create: true, truncate: true }) + ); + yield* Effect.addFinalizer(() => + Effect.promise(() => Promise.resolve(logFile.close())) + ); + + const result = yield* Effect.tryPromise({ + try: async () => { + const child = $`uv ${cmdArgs}` + .cwd(s3TestsDir) + .env({ + S3TEST_CONF: confPath, + UV_PYTHON: "3.11", + PYTHONUNBUFFERED: "1", + }) + .noThrow() + .stdout("piped") + .stderr("piped") + .spawn(); + + const decoder = new TextDecoder(); + async function streamToLog(stream: ReadableStream) { + const reader = stream.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) break; + await logFile.write(value); + Deno.stdout.writeSync(value); // Echo to console for now + } + } + + const [procResult] = await Promise.all([ + child, + streamToLog(child.stdout()), + streamToLog(child.stderr()), + ]); + + return procResult; + }, + catch: (e) => new Error(`Failed to run pytest: ${e}`), + }); + + if (result.code !== 0) { + yield* Effect.fail(new Error(`s3-tests failed with code ${result.code}`)); + } + + console.log(colors.green(`\n✓ s3-tests completed successfully.`)); +}).pipe( + Effect.scoped, + Effect.provide(Logger.minimumLogLevel(LogLevel.Info)), +); + +if (import.meta.main) { + Effect.runPromiseExit(program).then((exit) => { + if (exit._tag === "Failure") { + console.error(colors.red(`Error: ${exit.cause}`)); + Deno.exit(1); + } + }); +} From 9d27f1c6695cd0aa93825e8e6481ec49c2c2d62f Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Tue, 20 Jan 2026 03:09:10 +0300 Subject: [PATCH 2/3] wip: clean symlinks --- .gitignore | 1 + ghjk | 1 - herald | 1 - s3proxy | 1 - sample-http | 1 - sample-rust | 1 - 6 files changed, 1 insertion(+), 5 deletions(-) delete mode 120000 ghjk delete mode 120000 herald delete mode 120000 s3proxy delete mode 120000 sample-http delete mode 120000 sample-rust diff --git a/.gitignore b/.gitignore index c8be0ce..7f896dd 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,4 @@ token *.db-shm *.db-wal .vscode +symlinks diff --git a/ghjk b/ghjk deleted file mode 120000 index 4855491..0000000 --- a/ghjk +++ /dev/null @@ -1 +0,0 @@ -../../rust/ghjk/ \ No newline at end of file diff --git a/herald b/herald deleted file mode 120000 index bfe949e..0000000 --- a/herald +++ /dev/null @@ -1 +0,0 @@ -../herald \ No newline at end of file diff --git a/s3proxy b/s3proxy deleted file mode 120000 index 27ea8bf..0000000 --- a/s3proxy +++ /dev/null @@ -1 +0,0 @@ -../../java/s3proxy/ \ No newline at end of file diff --git a/sample-http b/sample-http deleted file mode 120000 index dfd9d33..0000000 --- a/sample-http +++ /dev/null @@ -1 +0,0 @@ -../sample-http/ \ No newline at end of file diff --git a/sample-rust b/sample-rust deleted file mode 120000 index b7cd242..0000000 --- a/sample-rust +++ /dev/null @@ -1 +0,0 @@ -../../rust/Yohe-Am-backend-1/ \ No newline at end of file From bff2cac297382de86d333f48a61b53f7e92f2d0b Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Tue, 20 Jan 2026 09:03:31 +0300 Subject: [PATCH 3/3] feat(s3): multipart support (#80) --- .github/workflows/checks.yml | 94 +++ .github/workflows/pre-commit.yml | 24 - .github/workflows/release-request.yml | 159 ---- .github/workflows/tests.yml | 119 --- .pre-commit-config.yaml | 9 +- AGENTS.md | 2 + CONTRIBUTING.md | 4 +- chart/values.yaml | 7 +- deno.jsonc | 5 + deno.lock | 2 + flake.nix | 4 +- src/Api.ts | 13 +- src/Backends/S3/Backend.ts | 668 +--------------- src/Backends/S3/Buckets.ts | 74 ++ src/Backends/S3/Client.ts | 174 ++-- src/Backends/S3/Objects.ts | 757 ++++++++++++++++++ src/Backends/S3/Utils.ts | 147 ++++ src/Backends/Swift/Backend.ts | 609 +------------- src/Backends/Swift/Buckets.ts | 143 ++++ src/Backends/Swift/Client.ts | 289 ++++--- src/Backends/Swift/Objects.ts | 529 ++++++++++++ src/Backends/Swift/Utils.ts | 74 ++ src/Config/Layer.ts | 8 +- src/Domain/Config.ts | 2 +- src/Frontend/Api.ts | 2 +- src/Frontend/Buckets/Create.ts | 63 +- src/Frontend/Buckets/Delete.ts | 16 +- src/Frontend/Buckets/Head.ts | 16 +- src/Frontend/Buckets/List.ts | 4 +- src/Frontend/Health/Api.ts | 2 +- src/Frontend/Health/Http.ts | 4 +- src/Frontend/Http.ts | 30 +- src/Frontend/Objects/Delete.ts | 24 +- src/Frontend/Objects/Get.ts | 43 +- src/Frontend/Objects/Head.ts | 31 +- src/Frontend/Objects/List.ts | 76 +- src/Frontend/Objects/Post.ts | 197 +++-- src/Frontend/Objects/Put.ts | 42 +- src/Frontend/Utils.ts | 152 +++- src/Http.ts | 19 +- src/Logging/Layer.ts | 4 +- src/Services/Backend.ts | 139 +++- src/Services/BackendResolver.ts | 125 ++- src/Services/S3Xml.ts | 116 +++ src/main.ts | 10 +- tests/config.test.ts | 8 +- tests/health.test.ts | 16 +- .../__snapshots__/buckets.test.ts.snap | 4 +- .../__snapshots__/objects.test.ts.snap | 94 ++- tests/integration/objects.test.ts | 179 +++++ tests/utils.ts | 10 +- tools/compose.yml | 2 + x/compose-down.ts | 4 +- x/compose-up.ts | 4 +- x/purge-minio.ts | 1 - x/s3-tests.ts | 3 - x/swift-debug.ts | 7 +- 57 files changed, 3228 insertions(+), 2135 deletions(-) create mode 100644 .github/workflows/checks.yml delete mode 100644 .github/workflows/pre-commit.yml delete mode 100644 .github/workflows/release-request.yml delete mode 100644 .github/workflows/tests.yml create mode 100644 src/Backends/S3/Buckets.ts create mode 100644 src/Backends/S3/Objects.ts create mode 100644 src/Backends/S3/Utils.ts create mode 100644 src/Backends/Swift/Buckets.ts create mode 100644 src/Backends/Swift/Objects.ts create mode 100644 src/Backends/Swift/Utils.ts diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..13c559e --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,94 @@ +name: checks + +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + +env: + DOCKER_CMD: docker + UV_CACHE_DIR: /tmp/.uv-cache + +jobs: + checks: + runs-on: ubuntu-latest + env: + HERALD_SWIFTTEST_OS_USERNAME: ${{ secrets.OPENSTACK_USERNAME }} + HERALD_SWIFTTEST_OS_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD }} + HERALD_SWIFTTEST_OS_PROJECT_NAME: ${{ secrets.OPENSTACK_PROJECT }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v16 + + - name: Set up Nix cache + uses: DeterminateSystems/magic-nix-cache-action@v9 + + - name: Run pre-commit hooks via prek + run: nix develop --command prek run --all-files + + - name: Cache Deno + uses: actions/cache@v4 + with: + path: ~/.cache/deno + key: ${{ runner.os }}-deno-${{ hashFiles('deno.lock') }} + restore-keys: | + ${{ runner.os }}-deno- + + - name: Restore uv cache + uses: actions/cache@v5 + with: + path: /tmp/.uv-cache + key: uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }} + restore-keys: | + uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }} + uv-${{ runner.os }} + + - name: Start services + run: nix develop --command deno run --allow-all x/compose-up.ts s3 db + + - name: Wait for MinIO + run: | + for i in {1..30}; do + if curl -f http://localhost:9000/minio/health/live; then + echo "MinIO is ready" + exit 0 + fi + echo "Waiting for MinIO..." + sleep 2 + done + echo "MinIO failed to start" + exit 1 + + - name: Integration tests + run: nix develop --command deno task test + + - name: S3 Compatibility (MinIO) + run: nix develop --command deno run --allow-all x/s3-tests.ts --backend minio + + - name: S3 Compatibility (Swift) + if: env.HERALD_SWIFTTEST_OS_USERNAME != '' + env: + HERALD_SWIFTTEST_OS_REGION_NAME: dc3-a + HERALD_SWIFTTEST_AUTH_URL: https://api.pub1.infomaniak.cloud/identity/v3 + run: nix develop --command deno run --allow-all x/s3-tests.ts --backend swift + + - name: Minimize uv cache + run: nix develop --command uv cache prune --ci + + - name: Dump logs on failure + if: failure() + run: | + echo "--- s3-tests/s3-tests.log ---" + cat s3-tests/s3-tests.log || true + echo "--- s3-tests/herald-proxy.log ---" + cat s3-tests/herald-proxy.log || true + echo "--- s3-tests/herald-proxy-swift.log ---" + cat s3-tests/herald-proxy-swift.log || true diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index 630ff40..0000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: pre-commit - -on: - push: - branches: [main] - pull_request: - branches: [main] - workflow_dispatch: - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install Nix - uses: DeterminateSystems/nix-installer-action@main - - - name: Set up Nix cache - uses: DeterminateSystems/magic-nix-cache-action@main - - - name: Run pre-commit hooks via prek - run: nix develop --command prek run --all-files diff --git a/.github/workflows/release-request.yml b/.github/workflows/release-request.yml deleted file mode 100644 index e11d30b..0000000 --- a/.github/workflows/release-request.yml +++ /dev/null @@ -1,159 +0,0 @@ -name: Prepare Release - -on: - workflow_dispatch: - push: - branches: - - main - -jobs: - check-version: - name: Check Commitizen Version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.version.outputs.version }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Configure Git - run: | - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - - - name: Get current version (without bumping or pushing) - id: version - uses: commitizen-tools/commitizen-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - push: false - dry_run: true - changelog: false - - prepare-release-pr: - name: Create Release Branch and PR - needs: check-version - if: ${{ needs.check-version.outputs.version != '' && github.ref == 'refs/heads/main' }} - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: main - - - name: Bump version using Commitizen - id: cz - uses: commitizen-tools/commitizen-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - git_name: ${{ github.actor }} - git_email: ${{ github.actor }}@users.noreply.github.com - push: false - changelog: true - dry_run: false - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v8 - with: - title: "Release ${{ steps.cz.outputs.version }}" - body: "Automated PR for version bump to ${{ steps.cz.outputs.version }}" - branch: "release-v${{ steps.cz.outputs.version }}" - delete-branch: true - - check-release: - runs-on: ubuntu-latest - # if: github.ref == 'refs/heads/main' && github.event_name == 'push' - outputs: - release: ${{ steps.check.outputs.release }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: main - - - name: Configure Git - run: | - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - - - name: Get current version - id: version - run: | - VERSION=$(yq '.commitizen.version' .cz.yaml) - echo "version=$VERSION" >> $GITHUB_OUTPUT - - - name: Check if GitHub release already exists - id: check - run: | - VERSION=${{ steps.version.outputs.version }} - echo "Detected version: $VERSION" - - RELEASE_EXISTS=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - https://api.github.com/repos/${{ github.repository }}/releases/tags/v$VERSION \ - | jq -r '.tag_name // empty') - - if [[ "$RELEASE_EXISTS" == "v$VERSION" ]]; then - echo "Release v$VERSION already exists." - echo "release=" >> $GITHUB_OUTPUT - else - echo "Release v$VERSION does not exist yet." - echo "release=$VERSION" >> $GITHUB_OUTPUT - fi - finalize-release: - name: Finalize Release - needs: check-release - if: ${{ needs.check-release.outputs.release != '' }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Tag and Push - run: | - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - git tag -a "v${{ needs.check-release.outputs.release }}" -m "Release v${{ needs.check-release.outputs.release }}" - git push origin "v${{ needs.check-release.outputs.release }}" - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - tag_name: "v${{ needs.check-release.outputs.release }}" - name: "Release v${{ needs.check-release.outputs.release }}" - body_path: "CHANGELOG.md" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - build-docker: - name: Build and Push Docker - needs: check-release - if: ${{ needs.check-release.outputs.release != '' }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and Push Docker - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ghcr.io/${{ github.repository_owner }}/herald:v${{ needs.check-release.outputs.release }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index f09fed5..0000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: test suite -run-name: test suite for ${{ github.event.pull_request.title || github.ref }} -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - types: - - opened - - reopened - - synchronize - - ready_for_review - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - DENO_V: 2.3.5 - GHJK_VERSION: "v0.3.2" - GHJK_ENV: "ci" - -jobs: - changes: - runs-on: ubuntu-latest - permissions: - pull-requests: read - steps: - - uses: actions/checkout@v6 - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - full: - - '.github/workflows/tests.yml' - - 'src/**' - - 'tests/**' - - 'examples/**' - outputs: - full: ${{ steps.filter.outputs.full }} - - pre-commit: - needs: changes - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - uses: denoland/setup-deno@v2 - with: - deno-version: ${{ env.DENO_V }} - - name: Install tofu - run: | - curl --proto '=https' --tlsv1.2 -fsSL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh - chmod +x install-opentofu.sh - ./install-opentofu.sh --install-method deb - rm -f install-opentofu.sh - - - shell: bash - run: | - python -m pip install --upgrade pip - pip install pre-commit - pre-commit install - deno --version - pre-commit run --all-files - - test-full: - needs: [changes] - if: ${{ needs.changes.outputs.full == 'true' && github.event.pull_request.draft == false }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: denoland/setup-deno@v2 - with: - deno-version: ${{ env.DENO_V }} - - name: Download Install Script - run: curl -fsSL "https://raw.github.com/metatypedev/ghjk/$GHJK_VERSION/install.sh" -o install.sh - - name: Execute Install Script - run: yes | bash install.sh - - run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - run: echo "BASH_ENV=$HOME/.local/share/ghjk/env.sh" >> "$GITHUB_ENV" - - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - name: Install tofu - run: | - curl --proto '=https' --tlsv1.2 -fsSL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh - chmod +x install-opentofu.sh - ./install-opentofu.sh --install-method deb - rm -f install-opentofu.sh - - uses: actions/setup-node@v6 - with: - node-version: 18 - - name: setup start-server-and-test - run: npm install -g start-server-and-test - - shell: bash - env: - AUTH_TYPE: "default" - LOG_LEVEL: "DEBUG" - ENV: "DEV" - S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} - S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} - OPENSTACK_USERNAME: ${{ secrets.OPENSTACK_USERNAME }} - OPENSTACK_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD }} - OPENSTACK_PROJECT: ${{ secrets.OPENSTACK_PROJECT }} - AWS_ACCESS_KEY_ID: ${{ secrets.OPENSTACK_USERNAME }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.OPENSTACK_PASSWORD }} - run: | - # run all tests - deno --version - ghjk x dev-compose s3 - sleep 20 - - deno install - - # ghjk x setup-auth - npx start-server-and-test 'deno serve -A --unstable-kv src/main.ts' http://0.0.0.0:8000/ 'deno test -A' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b600640..96b71d2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v6.0.0 hooks: - id: check-added-large-files exclude: tests/res @@ -39,9 +39,14 @@ repos: types: - ts - repo: https://github.com/tofuutils/pre-commit-opentofu - rev: v1.0.3 + rev: v2.2.2 hooks: - id: tofu_fmt + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.36.0 + hooks: + - id: check-dependabot + - id: check-github-workflows # - repo: https://github.com/shellcheck-py/shellcheck-py # rev: v0.10.0.1 # hooks: diff --git a/AGENTS.md b/AGENTS.md index fbc11d0..4410833 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,7 @@ - We're using the effects library https://effect.website/llms.txt - Their HTTP implementation is described in ./HTTP_PLATFORM.md + - **ALWAYS** use `@effect/platform/HttpClient` instead of native `fetch` for + all HTTP requests. - Prefer generators over effect piping. - Use methods on `Effect.Option` like `Option.isNone` instead of looking at _tag. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96c5f88..219781a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,8 +5,8 @@ - `src/Domain`: Core logic and data models. Contains Effect Schemas for global configuration and logic for bucket matching. -- `src/Config`: Application configuration loading. Defines the AppConfig service - layer. +- `src/Config`: Application configuration loading. Defines the HeraldConfig + service layer. - `src/Services`: Shared service abstractions and implementations. diff --git a/chart/values.yaml b/chart/values.yaml index e55ba44..1691974 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1,4 +1,3 @@ - name: herald namespace: herald @@ -87,9 +86,9 @@ ingress: - path: / pathType: ImplementationSpecific tls: [] - # - secretName: web-tls - # hosts: - # - chart-example.local +# - secretName: web-tls +# hosts: +# - chart-example.local volumeMounts: - name: herald diff --git a/deno.jsonc b/deno.jsonc index c600c50..646ee69 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -30,6 +30,11 @@ "cliffy/ansi/": "https://deno.land/x/cliffy@v1.0.0-rc.3/ansi/" }, "compilerOptions": {}, + "fmt": { + "exclude": [ + "./chart/" + ] + }, "lint": { "exclude": [ "x", diff --git a/deno.lock b/deno.lock index da816a9..766c865 100644 --- a/deno.lock +++ b/deno.lock @@ -26,12 +26,14 @@ "npm:@aws-sdk/client-s3@3": "3.937.0", "npm:@effect/opentelemetry@~0.56.2": "0.56.6_@effect+platform@0.90.10__effect@3.19.14_@opentelemetry+sdk-trace-base@2.3.0__@opentelemetry+api@1.9.0_@opentelemetry+sdk-trace-node@2.3.0__@opentelemetry+api@1.9.0_@opentelemetry+semantic-conventions@1.38.0_effect@3.19.14", "npm:@effect/platform-node@0.96": "0.96.1_@effect+cluster@0.48.16__@effect+platform@0.90.10___effect@3.19.14__@effect+rpc@0.69.5___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+sql@0.44.2___@effect+experimental@0.54.6____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+workflow@0.9.6___@effect+platform@0.90.10____effect@3.19.14___@effect+rpc@0.69.5____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___effect@3.19.14__effect@3.19.14_@effect+platform@0.90.10__effect@3.19.14_@effect+rpc@0.69.5__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_@effect+sql@0.44.2__@effect+experimental@0.54.6___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_effect@3.19.14", + "npm:@effect/platform@*": "0.90.10_effect@3.19.14", "npm:@effect/platform@~0.90.3": "0.90.10_effect@3.19.14", "npm:@opentelemetry/exporter-trace-otlp-http@0.203": "0.203.0_@opentelemetry+api@1.9.0", "npm:@opentelemetry/sdk-trace-base@^2.0.1": "2.3.0_@opentelemetry+api@1.9.0", "npm:@opentelemetry/sdk-trace-node@^2.0.1": "2.3.0_@opentelemetry+api@1.9.0", "npm:@smithy/signature-v4@^4.2.0": "4.2.4", "npm:@smithy/types@^3.7.0": "3.7.2", + "npm:effect@*": "3.19.14", "npm:effect@^3.17.7": "3.19.14", "npm:jest-diff@*": "29.7.0", "npm:jest-diff@^29.7.0": "29.7.0", diff --git a/flake.nix b/flake.nix index 1e0d33f..a1e07ab 100644 --- a/flake.nix +++ b/flake.nix @@ -41,7 +41,9 @@ ]; shellHook = '' export PATH=$PATH:$PWD/x/ - exec $(getent passwd $USER | cut -d: -f7) + if [[ -t 0 ]]; then + exec $(getent passwd $USER | cut -d: -f7) + fi ''; }; diff --git a/src/Api.ts b/src/Api.ts index 2b3afe2..b7a62ec 100644 --- a/src/Api.ts +++ b/src/Api.ts @@ -1,8 +1,11 @@ import { HttpApi, OpenApi } from "@effect/platform"; -import { HealthApi } from "./Frontend/Health/Api.ts"; -import { S3Api } from "./Frontend/Api.ts"; +import { HealthHttpApi } from "./Frontend/Health/Api.ts"; +import { HttpS3Api } from "./Frontend/Api.ts"; -export class Api extends HttpApi.make("api") - .add(HealthApi) - .add(S3Api) +// the http interface is declared first and separately +// and the impl is to adhere to it +// used for openAPI +export class HttpHeraldApi extends HttpApi.make("HeraldHttpApi") + .add(HealthHttpApi) + .add(HttpS3Api) .annotate(OpenApi.Title, "Herald API") {} diff --git a/src/Backends/S3/Backend.ts b/src/Backends/S3/Backend.ts index e888527..fc930c7 100644 --- a/src/Backends/S3/Backend.ts +++ b/src/Backends/S3/Backend.ts @@ -1,658 +1,24 @@ -import { Chunk, Effect, Option, Stream } from "effect"; -import { - CreateBucketCommand, - DeleteBucketCommand, - DeleteObjectCommand, - DeleteObjectsCommand, - GetObjectCommand, - HeadBucketCommand, - HeadObjectCommand, - ListBucketsCommand, - type ListBucketsCommandOutput, - ListObjectsCommand, - type ListObjectsCommandOutput, - ListObjectsV2Command, - type ListObjectsV2CommandOutput, - ListObjectVersionsCommand, - PutObjectCommand, -} from "@aws-sdk/client-s3"; +import { Effect } from "effect"; import type { MaterializedBucket } from "../../Domain/Config.ts"; -import { AppConfig } from "../../Config/Layer.ts"; -import { - AccessDenied, - type BackendError, - type BackendService, - BucketAlreadyExists, - BucketAlreadyOwnedByYou, - type BucketInfo, - BucketNotEmpty, - type CommonPrefix, - type DeleteObjectsResult, - InternalError, - type ListObjectsResult, - NoSuchBucket, - NoSuchKey, - type ObjectInfo, -} from "../../Services/Backend.ts"; -import { S3Client } from "./Client.ts"; - -/** - * Strips MinIO metadata suffixes like [minio_cache:v2,return:] from strings. - */ -function stripMinioMetadata(s: string): string { - return s.replace(/\[minio_cache:[^\]]+\]/g, ""); -} - -/** - * Maps S3 SDK exceptions to internal BackendError types. - */ -function mapS3Error(e: unknown, bucketName?: string): BackendError { - const err = e as { - name?: string; - Code?: string; - Message?: string; - message?: string; - $metadata?: { httpStatusCode?: number }; - }; - const name = err?.name || err?.Code || - (e instanceof Error ? e.name : "UnknownError"); - const message = err?.message || err?.Message || - "An unknown S3 error occurred"; - const bucket = bucketName ?? "unknown-bucket"; - - switch (name) { - case "NoSuchBucket": - case "NotFound": - return new NoSuchBucket({ bucketName: bucket, message }); - case "NoSuchKey": - return new NoSuchKey({ - bucketName: bucket, - key: "unknown", - message: message, - }); - case "BucketAlreadyExists": - return new BucketAlreadyExists({ bucketName: bucket, message }); - case "BucketAlreadyOwnedByYou": - return new BucketAlreadyOwnedByYou({ bucketName: bucket, message }); - case "AccessDenied": - case "Forbidden": - return new AccessDenied({ message }); - case "BucketNotEmpty": - case "Conflict": - return new BucketNotEmpty({ bucketName: bucket, message }); - } - - // Handle case where it might be a raw 404 from HEAD request - if (err?.$metadata?.httpStatusCode === 404) { - return new NoSuchKey({ - bucketName: bucket, - key: "unknown", - message: "Not Found", - }); - } - - return new InternalError({ - message: e instanceof Error ? `${e.name}: ${e.message}` : String(e), - }); -} +import type { BackendError, BackendService } from "../../Services/Backend.ts"; +import { makeBucketOps } from "./Buckets.ts"; +import { makeObjectOps } from "./Objects.ts"; +import { getTarget } from "./Utils.ts"; +import type { S3Client } from "./Client.ts"; +import type { HeraldConfig } from "../../Config/Layer.ts"; /** * Creates an S3-specific Backend implementation for a given configuration context. + * Composes bucket and object operations modularly. + * Resolves the target once per backend creation (request-scoped). */ export const makeS3Backend = ( bucket: MaterializedBucket | { backend_id: string }, -): Effect.Effect => - Effect.all({ - s3Service: S3Client, - config: AppConfig, - }).pipe( - Effect.map(({ s3Service, config }) => { - const getTargetBucket = (): MaterializedBucket => { - if ("bucket_name" in bucket) return bucket as MaterializedBucket; - - const backendConfig = config.raw.backends[bucket.backend_id]; - if (backendConfig && backendConfig.protocol === "s3") { - return { - name: "", - backend_id: bucket.backend_id, - protocol: "s3" as const, - endpoint: backendConfig.endpoint, - region: backendConfig.region, - bucket_name: "", - credentials: backendConfig.credentials, - }; - } - throw new Error(`Backend ${bucket.backend_id} is not an S3 backend`); - }; - - const targetBucket = getTargetBucket(); - - const service: BackendService = { - listBuckets: () => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send(new ListBucketsCommand({})) as Promise< - ListBucketsCommandOutput - >, - catch: (e) => mapS3Error(e, targetBucket.name), - }) - ), - Effect.flatMap((result) => { - const buckets: BucketInfo[] = []; - for (const b of (result.Buckets ?? [])) { - if (b.Name === undefined) { - return Effect.fail( - new InternalError({ - message: "S3 returned bucket without Name", - }), - ); - } - buckets.push({ - name: b.Name, - creationDate: b.CreationDate, - }); - } - - return Effect.succeed({ - buckets, - owner: { - id: result.Owner?.ID ?? "unknown-owner-id", - displayName: result.Owner?.DisplayName ?? - "unknown-owner-name", - }, - }); - }), - ), - - createBucket: () => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new CreateBucketCommand({ - Bucket: targetBucket.bucket_name, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map(() => undefined), - ), - - deleteBucket: () => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new DeleteBucketCommand({ - Bucket: targetBucket.bucket_name, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map(() => undefined), - ), - - headBucket: () => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new HeadBucketCommand({ Bucket: targetBucket.bucket_name }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map(() => undefined), - ), - - listObjects: (args) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => { - if (args.listType === 2) { - return Effect.tryPromise({ - try: () => - client.send( - new ListObjectsV2Command({ - Bucket: targetBucket.bucket_name, - Prefix: args.prefix, - Delimiter: args.delimiter, - MaxKeys: args.maxKeys, - ContinuationToken: args.continuationToken, - StartAfter: args.startAfter, - }), - ) as Promise, - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }).pipe( - Effect.map((result): ListObjectsResult => ({ - name: result.Name ?? targetBucket.bucket_name, - prefix: result.Prefix, - maxKeys: result.MaxKeys ?? 1000, - delimiter: result.Delimiter, - isTruncated: result.IsTruncated ?? false, - encodingType: args.encodingType, - continuationToken: result.ContinuationToken, - nextContinuationToken: result.NextContinuationToken, - keyCount: result.KeyCount, - listType: 2, - contents: (result.Contents ?? []).map((c): ObjectInfo => ({ - key: stripMinioMetadata(c.Key ?? ""), - lastModified: c.LastModified ?? new Date(), - etag: c.ETag ?? "", - size: c.Size ?? 0, - storageClass: c.StorageClass, - owner: c.Owner - ? { - id: c.Owner.ID ?? "unknown", - displayName: c.Owner.DisplayName ?? "unknown", - } - : undefined, - })), - commonPrefixes: (result.CommonPrefixes ?? []).map(( - cp, - ): CommonPrefix => ({ - prefix: stripMinioMetadata(cp.Prefix ?? ""), - })), - })), - ); - } else { - return Effect.tryPromise({ - try: () => - client.send( - new ListObjectsCommand({ - Bucket: targetBucket.bucket_name, - Prefix: args.prefix, - Delimiter: args.delimiter, - Marker: args.marker, - MaxKeys: args.maxKeys, - }), - ) as Promise, - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }).pipe( - Effect.map((result): ListObjectsResult => ({ - name: result.Name ?? targetBucket.bucket_name, - prefix: result.Prefix, - marker: result.Marker, - nextMarker: result.NextMarker, - maxKeys: result.MaxKeys ?? 1000, - delimiter: result.Delimiter, - isTruncated: result.IsTruncated ?? false, - encodingType: args.encodingType, - listType: 1, - contents: (result.Contents ?? []).map((c): ObjectInfo => ({ - key: stripMinioMetadata(c.Key ?? ""), - lastModified: c.LastModified ?? new Date(), - etag: c.ETag ?? "", - size: c.Size ?? 0, - storageClass: c.StorageClass, - owner: c.Owner - ? { - id: c.Owner.ID ?? "unknown", - displayName: c.Owner.DisplayName ?? "unknown", - } - : undefined, - })), - commonPrefixes: (result.CommonPrefixes ?? []).map(( - cp, - ): CommonPrefix => ({ - prefix: stripMinioMetadata(cp.Prefix ?? ""), - })), - })), - ); - } - }), - ), - - listVersions: (args) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new ListObjectVersionsCommand({ - Bucket: targetBucket.bucket_name, - Prefix: args.prefix, - Delimiter: args.delimiter, - KeyMarker: args.keyMarker, - VersionIdMarker: args.versionIdMarker, - MaxKeys: args.maxKeys, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map((result): ListObjectsResult => ({ - name: result.Name ?? targetBucket.bucket_name, - prefix: result.Prefix, - marker: result.KeyMarker, - nextMarker: result.NextKeyMarker, - maxKeys: result.MaxKeys ?? 1000, - delimiter: result.Delimiter, - isTruncated: result.IsTruncated ?? false, - encodingType: args.encodingType, - listType: 1, // listVersions is similar to V1 - contents: [ - ...(result.Versions ?? []).map((v): ObjectInfo => ({ - key: stripMinioMetadata(v.Key ?? ""), - lastModified: v.LastModified ?? new Date(), - etag: v.ETag ?? "", - size: v.Size ?? 0, - storageClass: v.StorageClass, - versionId: v.VersionId, - isDeleteMarker: false, - isLatest: v.IsLatest, - owner: v.Owner - ? { - id: v.Owner.ID ?? "unknown", - displayName: v.Owner.DisplayName ?? "unknown", - } - : undefined, - })), - ...(result.DeleteMarkers ?? []).map((dm): ObjectInfo => ({ - key: stripMinioMetadata(dm.Key ?? ""), - lastModified: dm.LastModified ?? new Date(), - etag: "", - size: 0, - versionId: dm.VersionId, - isDeleteMarker: true, - isLatest: dm.IsLatest, - owner: dm.Owner - ? { - id: dm.Owner.ID ?? "unknown", - displayName: dm.Owner.DisplayName ?? "unknown", - } - : undefined, - })), - ], - commonPrefixes: (result.CommonPrefixes ?? []).map(( - cp, - ): CommonPrefix => ({ - prefix: stripMinioMetadata(cp.Prefix ?? ""), - })), - })), - ), - - getObject: (key) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new GetObjectCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.flatMap((result) => { - const body = result.Body; - if (!body) { - return Effect.fail( - new InternalError({ - message: "S3 returned empty body for GetObject", - }), - ); - } - - // AWS SDK Body can be many things. In Deno/Browser it has transformToWebStream() - // Use a type-safe check to avoid 'any' - const getWebStream = (): ReadableStream => { - if ( - body && typeof body === "object" && - "transformToWebStream" in body - ) { - const b = body as { transformToWebStream: unknown }; - if (typeof b.transformToWebStream === "function") { - return b.transformToWebStream() as ReadableStream< - Uint8Array - >; - } - } - return body as ReadableStream; - }; - - const stream = Stream.fromReadableStream( - getWebStream, - (e) => new Error(String(e)), - ); - - const metadata: Record = {}; - if (result.Metadata) { - for (const [k, v] of Object.entries(result.Metadata)) { - metadata[k] = Option.liftThrowable(decodeURIComponent)( - v ?? "", - ).pipe( - Option.getOrElse(() => v ?? ""), - ); - } - } - - const headers: Record = {}; - if (result.ContentType) { - headers["content-type"] = result.ContentType; - } - if (result.ETag) headers["etag"] = result.ETag; - if (result.LastModified) { - headers["last-modified"] = result.LastModified.toUTCString(); - } - - for (const [k, v] of Object.entries(metadata)) { - headers[`x-amz-meta-${k}`] = v; - } - - // Buffer the entire stream to ensure it's fully read and connection is closed - // This also addresses issues where the SDK's Body might not be a standard ReadableStream - return Stream.runCollect(stream).pipe( - Effect.mapError((e) => - new InternalError({ message: String(e) }) - ), - Effect.map((chunks) => { - const totalLength = Chunk.reduce( - chunks, - 0, - (acc, chunk) => acc + chunk.length, - ); - const all = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of chunks) { - all.set(chunk, offset); - offset += chunk.length; - } - return { - stream: Stream.succeed(all), - contentType: result.ContentType, - contentLength: all.length, - etag: result.ETag, - lastModified: result.LastModified, - metadata, - headers, - }; - }), - ); - }), - ), - - headObject: (key) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new HeadObjectCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map((result) => { - const metadata: Record = {}; - if (result.Metadata) { - for (const [k, v] of Object.entries(result.Metadata)) { - metadata[k] = Option.liftThrowable(decodeURIComponent)( - v ?? "", - ).pipe( - Option.getOrElse(() => v ?? ""), - ); - } - } - - const headers: Record = {}; - if (result.ContentType) { - headers["content-type"] = result.ContentType; - } - if (result.ContentLength !== undefined) { - headers["content-length"] = String(result.ContentLength); - } - if (result.ETag) headers["etag"] = result.ETag; - if (result.LastModified) { - headers["last-modified"] = result - .LastModified.toUTCString(); - } - - for (const [k, v] of Object.entries(metadata)) { - headers[`x-amz-meta-${k}`] = v; - } - - return { - contentType: result.ContentType, - contentLength: result.ContentLength, - etag: result.ETag, - lastModified: result.LastModified, - metadata, - headers, - }; - }), - ), - - putObject: (key, bodyStream, headers) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Stream.runCollect(bodyStream).pipe( - Effect.mapError((e) => - new InternalError({ message: String(e) }) - ), - Effect.flatMap((chunks) => { - const totalLength = Chunk.reduce( - chunks, - 0, - (acc, chunk) => acc + chunk.length, - ); - const body = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of chunks) { - body.set(chunk, offset); - offset += chunk.length; - } - - const metadata: Record = {}; - for (const [k, v] of Object.entries(headers)) { - if (k.toLowerCase().startsWith("x-amz-meta-")) { - const metaKey = k.substring("x-amz-meta-".length); - const value = String(v); - metadata[metaKey] = /[^\x20-\x7E]/.test(value) - ? encodeURIComponent(value) - : value; - } - } - - const contentType = headers["content-type"]; - - return Effect.tryPromise({ - try: () => - client.send( - new PutObjectCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - Body: body, - ContentType: contentType - ? String(contentType) - : undefined, - Metadata: metadata, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }); - }), - ) - ), - Effect.map((result) => ({ - etag: result.ETag, - versionId: result.VersionId, - })), - ), - - deleteObject: (key) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new DeleteObjectCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map(() => undefined), - ), - - deleteObjects: ( - objects, - ): Effect.Effect => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new DeleteObjectsCommand({ - Bucket: targetBucket.bucket_name, - Delete: { - Objects: objects.map((o) => ({ - Key: o.key, - VersionId: o.versionId === "null" - ? undefined - : o.versionId, - })), - }, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map((result) => ({ - deleted: (result.Deleted ?? []).map((d) => d.Key ?? ""), - errors: (result.Errors ?? []).map((e) => ({ - key: e.Key ?? "unknown", - code: e.Code ?? "InternalError", - message: e.Message ?? "Unknown error", - })), - })), - ), - }; - - return service; - }), - ); +): Effect.Effect => + Effect.gen(function* () { + const target = yield* getTarget(bucket); + return { + ...makeBucketOps(target), + ...makeObjectOps(target), + } satisfies BackendService; + }); diff --git a/src/Backends/S3/Buckets.ts b/src/Backends/S3/Buckets.ts new file mode 100644 index 0000000..b0ff891 --- /dev/null +++ b/src/Backends/S3/Buckets.ts @@ -0,0 +1,74 @@ +import { Effect } from "effect"; +import { + CreateBucketCommand, + DeleteBucketCommand, + HeadBucketCommand, + ListBucketsCommand, + type ListBucketsCommandOutput, +} from "@aws-sdk/client-s3"; +import { type BucketInfo, InternalError } from "../../Services/Backend.ts"; +import { mapS3Error, type S3Target } from "./Utils.ts"; + +export const makeBucketOps = (target: S3Target) => ({ + listBuckets: () => + Effect.gen(function* () { + const { client, name } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send(new ListBucketsCommand({})) as Promise< + ListBucketsCommandOutput + >, + catch: (e) => mapS3Error(e, name), + }); + + const buckets: BucketInfo[] = []; + for (const b of (result.Buckets ?? [])) { + if (b.Name === undefined) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned bucket without Name", + }), + ); + } + buckets.push({ + name: b.Name, + creationDate: b.CreationDate, + }); + } + + return { + buckets, + owner: { + id: result.Owner?.ID ?? "unknown-owner-id", + displayName: result.Owner?.DisplayName ?? "unknown-owner-name", + }, + }; + }), + + createBucket: () => + Effect.gen(function* () { + const { client, bucketName, name } = target; + yield* Effect.tryPromise({ + try: () => client.send(new CreateBucketCommand({ Bucket: bucketName })), + catch: (e) => mapS3Error(e, bucketName || name), + }); + }), + + deleteBucket: () => + Effect.gen(function* () { + const { client, bucketName, name } = target; + yield* Effect.tryPromise({ + try: () => client.send(new DeleteBucketCommand({ Bucket: bucketName })), + catch: (e) => mapS3Error(e, bucketName || name), + }); + }), + + headBucket: () => + Effect.gen(function* () { + const { client, bucketName, name } = target; + yield* Effect.tryPromise({ + try: () => client.send(new HeadBucketCommand({ Bucket: bucketName })), + catch: (e) => mapS3Error(e, bucketName || name), + }); + }), +}); diff --git a/src/Backends/S3/Client.ts b/src/Backends/S3/Client.ts index bf79c56..376de09 100644 --- a/src/Backends/S3/Client.ts +++ b/src/Backends/S3/Client.ts @@ -1,7 +1,7 @@ -import { Context, Effect, Layer } from "effect"; +import { Cache, Context, Effect, Layer } from "effect"; import { S3Client as S3ClientSDK } from "@aws-sdk/client-s3"; import type { MaterializedBucket } from "../../Domain/Config.ts"; -import { AppConfig } from "../../Config/Layer.ts"; +import { HeraldConfig } from "../../Config/Layer.ts"; export class S3Client extends Context.Tag("S3Client")< S3Client, @@ -14,106 +14,100 @@ export class S3Client extends Context.Tag("S3Client")< export const S3ClientLive = Layer.effect( S3Client, - AppConfig.pipe( - Effect.flatMap((appConfig) => { - // A simple cache for SDK clients - const clients = new Map(); + Effect.gen(function* () { + const appConfig = yield* HeraldConfig; - return Effect.succeed( - S3Client.of({ - getClient: (bucket: MaterializedBucket | { backend_id: string }) => { - // Resolve full bucket if only backend_id provided - let resolved: MaterializedBucket; - if ("bucket_name" in bucket) { - resolved = bucket; - } else { - const backendConfig = appConfig.raw.backends[bucket.backend_id]; - if (backendConfig && backendConfig.protocol === "s3") { - resolved = { - name: "", - backend_id: bucket.backend_id, - protocol: "s3" as const, - endpoint: backendConfig.endpoint, - region: backendConfig.region, - bucket_name: "", - credentials: backendConfig.credentials, - }; - } else { - return Effect.fail( - new Error( - `Backend ${bucket.backend_id} is not an S3 backend or not found`, - ), - ); - } - } + const cache = yield* Cache.make({ + capacity: 100, + timeToLive: "24 hours", // S3 clients can live a long time + lookup: (resolved: MaterializedBucket) => + Effect.gen(function* () { + if (resolved.endpoint === undefined) { + return yield* Effect.fail( + new Error( + `Missing endpoint for backend ${resolved.backend_id}`, + ), + ); + } + + if (resolved.region === undefined) { + return yield* Effect.fail( + new Error(`Missing region for backend ${resolved.backend_id}`), + ); + } + + let accessKeyId: string | undefined; + let secretAccessKey: string | undefined; - const key = - `${resolved.backend_id}:${resolved.endpoint}:${resolved.region}`; - const existing = clients.get(key); - if (existing) { - return Effect.succeed(existing); + if (resolved.credentials) { + const creds = resolved.credentials; + if ("accessKeyId" in creds) { + accessKeyId = creds.accessKeyId; + secretAccessKey = creds.secretAccessKey; + } else if ("username" in creds) { + accessKeyId = creds.username; + secretAccessKey = creds.password; } - if (resolved.endpoint === undefined) { - return Effect.fail( + if (accessKeyId === undefined) { + return yield* Effect.fail( new Error( - `Missing endpoint for backend ${resolved.backend_id}`, + `Missing accessKeyId/username for backend ${resolved.backend_id}`, ), ); } - - if (resolved.region === undefined) { - return Effect.fail( - new Error(`Missing region for backend ${resolved.backend_id}`), + if (secretAccessKey === undefined) { + return yield* Effect.fail( + new Error( + `Missing secretAccessKey/password for backend ${resolved.backend_id}`, + ), ); } + } - let accessKeyId: string | undefined; - let secretAccessKey: string | undefined; - - if (resolved.credentials) { - const creds = resolved.credentials; - if ("accessKeyId" in creds) { - accessKeyId = creds.accessKeyId; - secretAccessKey = creds.secretAccessKey; - } else if ("username" in creds) { - accessKeyId = creds.username; - secretAccessKey = creds.password; - } - - if (accessKeyId === undefined) { - return Effect.fail( - new Error( - `Missing accessKeyId/username for backend ${resolved.backend_id}`, - ), - ); - } - if (secretAccessKey === undefined) { - return Effect.fail( - new Error( - `Missing secretAccessKey/password for backend ${resolved.backend_id}`, - ), - ); + return new S3ClientSDK({ + endpoint: resolved.endpoint, + region: resolved.region, + credentials: accessKeyId && secretAccessKey + ? { + accessKeyId, + secretAccessKey, } - } + : undefined, + forcePathStyle: true, + }); + }), + }); - const sdkClient = new S3ClientSDK({ - endpoint: resolved.endpoint, - region: resolved.region, - credentials: accessKeyId && secretAccessKey - ? { - accessKeyId, - secretAccessKey, - } - : undefined, - forcePathStyle: true, - }); + return S3Client.of({ + getClient: (bucket: MaterializedBucket | { backend_id: string }) => { + // Resolve full bucket if only backend_id provided + let resolved: MaterializedBucket; + if ("bucket_name" in bucket) { + resolved = bucket; + } else { + const backendConfig = appConfig.raw.backends[bucket.backend_id]; + if (backendConfig && backendConfig.protocol === "s3") { + resolved = { + name: "", + backend_id: bucket.backend_id, + protocol: "s3" as const, + endpoint: backendConfig.endpoint, + region: backendConfig.region, + bucket_name: "", + credentials: backendConfig.credentials, + }; + } else { + return Effect.fail( + new Error( + `Backend ${bucket.backend_id} is not an S3 backend or not found`, + ), + ); + } + } - clients.set(key, sdkClient); - return Effect.succeed(sdkClient); - }, - }), - ); - }), - ), + return cache.get(resolved); + }, + }); + }), ); diff --git a/src/Backends/S3/Objects.ts b/src/Backends/S3/Objects.ts new file mode 100644 index 0000000..58a1f06 --- /dev/null +++ b/src/Backends/S3/Objects.ts @@ -0,0 +1,757 @@ +import { Chunk, Effect, Option, Stream } from "effect"; +import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CreateMultipartUploadCommand, + DeleteObjectCommand, + DeleteObjectsCommand, + GetObjectCommand, + HeadObjectCommand, + ListMultipartUploadsCommand, + ListObjectsCommand, + type ListObjectsCommandOutput, + ListObjectsV2Command, + type ListObjectsV2CommandOutput, + ListObjectVersionsCommand, + ListPartsCommand, + PutObjectCommand, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import { + type CommonPrefix, + InternalError, + type ListObjectsResult, + type ObjectInfo, + type ObjectResponse, +} from "../../Services/Backend.ts"; +import { mapS3Error, type S3Target, stripMinioMetadata } from "./Utils.ts"; + +export const makeObjectOps = (target: S3Target) => ({ + listObjects: (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + encodingType?: string; + continuationToken?: string; + startAfter?: string; + listType?: 1 | 2; + }) => + Effect.gen(function* () { + const { client, bucketName } = target; + if (args.listType === 2) { + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + MaxKeys: args.maxKeys, + ContinuationToken: args.continuationToken, + StartAfter: args.startAfter, + }), + ) as Promise, + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + name: result.Name ?? bucketName, + prefix: result.Prefix, + maxKeys: result.MaxKeys ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: args.encodingType, + continuationToken: result.ContinuationToken, + nextContinuationToken: result.NextContinuationToken, + keyCount: result.KeyCount, + listType: 2, + contents: (result.Contents ?? []).map((c): ObjectInfo => ({ + key: stripMinioMetadata(c.Key ?? ""), + lastModified: c.LastModified ?? new Date(), + etag: c.ETag ?? "", + size: c.Size ?? 0, + storageClass: c.StorageClass, + owner: c.Owner + ? { + id: c.Owner.ID ?? "unknown", + displayName: c.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + commonPrefixes: (result.CommonPrefixes ?? []).map(( + cp, + ): CommonPrefix => ({ + prefix: stripMinioMetadata(cp.Prefix ?? ""), + })), + } satisfies ListObjectsResult; + } else { + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListObjectsCommand({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + Marker: args.marker, + MaxKeys: args.maxKeys, + }), + ) as Promise, + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + name: result.Name ?? bucketName, + prefix: result.Prefix, + marker: result.Marker, + nextMarker: result.NextMarker, + maxKeys: result.MaxKeys ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: args.encodingType, + listType: 1, + contents: (result.Contents ?? []).map((c): ObjectInfo => ({ + key: stripMinioMetadata(c.Key ?? ""), + lastModified: c.LastModified ?? new Date(), + etag: c.ETag ?? "", + size: c.Size ?? 0, + storageClass: c.StorageClass, + owner: c.Owner + ? { + id: c.Owner.ID ?? "unknown", + displayName: c.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + commonPrefixes: (result.CommonPrefixes ?? []).map(( + cp, + ): CommonPrefix => ({ + prefix: stripMinioMetadata(cp.Prefix ?? ""), + })), + } satisfies ListObjectsResult; + } + }), + + listVersions: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + versionIdMarker?: string; + maxKeys?: number; + encodingType?: string; + }) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListObjectVersionsCommand({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + KeyMarker: args.keyMarker, + VersionIdMarker: args.versionIdMarker, + MaxKeys: args.maxKeys, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + name: result.Name ?? bucketName, + prefix: result.Prefix, + marker: result.KeyMarker, + nextMarker: result.NextKeyMarker, + maxKeys: result.MaxKeys ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: args.encodingType, + listType: 1, + contents: [ + ...(result.Versions ?? []).map((v): ObjectInfo => ({ + key: stripMinioMetadata(v.Key ?? ""), + lastModified: v.LastModified ?? new Date(), + etag: v.ETag ?? "", + size: v.Size ?? 0, + storageClass: v.StorageClass, + versionId: v.VersionId, + isDeleteMarker: false, + isLatest: v.IsLatest, + owner: v.Owner + ? { + id: v.Owner.ID ?? "unknown", + displayName: v.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + ...(result.DeleteMarkers ?? []).map((dm): ObjectInfo => ({ + key: stripMinioMetadata(dm.Key ?? ""), + lastModified: dm.LastModified ?? new Date(), + etag: "", + size: 0, + versionId: dm.VersionId, + isDeleteMarker: true, + isLatest: dm.IsLatest, + owner: dm.Owner + ? { + id: dm.Owner.ID ?? "unknown", + displayName: dm.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + ], + commonPrefixes: (result.CommonPrefixes ?? []).map(( + cp, + ): CommonPrefix => ({ + prefix: stripMinioMetadata(cp.Prefix ?? ""), + })), + } satisfies ListObjectsResult; + }), + + getObject: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: key, + Range: (headers["range"] || headers["Range"]) as string, + PartNumber: (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) + ? parseInt( + (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) as string, + ) + : undefined, + IfMatch: (headers["if-match"] || headers["If-Match"]) as string, + IfNoneMatch: (headers["if-none-match"] || + headers["If-None-Match"]) as string, + IfModifiedSince: (headers["if-modified-since"] || + headers["If-Modified-Since"]) + ? new Date( + (headers["if-modified-since"] || + headers["If-Modified-Since"]) as string, + ) + : undefined, + IfUnmodifiedSince: (headers["if-unmodified-since"] || + headers["If-Unmodified-Since"]) + ? new Date( + (headers["if-unmodified-since"] || + headers["If-Unmodified-Since"]) as string, + ) + : undefined, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + const body = result.Body; + if (!body) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned empty body for GetObject", + }), + ); + } + + const getWebStream = (): ReadableStream => { + if ( + body && typeof body === "object" && + "transformToWebStream" in body + ) { + const b = body as { transformToWebStream: unknown }; + if (typeof b.transformToWebStream === "function") { + return b.transformToWebStream() as ReadableStream< + Uint8Array + >; + } + } + return body as ReadableStream; + }; + + const stream = Stream.fromReadableStream( + getWebStream, + (e) => new Error(String(e)), + ); + + const metadata: Record = {}; + if (result.Metadata) { + for (const [k, v] of Object.entries(result.Metadata)) { + metadata[k] = Option.liftThrowable(decodeURIComponent)( + v ?? "", + ).pipe( + Option.getOrElse(() => v ?? ""), + ); + } + } + + const s3Headers: Record = {}; + if (result.ContentType) { + s3Headers["content-type"] = result.ContentType; + } + if (result.ContentLength !== undefined) { + s3Headers["content-length"] = String(result.ContentLength); + } + if (result.ETag) s3Headers["etag"] = result.ETag; + if (result.PartsCount !== undefined) { + s3Headers["x-amz-mp-parts-count"] = String(result.PartsCount); + } + if (result.VersionId) { + s3Headers["x-amz-version-id"] = result.VersionId; + } + if (result.LastModified) { + s3Headers["last-modified"] = result.LastModified.toUTCString(); + } + + for (const [k, v] of Object.entries(metadata)) { + s3Headers[`x-amz-meta-${k}`] = v; + } + + return yield* Stream.runCollect(stream).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + Effect.map((chunks) => { + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const all = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + all.set(chunk, offset); + offset += chunk.length; + } + return { + stream: Stream.succeed(all), + contentType: result.ContentType, + contentLength: all.length, + etag: result.ETag, + lastModified: result.LastModified, + metadata, + headers: s3Headers, + } satisfies ObjectResponse; + }), + ); + }), + + headObject: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const commandInput = { + Bucket: bucketName, + Key: key, + PartNumber: (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) + ? parseInt( + (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) as string, + ) + : undefined, + }; + const result = yield* Effect.tryPromise({ + try: () => client.send(new HeadObjectCommand(commandInput)), + catch: (e) => mapS3Error(e, bucketName), + }); + + const metadata: Record = {}; + if (result.Metadata) { + for (const [k, v] of Object.entries(result.Metadata)) { + metadata[k] = Option.liftThrowable(decodeURIComponent)( + v ?? "", + ).pipe( + Option.getOrElse(() => v ?? ""), + ); + } + } + + const s3Headers: Record = {}; + if (result.ContentType) { + s3Headers["content-type"] = result.ContentType; + } + if (result.ContentLength !== undefined) { + s3Headers["content-length"] = String(result.ContentLength); + } + if (result.ETag) s3Headers["etag"] = result.ETag; + if (result.PartsCount !== undefined) { + s3Headers["x-amz-mp-parts-count"] = String(result.PartsCount); + } + if (result.VersionId) { + s3Headers["x-amz-version-id"] = result.VersionId; + } + if (result.LastModified) { + s3Headers["last-modified"] = result + .LastModified.toUTCString(); + } + + for (const [k, v] of Object.entries(metadata)) { + s3Headers[`x-amz-meta-${k}`] = v; + } + + return { + contentType: result.ContentType, + contentLength: result.ContentLength, + etag: result.ETag, + lastModified: result.LastModified, + metadata, + headers: s3Headers, + }; + }), + + putObject: ( + key: string, + bodyStream: Stream.Stream, + headers: Record, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const chunks = yield* Stream.runCollect(bodyStream).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + ); + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const body = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.length; + } + + const metadata: Record = {}; + for (const [k, v] of Object.entries(headers)) { + if (k.toLowerCase().startsWith("x-amz-meta-")) { + const metaKey = k.substring("x-amz-meta-".length); + const value = String(v); + metadata[metaKey] = /[^\x20-\x7E]/.test(value) + ? encodeURIComponent(value) + : value; + } + } + + const contentType = headers["content-type"]; + + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: key, + Body: body, + ContentType: contentType ? String(contentType) : undefined, + Metadata: metadata, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + etag: result.ETag, + versionId: result.VersionId, + }; + }), + + deleteObject: (key: string) => + Effect.gen(function* () { + const { client, bucketName } = target; + yield* Effect.tryPromise({ + try: () => + client.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: key, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + }), + + deleteObjects: (objects: readonly { key: string; versionId?: string }[]) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects.map((o) => ({ + Key: o.key, + VersionId: o.versionId === "null" ? undefined : o.versionId, + })), + }, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + deleted: (result.Deleted ?? []).map((d) => d.Key ?? ""), + errors: (result.Errors ?? []).map((e) => ({ + key: e.Key ?? "unknown", + code: e.Code ?? "InternalError", + message: e.Message ?? "Unknown error", + })), + }; + }), + + createMultipartUpload: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const metadata: Record = {}; + for (const [k, v] of Object.entries(headers)) { + if (k.toLowerCase().startsWith("x-amz-meta-")) { + const metaKey = k.substring("x-amz-meta-".length); + metadata[metaKey] = String(v); + } + } + const contentType = headers["content-type"]; + + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + Metadata: metadata, + ContentType: contentType ? String(contentType) : undefined, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + if (!result.UploadId) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned empty UploadId", + }), + ); + } + return { uploadId: result.UploadId }; + }), + + uploadPart: ( + key: string, + uploadId: string, + partNumber: number, + bodyStream: Stream.Stream, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const chunks = yield* Stream.runCollect(bodyStream).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + ); + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const body = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.length; + } + + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + PartNumber: partNumber, + Body: body, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + if (!result.ETag) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned empty ETag for UploadPart", + }), + ); + } + return { etag: result.ETag }; + }), + + completeMultipartUpload: ( + key: string, + uploadId: string, + parts: readonly { etag: string; partNumber: number }[], + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: parts.map((p) => ({ + ETag: p.etag, + PartNumber: p.partNumber, + })), + }, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + if ( + !result.Location || !result.Bucket || !result.Key || + !result.ETag + ) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned incomplete CompleteMultipartUploadResult", + }), + ); + } + return { + location: result.Location, + bucket: result.Bucket, + key: result.Key, + etag: result.ETag, + versionId: result.VersionId, + }; + }), + + abortMultipartUpload: (key: string, uploadId: string) => + Effect.gen(function* () { + const { client, bucketName } = target; + yield* Effect.tryPromise({ + try: () => + client.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + }), + + listMultipartUploads: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + uploadIdMarker?: string; + maxUploads?: number; + encodingType?: string; + }) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListMultipartUploadsCommand({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + KeyMarker: args.keyMarker, + UploadIdMarker: args.uploadIdMarker, + MaxUploads: args.maxUploads, + EncodingType: args.encodingType as "url" | undefined, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + bucket: result.Bucket ?? bucketName, + prefix: result.Prefix, + keyMarker: result.KeyMarker, + uploadIdMarker: result.UploadIdMarker, + nextKeyMarker: result.NextKeyMarker, + nextUploadIdMarker: result.NextUploadIdMarker, + maxUploads: result.MaxUploads ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: result.EncodingType as string, + uploads: (result.Uploads ?? []).map((u) => ({ + key: u.Key ?? "", + uploadId: u.UploadId ?? "", + owner: { + id: u.Owner?.ID ?? "", + displayName: u.Owner?.DisplayName ?? "", + }, + initiator: { + id: u.Initiator?.ID ?? "", + displayName: u.Initiator?.DisplayName ?? "", + }, + storageClass: u.StorageClass ?? "STANDARD", + initiated: u.Initiated ?? new Date(), + })), + commonPrefixes: (result.CommonPrefixes ?? []).map((cp) => ({ + prefix: cp.Prefix ?? "", + })), + }; + }), + + listParts: (key: string, uploadId: string) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListPartsCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + bucket: result.Bucket ?? bucketName, + key: result.Key ?? key, + uploadId: result.UploadId ?? uploadId, + owner: { + id: result.Owner?.ID ?? "", + displayName: result.Owner?.DisplayName ?? "", + }, + initiator: { + id: result.Initiator?.ID ?? "", + displayName: result.Initiator?.DisplayName ?? "", + }, + storageClass: result.StorageClass ?? "STANDARD", + partNumberMarker: result.PartNumberMarker + ? parseInt(String(result.PartNumberMarker)) + : 0, + nextPartNumberMarker: result.NextPartNumberMarker + ? parseInt(String(result.NextPartNumberMarker)) + : 0, + maxParts: result.MaxParts ?? 1000, + isTruncated: result.IsTruncated ?? false, + parts: (result.Parts ?? []).map((p) => ({ + partNumber: p.PartNumber ?? 0, + lastModified: p.LastModified ?? new Date(), + etag: p.ETag ?? "", + size: p.Size ?? 0, + })), + }; + }), +}); diff --git a/src/Backends/S3/Utils.ts b/src/Backends/S3/Utils.ts new file mode 100644 index 0000000..f11d486 --- /dev/null +++ b/src/Backends/S3/Utils.ts @@ -0,0 +1,147 @@ +import { Effect } from "effect"; +import type { S3Client as S3ClientSDK } from "@aws-sdk/client-s3"; +import type { MaterializedBucket } from "../../Domain/Config.ts"; +import { HeraldConfig } from "../../Config/Layer.ts"; +import { + AccessDenied, + type BackendError, + BucketAlreadyExists, + BucketAlreadyOwnedByYou, + BucketNotEmpty, + EntityTooSmall, + InternalError, + InvalidPart, + InvalidPartOrder, + InvalidRequest, + MalformedXML, + NoSuchBucket, + NoSuchKey, + NoSuchUpload, +} from "../../Services/Backend.ts"; +import { S3Client } from "./Client.ts"; + +export interface S3Target { + readonly client: S3ClientSDK; + readonly bucketName: string; + readonly name: string; +} + +/** + * Strips MinIO metadata suffixes like [minio_cache:v2,return:] from strings. + */ +export function stripMinioMetadata(s: string): string { + return s.replace(/\[minio_cache:[^\]]+\]/g, ""); +} + +/** + * Maps S3 SDK exceptions to internal BackendError types. + */ +export function mapS3Error(e: unknown, bucketName?: string): BackendError { + const err = e as { + name?: string; + Code?: string; + Message?: string; + message?: string; + $metadata?: { httpStatusCode?: number }; + }; + const name = err?.name || err?.Code || + (e instanceof Error ? e.name : "UnknownError"); + const message = err?.message || err?.Message || + "An unknown S3 error occurred"; + const bucket = bucketName ?? "unknown-bucket"; + + switch (name) { + case "NoSuchBucket": + case "NotFound": + return new NoSuchBucket({ bucketName: bucket, message }); + case "NoSuchKey": + return new NoSuchKey({ + bucketName: bucket, + key: "unknown", + message: message, + }); + case "NoSuchUpload": + return new NoSuchUpload({ + uploadId: "unknown", + message: message, + }); + case "InvalidPart": + case "InvalidPartNumber": + return new InvalidPart({ message }); + case "InvalidPartOrder": + return new InvalidPartOrder({ message }); + case "EntityTooSmall": + return new EntityTooSmall({ message }); + case "InvalidRequest": + if (message.includes("at least one part")) { + return new MalformedXML({ message }); + } + return new InvalidRequest({ message }); + case "MalformedXML": + return new MalformedXML({ message }); + case "BucketAlreadyExists": + return new BucketAlreadyExists({ bucketName: bucket, message }); + case "BucketAlreadyOwnedByYou": + return new BucketAlreadyOwnedByYou({ bucketName: bucket, message }); + case "AccessDenied": + case "Forbidden": + return new AccessDenied({ message }); + case "BucketNotEmpty": + case "Conflict": + return new BucketNotEmpty({ bucketName: bucket, message }); + } + + // Handle case where it might be a raw 404 from HEAD request + if (err?.$metadata?.httpStatusCode === 404) { + return new NoSuchKey({ + bucketName: bucket, + key: "unknown", + message: "Not Found", + }); + } + + return new InternalError({ + message: e instanceof Error ? `${e.name}: ${e.message}` : String(e), + }); +} + +/** + * Resolves the target bucket configuration and acquires the S3 client. + * This ensures the backend remains a stateless proxy that picks up request-local configuration and clients. + */ +export const getTarget = ( + bucket: MaterializedBucket | { backend_id: string }, +): Effect.Effect => + Effect.gen(function* () { + const s3Service = yield* S3Client; + const config = yield* HeraldConfig; + + const resolveTargetBucket = (): MaterializedBucket => { + if ("bucket_name" in bucket) return bucket as MaterializedBucket; + + const backendConfig = config.raw.backends[bucket.backend_id]; + if (backendConfig && backendConfig.protocol === "s3") { + return { + name: "", + backend_id: bucket.backend_id, + protocol: "s3" as const, + endpoint: backendConfig.endpoint, + region: backendConfig.region, + bucket_name: "", + credentials: backendConfig.credentials, + }; + } + throw new Error(`Backend ${bucket.backend_id} is not an S3 backend`); + }; + + const targetBucket = resolveTargetBucket(); + const client = yield* s3Service.getClient(targetBucket).pipe( + Effect.mapError((e) => mapS3Error(e, targetBucket.name)), + ); + + return { + client, + bucketName: targetBucket.bucket_name, + name: targetBucket.name, + }; + }); diff --git a/src/Backends/Swift/Backend.ts b/src/Backends/Swift/Backend.ts index 32079da..1c50070 100644 --- a/src/Backends/Swift/Backend.ts +++ b/src/Backends/Swift/Backend.ts @@ -1,592 +1,29 @@ -import { Effect, Option, Stream } from "effect"; -import { - type BackendError, - type BackendService, - BucketAlreadyExists, - BucketAlreadyOwnedByYou, - type BucketInfo, - BucketNotEmpty, - type CommonPrefix, - type DeleteObjectsResult, - type HeadObjectResult, - InternalError, - type ListObjectsResult, - NoSuchBucket, - NoSuchKey, - type ObjectInfo, - type ObjectResponse, - type OwnerInfo, - type PutObjectResult, -} from "../../Services/Backend.ts"; +import { Effect } from "effect"; +import { HttpClient } from "@effect/platform"; +import type { BackendError, BackendService } from "../../Services/Backend.ts"; import type { MaterializedBucket } from "../../Domain/Config.ts"; -import { SwiftClient } from "./Client.ts"; -import { fixHeaderEncoding } from "../../Frontend/Utils.ts"; - -interface SwiftContainer { - readonly name: string; - readonly last_modified?: string; -} - -interface SwiftObject { - readonly name?: string; - readonly hash?: string; - readonly bytes?: number; - readonly content_type?: string; - readonly last_modified?: string; - readonly subdir?: string; -} - +import { makeBucketOps } from "./Buckets.ts"; +import { makeObjectOps } from "./Objects.ts"; +import { getTarget } from "./Utils.ts"; +import type { SwiftClient } from "./Client.ts"; + +/** + * Creates a Swift-specific Backend implementation for a given configuration context. + * Composes bucket and object operations modularly. + * Resolves the target and client once per backend creation (request-scoped). + */ export const makeSwiftBackend = ( bucket: MaterializedBucket | { backend_id: string }, -): Effect.Effect => +): Effect.Effect< + BackendService, + BackendError, + SwiftClient | HttpClient.HttpClient +> => Effect.gen(function* () { - const swiftClient = yield* SwiftClient; - - const getTarget = () => - Effect.gen(function* () { - const auth = yield* swiftClient.getAuthMeta(bucket).pipe( - Effect.mapError((e) => new InternalError({ message: e.message })), - ); - const container = "bucket_name" in bucket ? bucket.bucket_name : ""; - const encodedContainer = container ? encodeURIComponent(container) : ""; - return { - storageUrl: auth.storageUrl, - token: auth.token, - container, - url: encodedContainer - ? `${auth.storageUrl}/${encodedContainer}` - : auth.storageUrl, - }; - }); - - const mapError = ( - status: number, - message: string, - bucketName: string, - method?: string, - key?: string, - ): BackendError => { - switch (status) { - case 404: - if (key) { - return new NoSuchKey({ bucketName, key, message }); - } - return new NoSuchBucket({ bucketName, message }); - case 409: - if (method === "DELETE") { - return new BucketNotEmpty({ bucketName, message }); - } - return new BucketAlreadyExists({ bucketName, message }); - case 202: - if (method === "PUT") { - return new BucketAlreadyOwnedByYou({ bucketName, message }); - } - return new InternalError({ - message: `Swift error (${status}): ${message}`, - }); - default: - return new InternalError({ - message: `Swift error (${status}): ${message}`, - }); - } - }; - - const listObjects = (args: { - prefix?: string; - delimiter?: string; - marker?: string; - maxKeys?: number; - encodingType?: string; - continuationToken?: string; - startAfter?: string; - listType?: 1 | 2; - }) => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const limit = args.maxKeys ?? 1000; - const query = new URLSearchParams({ format: "json" }); - if (args.prefix) query.set("prefix", args.prefix); - if (args.delimiter) query.set("delimiter", args.delimiter); - if (args.marker) query.set("marker", args.marker); - query.set("limit", String(limit + 1)); - if (args.continuationToken) query.set("marker", args.continuationToken); - if (args.startAfter) query.set("marker", args.startAfter); - - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${url}?${query.toString()}`, { - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - yield* Effect.logDebug( - `Swift listObjects query=[${query.toString()}] status=${response.status}`, - ); - - if (!response.ok) { - return yield* Effect.fail( - mapError(response.status, response.statusText, container, "GET"), - ); - } - - const rawObjects = (yield* Effect.tryPromise({ - try: () => response.json(), - catch: (e) => - new InternalError({ - message: `Failed to parse Swift response: ${e}`, - }), - })) as readonly SwiftObject[]; - - const isTruncated = rawObjects.length > limit; - const objects = isTruncated ? rawObjects.slice(0, limit) : rawObjects; - - const contents: ObjectInfo[] = []; - const commonPrefixes: CommonPrefix[] = []; - - for (const obj of objects) { - if (obj.subdir) { - commonPrefixes.push({ prefix: obj.subdir }); - } else if (obj.name) { - contents.push({ - key: obj.name, - lastModified: obj.last_modified - ? new Date(obj.last_modified) - : new Date(), - etag: obj.hash ? `"${obj.hash}"` : "", - size: obj.bytes ?? 0, - storageClass: "STANDARD", - owner: { id: "swift", displayName: "Swift User" }, - }); - } - } - - const nextMarker = isTruncated && objects.length > 0 - ? objects[objects.length - 1].name || - objects[objects.length - 1].subdir - : undefined; - - return { - name: container, - prefix: args.prefix, - maxKeys: limit, - delimiter: args.delimiter, - isTruncated, - marker: args.marker, - nextMarker, - contents, - commonPrefixes, - encodingType: args.encodingType, - listType: args.listType ?? 1, - nextContinuationToken: args.listType === 2 ? nextMarker : undefined, - keyCount: contents.length + commonPrefixes.length, - } satisfies ListObjectsResult; - }); - + const target = yield* getTarget(bucket); + const client = yield* HttpClient.HttpClient; return { - listBuckets: () => - Effect.gen(function* () { - const { storageUrl, token } = yield* getTarget(); - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${storageUrl}?format=json`, { - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - if (!response.ok) { - return yield* Effect.fail( - mapError(response.status, response.statusText, "", "GET"), - ); - } - - const buckets = (yield* Effect.tryPromise({ - try: () => response.json(), - catch: (e) => - new InternalError({ - message: `Failed to parse Swift response: ${e}`, - }), - })) as readonly SwiftContainer[]; - - const bucketInfos: BucketInfo[] = buckets.map((b) => ({ - name: b.name, - creationDate: b.last_modified - ? new Date(b.last_modified) - : undefined, - })); - - const owner: OwnerInfo = { id: "swift", displayName: "Swift User" }; - - return { buckets: bucketInfos, owner }; - }), - - createBucket: () => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const response = yield* Effect.tryPromise({ - try: () => - fetch(url, { - method: "PUT", - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - if (response.status === 201) { - return yield* Effect.void; - } - - if (response.status === 202) { - return yield* Effect.fail( - new BucketAlreadyOwnedByYou({ - bucketName: container, - message: "Bucket already exists", - }), - ); - } - - if (!response.ok) { - return yield* Effect.fail( - mapError(response.status, response.statusText, container, "PUT"), - ); - } - - return yield* Effect.void; - }), - - deleteBucket: () => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const response = yield* Effect.tryPromise({ - try: () => - fetch(url, { - method: "DELETE", - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - yield* Effect.logDebug( - `Swift deleteBucket container=[${container}] status=${response.status}`, - ); - - if (response.status === 204) { - return yield* Effect.void; - } - - if (!response.ok) { - return yield* Effect.fail( - mapError( - response.status, - response.statusText, - container, - "DELETE", - ), - ); - } - - return yield* Effect.void; - }), - - headBucket: () => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const response = yield* Effect.tryPromise({ - try: () => - fetch(url, { - method: "HEAD", - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - if (!response.ok) { - return yield* Effect.fail( - mapError(response.status, response.statusText, container, "HEAD"), - ); - } - - return yield* Effect.void; - }), - - listObjects, - - listVersions: (args) => - Effect.gen(function* () { - const result = yield* listObjects({ - prefix: args.prefix, - delimiter: args.delimiter, - marker: args.keyMarker, - maxKeys: args.maxKeys, - }); - return { - ...result, - contents: result.contents.map((c) => ({ - ...c, - versionId: "null", - isLatest: true, - })), - }; - }), - - getObject: (key: string) => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const encodedKey = key.split("/").map(encodeURIComponent).join("/"); - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${url}/${encodedKey}`, { - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - if (!response.ok) { - return yield* Effect.fail( - mapError( - response.status, - response.statusText, - container, - "GET", - key, - ), - ); - } - - const metadata: Record = {}; - const s3Headers: Record = {}; - response.headers.forEach((v, k) => { - const lowK = k.toLowerCase(); - if (lowK.startsWith("x-object-meta-")) { - const metaKey = lowK.substring("x-object-meta-".length); - const value = (v.includes("%")) - ? Option.liftThrowable(decodeURIComponent)(v).pipe( - Option.getOrElse(() => v), - ) - : v; - metadata[metaKey] = value; - s3Headers[`x-amz-meta-${metaKey}`] = value; - } else if (lowK === "content-type") { - s3Headers["Content-Type"] = v; - } else if (lowK === "content-length") { - s3Headers["Content-Length"] = v; - } else if (lowK === "etag") { - s3Headers["ETag"] = v; - } else if (lowK === "last-modified") { - s3Headers["Last-Modified"] = v; - } - }); - - return { - stream: Stream.fromReadableStream( - () => response.body!, - (e) => new InternalError({ message: String(e) }), - ), - contentType: response.headers.get("Content-Type") || undefined, - contentLength: parseInt( - response.headers.get("Content-Length") || "0", - ), - etag: response.headers.get("ETag") || undefined, - lastModified: response.headers.get("Last-Modified") - ? new Date(response.headers.get("Last-Modified")!) - : undefined, - metadata, - headers: s3Headers, - } satisfies ObjectResponse; - }), - - headObject: (key: string) => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const encodedKey = key.split("/").map(encodeURIComponent).join("/"); - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${url}/${encodedKey}`, { - method: "HEAD", - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - if (!response.ok) { - return yield* Effect.fail( - mapError( - response.status, - response.statusText, - container, - "HEAD", - key, - ), - ); - } - - const metadata: Record = {}; - const s3Headers: Record = {}; - response.headers.forEach((v, k) => { - const lowK = k.toLowerCase(); - if (lowK.startsWith("x-object-meta-")) { - const metaKey = lowK.substring("x-object-meta-".length); - const value = (v.includes("%")) - ? Option.liftThrowable(decodeURIComponent)(v).pipe( - Option.getOrElse(() => v), - ) - : v; - metadata[metaKey] = value; - s3Headers[`x-amz-meta-${metaKey}`] = value; - } else if (lowK === "content-type") { - s3Headers["Content-Type"] = v; - } else if (lowK === "content-length") { - s3Headers["Content-Length"] = v; - } else if (lowK === "etag") { - s3Headers["ETag"] = v; - } else if (lowK === "last-modified") { - s3Headers["Last-Modified"] = v; - } - }); - - return { - contentType: response.headers.get("Content-Type") || undefined, - contentLength: parseInt( - response.headers.get("Content-Length") || "0", - ), - etag: response.headers.get("ETag") || undefined, - lastModified: response.headers.get("Last-Modified") - ? new Date(response.headers.get("Last-Modified")!) - : undefined, - metadata, - headers: s3Headers, - } satisfies HeadObjectResult; - }), - - putObject: (key, stream, headers) => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const encodedKey = key.split("/").map(encodeURIComponent).join("/"); - const contentLength = headers["content-length"] || - headers["Content-Length"]; - - const swiftHeaders: Record = { - "X-Auth-Token": token, - "Content-Type": - (headers["content-type"] || headers["Content-Type"] || - "application/octet-stream") as string, - ...(contentLength - ? { "Content-Length": String(contentLength) } - : {}), - }; - - for (const [k, v] of Object.entries(headers)) { - const lowK = k.toLowerCase(); - if (lowK.startsWith("x-amz-meta-")) { - const metaKey = lowK.substring("x-amz-meta-".length); - const value = fixHeaderEncoding(String(v)); - swiftHeaders[`X-Object-Meta-${metaKey}`] = - /[^\x20-\x7E]/.test(value) ? encodeURIComponent(value) : value; - } - } - - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${url}/${encodedKey}`, { - method: "PUT", - headers: swiftHeaders, - body: Stream.toReadableStream(stream), - // @ts-ignore: duplex is required for streaming body in fetch - duplex: "half", - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - yield* Effect.logDebug( - `Swift putObject key=[${key}] status=${response.status}`, - ); - - if (!response.ok) { - return yield* Effect.fail( - mapError( - response.status, - response.statusText, - container, - "PUT", - key, - ), - ); - } - - return { - etag: response.headers.get("ETag") || undefined, - } satisfies PutObjectResult; - }), - - deleteObject: (key: string) => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const encodedKey = key.split("/").map(encodeURIComponent).join("/"); - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${url}/${encodedKey}`, { - method: "DELETE", - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - if (!response.ok && response.status !== 204) { - return yield* Effect.fail( - mapError( - response.status, - response.statusText, - container, - "DELETE", - key, - ), - ); - } - - return yield* Effect.void; - }), - - deleteObjects: (objects) => - Effect.gen(function* () { - const { url, token, container: _container } = yield* getTarget(); - const deleted: string[] = []; - const errors: { key: string; code: string; message: string }[] = []; - - for (const obj of objects) { - const encodedKey = obj.key.split("/").map(encodeURIComponent).join( - "/", - ); - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${url}/${encodedKey}`, { - method: "DELETE", - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - yield* Effect.logDebug( - `Swift deleteObject key=[${obj.key}] status=${response.status}`, - ); - - if ( - response.ok || response.status === 204 || response.status === 404 - ) { - deleted.push(obj.key); - } else { - const errorBody = yield* Effect.tryPromise(() => response.text()) - .pipe( - Effect.orElseSucceed(() => "Unknown error"), - ); - errors.push({ - key: obj.key, - code: String(response.status), - message: errorBody, - }); - } - } - - return { deleted, errors } satisfies DeleteObjectsResult; - }), - }; + ...makeBucketOps(target, client), + ...makeObjectOps(target, client), + } satisfies BackendService; }); diff --git a/src/Backends/Swift/Buckets.ts b/src/Backends/Swift/Buckets.ts new file mode 100644 index 0000000..c6371fd --- /dev/null +++ b/src/Backends/Swift/Buckets.ts @@ -0,0 +1,143 @@ +import { Effect } from "effect"; +import { type HttpClient, HttpClientRequest } from "@effect/platform"; +import { + BucketAlreadyOwnedByYou, + type BucketInfo, + type OwnerInfo, +} from "../../Services/Backend.ts"; +import { mapError, type SwiftTarget } from "./Utils.ts"; + +export interface SwiftContainer { + readonly name: string; + readonly last_modified?: string; +} + +export const makeBucketOps = ( + target: SwiftTarget, + client: HttpClient.HttpClient, +) => ({ + listBuckets: () => + Effect.gen(function* () { + const { storageUrl, token } = target; + const response = yield* client.execute( + HttpClientRequest.get(`${storageUrl}?format=json`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), "")), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", "", "GET"), + ); + } + + const containers = (yield* response.json.pipe( + Effect.mapError((e) => + mapError(500, `Failed to parse Swift response: ${e}`, "") + ), + )) as readonly SwiftContainer[]; + + const bucketInfos: BucketInfo[] = containers.map((b) => ({ + name: b.name, + creationDate: b.last_modified ? new Date(b.last_modified) : undefined, + })); + + const owner: OwnerInfo = { id: "swift", displayName: "Swift User" }; + + return { buckets: bucketInfos, owner }; + }), + + createBucket: () => + Effect.gen(function* () { + const { url, token, container } = target; + const response = yield* client.execute( + HttpClientRequest.put(url).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status === 201) { + return; + } + + if (response.status === 202) { + return yield* Effect.fail( + new BucketAlreadyOwnedByYou({ + bucketName: container, + message: "Bucket already exists", + }), + ); + } + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", container, "PUT"), + ); + } + }), + + deleteBucket: () => + Effect.gen(function* () { + const { url, token, container } = target; + const response = yield* client.execute( + HttpClientRequest.del(url).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + yield* Effect.logDebug( + `Swift deleteBucket container=[${container}] status=${response.status}`, + ); + + if (response.status === 204) { + return; + } + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "DELETE", + ), + ); + } + }), + + headBucket: () => + Effect.gen(function* () { + const { url, token, container } = target; + const response = yield* client.execute( + HttpClientRequest.head(url).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", container, "HEAD"), + ); + } + }), +}); diff --git a/src/Backends/Swift/Client.ts b/src/Backends/Swift/Client.ts index 7be8123..0feda08 100644 --- a/src/Backends/Swift/Client.ts +++ b/src/Backends/Swift/Client.ts @@ -1,6 +1,7 @@ -import { Context, Effect, Layer, type Schema } from "effect"; +import { Cache, Context, Effect, Layer, type Schema } from "effect"; +import { HttpClient, HttpClientRequest } from "@effect/platform"; import type { MaterializedBucket, SwiftConfig } from "../../Domain/Config.ts"; -import { AppConfig } from "../../Config/Layer.ts"; +import { HeraldConfig } from "../../Config/Layer.ts"; export interface SwiftAuthMeta { readonly token: string; @@ -35,160 +36,152 @@ interface SwiftTokenResponse { export const SwiftClientLive = Layer.effect( SwiftClient, - AppConfig.pipe( - Effect.flatMap((appConfig) => { - const cache = new Map(); + Effect.gen(function* () { + const appConfig = yield* HeraldConfig; + const client = yield* HttpClient.HttpClient; + + const fetchAuthMeta = ( + config: Schema.Schema.Type, + ): Effect.Effect => { + const { auth_url, credentials, region } = config; + + if (!credentials || !("username" in credentials)) { + return Effect.fail( + new Error( + "Swift credentials (username, password, etc.) are required", + ), + ); + } + + const { + username, + password, + project_name, + user_domain_name = "Default", + project_domain_name = "Default", + } = credentials; + + const requestBody = { + auth: { + identity: { + methods: ["password"], + password: { + user: { + name: username, + domain: { name: user_domain_name }, + password: password, + }, + }, + }, + scope: { + project: { + domain: { name: project_domain_name }, + name: project_name, + }, + }, + }, + }; - const fetchAuthMeta = ( - config: Schema.Schema.Type, - ): Effect.Effect => { - const { auth_url, credentials, region } = config; + return Effect.gen(function* () { + const request = yield* HttpClientRequest.post(`${auth_url}/auth/tokens`) + .pipe( + HttpClientRequest.bodyJson(requestBody), + Effect.mapError((e) => new Error(String(e))), + ); + const response = yield* client.execute(request).pipe( + Effect.mapError((e) => new Error(String(e))), + ); - if (!auth_url) { - return Effect.fail( - new Error("auth_url is required for Swift backend"), + if (response.status < 200 || response.status >= 300) { + const msg = yield* response.text.pipe( + Effect.orElseSucceed(() => "Unknown error"), + ); + return yield* Effect.fail( + new Error(`Failed to authenticate with Swift: ${msg}`), ); } - if (!credentials || !("username" in credentials)) { - return Effect.fail( + + const token = response.headers["x-subject-token"]; + const tokenStr = Array.isArray(token) ? token[0] : token; + + if (!tokenStr) { + return yield* Effect.fail( new Error( - "Swift credentials (username, password, etc.) are required", + "X-Subject-Token header missing from Swift response", ), ); } - const { - username, - password, - project_name, - user_domain_name = "Default", - project_domain_name = "Default", - } = credentials; - - const requestBody = JSON.stringify({ - auth: { - identity: { - methods: ["password"], - password: { - user: { - name: username, - domain: { name: user_domain_name }, - password: password, - }, - }, - }, - scope: { - project: { - domain: { name: project_domain_name }, - name: project_name, - }, - }, - }, - }); - - return Effect.tryPromise({ - try: async () => { - const response = await fetch(`${auth_url}/auth/tokens`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: requestBody, - }); - - if (!response.ok) { - const msg = await response.text(); - throw new Error(`Failed to authenticate with Swift: ${msg}`); - } - - const token = response.headers.get("X-Subject-Token"); - if (!token) { - throw new Error( - "X-Subject-Token header missing from Swift response", - ); - } - - const body = (await response.json()) as SwiftTokenResponse; - const catalog = body.token.catalog; - const storageService = catalog.find((s) => - s.type === "object-store" - ); - - if (!storageService) { - throw new Error( - "Object Store service not found in Swift catalog", - ); - } - - const endpoint = storageService.endpoints.find( - (e) => - (region ? e.region === region : true) && - e.interface === "public", - ); - - if (!endpoint) { - throw new Error( - `Public Swift endpoint not found (region: ${region ?? "any"})`, - ); - } - - return { - token, - storageUrl: endpoint.url, - }; - }, - catch: (e) => e as Error, - }); - }; + const body = (yield* response.json.pipe( + Effect.mapError((e) => new Error(String(e))), + )) as SwiftTokenResponse; - return Effect.succeed( - SwiftClient.of({ - getAuthMeta: ( - bucket: MaterializedBucket | { backend_id: string }, - ) => { - let backend_id: string; - let config: Schema.Schema.Type; - - if ("protocol" in bucket) { - backend_id = bucket.backend_id; - config = appConfig.raw.backends[backend_id] as Schema.Schema.Type< - typeof SwiftConfig - >; - } else { - backend_id = bucket.backend_id; - config = appConfig.raw.backends[backend_id] as Schema.Schema.Type< - typeof SwiftConfig - >; - } - - if (!config || config.protocol !== "swift") { - return Effect.fail( - new Error(`Backend ${backend_id} is not a Swift backend`), - ); - } - - const cacheKey = - `${backend_id}:${config.auth_url}:${config.region}`; - const cached = cache.get(cacheKey); - const now = Date.now(); - - if (cached && cached.expires > now) { - return Effect.succeed({ - token: cached.token, - storageUrl: cached.storageUrl, - }); - } - - return fetchAuthMeta(config).pipe( - Effect.tap((meta) => { - // Cache for 50 minutes (Swift tokens usually last 1h) - cache.set(cacheKey, { - ...meta, - expires: now + 50 * 60 * 1000, - }); - }), - ); - }, - }), - ); - }), - ), + const catalog = body.token.catalog; + const storageService = catalog.find((s) => s.type === "object-store"); + + if (!storageService) { + return yield* Effect.fail( + new Error( + "Object Store service not found in Swift catalog", + ), + ); + } + + const endpoint = storageService.endpoints.find( + (e) => + (region ? e.region === region : true) && + e.interface === "public", + ); + + if (!endpoint) { + return yield* Effect.fail( + new Error( + `Public Swift endpoint not found (region: ${region ?? "any"})`, + ), + ); + } + + return { + token: tokenStr, + storageUrl: endpoint.url, + }; + }); + }; + + const cache = yield* Cache.make({ + capacity: 100, + timeToLive: "50 minutes", // Swift tokens usually last 1h + lookup: (config: Schema.Schema.Type) => + fetchAuthMeta(config), + }); + + return SwiftClient.of({ + getAuthMeta: ( + bucket: MaterializedBucket | { backend_id: string }, + ) => { + let backend_id: string; + let config: Schema.Schema.Type; + + if ("protocol" in bucket) { + backend_id = bucket.backend_id; + config = appConfig.raw.backends[backend_id] as Schema.Schema.Type< + typeof SwiftConfig + >; + } else { + backend_id = bucket.backend_id; + config = appConfig.raw.backends[backend_id] as Schema.Schema.Type< + typeof SwiftConfig + >; + } + + if (!config || config.protocol !== "swift") { + return Effect.fail( + new Error(`Backend ${backend_id} is not a Swift backend`), + ); + } + + return cache.get(config); + }, + }); + }), ); diff --git a/src/Backends/Swift/Objects.ts b/src/Backends/Swift/Objects.ts new file mode 100644 index 0000000..431394b --- /dev/null +++ b/src/Backends/Swift/Objects.ts @@ -0,0 +1,529 @@ +import { Effect, Option, type Stream } from "effect"; +import { type HttpClient, HttpClientRequest } from "@effect/platform"; +import { + type CommonPrefix, + type DeleteObjectsResult, + InternalError, + type ListObjectsResult, + type ObjectInfo, + type ObjectResponse, + type PutObjectResult, +} from "../../Services/Backend.ts"; +import { mapError, type SwiftTarget } from "./Utils.ts"; +import { fixHeaderEncoding } from "../../Frontend/Utils.ts"; + +export interface SwiftObject { + readonly name?: string; + readonly hash?: string; + readonly bytes?: number; + readonly content_type?: string; + readonly last_modified?: string; + readonly subdir?: string; +} + +export const makeObjectOps = ( + target: SwiftTarget, + client: HttpClient.HttpClient, +) => { + const listObjects = (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + encodingType?: string; + continuationToken?: string; + startAfter?: string; + listType?: 1 | 2; + }) => + Effect.gen(function* () { + const { url, token, container } = target; + const limit = args.maxKeys ?? 1000; + const query = new URLSearchParams({ format: "json" }); + if (args.prefix) query.set("prefix", args.prefix); + if (args.delimiter) query.set("delimiter", args.delimiter); + if (args.marker) query.set("marker", args.marker); + query.set("limit", String(limit + 1)); + if (args.continuationToken) query.set("marker", args.continuationToken); + if (args.startAfter) query.set("marker", args.startAfter); + + const response = yield* client.execute( + HttpClientRequest.get(`${url}?${query.toString()}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + yield* Effect.logDebug( + `Swift listObjects query=[${query.toString()}] status=${response.status}`, + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", container, "GET"), + ); + } + + const rawObjects = (yield* response.json.pipe( + Effect.mapError((e) => + mapError(500, `Failed to parse Swift response: ${e}`, container) + ), + )) as readonly SwiftObject[]; + + const isTruncated = rawObjects.length > limit; + const objects = isTruncated ? rawObjects.slice(0, limit) : rawObjects; + + const contents: ObjectInfo[] = []; + const commonPrefixes: CommonPrefix[] = []; + + for (const obj of objects) { + if (obj.subdir) { + commonPrefixes.push({ prefix: obj.subdir }); + } else if (obj.name) { + contents.push({ + key: obj.name, + lastModified: obj.last_modified + ? new Date(obj.last_modified) + : new Date(), + etag: obj.hash ? `"${obj.hash}"` : "", + size: obj.bytes ?? 0, + storageClass: "STANDARD", + owner: { id: "swift", displayName: "Swift User" }, + }); + } + } + + const nextMarker = isTruncated && objects.length > 0 + ? objects[objects.length - 1].name || + objects[objects.length - 1].subdir + : undefined; + + return { + name: container, + prefix: args.prefix, + maxKeys: limit, + delimiter: args.delimiter, + isTruncated, + marker: args.marker, + nextMarker, + contents, + commonPrefixes, + encodingType: args.encodingType, + listType: args.listType ?? 1, + nextContinuationToken: args.listType === 2 ? nextMarker : undefined, + keyCount: contents.length + commonPrefixes.length, + } satisfies ListObjectsResult; + }); + + return { + listObjects: (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + encodingType?: string; + continuationToken?: string; + startAfter?: string; + listType?: 1 | 2; + }) => listObjects(args), + + listVersions: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + versionIdMarker?: string; + maxKeys?: number; + encodingType?: string; + }) => + Effect.gen(function* () { + const result = yield* listObjects({ + prefix: args.prefix, + delimiter: args.delimiter, + marker: args.keyMarker, + maxKeys: args.maxKeys, + }); + return { + ...result, + contents: result.contents.map((c) => ({ + ...c, + versionId: "null", + isLatest: true, + })), + }; + }), + + getObject: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const swiftHeaders: Record = { + "X-Auth-Token": token, + }; + if (headers["range"] || headers["Range"]) { + swiftHeaders["Range"] = String( + headers["range"] || headers["Range"], + ); + } + if (headers["if-match"] || headers["If-Match"]) { + swiftHeaders["If-Match"] = String( + headers["if-match"] || + headers["If-Match"], + ); + } + if (headers["if-none-match"] || headers["If-None-Match"]) { + swiftHeaders["If-None-Match"] = String( + headers["if-none-match"] || + headers["If-None-Match"], + ); + } + if (headers["if-modified-since"] || headers["If-Modified-Since"]) { + swiftHeaders["If-Modified-Since"] = String( + headers["if-modified-since"] || + headers["If-Modified-Since"], + ); + } + if ( + headers["if-unmodified-since"] || headers["If-Unmodified-Since"] + ) { + swiftHeaders["If-Unmodified-Since"] = String( + headers["if-unmodified-since"] || + headers["If-Unmodified-Since"], + ); + } + + const response = yield* client.execute( + HttpClientRequest.get(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders(swiftHeaders), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "GET", + key, + ), + ); + } + + const metadata: Record = {}; + const s3Headers: Record = {}; + + for (const [k, v] of Object.entries(response.headers)) { + const lowK = k.toLowerCase(); + const value = Array.isArray(v) ? v.join(", ") : v; + if (lowK.startsWith("x-object-meta-")) { + const metaKey = lowK.substring("x-object-meta-".length); + const decodedValue = (value.includes("%")) + ? Option.liftThrowable(decodeURIComponent)(value).pipe( + Option.getOrElse(() => value), + ) + : value; + metadata[metaKey] = decodedValue; + s3Headers[`x-amz-meta-${metaKey}`] = decodedValue; + } else if (lowK === "content-type") { + s3Headers["Content-Type"] = value; + } else if (lowK === "content-length") { + s3Headers["Content-Length"] = value; + } else if (lowK === "etag") { + s3Headers["ETag"] = value; + } else if (lowK === "last-modified") { + s3Headers["Last-Modified"] = value; + } + } + + const contentLengthHeader = response.headers["content-length"]; + const contentLength = Array.isArray(contentLengthHeader) + ? parseInt(contentLengthHeader[0] || "0") + : parseInt(contentLengthHeader || "0"); + + const etagHeader = response.headers["etag"]; + const etag = Array.isArray(etagHeader) ? etagHeader[0] : etagHeader; + + const lastModifiedHeader = response.headers["last-modified"]; + const lastModified = Array.isArray(lastModifiedHeader) + ? lastModifiedHeader[0] + : lastModifiedHeader; + + return { + stream: response.stream, + contentType: (Array.isArray(response.headers["content-type"]) + ? response.headers["content-type"][0] + : response.headers["content-type"]) || undefined, + contentLength, + etag: etag || undefined, + lastModified: lastModified ? new Date(lastModified) : undefined, + metadata, + headers: s3Headers, + } satisfies ObjectResponse; + }), + + headObject: ( + key: string, + _headers: Record, + ) => + Effect.gen(function* () { + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const swiftHeaders: Record = { + "X-Auth-Token": token, + }; + // ... handle headers if needed + const response = yield* client.execute( + HttpClientRequest.head(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders(swiftHeaders), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "HEAD", + key, + ), + ); + } + + const metadata: Record = {}; + const s3Headers: Record = {}; + + for (const [k, v] of Object.entries(response.headers)) { + const lowK = k.toLowerCase(); + const value = Array.isArray(v) ? v.join(", ") : v; + if (lowK.startsWith("x-object-meta-")) { + const metaKey = lowK.substring("x-object-meta-".length); + const decodedValue = (value.includes("%")) + ? Option.liftThrowable(decodeURIComponent)(value).pipe( + Option.getOrElse(() => value), + ) + : value; + metadata[metaKey] = decodedValue; + s3Headers[`x-amz-meta-${metaKey}`] = decodedValue; + } else if (lowK === "content-type") { + s3Headers["Content-Type"] = value; + } else if (lowK === "content-length") { + s3Headers["Content-Length"] = value; + } else if (lowK === "etag") { + s3Headers["ETag"] = value; + } else if (lowK === "last-modified") { + s3Headers["Last-Modified"] = value; + } + } + + const contentLengthHeader = response.headers["content-length"]; + const contentLength = Array.isArray(contentLengthHeader) + ? parseInt(contentLengthHeader[0] || "0") + : parseInt(contentLengthHeader || "0"); + + const etagHeader = response.headers["etag"]; + const etag = Array.isArray(etagHeader) ? etagHeader[0] : etagHeader; + + const lastModifiedHeader = response.headers["last-modified"]; + const lastModified = Array.isArray(lastModifiedHeader) + ? lastModifiedHeader[0] + : lastModifiedHeader; + + return { + contentType: (Array.isArray(response.headers["content-type"]) + ? response.headers["content-type"][0] + : response.headers["content-type"]) || undefined, + contentLength, + etag: etag || undefined, + lastModified: lastModified ? new Date(lastModified) : undefined, + metadata, + headers: s3Headers, + }; + }), + + putObject: ( + key: string, + stream: Stream.Stream, + headers: Record, + ) => + Effect.gen(function* () { + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const contentLength = headers["content-length"] || + headers["Content-Length"]; + + const swiftHeaders: Record = { + "X-Auth-Token": token, + "Content-Type": (headers["content-type"] || headers["Content-Type"] || + "application/octet-stream") as string, + ...(contentLength ? { "Content-Length": String(contentLength) } : {}), + }; + + for (const [k, v] of Object.entries(headers)) { + const lowK = k.toLowerCase(); + if (lowK.startsWith("x-amz-meta-")) { + const metaKey = lowK.substring("x-amz-meta-".length); + const value = fixHeaderEncoding(String(v)); + swiftHeaders[`X-Object-Meta-${metaKey}`] = + /[^\x20-\x7E]/.test(value) + ? encodeURIComponent(value) + : value; + } + } + + const request = HttpClientRequest.put(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders(swiftHeaders), + HttpClientRequest.bodyStream(stream), + ); + + const response = yield* client.execute(request).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + yield* Effect.logDebug( + `Swift putObject key=[${key}] status=${response.status}`, + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "PUT", + key, + ), + ); + } + + const etagHeader = response.headers["etag"]; + const etagValue = Array.isArray(etagHeader) + ? etagHeader[0] + : etagHeader; + + return { + etag: etagValue || undefined, + } satisfies PutObjectResult; + }), + + deleteObject: (key: string) => + Effect.gen(function* () { + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const response = yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status < 200 || response.status >= 300) { + if (response.status === 404) { + return; + } + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "DELETE", + key, + ), + ); + } + }), + + deleteObjects: (objects: readonly { key: string; versionId?: string }[]) => + Effect.gen(function* () { + const { url, token, container } = target; + const deleted: string[] = []; + const errors: { key: string; code: string; message: string }[] = []; + + for (const obj of objects) { + const encodedKey = obj.key.split("/").map(encodeURIComponent).join( + "/", + ); + const response = yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + yield* Effect.logDebug( + `Swift deleteObject key=[${obj.key}] status=${response.status}`, + ); + + if ( + (response.status >= 200 && response.status < 300) || + response.status === 204 || response.status === 404 + ) { + deleted.push(obj.key); + } else { + const errorBody = yield* response.text.pipe( + Effect.orElseSucceed(() => "Unknown error"), + ); + errors.push({ + key: obj.key, + code: String(response.status), + message: errorBody, + }); + } + } + + return { deleted, errors } satisfies DeleteObjectsResult; + }), + + createMultipartUpload: ( + _key: string, + _headers: Record, + ) => Effect.fail(new InternalError({ message: "Not implemented" })), + uploadPart: ( + _key: string, + _uploadId: string, + _partNumber: number, + _body: Stream.Stream, + ) => Effect.fail(new InternalError({ message: "Not implemented" })), + completeMultipartUpload: ( + _key: string, + _uploadId: string, + _parts: readonly { etag: string; partNumber: number }[], + ) => Effect.fail(new InternalError({ message: "Not implemented" })), + abortMultipartUpload: (_key: string, _uploadId: string) => + Effect.fail(new InternalError({ message: "Not implemented" })), + listMultipartUploads: (_args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + uploadIdMarker?: string; + maxUploads?: number; + encodingType?: string; + }) => Effect.fail(new InternalError({ message: "Not implemented" })), + listParts: (_key: string, _uploadId: string) => + Effect.fail(new InternalError({ message: "Not implemented" })), + }; +}; diff --git a/src/Backends/Swift/Utils.ts b/src/Backends/Swift/Utils.ts new file mode 100644 index 0000000..62295b0 --- /dev/null +++ b/src/Backends/Swift/Utils.ts @@ -0,0 +1,74 @@ +import { Effect } from "effect"; +import { + type BackendError, + BucketAlreadyExists, + BucketAlreadyOwnedByYou, + BucketNotEmpty, + InternalError, + NoSuchBucket, + NoSuchKey, +} from "../../Services/Backend.ts"; +import type { MaterializedBucket } from "../../Domain/Config.ts"; +import { SwiftClient } from "./Client.ts"; + +export interface SwiftTarget { + readonly storageUrl: string; + readonly token: string; + readonly container: string; + readonly url: string; +} + +export const mapError = ( + status: number, + message: string, + bucketName: string, + method?: string, + key?: string, +): BackendError => { + switch (status) { + case 404: + if (key) { + return new NoSuchKey({ bucketName, key, message }); + } + return new NoSuchBucket({ bucketName, message }); + case 409: + if (method === "DELETE") { + return new BucketNotEmpty({ bucketName, message }); + } + return new BucketAlreadyExists({ bucketName, message }); + case 202: + if (method === "PUT") { + return new BucketAlreadyOwnedByYou({ bucketName, message }); + } + return new InternalError({ + message: `Swift error (${status}): ${message}`, + }); + default: + return new InternalError({ + message: `Swift error (${status}): ${message}`, + }); + } +}; + +/** + * Resolves the target container and acquires the Swift token dynamically. + */ +export const getTarget = ( + bucket: MaterializedBucket | { backend_id: string }, +): Effect.Effect => + Effect.gen(function* () { + const swiftClient = yield* SwiftClient; + const auth = yield* swiftClient.getAuthMeta(bucket).pipe( + Effect.mapError((e) => new InternalError({ message: e.message })), + ); + const container = "bucket_name" in bucket ? bucket.bucket_name : ""; + const encodedContainer = container ? encodeURIComponent(container) : ""; + return { + storageUrl: auth.storageUrl, + token: auth.token, + container, + url: encodedContainer + ? `${auth.storageUrl}/${encodedContainer}` + : auth.storageUrl, + }; + }); diff --git a/src/Config/Layer.ts b/src/Config/Layer.ts index 4acdfeb..8efd2d0 100644 --- a/src/Config/Layer.ts +++ b/src/Config/Layer.ts @@ -6,8 +6,8 @@ import { type MaterializedBucket, } from "../Domain/Config.ts"; -export class AppConfig extends Context.Tag("AppConfig")< - AppConfig, +export class HeraldConfig extends Context.Tag("HeraldConfig")< + HeraldConfig, { readonly raw: GlobalConfig; readonly lookupBucket: (name: string) => Option.Option; @@ -109,8 +109,8 @@ export function parseConfig( return Schema.decodeUnknownSync(GlobalConfig)({ backends }); } -export const AppConfigLive = Layer.effect( - AppConfig, +export const HeraldConfigLive = Layer.effect( + HeraldConfig, Effect.gen(function* () { const configPath = yield* Config.string("HERALD_CONFIG_PATH").pipe( Config.orElse(() => Config.string("CONFIG_PATH")), diff --git a/src/Domain/Config.ts b/src/Domain/Config.ts index 337f560..77f32a0 100644 --- a/src/Domain/Config.ts +++ b/src/Domain/Config.ts @@ -41,7 +41,7 @@ export const S3Config = Schema.Struct({ export const SwiftConfig = Schema.Struct({ protocol: Schema.Literal("swift"), - auth_url: Schema.optional(Schema.String), + auth_url: Schema.String, region: Schema.optional(Schema.String), container: Schema.optional(Schema.String), credentials: Schema.optional(SwiftCredentials), diff --git a/src/Frontend/Api.ts b/src/Frontend/Api.ts index 91a78de..f4a7fdc 100644 --- a/src/Frontend/Api.ts +++ b/src/Frontend/Api.ts @@ -5,7 +5,7 @@ export class BadGateway extends Schema.TaggedError()("BadGateway", { message: Schema.String, }) {} -export const S3Api = HttpApiGroup.make("s3") +export const HttpS3Api = HttpApiGroup.make("s3") .add( HttpApiEndpoint.post("postRoot", "/") .addError(BadGateway, { status: 502 }), diff --git a/src/Frontend/Buckets/Create.ts b/src/Frontend/Buckets/Create.ts index 7d42506..68c32a9 100644 --- a/src/Frontend/Buckets/Create.ts +++ b/src/Frontend/Buckets/Create.ts @@ -1,40 +1,37 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; -export const createBucket = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const url = new URL(request.url, "http://localhost"); - yield* Effect.logDebug( - `createBucket bucket=[${bucket}] url=[${request.url}]`, - ); +export const createBucket = () => + Effect.gen(function* () { + const { backend, bucket, params, request } = yield* RequestContext; - if (url.searchParams.has("acl")) { - // PutBucketAcl - // Check for canned ACL validity if present - const cannedAcl = request.headers["x-amz-acl"]; - const validCannedAcls = [ - "private", - "public-read", - "public-read-write", - "authenticated-read", - ]; - if (cannedAcl && !validCannedAcls.includes(cannedAcl)) { - return HttpServerResponse.text( - `InvalidArgumentArgument x-amz-acl is invalid.`, - { status: 400, headers: { "Content-Type": "application/xml" } }, - ); - } + yield* Effect.logDebug( + `createBucket bucket=[${bucket}] url=[${request.url}]`, + ); - // For now, we just return 200 OK if the bucket exists - yield* backend.headBucket(); - return HttpServerResponse.text("", { status: 200 }); + if (params.acl !== undefined) { + // PutBucketAcl + // Check for canned ACL validity if present + const cannedAcl = request.headers["x-amz-acl"]; + const validCannedAcls = [ + "private", + "public-read", + "public-read-write", + "authenticated-read", + ]; + if (cannedAcl && !validCannedAcls.includes(cannedAcl)) { + return HttpServerResponse.text( + `InvalidArgumentArgument x-amz-acl is invalid.`, + { status: 400, headers: { "Content-Type": "application/xml" } }, + ); } - yield* backend.createBucket(); + // For now, we just return 200 OK if the bucket exists + yield* backend.headBucket(); return HttpServerResponse.text("", { status: 200 }); - })); + } + + yield* backend.createBucket(); + return HttpServerResponse.text("", { status: 200 }); + }); diff --git a/src/Frontend/Buckets/Delete.ts b/src/Frontend/Buckets/Delete.ts index de3a301..6c7fbe1 100644 --- a/src/Frontend/Buckets/Delete.ts +++ b/src/Frontend/Buckets/Delete.ts @@ -1,12 +1,10 @@ import { Effect } from "effect"; import { HttpServerResponse } from "@effect/platform"; -import { resolveBucket } from "../Utils.ts"; +import { RequestContext } from "../Utils.ts"; -export const deleteBucket = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - yield* backend.deleteBucket(); - return HttpServerResponse.empty({ status: 204 }); - })); +export const deleteBucket = () => + Effect.gen(function* () { + const { backend } = yield* RequestContext; + yield* backend.deleteBucket(); + return HttpServerResponse.empty({ status: 204 }); + }); diff --git a/src/Frontend/Buckets/Head.ts b/src/Frontend/Buckets/Head.ts index fb371d8..a076d71 100644 --- a/src/Frontend/Buckets/Head.ts +++ b/src/Frontend/Buckets/Head.ts @@ -1,12 +1,10 @@ import { Effect } from "effect"; import { HttpServerResponse } from "@effect/platform"; -import { resolveBucket } from "../Utils.ts"; +import { RequestContext } from "../Utils.ts"; -export const headBucket = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - yield* backend.headBucket(); - return HttpServerResponse.empty({ status: 200 }); - })); +export const headBucket = () => + Effect.gen(function* () { + const { backend } = yield* RequestContext; + yield* backend.headBucket(); + return HttpServerResponse.empty({ status: 200 }); + }); diff --git a/src/Frontend/Buckets/List.ts b/src/Frontend/Buckets/List.ts index a6759fc..4bb13f5 100644 --- a/src/Frontend/Buckets/List.ts +++ b/src/Frontend/Buckets/List.ts @@ -1,11 +1,11 @@ import { Effect } from "effect"; -import { AppConfig } from "../../Config/Layer.ts"; +import { HeraldConfig } from "../../Config/Layer.ts"; import { S3Xml } from "../../Services/S3Xml.ts"; import { resolveBackend } from "../Utils.ts"; export const listBuckets = () => Effect.gen(function* () { - const config = yield* AppConfig; + const config = yield* HeraldConfig; // For ListBuckets, we need to decide which backend to proxy to. // We prefer an S3 backend if available, otherwise we take the first one. diff --git a/src/Frontend/Health/Api.ts b/src/Frontend/Health/Api.ts index 70032e5..829ae5f 100644 --- a/src/Frontend/Health/Api.ts +++ b/src/Frontend/Health/Api.ts @@ -1,7 +1,7 @@ import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "@effect/platform"; import { Schema } from "effect"; -export class HealthApi extends HttpApiGroup.make("health") +export class HealthHttpApi extends HttpApiGroup.make("health") .add( HttpApiEndpoint.get("getStatus", "/health") .addSuccess(Schema.Struct({ status: Schema.Literal("ok") })), diff --git a/src/Frontend/Health/Http.ts b/src/Frontend/Health/Http.ts index 52e64f2..0f186df 100644 --- a/src/Frontend/Health/Http.ts +++ b/src/Frontend/Health/Http.ts @@ -1,9 +1,9 @@ import { HttpApiBuilder } from "@effect/platform"; import { Effect } from "effect"; -import { Api } from "../../Api.ts"; +import { HttpHeraldApi } from "../../Api.ts"; export const HttpHealthLive = HttpApiBuilder.group( - Api, + HttpHeraldApi, "health", (handlers) => handlers.handle( diff --git a/src/Frontend/Http.ts b/src/Frontend/Http.ts index 28825cc..90b4541 100644 --- a/src/Frontend/Http.ts +++ b/src/Frontend/Http.ts @@ -1,6 +1,6 @@ import { HttpApiBuilder, HttpServerResponse } from "@effect/platform"; import { Effect, Layer } from "effect"; -import { Api } from "../Api.ts"; +import { HttpHeraldApi } from "../Api.ts"; import { listBuckets } from "./Buckets/List.ts"; import { createBucket } from "./Buckets/Create.ts"; import { deleteBucket } from "./Buckets/Delete.ts"; @@ -15,28 +15,34 @@ import { S3ClientLive } from "../Backends/S3/Client.ts"; import { SwiftClientLive } from "../Backends/Swift/Client.ts"; import { S3XmlLive } from "../Services/S3Xml.ts"; import { BackendResolverLive } from "../Services/BackendResolver.ts"; +import { provideRequestContext } from "./Utils.ts"; export const HttpS3Live = HttpApiBuilder.group( - Api, + HttpHeraldApi, "s3", (handlers) => handlers + // handleRaw is preferred througout since + // we want to return XML directly + // after setting our own .handleRaw("postRoot", (_handlers) => Effect.gen(function* () { yield* Effect.logDebug("POST / received"); + // FIXME: what's the purose of this handler? + // 200 diverges from 502 as defiend in the openapi return HttpServerResponse.text("", { status: 200 }); })) .handleRaw("listBuckets", listBuckets) - .handleRaw("createBucket", createBucket) - .handleRaw("deleteBucket", deleteBucket) - .handleRaw("headBucket", headBucket) - .handleRaw("listObjects", listObjects) - .handleRaw("postBucket", postObject) - .handleRaw("getObject", getObject) - .handleRaw("putObject", putObject) - .handleRaw("postObject", postObject) - .handleRaw("deleteObject", deleteObject) - .handleRaw("headObject", headObject), + .handleRaw("createBucket", provideRequestContext(createBucket)) + .handleRaw("deleteBucket", provideRequestContext(deleteBucket)) + .handleRaw("headBucket", provideRequestContext(headBucket)) + .handleRaw("listObjects", provideRequestContext(listObjects)) + .handleRaw("postBucket", provideRequestContext(postObject)) + .handleRaw("getObject", provideRequestContext(getObject)) + .handleRaw("putObject", provideRequestContext(putObject)) + .handleRaw("postObject", provideRequestContext(postObject)) + .handleRaw("deleteObject", provideRequestContext(deleteObject)) + .handleRaw("headObject", provideRequestContext(headObject)), ).pipe( Layer.provide(BackendResolverLive), Layer.provide(S3ClientLive), diff --git a/src/Frontend/Objects/Delete.ts b/src/Frontend/Objects/Delete.ts index f2c9270..3e1856b 100644 --- a/src/Frontend/Objects/Delete.ts +++ b/src/Frontend/Objects/Delete.ts @@ -1,18 +1,20 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; /** * Handler for DeleteObject (DELETE /:bucket/*) */ -export const deleteObject = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const key = extractKey(request.url, bucket); +export const deleteObject = () => + Effect.gen(function* () { + const { backend, key, params } = yield* RequestContext; - yield* backend.deleteObject(key); + if (params.uploadId) { + // Abort Multipart Upload + yield* backend.abortMultipartUpload(key, params.uploadId); return HttpServerResponse.empty({ status: 204 }); - })); + } + + yield* backend.deleteObject(key); + return HttpServerResponse.empty({ status: 204 }); + }); diff --git a/src/Frontend/Objects/Get.ts b/src/Frontend/Objects/Get.ts index fff8504..c223fec 100644 --- a/src/Frontend/Objects/Get.ts +++ b/src/Frontend/Objects/Get.ts @@ -1,20 +1,35 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; /** * Handler for GetObject (GET /:bucket/*) + * Also handles ListParts (?uploadId=...). */ -export const getObject = ({ path: { bucket } }: { path: { bucket: string } }) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const key = extractKey(request.url, bucket); +export const getObject = () => + Effect.gen(function* () { + const { backend, key, params, request } = yield* RequestContext; + const s3Xml = yield* S3Xml; - const result = yield* backend.getObject(key); - return HttpServerResponse.stream(result.stream, { - status: 200, - headers: result.headers, - contentType: result.contentType, - }); - })); + if (params.uploadId) { + // List Parts + const result = yield* backend.listParts(key, params.uploadId); + return s3Xml.formatListParts(result); + } + + const combinedHeaders = { ...request.headers }; + if (params.partNumber) { + combinedHeaders["x-amz-part-number"] = String(params.partNumber); + } + + const result = yield* backend.getObject(key, combinedHeaders); + const status = (request.headers["range"] || request.headers["Range"]) + ? 206 + : 200; + return HttpServerResponse.stream(result.stream, { + status, + headers: result.headers, + contentType: result.contentType, + }); + }); diff --git a/src/Frontend/Objects/Head.ts b/src/Frontend/Objects/Head.ts index b91a0a2..b3daa57 100644 --- a/src/Frontend/Objects/Head.ts +++ b/src/Frontend/Objects/Head.ts @@ -1,21 +1,22 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; /** * Handler for HeadObject (HEAD /:bucket/*) */ -export const headObject = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const key = extractKey(request.url, bucket); +export const headObject = () => + Effect.gen(function* () { + const { backend, key, params, request } = yield* RequestContext; - const result = yield* backend.headObject(key); - return HttpServerResponse.empty({ - status: 200, - headers: result.headers, - }); - })); + const combinedHeaders = { ...request.headers }; + if (params.partNumber) { + combinedHeaders["x-amz-part-number"] = String(params.partNumber); + } + + const result = yield* backend.headObject(key, combinedHeaders); + return HttpServerResponse.empty({ + status: 200, + headers: result.headers, + }); + }); diff --git a/src/Frontend/Objects/List.ts b/src/Frontend/Objects/List.ts index 3552e2e..883f247 100644 --- a/src/Frontend/Objects/List.ts +++ b/src/Frontend/Objects/List.ts @@ -1,47 +1,49 @@ import { Effect } from "effect"; -import { HttpServerRequest } from "@effect/platform"; -import { resolveBucket } from "../Utils.ts"; +import { RequestContext } from "../Utils.ts"; import { S3Xml } from "../../Services/S3Xml.ts"; /** * Handler for ListObjects (GET /:bucket) */ -export const listObjects = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const s3Xml = yield* S3Xml; - const url = new URL(request.url, "http://localhost"); - const searchParams = url.searchParams; +export const listObjects = () => + Effect.gen(function* () { + const { backend, params } = yield* RequestContext; + const s3Xml = yield* S3Xml; - if (searchParams.has("versions")) { - const result = yield* backend.listVersions({ - prefix: searchParams.get("prefix") ?? undefined, - delimiter: searchParams.get("delimiter") ?? undefined, - keyMarker: searchParams.get("key-marker") ?? undefined, - versionIdMarker: searchParams.get("version-id-marker") ?? undefined, - maxKeys: searchParams.has("max-keys") - ? parseInt(searchParams.get("max-keys")!) - : undefined, - encodingType: searchParams.get("encoding-type") ?? undefined, - }); - return s3Xml.formatListVersions(result); - } + if (params.versions !== undefined) { + const result = yield* backend.listVersions({ + prefix: params.prefix, + delimiter: params.delimiter, + keyMarker: params["key-marker"], + versionIdMarker: params["version-id-marker"], + maxKeys: params["max-keys"], + encodingType: params["encoding-type"], + }); + return s3Xml.formatListVersions(result); + } - const result = yield* backend.listObjects({ - prefix: searchParams.get("prefix") ?? undefined, - delimiter: searchParams.get("delimiter") ?? undefined, - marker: searchParams.get("marker") ?? undefined, - maxKeys: searchParams.has("max-keys") - ? parseInt(searchParams.get("max-keys")!) - : undefined, - encodingType: searchParams.get("encoding-type") ?? undefined, - continuationToken: searchParams.get("continuation-token") ?? undefined, - startAfter: searchParams.get("start-after") ?? undefined, - listType: searchParams.get("list-type") === "2" ? 2 : 1, + if (params.uploads !== undefined) { + const result = yield* backend.listMultipartUploads({ + prefix: params.prefix, + delimiter: params.delimiter, + keyMarker: params["key-marker"], + uploadIdMarker: params["upload-id-marker"], + maxUploads: params["max-uploads"], + encodingType: params["encoding-type"], }); + return s3Xml.formatListMultipartUploads(result); + } + + const result = yield* backend.listObjects({ + prefix: params.prefix, + delimiter: params.delimiter, + marker: params.marker, + maxKeys: params["max-keys"], + encodingType: params["encoding-type"], + continuationToken: params["continuation-token"], + startAfter: params["start-after"], + listType: params["list-type"] === "2" ? 2 : 1, + }); - return s3Xml.formatListObjects(result); - })); + return s3Xml.formatListObjects(result); + }); diff --git a/src/Frontend/Objects/Post.ts b/src/Frontend/Objects/Post.ts index c61c6ba..dffade7 100644 --- a/src/Frontend/Objects/Post.ts +++ b/src/Frontend/Objects/Post.ts @@ -1,82 +1,151 @@ import { Effect, Option, Stream } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; /** * Handler for POST requests on buckets or objects. * Primarily used for Multi-Object Delete (POST /:bucket?delete). + * Also handles InitiateMultipartUpload (?uploads) and CompleteMultipartUpload (?uploadId=...). */ -export const postObject = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const url = new URL(request.url, "http://localhost"); - const searchParams = url.searchParams; - const key = extractKey(request.url, bucket); +export const postObject = () => + Effect.gen(function* () { + const { backend, bucket, key, params, request } = yield* RequestContext; + const s3Xml = yield* S3Xml; - if (searchParams.has("delete")) { - // Multi-Object Delete - const bodyChunks = yield* Stream.runCollect(request.stream); - let totalLength = 0; - for (const chunk of Array.from(bodyChunks)) { - totalLength += chunk.length; - } - const bodyBytes = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of Array.from(bodyChunks)) { - bodyBytes.set(chunk, offset); - offset += chunk.length; - } - const bodyText = new TextDecoder().decode(bodyBytes); + if (params.delete !== undefined) { + // Multi-Object Delete + const bodyChunks = yield* Stream.runCollect(request.stream); + let totalLength = 0; + for (const chunk of Array.from(bodyChunks)) { + totalLength += chunk.length; + } + const bodyBytes = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of Array.from(bodyChunks)) { + bodyBytes.set(chunk, offset); + offset += chunk.length; + } + const bodyText = new TextDecoder().decode(bodyBytes); - const objects: { key: string; versionId?: string }[] = []; - // Simple XML parsing for Multi-Object Delete - const objectMatches = Array.from( - bodyText.matchAll(/(.*?)<\/Object>/gs), - ); - for (const match of objectMatches) { - const content = match[1]; - const keyMatch = content.match(/(.*?)<\/Key>/); - const versionIdMatch = content.match(/(.*?)<\/VersionId>/); - if (keyMatch) { - const rawKey = keyMatch[1]; - const key = Option.liftThrowable(decodeURIComponent)(rawKey).pipe( - Option.getOrElse(() => rawKey), - ); - yield* Effect.logDebug(`DeleteObjects extracted key=[${key}]`); - objects.push({ - key, - versionId: versionIdMatch ? versionIdMatch[1] : undefined, - }); - } + const objects: { key: string; versionId?: string }[] = []; + // Simple XML parsing for Multi-Object Delete + const objectMatches = Array.from( + bodyText.matchAll(/(.*?)<\/Object>/gs), + ); + for (const match of objectMatches) { + const content = match[1]; + const keyMatch = content.match(/(.*?)<\/Key>/); + const versionIdMatch = content.match(/(.*?)<\/VersionId>/); + if (keyMatch) { + const rawKey = keyMatch[1]; + const key = Option.liftThrowable(decodeURIComponent)(rawKey).pipe( + Option.getOrElse(() => rawKey), + ); + yield* Effect.logDebug(`DeleteObjects extracted key=[${key}]`); + objects.push({ + key, + versionId: versionIdMatch ? versionIdMatch[1] : undefined, + }); } + } - if (objects.length > 0) { - const deleteResult = yield* backend.deleteObjects(objects); - const deletedXml = deleteResult.deleted.map((k) => - `${k}` - ).join(""); - const errorsXml = deleteResult.errors.map((e) => - `${e.key}${e.code}${e.message}` - ).join(""); + if (objects.length > 0) { + const deleteResult = yield* backend.deleteObjects(objects); + const deletedXml = deleteResult.deleted.map((k) => + `${k}` + ).join(""); + const errorsXml = deleteResult.errors.map((e) => + `${e.key}${e.code}${e.message}` + ).join(""); - const xml = - `${deletedXml}${errorsXml}`; - return HttpServerResponse.text(xml, { - headers: { "Content-Type": "application/xml" }, - }); - } - // If no keys, still return empty result const xml = - ``; + `${deletedXml}${errorsXml}`; return HttpServerResponse.text(xml, { headers: { "Content-Type": "application/xml" }, }); } + // If no keys, still return empty result + const xml = + ``; + return HttpServerResponse.text(xml, { + headers: { "Content-Type": "application/xml" }, + }); + } - return yield* Effect.fail( - new Error(`Method POST for key [${key}] not implemented`), + if (params.uploads !== undefined) { + // Initiate Multipart Upload + const result = yield* backend.createMultipartUpload( + key, + request.headers, + ); + return s3Xml.formatInitiateMultipartUpload( + bucket, + key, + result.uploadId, ); - })); + } + + if (params.uploadId) { + // Complete Multipart Upload + const bodyChunks = yield* Stream.runCollect(request.stream); + let totalLength = 0; + for (const chunk of Array.from(bodyChunks)) { + totalLength += chunk.length; + } + const bodyBytes = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of Array.from(bodyChunks)) { + bodyBytes.set(chunk, offset); + offset += chunk.length; + } + const bodyText = new TextDecoder().decode(bodyBytes); + + const parts: { etag: string; partNumber: number }[] = []; + const partMatches = Array.from( + bodyText.matchAll(/(.*?)<\/Part>/gs), + ); + for (const match of partMatches) { + const content = match[1]; + const partNumberMatch = content.match( + /(.*?)<\/PartNumber>/, + ); + const etagMatch = content.match(/(.*?)<\/ETag>/); + if (partNumberMatch && etagMatch) { + parts.push({ + partNumber: parseInt(partNumberMatch[1]), + etag: etagMatch[1].replace(/"/g, '"'), + }); + } + } + + const result = yield* backend.completeMultipartUpload( + key, + params.uploadId, + parts, + ).pipe( + Effect.catchTag("NoSuchUpload", (e) => + Effect.gen(function* () { + // Idempotency: check if object already exists + const head = yield* backend.headObject(key, {}).pipe( + Effect.orElseFail(() => e), + ); + if (head.etag) { + return { + location: `http://localhost/${bucket}/${key}`, // Approximate + bucket, + key, + etag: head.etag, + versionId: head.headers["x-amz-version-id"], + }; + } + return yield* Effect.fail(e); + })), + ); + return s3Xml.formatCompleteMultipartUpload(result); + } + + return yield* Effect.fail( + new Error(`Method POST for key [${key}] not implemented`), + ); + }); diff --git a/src/Frontend/Objects/Put.ts b/src/Frontend/Objects/Put.ts index 62894cc..08421cd 100644 --- a/src/Frontend/Objects/Put.ts +++ b/src/Frontend/Objects/Put.ts @@ -1,27 +1,39 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; /** * Handler for PutObject (PUT /:bucket/*) */ -export const putObject = ({ path: { bucket } }: { path: { bucket: string } }) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const key = extractKey(request.url, bucket); +export const putObject = () => + Effect.gen(function* () { + const { backend, key, params, request } = yield* RequestContext; - const result = yield* backend.putObject( + if (params.partNumber && params.uploadId) { + // Upload Part + const result = yield* backend.uploadPart( key, + params.uploadId, + params.partNumber, request.stream, - request.headers, ); - const headers: Record = {}; - if (result.etag) headers["etag"] = result.etag; - if (result.versionId) headers["x-amz-version-id"] = result.versionId; - return HttpServerResponse.empty({ status: 200, - headers, + headers: { ETag: result.etag }, }); - })); + } + + const result = yield* backend.putObject( + key, + request.stream, + request.headers, + ); + const headers: Record = {}; + if (result.etag) headers["ETag"] = result.etag; + if (result.versionId) headers["x-amz-version-id"] = result.versionId; + + return HttpServerResponse.empty({ + status: 200, + headers, + }); + }); diff --git a/src/Frontend/Utils.ts b/src/Frontend/Utils.ts index 9e08b74..972743c 100644 --- a/src/Frontend/Utils.ts +++ b/src/Frontend/Utils.ts @@ -1,4 +1,4 @@ -import { Effect, Option } from "effect"; +import { Context, Effect, Either, Option, Schema } from "effect"; import { BackendResolver } from "../Services/BackendResolver.ts"; import { S3Xml } from "../Services/S3Xml.ts"; import { @@ -7,12 +7,23 @@ import { BucketAlreadyExists, BucketAlreadyOwnedByYou, BucketNotEmpty, + DeleteObjectsError, + EntityTooSmall, InternalError, + InvalidPart, + InvalidPartOrder, + InvalidRequest, + MalformedXML, NoSuchBucket, NoSuchKey, + NoSuchUpload, } from "../Services/Backend.ts"; -import { HttpServerRequest, type HttpServerResponse } from "@effect/platform"; -import type { AppConfig } from "../Config/Layer.ts"; +import { + HttpServerRequest, + type HttpServerResponse, + Url, +} from "@effect/platform"; +import type { HeraldConfig } from "../Config/Layer.ts"; import type { S3Client } from "../Backends/S3/Client.ts"; import type { SwiftClient } from "../Backends/Swift/Client.ts"; import { BadGateway } from "./Api.ts"; @@ -38,9 +49,10 @@ export function fixHeaderEncoding(value: string): string { * Extracts the object key from the request URL, given the bucket name. */ export function extractKey(requestUrl: string, bucket: string): string { - const pathname = requestUrl.startsWith("/") - ? requestUrl - : new URL(requestUrl).pathname; + const urlResult = Url.fromString(requestUrl, "http://localhost"); + const pathname = Either.isRight(urlResult) + ? urlResult.right.pathname + : requestUrl; const [pathOnly] = pathname.split("?"); const bucketPrefixWithSlash = `/${bucket}/`; @@ -54,6 +66,112 @@ export function extractKey(requestUrl: string, bucket: string): string { return ""; } +/** + * Context for S3 operations (bucket or object). + */ +export class RequestContext extends Context.Tag("RequestContext")< + RequestContext, + { + readonly backend: typeof Backend.Service; + readonly bucket: string; + readonly key: string; + readonly params: S3QueryParams; + readonly request: HttpServerRequest.HttpServerRequest; + } +>() {} + +/** + * Higher-order function to handle S3 context. + */ +export function provideRequestContext< + A extends HttpServerResponse.HttpServerResponse, + E, + R, +>( + fn: () => Effect.Effect, +): ( + args: { path: { bucket: string } }, +) => Effect.Effect< + HttpServerResponse.HttpServerResponse, + BadGateway, + | Exclude + | BackendResolver + | S3Xml + | HeraldConfig + | S3Client + | SwiftClient + | HttpServerRequest.HttpServerRequest +> { + return ({ path: { bucket } }) => + resolveBucket(bucket, (backend) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const urlResult = Url.fromString(request.url, "http://localhost"); + if (Either.isLeft(urlResult)) { + return yield* Effect.fail( + new InternalError({ message: String(urlResult.left) }), + ); + } + const url = urlResult.right; + const key = extractKey(request.url, bucket); + const params = yield* parseQueryParams(url.searchParams, S3QueryParams); + const ctx = { + backend, + bucket, + key, + params, + request, + }; + return yield* fn().pipe(Effect.provideService(RequestContext, ctx)); + }) as unknown as Effect.Effect< + HttpServerResponse.HttpServerResponse, + BadGateway, + Exclude + >); +} + +/** + * Common S3 Query Parameters Schema + */ +export const S3QueryParams = Schema.Struct({ + uploadId: Schema.optional(Schema.String), + partNumber: Schema.optional(Schema.NumberFromString), + prefix: Schema.optional(Schema.String), + delimiter: Schema.optional(Schema.String), + marker: Schema.optional(Schema.String), + "max-keys": Schema.optional(Schema.NumberFromString), + "max-uploads": Schema.optional(Schema.NumberFromString), + "encoding-type": Schema.optional(Schema.String), + "continuation-token": Schema.optional(Schema.String), + "start-after": Schema.optional(Schema.String), + "list-type": Schema.optional(Schema.String), + "version-id-marker": Schema.optional(Schema.String), + "key-marker": Schema.optional(Schema.String), + "upload-id-marker": Schema.optional(Schema.String), + versions: Schema.optional(Schema.String), + uploads: Schema.optional(Schema.String), + delete: Schema.optional(Schema.String), + acl: Schema.optional(Schema.String), +}); + +export type S3QueryParams = Schema.Schema.Type; + +/** + * Utility to parse search params using a Schema. + */ +export function parseQueryParams( + searchParams: URLSearchParams, + schema: Schema.Schema, +): Effect.Effect { + const paramsRecord: Record = {}; + searchParams.forEach((value, key) => { + paramsRecord[key] = value; + }); + return Schema.decodeUnknown(schema)(paramsRecord).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + ); +} + /** * Resolves a bucket by name and runs the provided effect with the resolved backend. * Centralizes error handling via S3Xml.formatError. @@ -71,7 +189,7 @@ export function resolveBucket< | R | BackendResolver | S3Xml - | AppConfig + | HeraldConfig | S3Client | SwiftClient | HttpServerRequest.HttpServerRequest @@ -120,7 +238,14 @@ export function resolveBucket< e instanceof BucketAlreadyOwnedByYou || e instanceof InternalError || e instanceof AccessDenied || - e instanceof BucketNotEmpty + e instanceof BucketNotEmpty || + e instanceof NoSuchUpload || + e instanceof InvalidPart || + e instanceof InvalidPartOrder || + e instanceof EntityTooSmall || + e instanceof InvalidRequest || + e instanceof MalformedXML || + e instanceof DeleteObjectsError ) { return Effect.succeed(s3Xml.formatError(e, isHead)); } @@ -157,7 +282,7 @@ export function resolveBackend< | R | BackendResolver | S3Xml - | AppConfig + | HeraldConfig | S3Client | SwiftClient | HttpServerRequest.HttpServerRequest @@ -186,7 +311,14 @@ export function resolveBackend< e instanceof BucketAlreadyOwnedByYou || e instanceof InternalError || e instanceof AccessDenied || - e instanceof BucketNotEmpty + e instanceof BucketNotEmpty || + e instanceof NoSuchUpload || + e instanceof InvalidPart || + e instanceof InvalidPartOrder || + e instanceof EntityTooSmall || + e instanceof InvalidRequest || + e instanceof MalformedXML || + e instanceof DeleteObjectsError ) { return Effect.succeed(s3Xml.formatError(e, isHead)); } diff --git a/src/Http.ts b/src/Http.ts index e77671f..3c3c693 100644 --- a/src/Http.ts +++ b/src/Http.ts @@ -9,33 +9,38 @@ import { Config, Effect, Layer } from "effect"; // deno-lint-ignore no-external-import import { createServer } from "node:http"; -export { Api } from "./Api.ts"; +export { HttpHeraldApi as HeraldHttpApi } from "./Api.ts"; export { HttpHealthLive } from "./Frontend/Health/Http.ts"; export { HttpS3Live } from "./Frontend/Http.ts"; -import { AppConfigLive } from "./Config/Layer.ts"; +import { HeraldConfigLive } from "./Config/Layer.ts"; import { HttpHealthLive } from "./Frontend/Health/Http.ts"; import { HttpS3Live } from "./Frontend/Http.ts"; -import { Api } from "./Api.ts"; +import { HttpHeraldApi } from "./Api.ts"; -export const ApiLive = HttpApiBuilder.api(Api).pipe( +export const HttpHeraldLive = HttpApiBuilder.api(HttpHeraldApi).pipe( Layer.provide(HttpHealthLive), Layer.provide(HttpS3Live), ); -export const HttpLive = Layer.unwrapEffect( +export const HttpServerHeraldLive = Layer.unwrapEffect( Effect.gen(function* () { const port = yield* Config.withDefault( Config.integer("PORT"), 3000, ); return HttpApiBuilder.serve(HttpMiddleware.logger).pipe( + // provides swagger ui for http api Layer.provide(HttpApiSwagger.layer()), + // provides openapi.json endpoint Layer.provide(HttpApiBuilder.middlewareOpenApi()), + // adds cors support + // FIXME: config support Layer.provide(HttpApiBuilder.middlewareCors()), - Layer.provide(ApiLive), + Layer.provide(HttpHeraldLive), + // log address at startup HttpServer.withLogAddress, Layer.provide(NodeHttpServer.layer(createServer, { port })), - Layer.provide(AppConfigLive), + Layer.provide(HeraldConfigLive), ); }), ); diff --git a/src/Logging/Layer.ts b/src/Logging/Layer.ts index ab0c5ed..1c2c233 100644 --- a/src/Logging/Layer.ts +++ b/src/Logging/Layer.ts @@ -1,4 +1,4 @@ -import { Config, Effect, Layer, Logger, LogLevel } from "effect"; +import { Config, Effect, Layer, Logger, LogLevel, Option } from "effect"; export const LoggingLive = Layer.mergeAll( Layer.unwrapEffect( @@ -7,7 +7,7 @@ export const LoggingLive = Layer.mergeAll( Config.string("HERALD_LOG_LEVEL"), ); - if (logLevelStr._tag === "None") { + if (Option.isNone(logLevelStr)) { return Logger.minimumLogLevel(LogLevel.Info); } diff --git a/src/Services/Backend.ts b/src/Services/Backend.ts index 8da9574..9f3f1f4 100644 --- a/src/Services/Backend.ts +++ b/src/Services/Backend.ts @@ -1,3 +1,7 @@ +/** + * The `Backend` service represents a single impl that herald can proxy to. + */ + import { Context, type Effect, Schema, type Stream } from "effect"; export interface BucketInfo { @@ -68,6 +72,67 @@ export interface PutObjectResult { readonly versionId?: string; } +export interface MultipartUploadResult { + readonly uploadId: string; +} + +export interface UploadPartResult { + readonly etag: string; +} + +export interface CompleteMultipartUploadResult { + readonly location: string; + readonly bucket: string; + readonly key: string; + readonly etag: string; + readonly versionId?: string; +} + +export interface PartInfo { + readonly partNumber: number; + readonly lastModified: Date; + readonly etag: string; + readonly size: number; +} + +export interface ListPartsResult { + readonly bucket: string; + readonly key: string; + readonly uploadId: string; + readonly owner: OwnerInfo; + readonly initiator: OwnerInfo; + readonly storageClass: string; + readonly partNumberMarker: number; + readonly nextPartNumberMarker: number; + readonly maxParts: number; + readonly isTruncated: boolean; + readonly parts: readonly PartInfo[]; +} + +export interface MultipartUploadInfo { + readonly key: string; + readonly uploadId: string; + readonly owner: OwnerInfo; + readonly initiator: OwnerInfo; + readonly storageClass: string; + readonly initiated: Date; +} + +export interface ListMultipartUploadsResult { + readonly bucket: string; + readonly prefix?: string; + readonly keyMarker?: string; + readonly uploadIdMarker?: string; + readonly nextKeyMarker?: string; + readonly nextUploadIdMarker?: string; + readonly maxUploads: number; + readonly delimiter?: string; + readonly isTruncated: boolean; + readonly uploads: readonly MultipartUploadInfo[]; + readonly commonPrefixes: readonly CommonPrefix[]; + readonly encodingType?: string; +} + export class NoSuchBucket extends Schema.TaggedError()("NoSuchBucket", { bucketName: Schema.String, @@ -111,6 +176,37 @@ export class BucketNotEmpty message: Schema.String, }) {} +export class NoSuchUpload + extends Schema.TaggedError()("NoSuchUpload", { + uploadId: Schema.String, + message: Schema.String, + }) {} + +export class InvalidPart + extends Schema.TaggedError()("InvalidPart", { + message: Schema.String, + }) {} + +export class InvalidPartOrder + extends Schema.TaggedError()("InvalidPartOrder", { + message: Schema.String, + }) {} + +export class EntityTooSmall + extends Schema.TaggedError()("EntityTooSmall", { + message: Schema.String, + }) {} + +export class InvalidRequest + extends Schema.TaggedError()("InvalidRequest", { + message: Schema.String, + }) {} + +export class MalformedXML + extends Schema.TaggedError()("MalformedXML", { + message: Schema.String, + }) {} + export interface DeleteError { readonly key: string; readonly code: string; @@ -141,7 +237,13 @@ export type BackendError = | AccessDenied | NoSuchKey | BucketNotEmpty - | DeleteObjectsError; + | DeleteObjectsError + | NoSuchUpload + | InvalidPart + | InvalidPartOrder + | EntityTooSmall + | InvalidRequest + | MalformedXML; export interface BackendService { readonly listBuckets: () => Effect.Effect< @@ -171,9 +273,11 @@ export interface BackendService { }) => Effect.Effect; readonly getObject: ( key: string, + headers: Record, ) => Effect.Effect; readonly headObject: ( key: string, + headers: Record, ) => Effect.Effect; readonly putObject: ( key: string, @@ -184,6 +288,39 @@ export interface BackendService { readonly deleteObjects: ( objects: readonly { key: string; versionId?: string }[], ) => Effect.Effect; + + // Multipart Upload + readonly createMultipartUpload: ( + key: string, + headers: Record, + ) => Effect.Effect; + readonly uploadPart: ( + key: string, + uploadId: string, + partNumber: number, + body: Stream.Stream, + ) => Effect.Effect; + readonly completeMultipartUpload: ( + key: string, + uploadId: string, + parts: readonly { etag: string; partNumber: number }[], + ) => Effect.Effect; + readonly abortMultipartUpload: ( + key: string, + uploadId: string, + ) => Effect.Effect; + readonly listMultipartUploads: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + uploadIdMarker?: string; + maxUploads?: number; + encodingType?: string; + }) => Effect.Effect; + readonly listParts: ( + key: string, + uploadId: string, + ) => Effect.Effect; } /** diff --git a/src/Services/BackendResolver.ts b/src/Services/BackendResolver.ts index 979866d..4ef9801 100644 --- a/src/Services/BackendResolver.ts +++ b/src/Services/BackendResolver.ts @@ -1,10 +1,11 @@ -import { Context, Effect, Layer, Option } from "effect"; -import { AppConfig } from "../Config/Layer.ts"; -import { Backend, type BackendService } from "./Backend.ts"; +import { Cache, Context, Effect, Layer, Option } from "effect"; +import { HeraldConfig } from "../Config/Layer.ts"; +import { Backend } from "./Backend.ts"; import type { S3Client } from "../Backends/S3/Client.ts"; import { makeS3Backend } from "../Backends/S3/Backend.ts"; import { makeSwiftBackend } from "../Backends/Swift/Backend.ts"; import type { SwiftClient } from "../Backends/Swift/Client.ts"; +import type { MaterializedBucket } from "../Domain/Config.ts"; /** * BackendResolver handles dynamic resolution and provisioning of Backend implementations @@ -19,7 +20,7 @@ export class BackendResolver extends Context.Tag("BackendResolver")< ) => Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client | SwiftClient + Exclude | HeraldConfig | S3Client | SwiftClient >; readonly provideForBackendId: ( @@ -28,7 +29,7 @@ export class BackendResolver extends Context.Tag("BackendResolver")< ) => Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client | SwiftClient + Exclude | HeraldConfig | S3Client | SwiftClient >; } >() {} @@ -36,52 +37,78 @@ export class BackendResolver extends Context.Tag("BackendResolver")< export const BackendResolverLive = Layer.effect( BackendResolver, Effect.gen(function* () { - const config = yield* AppConfig; + const config = yield* HeraldConfig; - // Dynamic provision logic with memoization. - const bucketCache = new Map(); - const backendCache = new Map(); + const makeBackend = ( + bucketConfig: MaterializedBucket | { backend_id: string }, + ) => + Effect.gen(function* () { + const protocol = "protocol" in bucketConfig + ? bucketConfig.protocol + : config.raw.backends[bucketConfig.backend_id]?.protocol; - return { - provideForBucket: ( - bucketName: string, - effect: Effect.Effect, - ) => - Effect.gen(function* () { - if (bucketCache.has(bucketName)) { - return yield* Effect.provideService( - effect, - Backend, - bucketCache.get(bucketName)!, - ); - } + if (protocol === "s3") { + return yield* makeS3Backend(bucketConfig); + } else if (protocol === "swift") { + return yield* makeSwiftBackend(bucketConfig); + } else { + return yield* Effect.fail( + new Error(`Unsupported protocol: ${protocol}`), + ); + } + }); + + // We cache by the string identifier (bucket name or backend ID). + // The BackendService itself is request-scoped because makeBackend yields requirements + // that are resolved from the current context when the cache is lookep up. + // Wait, Cache.get(key) will execute the lookup if not present. + // If we want the BackendService to be truly request-scoped but cached, + // we have a conflict if the requirements (like HeraldConfig) change per request. + // However, in Herald, HeraldConfig is usually a singleton for the app. + // If it's a singleton, then caching the BackendService is fine. + const bucketCache = yield* Cache.make({ + capacity: 100, + timeToLive: "24 hours", + lookup: (bucketName: string) => + Effect.gen(function* () { const matched = config.lookupBucket(bucketName); if (Option.isNone(matched)) { return yield* Effect.fail( new Error(`No configuration found for bucket: ${bucketName}`), ); } + return yield* makeBackend(matched.value); + }), + }); - const bucketConfig = matched.value; - let backendImpl: BackendService; - - if (bucketConfig.protocol === "s3") { - backendImpl = yield* makeS3Backend(bucketConfig); - } else if (bucketConfig.protocol === "swift") { - backendImpl = yield* makeSwiftBackend(bucketConfig); - } else { + const backendCache = yield* Cache.make({ + capacity: 100, + timeToLive: "24 hours", + lookup: (backendId: string) => + Effect.gen(function* () { + const backendConfig = config.raw.backends[backendId]; + if (!backendConfig) { return yield* Effect.fail( - new Error(`Unsupported protocol: ${bucketConfig.protocol}`), + new Error(`No configuration found for backend: ${backendId}`), ); } + return yield* makeBackend({ backend_id: backendId }); + }), + }); - bucketCache.set(bucketName, backendImpl); + return { + provideForBucket: ( + bucketName: string, + effect: Effect.Effect, + ) => + Effect.gen(function* () { + const backendImpl = yield* bucketCache.get(bucketName); return yield* Effect.provideService(effect, Backend, backendImpl); }) as Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client | SwiftClient + Exclude | HeraldConfig | S3Client | SwiftClient >, provideForBackendId: ( @@ -89,40 +116,12 @@ export const BackendResolverLive = Layer.effect( effect: Effect.Effect, ) => Effect.gen(function* () { - if (backendCache.has(backendId)) { - return yield* Effect.provideService( - effect, - Backend, - backendCache.get(backendId)!, - ); - } - - const backendConfig = config.raw.backends[backendId]; - if (!backendConfig) { - return yield* Effect.fail( - new Error(`No configuration found for backend: ${backendId}`), - ); - } - - let backendImpl: BackendService; - - if (backendConfig.protocol === "s3") { - backendImpl = yield* makeS3Backend({ backend_id: backendId }); - } else if (backendConfig.protocol === "swift") { - backendImpl = yield* makeSwiftBackend({ backend_id: backendId }); - } else { - const protocol = (backendConfig as { protocol: string }).protocol; - return yield* Effect.fail( - new Error(`Unsupported protocol: ${protocol}`), - ); - } - - backendCache.set(backendId, backendImpl); + const backendImpl = yield* backendCache.get(backendId); return yield* Effect.provideService(effect, Backend, backendImpl); }) as Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client | SwiftClient + Exclude | HeraldConfig | S3Client | SwiftClient >, }; }), diff --git a/src/Services/S3Xml.ts b/src/Services/S3Xml.ts index 444cfb5..06815fb 100644 --- a/src/Services/S3Xml.ts +++ b/src/Services/S3Xml.ts @@ -6,13 +6,24 @@ import { BucketAlreadyOwnedByYou, type BucketInfo, BucketNotEmpty, + EntityTooSmall, InternalError, + InvalidPart, + InvalidPartOrder, + InvalidRequest, + type ListMultipartUploadsResult, type ListObjectsResult, + type ListPartsResult, + MalformedXML, NoSuchBucket, NoSuchKey, + NoSuchUpload, type OwnerInfo, } from "./Backend.ts"; +/** + * This service centeralizes XML authoring logic. + */ export class S3Xml extends Context.Tag("S3Xml")< S3Xml, { @@ -30,6 +41,25 @@ export class S3Xml extends Context.Tag("S3Xml")< readonly formatListVersions: ( result: ListObjectsResult, ) => HttpServerResponse.HttpServerResponse; + readonly formatListMultipartUploads: ( + result: ListMultipartUploadsResult, + ) => HttpServerResponse.HttpServerResponse; + readonly formatInitiateMultipartUpload: ( + bucket: string, + key: string, + uploadId: string, + ) => HttpServerResponse.HttpServerResponse; + readonly formatCompleteMultipartUpload: ( + result: { + location: string; + bucket: string; + key: string; + etag: string; + }, + ) => HttpServerResponse.HttpServerResponse; + readonly formatListParts: ( + result: ListPartsResult, + ) => HttpServerResponse.HttpServerResponse; } >() {} @@ -66,6 +96,30 @@ export const S3XmlLive = Layer.succeed( code = "BucketNotEmpty"; message = e.message; status = 409; + } else if (e instanceof NoSuchUpload) { + code = "NoSuchUpload"; + message = e.message; + status = 404; + } else if (e instanceof InvalidPart) { + code = "InvalidPart"; + message = e.message; + status = 400; + } else if (e instanceof InvalidPartOrder) { + code = "InvalidPartOrder"; + message = e.message; + status = 400; + } else if (e instanceof EntityTooSmall) { + code = "EntityTooSmall"; + message = e.message; + status = 400; + } else if (e instanceof InvalidRequest) { + code = "InvalidRequest"; + message = e.message; + status = 400; + } else if (e instanceof MalformedXML) { + code = "MalformedXML"; + message = e.message; + status = 400; } else if (e instanceof InternalError) { code = "InternalError"; message = e.message; @@ -260,5 +314,67 @@ export const S3XmlLive = Layer.succeed( }, }); }, + + formatListMultipartUploads: (result) => { + const uploadsXml = result.uploads.map((u) => + `${u.key}${u.uploadId}${u.initiator.id}${u.initiator.displayName}${u.owner.id}${u.owner.displayName}${u.storageClass}${u.initiated.toISOString()}` + ).join(""); + + const commonPrefixesXml = result.commonPrefixes.map((cp) => + `${cp.prefix}` + ).join(""); + + const xml = + `${result.bucket}${ + result.keyMarker ?? "" + }${ + result.uploadIdMarker ?? "" + }${ + result.nextKeyMarker ?? "" + }${ + result.nextUploadIdMarker ?? "" + }${result.maxUploads}${result.isTruncated}${uploadsXml}${commonPrefixesXml}`; + + return HttpServerResponse.text(xml, { + headers: { "Content-Type": "application/xml" }, + }); + }, + + formatInitiateMultipartUpload: (bucket, key, uploadId) => { + const xml = + `${bucket}${key}${uploadId}`; + + return HttpServerResponse.text(xml, { + headers: { + "Content-Type": "application/xml", + }, + }); + }, + + formatCompleteMultipartUpload: (result) => { + const xml = + `${result.location}${result.bucket}${result.key}${result.etag}`; + + return HttpServerResponse.text(xml, { + headers: { + "Content-Type": "application/xml", + }, + }); + }, + + formatListParts: (result) => { + const partsXml = result.parts.map((p) => + `${p.partNumber}${p.lastModified.toISOString()}${p.etag}${p.size}` + ).join(""); + + const xml = + `${result.bucket}${result.key}${result.uploadId}${result.initiator.id}${result.initiator.displayName}${result.owner.id}${result.owner.displayName}${result.storageClass}${result.partNumberMarker}${result.nextPartNumberMarker}${result.maxParts}${result.isTruncated}${partsXml}`; + + return HttpServerResponse.text(xml, { + headers: { + "Content-Type": "application/xml", + }, + }); + }, }), ); diff --git a/src/main.ts b/src/main.ts index cff9dea..56ccc3b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,13 +2,19 @@ import { FetchHttpClient } from "@effect/platform"; import { NodeRuntime } from "@effect/platform-node"; import { Layer } from "effect"; // our http server impl layer -import { HttpLive } from "./Http.ts"; +import { HttpServerHeraldLive } from "./Http.ts"; // otel tracing layer import { TracingLive } from "./Tracing.ts"; -HttpLive.pipe( +HttpServerHeraldLive.pipe( Layer.provide(TracingLive), + // provider an HttpClient impl based on `fetch` + // used to talk the the swift impl Layer.provide(FetchHttpClient.layer), + // run layer until interrupted Layer.launch, + // add support for Cli goodies like + // signal mgmt, teardown, exit codes and stdio impl + // for Logger NodeRuntime.runMain, ); diff --git a/tests/config.test.ts b/tests/config.test.ts index 2e1f485..eb5981a 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -6,7 +6,7 @@ import { BackendResolver, BackendResolverLive, } from "../src/Services/BackendResolver.ts"; -import { AppConfig, parseConfig } from "../src/Config/Layer.ts"; +import { HeraldConfig, parseConfig } from "../src/Config/Layer.ts"; import { S3Client } from "../src/Backends/S3/Client.ts"; import { SwiftClient } from "../src/Backends/Swift/Client.ts"; import type { S3Client as S3ClientSDK } from "@aws-sdk/client-s3"; @@ -333,7 +333,7 @@ interface ResolverTestCase { config: GlobalConfig; op: ( resolver: Context.Tag.Service, - ) => Effect.Effect; + ) => Effect.Effect; expectedError?: string; } @@ -404,7 +404,7 @@ const resolverCases: ResolverTestCase[] = [ for (const tc of resolverCases) { testEffect(`resolver/${tc.id}`, () => Effect.gen(function* () { - const AppConfigLive = Layer.succeed(AppConfig, { + const HeraldConfigLive = Layer.succeed(HeraldConfig, { raw: tc.config, lookupBucket: (name: string) => lookupBucket(tc.config, name), }); @@ -425,7 +425,7 @@ for (const tc of resolverCases) { return yield* tc.op(resolver); }).pipe( Effect.provide(BackendResolverLive), - Effect.provide(AppConfigLive), + Effect.provide(HeraldConfigLive), Effect.provide(S3ClientLive), Effect.provide(SwiftClientLive), Effect.either, diff --git a/tests/health.test.ts b/tests/health.test.ts index 8ce8e58..296d6c5 100644 --- a/tests/health.test.ts +++ b/tests/health.test.ts @@ -5,27 +5,29 @@ import { HttpApiClient, HttpServer, } from "@effect/platform"; -import { Api, HttpHealthLive, HttpS3Live } from "../src/Http.ts"; -import { AppConfig } from "../src/Config/Layer.ts"; +import { HeraldHttpApi, HttpHealthLive, HttpS3Live } from "../src/Http.ts"; +import { HeraldConfig } from "../src/Config/Layer.ts"; import { S3ClientLive } from "../src/Backends/S3/Client.ts"; +import { SwiftClientLive } from "../src/Backends/Swift/Client.ts"; import { S3XmlLive } from "../src/Services/S3Xml.ts"; import { BackendResolverLive } from "../src/Services/BackendResolver.ts"; import { EffectAssert, testEffect } from "./utils.ts"; testEffect("health/getStatus", () => Effect.gen(function* () { - const AppConfigLive = Layer.succeed(AppConfig, { + const HeraldConfigLive = Layer.succeed(HeraldConfig, { raw: { backends: {} }, lookupBucket: () => Option.none(), }); - const ApiWithRequirements = HttpApiBuilder.api(Api).pipe( + const ApiWithRequirements = HttpApiBuilder.api(HeraldHttpApi).pipe( Layer.provide(HttpHealthLive), Layer.provide(HttpS3Live), - Layer.provide(S3ClientLive), Layer.provide(BackendResolverLive), + Layer.provide(S3ClientLive), + Layer.provide(SwiftClientLive), Layer.provide(S3XmlLive), - Layer.provide(AppConfigLive), + Layer.provide(HeraldConfigLive), Layer.provide(FetchHttpClient.layer), Layer.provideMerge(HttpServer.layerContext), ); @@ -34,7 +36,7 @@ testEffect("health/getStatus", () => const webHandler = HttpApiBuilder.toWebHandler(ApiWithRequirements); const clientProgram = Effect.gen(function* () { - const client = yield* HttpApiClient.make(Api, { + const client = yield* HttpApiClient.make(HeraldHttpApi, { baseUrl: "http://localhost", }); return yield* client.health.getStatus(); diff --git a/tests/integration/__snapshots__/buckets.test.ts.snap b/tests/integration/__snapshots__/buckets.test.ts.snap index c982eb5..76add56 100644 --- a/tests/integration/__snapshots__/buckets.test.ts.snap +++ b/tests/integration/__snapshots__/buckets.test.ts.snap @@ -123,7 +123,7 @@ snapshot[`Swift/buckets/delete/non-existent metadata 1`] = ` } `; -snapshot[`Swift/buckets/delete/non-existent body 1`] = `'NoSuchBucketNot Found'`; +snapshot[`Swift/buckets/delete/non-existent body 1`] = `'NoSuchBucket

Not Found

The resource could not be found.

'`; snapshot[`Baseline/buckets/head/existing metadata 1`] = ` { @@ -222,4 +222,4 @@ snapshot[`Swift/buckets/list metadata 1`] = ` } `; -snapshot[`Swift/buckets/list body 1`] = `'swiftSwift User192.168.5.1232026-01-15T00:00:00.000Za2026-01-15T00:00:00.000Zaa2026-01-15T00:00:00.000Zbuilds2026-01-15T00:00:00.000Zfoo-2026-01-15T00:00:00.000Zfoo-.bar2026-01-15T00:00:00.000Zfoo.-bar2026-01-15T00:00:00.000Zfoo..bar2026-01-15T00:00:00.000Zfoo_bar2026-01-15T00:00:00.000Zherald-swift-2w97l75ompcxiypo-12026-01-15T00:00:00.000Zherald-swift-5dfyoor543wddfpb-12026-01-15T00:00:00.000Zherald-swift-5m8pru9nzpno98zp-1572026-01-15T00:00:00.000Zherald-swift-5txup4vs19i8tr6s-292026-01-15T00:00:00.000Zherald-swift-cze1vw7y05q33782-1462026-01-15T00:00:00.000Zherald-swift-fd0oi5radob46p39-12026-01-15T00:00:00.000Zherald-swift-oda2k1hu2ds0wir6-22026-01-15T00:00:00.000Zherald-swift-sy6d1ftl2i7g78jj-12026-01-15T00:00:00.000Zherald-swift-yx0xlhaeebv1g9c7-12026-01-15T00:00:00.000Zherald-task-store-mr-120-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-127-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-130-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-131-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-132-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-137-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-139-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-143-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-144-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-145-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-146-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-147-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-149-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-150-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-151-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-154-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-155-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-157-vivavox2026-01-15T00:00:00.000Zherald-task-store-prd-vivavox2026-01-15T00:00:00.000Zherald-task-store-stg-vivavox2026-01-15T00:00:00.000Ziac-swift2026-01-15T00:00:00.000Zmr-101-vivavox2026-01-15T00:00:00.000Zmr-109-vivavox2026-01-15T00:00:00.000Zmr-111-vivavox2026-01-15T00:00:00.000Zmr-115-vivavox2026-01-15T00:00:00.000Zmr-116-vivavox2026-01-15T00:00:00.000Zmr-120-vivavox2026-01-15T00:00:00.000Zmr-121-vivavox2026-01-15T00:00:00.000Zmr-122-vivavox2026-01-15T00:00:00.000Zmr-124-vivavox2026-01-15T00:00:00.000Zmr-126-vivavox2026-01-15T00:00:00.000Zmr-127-vivavox2026-01-15T00:00:00.000Zmr-130-vivavox2026-01-15T00:00:00.000Zmr-131-vivavox2026-01-15T00:00:00.000Zmr-132-vivavox2026-01-15T00:00:00.000Zmr-137-vivavox2026-01-15T00:00:00.000Zmr-139-vivavox2026-01-15T00:00:00.000Zmr-143-vivavox2026-01-15T00:00:00.000Zmr-144-vivavox2026-01-15T00:00:00.000Zmr-145-vivavox2026-01-15T00:00:00.000Zmr-146-vivavox2026-01-15T00:00:00.000Zmr-147-vivavox2026-01-15T00:00:00.000Zmr-149-vivavox2026-01-15T00:00:00.000Zmr-150-vivavox2026-01-15T00:00:00.000Zmr-151-vivavox2026-01-15T00:00:00.000Zmr-154-vivavox2026-01-15T00:00:00.000Zmr-155-vivavox2026-01-15T00:00:00.000Zmr-157-vivavox2026-01-15T00:00:00.000Zprd-vivavox2026-01-15T00:00:00.000Zstg-vivavox2026-01-15T00:00:00.000Zstg-vivavox+segments2026-01-15T00:00:00.000Z'`; +snapshot[`Swift/buckets/list body 1`] = `'swiftSwift User192.168.5.1232026-01-15T00:00:00.000Za2026-01-15T00:00:00.000Zaa2026-01-15T00:00:00.000Zbuilds2026-01-15T00:00:00.000Zfoo-2026-01-15T00:00:00.000Zfoo-.bar2026-01-15T00:00:00.000Zfoo.-bar2026-01-15T00:00:00.000Zfoo..bar2026-01-15T00:00:00.000Zfoo_bar2026-01-15T00:00:00.000Zherald-swift-2w97l75ompcxiypo-12026-01-15T00:00:00.000Zherald-swift-5dfyoor543wddfpb-12026-01-15T00:00:00.000Zherald-swift-5m8pru9nzpno98zp-1572026-01-15T00:00:00.000Zherald-swift-5txup4vs19i8tr6s-292026-01-15T00:00:00.000Zherald-swift-cze1vw7y05q33782-1462026-01-15T00:00:00.000Zherald-swift-fd0oi5radob46p39-12026-01-15T00:00:00.000Zherald-swift-m55m3lqytoaxuxro-132026-01-15T00:00:00.000Zherald-swift-oda2k1hu2ds0wir6-22026-01-15T00:00:00.000Zherald-swift-sy6d1ftl2i7g78jj-12026-01-15T00:00:00.000Zherald-swift-yx0xlhaeebv1g9c7-12026-01-15T00:00:00.000Zherald-task-store-mr-120-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-127-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-130-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-131-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-132-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-137-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-139-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-143-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-144-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-145-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-146-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-147-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-149-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-150-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-151-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-154-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-155-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-157-vivavox2026-01-15T00:00:00.000Zherald-task-store-prd-vivavox2026-01-15T00:00:00.000Zherald-task-store-stg-vivavox2026-01-15T00:00:00.000Ziac-swift2026-01-15T00:00:00.000Zmr-101-vivavox2026-01-15T00:00:00.000Zmr-109-vivavox2026-01-15T00:00:00.000Zmr-111-vivavox2026-01-15T00:00:00.000Zmr-115-vivavox2026-01-15T00:00:00.000Zmr-116-vivavox2026-01-15T00:00:00.000Zmr-120-vivavox2026-01-15T00:00:00.000Zmr-121-vivavox2026-01-15T00:00:00.000Zmr-122-vivavox2026-01-15T00:00:00.000Zmr-124-vivavox2026-01-15T00:00:00.000Zmr-126-vivavox2026-01-15T00:00:00.000Zmr-127-vivavox2026-01-15T00:00:00.000Zmr-130-vivavox2026-01-15T00:00:00.000Zmr-131-vivavox2026-01-15T00:00:00.000Zmr-132-vivavox2026-01-15T00:00:00.000Zmr-137-vivavox2026-01-15T00:00:00.000Zmr-139-vivavox2026-01-15T00:00:00.000Zmr-143-vivavox2026-01-15T00:00:00.000Zmr-144-vivavox2026-01-15T00:00:00.000Zmr-145-vivavox2026-01-15T00:00:00.000Zmr-146-vivavox2026-01-15T00:00:00.000Zmr-147-vivavox2026-01-15T00:00:00.000Zmr-149-vivavox2026-01-15T00:00:00.000Zmr-150-vivavox2026-01-15T00:00:00.000Zmr-151-vivavox2026-01-15T00:00:00.000Zmr-154-vivavox2026-01-15T00:00:00.000Zmr-155-vivavox2026-01-15T00:00:00.000Zmr-157-vivavox2026-01-15T00:00:00.000Zprd-vivavox2026-01-15T00:00:00.000Zstg-vivavox2026-01-15T00:00:00.000Zstg-vivavox+segments2026-01-15T00:00:00.000Ztest-objects-bucket2026-01-15T00:00:00.000Z'`; diff --git a/tests/integration/__snapshots__/objects.test.ts.snap b/tests/integration/__snapshots__/objects.test.ts.snap index ba27552..63b70f8 100644 --- a/tests/integration/__snapshots__/objects.test.ts.snap +++ b/tests/integration/__snapshots__/objects.test.ts.snap @@ -96,7 +96,7 @@ snapshot[`Swift/objects/get/non-existent metadata 1`] = ` } `; -snapshot[`Swift/objects/get/non-existent body 1`] = `'NoSuchKeyNot Found'`; +snapshot[`Swift/objects/get/non-existent body 1`] = `'NoSuchKey

Not Found

The resource could not be found.

'`; snapshot[`Baseline/objects/head/existing metadata 1`] = ` { @@ -181,3 +181,95 @@ snapshot[`Swift/objects/delete/existing metadata 1`] = ` status: 204, } `; + +snapshot[`Baseline/objects/multipart/basic metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/multipart/basic metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/multipart/abort metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "479", + "content-type": "application/xml", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Baseline/objects/multipart/abort body 1`] = ` +' +NoSuchUploadThe specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.multipart-abort.txttest-objects-bucket/test-objects-bucket/multipart-abort.txtIDHOST' +`; + +snapshot[`Proxy/objects/multipart/abort metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/objects/multipart/abort body 1`] = `'NoSuchUploadThe specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.'`; + +snapshot[`Baseline/objects/multipart/list-parts metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/multipart/list-parts metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/multipart/empty metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/multipart/empty metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; diff --git a/tests/integration/objects.test.ts b/tests/integration/objects.test.ts index e2799f1..a4f2de9 100644 --- a/tests/integration/objects.test.ts +++ b/tests/integration/objects.test.ts @@ -1,12 +1,17 @@ import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, CreateBucketCommand, + CreateMultipartUploadCommand, DeleteBucketCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, + ListPartsCommand, PutObjectCommand, type S3Client, S3ServiceException, + UploadPartCommand, } from "@aws-sdk/client-s3"; import { harness, type ProxyTestCase } from "../utils.ts"; import type { GlobalConfig } from "../../src/Domain/Config.ts"; @@ -121,6 +126,180 @@ const specs: ObjectTestSpec[] = [ ); }, }, + { + name: "objects/multipart/basic", + fn: async (c) => { + const key = "multipart-basic.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + const partSize = 5 * 1024 * 1024 + 1; + const body1 = new Uint8Array(partSize).fill(97); // 'a' + const body2 = new Uint8Array(10).fill(98); // 'b' + + const { ETag: etag1 } = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: body1, + }), + ); + const { ETag: etag2 } = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 2, + Body: body2, + }), + ); + + await c.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { + Parts: [ + { ETag: etag1, PartNumber: 1 }, + { ETag: etag2, PartNumber: 2 }, + ], + }, + }), + ); + + const { ContentLength } = await c.send( + new HeadObjectCommand({ Bucket: BUCKET, Key: key }), + ); + if (ContentLength !== partSize + 10) { + throw new Error( + `Size mismatch: expected ${partSize + 10}, got ${ContentLength}`, + ); + } + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ + Bucket: BUCKET, + Key: "multipart-basic.txt", + }), + ); + } catch { /* ignore */ } + }, + }, + { + name: "objects/multipart/abort", + fn: async (c) => { + const key = "multipart-abort.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: "part 1", + }), + ); + + await c.send( + new AbortMultipartUploadCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + + try { + await c.send( + new ListPartsCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + throw new Error("ListParts should have failed after Abort"); + } catch (e) { + if (!(e instanceof S3ServiceException && e.name === "NoSuchUpload")) { + throw e; + } + } + }, + }, + { + name: "objects/multipart/list-parts", + fn: async (c) => { + const key = "multipart-list.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: "part 1", + }), + ); + + const { Parts } = await c.send( + new ListPartsCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + + if (!Parts || Parts.length !== 1 || Parts[0].PartNumber !== 1) { + throw new Error(`Unexpected parts list: ${JSON.stringify(Parts)}`); + } + + await c.send( + new AbortMultipartUploadCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + }, + }, + { + name: "objects/multipart/empty", + fn: async (c) => { + const key = "multipart-empty.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + try { + await c.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { Parts: [] }, + }), + ); + throw new Error("Complete should have failed for empty parts"); + } catch (e) { + if ( + e instanceof S3ServiceException && + (e.name === "MalformedXML" || e.name === "InvalidPart" || + e.name === "InvalidRequest") + ) { + return; + } + throw e; + } finally { + try { + await c.send( + new AbortMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + }), + ); + } catch { /* ignore */ } + } + }, + }, ]; async function runObjectTest(tc: ObjectTestSpec, client: S3Client) { diff --git a/tests/utils.ts b/tests/utils.ts index 6748cd1..e5446b5 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -1,7 +1,7 @@ import { S3Client } from "@aws-sdk/client-s3"; import { Config, Effect, Layer, Logger, LogLevel, Option } from "effect"; -import { ApiLive } from "../src/Http.ts"; -import { AppConfig } from "../src/Config/Layer.ts"; +import { HttpHeraldLive } from "../src/Http.ts"; +import { HeraldConfig } from "../src/Config/Layer.ts"; import { lookupBucket } from "../src/Domain/Config.ts"; import { BackendResolverLive } from "../src/Services/BackendResolver.ts"; import { S3ClientLive } from "../src/Backends/S3/Client.ts"; @@ -39,17 +39,17 @@ export const makeTestHarness = ( ), ) => Effect.gen(function* () { - const AppConfigLive = Layer.succeed(AppConfig, { + const HeraldConfigLive = Layer.succeed(HeraldConfig, { raw: config, lookupBucket: (name: string) => lookupBucket(config, name), }); - const ApiWithRequirements = ApiLive.pipe( + const ApiWithRequirements = HttpHeraldLive.pipe( Layer.provide(BackendResolverLive), Layer.provide(S3ClientLive), Layer.provide(SwiftClientLive), Layer.provide(S3XmlLive), - Layer.provide(AppConfigLive), + Layer.provide(HeraldConfigLive), Layer.provide(FetchHttpClient.layer), Layer.provideMerge(HttpServer.layerContext), Layer.provideMerge(loggingLayer), diff --git a/tools/compose.yml b/tools/compose.yml index 0b5de2d..0067818 100644 --- a/tools/compose.yml +++ b/tools/compose.yml @@ -1,6 +1,7 @@ name: herald services: redis: + profiles: ["db"] image: docker.io/library/redis:alpine command: --save 60 1 --loglevel warning healthcheck: @@ -15,6 +16,7 @@ services: - redisdata:/data minio: + profiles: ["s3"] image: docker.io/minio/minio:latest command: server /data --console-address ":9001" ports: diff --git a/x/compose-down.ts b/x/compose-down.ts index 4094166..04f52d5 100755 --- a/x/compose-down.ts +++ b/x/compose-down.ts @@ -2,4 +2,6 @@ import { $, DOCKER_CMD } from "./utils.ts"; -await $.raw`${DOCKER_CMD} compose down`.cwd($.relativeDir("../tools/")); +await $.raw`${DOCKER_CMD} compose -f compose.yml down`.cwd( + $.path(import.meta.resolve("../tools/")), +); diff --git a/x/compose-up.ts b/x/compose-up.ts index 367e9a6..e6cfe3d 100755 --- a/x/compose-up.ts +++ b/x/compose-up.ts @@ -6,6 +6,6 @@ const profiles = $.argv .map((prof) => `--profile ${prof}`) .join(" "); -await $.raw`${DOCKER_CMD} compose ${profiles} up -d`.cwd( - $.relativeDir("../tools/"), +await $.raw`${DOCKER_CMD} compose -f compose.yml ${profiles} up -d`.cwd( + $.path(import.meta.resolve("../tools/")), ); diff --git a/x/purge-minio.ts b/x/purge-minio.ts index 204714a..3bf8ccd 100644 --- a/x/purge-minio.ts +++ b/x/purge-minio.ts @@ -2,7 +2,6 @@ import { DeleteBucketCommand, DeleteObjectsCommand, ListBucketsCommand, - ListObjectsV2Command, ListObjectVersionsCommand, S3Client, } from "npm:@aws-sdk/client-s3"; diff --git a/x/s3-tests.ts b/x/s3-tests.ts index 9c9bc41..d4e958e 100755 --- a/x/s3-tests.ts +++ b/x/s3-tests.ts @@ -179,9 +179,6 @@ const program = Effect.gen(function* () { const FileLoggingLive = Logger.replace( Logger.defaultLogger, Logger.make(({ message, logLevel: currentLogLevel }) => { - if (currentLogLevel.syslog > minLogLevel.syslog) { - return; - } const timestamp = new Date().toISOString(); const level = currentLogLevel.label; const msg = typeof message === "string" ? message : String(message); diff --git a/x/swift-debug.ts b/x/swift-debug.ts index 986d71b..76e888c 100644 --- a/x/swift-debug.ts +++ b/x/swift-debug.ts @@ -1,9 +1,9 @@ #!/usr/bin/env -S deno run --allow-all import { Effect, Logger, LogLevel } from "effect"; import { SwiftClient, SwiftClientLive } from "../src/Backends/Swift/Client.ts"; -import { AppConfigLive } from "../src/Config/Layer.ts"; +import { HeraldConfigLive } from "../src/Config/Layer.ts"; import { makeSwiftBackend } from "../src/Backends/Swift/Backend.ts"; -import { Backend } from "../src/Services/Backend.ts"; +import { FetchHttpClient } from "@effect/platform"; const program = Effect.gen(function* () { console.log("Checking Swift connection..."); @@ -27,7 +27,8 @@ const program = Effect.gen(function* () { } }).pipe( Effect.provide(SwiftClientLive), - Effect.provide(AppConfigLive), + Effect.provide(HeraldConfigLive), + Effect.provide(FetchHttpClient.layer), Effect.provide(Logger.minimumLogLevel(LogLevel.Debug)), );