From 56b855df0e1fdaf318c00f522e815f32c4b49759 Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Thu, 22 Jan 2026 01:41:39 +0300 Subject: [PATCH 01/10] feat(swift): multipart support --- .github/workflows/build-image.yml | 1 - AGENTS.md | 5 +- CONTRIBUTING.md | 8 + TODO.md | 3 +- src/Backends/S3/Backend.ts | 2 + src/Backends/S3/Objects.ts | 1 + src/Backends/Swift/Backend.ts | 22 +- src/Backends/Swift/Buckets.ts | 24 + src/Backends/Swift/Objects.ts | 419 ++++++++++++++++-- src/Backends/Swift/Utils.ts | 13 +- src/Frontend/Objects/Delete.ts | 3 + src/Frontend/Objects/Post.ts | 114 +++-- src/Frontend/Utils.ts | 60 +-- src/Http.ts | 5 - src/Services/Backend.ts | 4 + src/Services/BackendKeyValueStore.ts | 125 ++++++ src/Services/NoopKeyValueStore.ts | 12 + .../__snapshots__/objects.test.ts.snap | 251 +---------- tests/utils.ts | 8 +- x/swift-s3-tests.ts | 48 +- 20 files changed, 750 insertions(+), 378 deletions(-) create mode 100644 src/Services/BackendKeyValueStore.ts create mode 100644 src/Services/NoopKeyValueStore.ts diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index d5f277e..d38e054 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -1,6 +1,5 @@ name: build image - on: push: branches: diff --git a/AGENTS.md b/AGENTS.md index 4410833..8a205ac 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ all HTTP requests. - Prefer generators over effect piping. - Use methods on `Effect.Option` like `Option.isNone` instead of looking at - _tag. + `_tag`. - **NEVER** use standard `try/catch` or `try/finally` blocks around `yield*` in Effect generators. Use `Effect.addFinalizer`, `Effect.try`, `Effect.catchAll`, or `Effect.orElse`. @@ -28,3 +28,6 @@ agent. - Maintain strict type safety. Avoid "any" casts or requirement hacks. - Use the structured `Logger` layer for all diagnostic output. + +- Always fix deno lint and deno check issues before running tests, the type + system is there to help. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 219781a..c73b2f9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,6 +68,14 @@ - `x/snapdiff.ts`: Tool for comparing Herald proxy snapshots against baseline responses. + - `x/swift-s3-tests.ts`: Orchestration script for running the ceph `s3-tests` + suite against the proxy with a Swift backend. Requires `infisical` for + secrets. + + ```bash + infisical run -- deno task test x/swift-s3-tests.ts + ``` + - `x/utils.ts`: Shell scripting utilities powered by `dax`. - `tools/`: Infrastructure and development tools. diff --git a/TODO.md b/TODO.md index e436adf..348c0b4 100644 --- a/TODO.md +++ b/TODO.md @@ -48,10 +48,11 @@ implementation. - [ ] **Multi-Object Delete**: Implementation of `POST /?delete`. _(Focus tests: `test_multi_object_delete`, `test_multi_object_delete_key_limit`)_ -- [ ] **Multipart Upload**: Support for `InitiateMultipartUpload`, `UploadPart`, +- [x] **Multipart Upload**: Support for `InitiateMultipartUpload`, `UploadPart`, `CompleteMultipartUpload`, `AbortMultipartUpload`, and `ListParts`. _(Focus tests: `test_multipart_upload`, `test_multipart_upload_empty`, `test_abort_multipart_upload`)_ + - [ ] **Swift Multipart Upload**: Implement S3 multipart mapping to Swift SLO. - [ ] **GetObject Attributes**: Implementation of `GET /bucket/key?attributes`. _(Focus tests: `test_get_object_attributes`)_ - [ ] **HeadObject Consistency**: Fix `404 Not Found` errors on existing objects diff --git a/src/Backends/S3/Backend.ts b/src/Backends/S3/Backend.ts index fc930c7..be8c125 100644 --- a/src/Backends/S3/Backend.ts +++ b/src/Backends/S3/Backend.ts @@ -6,6 +6,7 @@ import { makeObjectOps } from "./Objects.ts"; import { getTarget } from "./Utils.ts"; import type { S3Client } from "./Client.ts"; import type { HeraldConfig } from "../../Config/Layer.ts"; +import { makeNoopKeyValueStore } from "../../Services/NoopKeyValueStore.ts"; /** * Creates an S3-specific Backend implementation for a given configuration context. @@ -20,5 +21,6 @@ export const makeS3Backend = ( return { ...makeBucketOps(target), ...makeObjectOps(target), + multipartMetadataStore: makeNoopKeyValueStore(), } satisfies BackendService; }); diff --git a/src/Backends/S3/Objects.ts b/src/Backends/S3/Objects.ts index 58a1f06..62761ba 100644 --- a/src/Backends/S3/Objects.ts +++ b/src/Backends/S3/Objects.ts @@ -597,6 +597,7 @@ export const makeObjectOps = (target: S3Target) => ({ key: string, uploadId: string, parts: readonly { etag: string; partNumber: number }[], + _metadata: Record, ) => Effect.gen(function* () { const { client, bucketName } = target; diff --git a/src/Backends/Swift/Backend.ts b/src/Backends/Swift/Backend.ts index 1c50070..930a1ad 100644 --- a/src/Backends/Swift/Backend.ts +++ b/src/Backends/Swift/Backend.ts @@ -6,6 +6,7 @@ import { makeBucketOps } from "./Buckets.ts"; import { makeObjectOps } from "./Objects.ts"; import { getTarget } from "./Utils.ts"; import type { SwiftClient } from "./Client.ts"; +import { makeBackendKeyValueStore } from "../../Services/BackendKeyValueStore.ts"; /** * Creates a Swift-specific Backend implementation for a given configuration context. @@ -22,8 +23,21 @@ export const makeSwiftBackend = ( Effect.gen(function* () { const target = yield* getTarget(bucket); const client = yield* HttpClient.HttpClient; - return { - ...makeBucketOps(target, client), - ...makeObjectOps(target, client), - } satisfies BackendService; + const objectOps = makeObjectOps(target, client); + const bucketOps = makeBucketOps(target, client, objectOps); + + const baseBackend = { + ...bucketOps, + ...objectOps, + }; + + const backend: BackendService = { + ...baseBackend, + multipartMetadataStore: makeBackendKeyValueStore( + objectOps, + ".herald/multipart-meta/", + ), + }; + + return backend; }); diff --git a/src/Backends/Swift/Buckets.ts b/src/Backends/Swift/Buckets.ts index c6371fd..dde6ea5 100644 --- a/src/Backends/Swift/Buckets.ts +++ b/src/Backends/Swift/Buckets.ts @@ -1,8 +1,10 @@ import { Effect } from "effect"; import { type HttpClient, HttpClientRequest } from "@effect/platform"; import { + type BackendService, BucketAlreadyOwnedByYou, type BucketInfo, + type ListObjectsResult, type OwnerInfo, } from "../../Services/Backend.ts"; import { mapError, type SwiftTarget } from "./Utils.ts"; @@ -15,6 +17,10 @@ export interface SwiftContainer { export const makeBucketOps = ( target: SwiftTarget, client: HttpClient.HttpClient, + objectOps: { + listObjects: BackendService["listObjects"]; + deleteObject: BackendService["deleteObject"]; + }, ) => ({ listBuckets: () => Effect.gen(function* () { @@ -89,6 +95,24 @@ export const makeBucketOps = ( deleteBucket: () => Effect.gen(function* () { const { url, token, container } = target; + + // 1. Cleanup .herald/ objects so bucket can be deleted + let marker: string | undefined = undefined; + while (true) { + const heraldObjects: ListObjectsResult = yield* objectOps.listObjects({ + prefix: ".herald/", + marker, + }); + for (const obj of heraldObjects.contents) { + yield* objectOps.deleteObject(obj.key).pipe(Effect.ignore); + } + if (!heraldObjects.isTruncated || !heraldObjects.nextMarker) { + break; + } + marker = heraldObjects.nextMarker; + } + + // 2. Delete the bucket const response = yield* client.execute( HttpClientRequest.del(url).pipe( HttpClientRequest.setHeaders({ "X-Auth-Token": token }), diff --git a/src/Backends/Swift/Objects.ts b/src/Backends/Swift/Objects.ts index 431394b..c2ec6c0 100644 --- a/src/Backends/Swift/Objects.ts +++ b/src/Backends/Swift/Objects.ts @@ -1,13 +1,23 @@ import { Effect, Option, type Stream } from "effect"; import { type HttpClient, HttpClientRequest } from "@effect/platform"; import { + type BackendError, type CommonPrefix, + type CompleteMultipartUploadResult, type DeleteObjectsResult, InternalError, + InvalidPart, + type ListMultipartUploadsResult, type ListObjectsResult, + type ListPartsResult, + type MultipartUploadInfo, + type MultipartUploadResult, + NoSuchUpload, type ObjectInfo, type ObjectResponse, + type PartInfo, type PutObjectResult, + type UploadPartResult, } from "../../Services/Backend.ts"; import { mapError, type SwiftTarget } from "./Utils.ts"; import { fixHeaderEncoding } from "../../Frontend/Utils.ts"; @@ -54,10 +64,6 @@ export const makeObjectOps = ( Effect.mapError((e) => mapError(500, String(e), container)), ); - yield* Effect.logDebug( - `Swift listObjects query=[${query.toString()}] status=${response.status}`, - ); - if (response.status < 200 || response.status >= 300) { const message = yield* response.text.pipe( Effect.orElseSucceed(() => "Error"), @@ -282,7 +288,6 @@ export const makeObjectOps = ( const swiftHeaders: Record = { "X-Auth-Token": token, }; - // ... handle headers if needed const response = yield* client.execute( HttpClientRequest.head(`${url}/${encodedKey}`).pipe( HttpClientRequest.setHeaders(swiftHeaders), @@ -361,29 +366,30 @@ export const makeObjectOps = ( key: string, stream: Stream.Stream, headers: Record, - ) => - Effect.gen(function* () { - const { url, token, container } = target; - const encodedKey = key.split("/").map(encodeURIComponent).join("/"); - const contentLength = headers["content-length"] || - headers["Content-Length"]; + ): Effect.Effect => { + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + return Effect.gen(function* () { const swiftHeaders: Record = { "X-Auth-Token": token, "Content-Type": (headers["content-type"] || headers["Content-Type"] || "application/octet-stream") as string, - ...(contentLength ? { "Content-Length": String(contentLength) } : {}), }; + const contentLength = headers["content-length"] || + headers["Content-Length"]; + if (contentLength) { + swiftHeaders["Content-Length"] = String(contentLength); + } + for (const [k, v] of Object.entries(headers)) { const lowK = k.toLowerCase(); if (lowK.startsWith("x-amz-meta-")) { const metaKey = lowK.substring("x-amz-meta-".length); const value = fixHeaderEncoding(String(v)); swiftHeaders[`X-Object-Meta-${metaKey}`] = - /[^\x20-\x7E]/.test(value) - ? encodeURIComponent(value) - : value; + /[^\x20-\x7E]/.test(value) ? encodeURIComponent(value) : value; } } @@ -393,11 +399,9 @@ export const makeObjectOps = ( ); const response = yield* client.execute(request).pipe( - Effect.mapError((e) => mapError(500, String(e), container)), - ); - - yield* Effect.logDebug( - `Swift putObject key=[${key}] status=${response.status}`, + Effect.mapError((e) => { + return mapError(500, String(e), container); + }), ); if (response.status < 200 || response.status >= 300) { @@ -423,20 +427,49 @@ export const makeObjectOps = ( return { etag: etagValue || undefined, } satisfies PutObjectResult; - }), + }); + }, deleteObject: (key: string) => Effect.gen(function* () { const { url, token, container } = target; const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + + // Try SLO delete first (recursive) const response = yield* client.execute( HttpClientRequest.del(`${url}/${encodedKey}`).pipe( - HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + HttpClientRequest.setHeaders({ + "X-Auth-Token": token, + "X-Static-Large-Object": "true", + }), + HttpClientRequest.setUrlParams({ "multipart-manifest": "delete" }), ), ).pipe( Effect.mapError((e) => mapError(500, String(e), container)), ); + if (response.status === 400) { + // Not an SLO, try regular delete + const regResponse = yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (regResponse.status < 200 || regResponse.status >= 300) { + if (regResponse.status === 404) return; + const message = yield* regResponse.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(regResponse.status, message, container, "DELETE", key), + ); + } + return; + } + if (response.status < 200 || response.status >= 300) { if (response.status === 404) { return; @@ -466,17 +499,30 @@ export const makeObjectOps = ( const encodedKey = obj.key.split("/").map(encodeURIComponent).join( "/", ); - const response = yield* client.execute( + let response = yield* client.execute( HttpClientRequest.del(`${url}/${encodedKey}`).pipe( - HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + HttpClientRequest.setHeaders({ + "X-Auth-Token": token, + "X-Static-Large-Object": "true", + }), + HttpClientRequest.setUrlParams({ + "multipart-manifest": "delete", + }), ), ).pipe( Effect.mapError((e) => mapError(500, String(e), container)), ); - yield* Effect.logDebug( - `Swift deleteObject key=[${obj.key}] status=${response.status}`, - ); + if (response.status === 400) { + // Not an SLO, try regular delete + response = yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + } if ( (response.status >= 200 && response.status < 300) || @@ -501,29 +547,320 @@ export const makeObjectOps = ( createMultipartUpload: ( _key: string, _headers: Record, - ) => Effect.fail(new InternalError({ message: "Not implemented" })), + ): Effect.Effect => + Effect.gen(function* () { + const uploadId = yield* Effect.try({ + try: () => crypto.randomUUID(), + catch: (e) => new InternalError({ message: String(e) }), + }); + return { uploadId } satisfies MultipartUploadResult; + }), + uploadPart: ( _key: string, - _uploadId: string, - _partNumber: number, - _body: Stream.Stream, - ) => Effect.fail(new InternalError({ message: "Not implemented" })), + uploadId: string, + partNumber: number, + body: Stream.Stream, + ): Effect.Effect => + Effect.gen(function* () { + const { url, token, container } = target; + const segmentKey = + `.herald/multipart-segments/${uploadId}/${partNumber}`; + const encodedSegmentKey = segmentKey.split("/").map(encodeURIComponent) + .join("/"); + + const response = yield* client.execute( + HttpClientRequest.put(`${url}/${encodedSegmentKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + HttpClientRequest.bodyStream(body), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "PUT", + segmentKey, + ), + ); + } + + const etagHeader = response.headers["etag"]; + const etagValue = Array.isArray(etagHeader) + ? etagHeader[0] + : etagHeader; + + return { + etag: etagValue || "", + } satisfies UploadPartResult; + }), + completeMultipartUpload: ( + key: string, + uploadId: string, + parts: readonly { etag: string; partNumber: number }[], + metadata: Record, + ): Effect.Effect => + Effect.gen(function* () { + if (parts.length === 0) { + return yield* Effect.fail( + new InvalidPart({ + message: "At least one part must be specified.", + }), + ); + } + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + + // Fetch segment info to get sizes + const segmentMap = new Map(); + let segmentMarker: string | undefined = undefined; + while (true) { + const segmentsResult: ListObjectsResult = yield* listObjects({ + prefix: `.herald/multipart-segments/${uploadId}/`, + marker: segmentMarker, + }); + for (const c of segmentsResult.contents) { + segmentMap.set(c.key, c); + } + if (!segmentsResult.isTruncated || !segmentsResult.nextMarker) { + break; + } + segmentMarker = segmentsResult.nextMarker; + } + + // 1. Build SLO manifest + const manifest = []; + for (const p of parts) { + const segmentKey = + `.herald/multipart-segments/${uploadId}/${p.partNumber}`; + const info = segmentMap.get(segmentKey); + if (!info) { + return yield* Effect.fail( + new NoSuchUpload({ + uploadId, + message: + `Part ${p.partNumber} not found. The upload might have already been completed or aborted.`, + }), + ); + } + manifest.push({ + path: `/${container}/${segmentKey}`, + etag: p.etag.replace(/"/g, ""), + size_bytes: info.size, + }); + } + + // 2. PUT SLO manifest + const swiftHeaders: Record = { + "X-Auth-Token": token, + "Content-Type": (metadata["content-type"] || + "application/octet-stream") as string, + }; + + for (const [k, v] of Object.entries(metadata)) { + const lowK = k.toLowerCase(); + if (lowK.startsWith("x-amz-meta-")) { + const metaKey = lowK.substring("x-amz-meta-".length); + const value = fixHeaderEncoding(String(v)); + swiftHeaders[`X-Object-Meta-${metaKey}`] = + /[^\x20-\x7E]/.test(value) ? encodeURIComponent(value) : value; + } + } + + const body = new TextEncoder().encode(JSON.stringify(manifest)); + + const request = HttpClientRequest.put(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setUrlParams({ "multipart-manifest": "put" }), + HttpClientRequest.bodyUint8Array(body), + HttpClientRequest.setHeaders({ + ...swiftHeaders, + "X-Static-Large-Object": "true", + "Content-Length": String(body.length), + }), + ); + + const response = yield* client.execute(request).pipe( + Effect.mapError((e) => { + return mapError(500, String(e), container); + }), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "PUT", + key, + ), + ); + } + + const etagHeader = response.headers["etag"]; + const etagValue = Array.isArray(etagHeader) + ? etagHeader[0] + : etagHeader; + + return { + location: `${url}/${encodedKey}`, + bucket: container, + key, + etag: etagValue || "", + } satisfies CompleteMultipartUploadResult; + }), + + abortMultipartUpload: ( _key: string, - _uploadId: string, - _parts: readonly { etag: string; partNumber: number }[], - ) => Effect.fail(new InternalError({ message: "Not implemented" })), - abortMultipartUpload: (_key: string, _uploadId: string) => - Effect.fail(new InternalError({ message: "Not implemented" })), - listMultipartUploads: (_args: { + uploadId: string, + ): Effect.Effect => + Effect.gen(function* () { + const { url, token } = target; + + // 1. Delete the segments + let marker: string | undefined = undefined; + while (true) { + const segmentsResult: ListObjectsResult = yield* listObjects({ + prefix: `.herald/multipart-segments/${uploadId}/`, + marker, + }); + + for (const content of segmentsResult.contents) { + const encodedKey = content.key.split("/").map(encodeURIComponent) + .join("/"); + yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe(Effect.ignore); + } + + if (!segmentsResult.isTruncated || !segmentsResult.nextMarker) { + break; + } + marker = segmentsResult.nextMarker; + } + + // 2. Delete the metadata object + const metaKey = `.herald/multipart-meta/${uploadId}`; + const encodedMetaKey = metaKey.split("/").map(encodeURIComponent).join( + "/", + ); + yield* client.execute( + HttpClientRequest.del(`${url}/${encodedMetaKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe(Effect.ignore); + }), + + listMultipartUploads: (args: { prefix?: string; delimiter?: string; keyMarker?: string; uploadIdMarker?: string; maxUploads?: number; encodingType?: string; - }) => Effect.fail(new InternalError({ message: "Not implemented" })), - listParts: (_key: string, _uploadId: string) => - Effect.fail(new InternalError({ message: "Not implemented" })), + }): Effect.Effect => + Effect.gen(function* () { + const { container } = target; + const metaResult = yield* listObjects({ + prefix: ".herald/multipart-meta/", + maxKeys: args.maxUploads, + marker: args.uploadIdMarker, + }); + + const uploads: MultipartUploadInfo[] = metaResult.contents.map((c) => { + const uploadId = c.key.substring(".herald/multipart-meta/".length); + return { + key: "unknown", + uploadId, + owner: { id: "swift", displayName: "Swift User" }, + initiator: { id: "swift", displayName: "Swift User" }, + storageClass: "STANDARD", + initiated: c.lastModified!, + }; + }); + + return { + bucket: container, + prefix: args.prefix, + keyMarker: args.keyMarker, + uploadIdMarker: args.uploadIdMarker, + maxUploads: args.maxUploads ?? 1000, + delimiter: args.delimiter, + isTruncated: metaResult.isTruncated, + uploads, + commonPrefixes: [], + encodingType: args.encodingType, + } satisfies ListMultipartUploadsResult; + }), + + listParts: ( + key: string, + uploadId: string, + ): Effect.Effect => + Effect.gen(function* () { + const { url, token, container } = target; + + // Check if upload exists by checking for metadata object + const metaKey = `.herald/multipart-meta/${uploadId}`; + const metaResponse = yield* client.execute( + HttpClientRequest.head(`${url}/${metaKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (metaResponse.status === 404) { + return yield* Effect.fail( + new NoSuchUpload({ + uploadId, + message: + `The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.`, + }), + ); + } + + const segmentsResult = yield* listObjects({ + prefix: `.herald/multipart-segments/${uploadId}/`, + }); + + const parts: PartInfo[] = segmentsResult.contents.map((c) => { + const partNumber = parseInt(c.key.split("/").pop() || "0"); + return { + partNumber, + lastModified: c.lastModified, + etag: c.etag, + size: c.size, + }; + }); + + return { + bucket: container, + key, + uploadId, + owner: { id: "swift", displayName: "Swift User" }, + initiator: { id: "swift", displayName: "Swift User" }, + storageClass: "STANDARD", + partNumberMarker: 0, + nextPartNumberMarker: 0, + maxParts: 1000, + isTruncated: false, + parts, + } satisfies ListPartsResult; + }), }; }; diff --git a/src/Backends/Swift/Utils.ts b/src/Backends/Swift/Utils.ts index 62295b0..81e3aa5 100644 --- a/src/Backends/Swift/Utils.ts +++ b/src/Backends/Swift/Utils.ts @@ -35,7 +35,12 @@ export const mapError = ( if (method === "DELETE") { return new BucketNotEmpty({ bucketName, message }); } - return new BucketAlreadyExists({ bucketName, message }); + if (method === "PUT" && !key) { + return new BucketAlreadyExists({ bucketName, message }); + } + return new InternalError({ + message: `Swift conflict error (${status}): ${message}`, + }); case 202: if (method === "PUT") { return new BucketAlreadyOwnedByYou({ bucketName, message }); @@ -63,7 +68,7 @@ export const getTarget = ( ); const container = "bucket_name" in bucket ? bucket.bucket_name : ""; const encodedContainer = container ? encodeURIComponent(container) : ""; - return { + const res = { storageUrl: auth.storageUrl, token: auth.token, container, @@ -71,4 +76,8 @@ export const getTarget = ( ? `${auth.storageUrl}/${encodedContainer}` : auth.storageUrl, }; + yield* Effect.logDebug( + `SwiftTarget resolved: url=[${res.url}] container=[${res.container}]`, + ); + return res; }); diff --git a/src/Frontend/Objects/Delete.ts b/src/Frontend/Objects/Delete.ts index 3e1856b..054a515 100644 --- a/src/Frontend/Objects/Delete.ts +++ b/src/Frontend/Objects/Delete.ts @@ -12,6 +12,9 @@ export const deleteObject = () => if (params.uploadId) { // Abort Multipart Upload yield* backend.abortMultipartUpload(key, params.uploadId); + yield* backend.multipartMetadataStore.remove(params.uploadId).pipe( + Effect.ignore, + ); return HttpServerResponse.empty({ status: 204 }); } diff --git a/src/Frontend/Objects/Post.ts b/src/Frontend/Objects/Post.ts index dffade7..a5165e9 100644 --- a/src/Frontend/Objects/Post.ts +++ b/src/Frontend/Objects/Post.ts @@ -1,7 +1,8 @@ -import { Effect, Option, Stream } from "effect"; +import { Effect, Option } from "effect"; import { HttpServerResponse } from "@effect/platform"; import { RequestContext } from "../Utils.ts"; import { S3Xml } from "../../Services/S3Xml.ts"; +import { NoSuchUpload } from "../../Services/Backend.ts"; /** * Handler for POST requests on buckets or objects. @@ -15,18 +16,7 @@ export const postObject = () => if (params.delete !== undefined) { // Multi-Object Delete - const bodyChunks = yield* Stream.runCollect(request.stream); - let totalLength = 0; - for (const chunk of Array.from(bodyChunks)) { - totalLength += chunk.length; - } - const bodyBytes = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of Array.from(bodyChunks)) { - bodyBytes.set(chunk, offset); - offset += chunk.length; - } - const bodyText = new TextDecoder().decode(bodyBytes); + const bodyText = yield* request.text; const objects: { key: string; versionId?: string }[] = []; // Simple XML parsing for Multi-Object Delete @@ -78,7 +68,28 @@ export const postObject = () => const result = yield* backend.createMultipartUpload( key, request.headers, + ).pipe( + Effect.tapError((e) => + Effect.logError(`createMultipartUpload failed: ${e}`) + ), + ); + // Save metadata + const metadata: Record = {}; + for (const [k, v] of Object.entries(request.headers)) { + const lowK = k.toLowerCase(); + if (lowK.startsWith("x-amz-meta-") || lowK === "content-type") { + metadata[lowK] = String(v); + } + } + yield* backend.multipartMetadataStore.set( + result.uploadId, + JSON.stringify(metadata), + ).pipe( + Effect.tapError((e) => + Effect.logError(`metadataStore.set failed: ${e}`) + ), ); + return s3Xml.formatInitiateMultipartUpload( bucket, key, @@ -88,18 +99,7 @@ export const postObject = () => if (params.uploadId) { // Complete Multipart Upload - const bodyChunks = yield* Stream.runCollect(request.stream); - let totalLength = 0; - for (const chunk of Array.from(bodyChunks)) { - totalLength += chunk.length; - } - const bodyBytes = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of Array.from(bodyChunks)) { - bodyBytes.set(chunk, offset); - offset += chunk.length; - } - const bodyText = new TextDecoder().decode(bodyBytes); + const bodyText = yield* request.text; const parts: { etag: string; partNumber: number }[] = []; const partMatches = Array.from( @@ -119,33 +119,61 @@ export const postObject = () => } } + // Retrieve metadata + const metadataOpt = yield* backend.multipartMetadataStore.get( + params.uploadId, + ); + + if (Option.isNone(metadataOpt)) { + const head = yield* backend.headObject(key, {}).pipe( + Effect.orElseFail(() => + new NoSuchUpload({ + uploadId: params.uploadId!, + message: "The specified upload does not exist.", + }) + ), + ); + if (head.etag) { + return s3Xml.formatCompleteMultipartUpload({ + location: `http://localhost/${bucket}/${key}`, // Approximate + bucket, + key, + etag: head.etag, + }); + } + return yield* Effect.fail( + new NoSuchUpload({ + uploadId: params.uploadId!, + message: "The specified upload does not exist.", + }), + ); + } + + const metadata = JSON.parse(metadataOpt.value); + const result = yield* backend.completeMultipartUpload( key, params.uploadId, parts, + metadata, ).pipe( - Effect.catchTag("NoSuchUpload", (e) => - Effect.gen(function* () { - // Idempotency: check if object already exists - const head = yield* backend.headObject(key, {}).pipe( - Effect.orElseFail(() => e), - ); - if (head.etag) { - return { - location: `http://localhost/${bucket}/${key}`, // Approximate - bucket, - key, - etag: head.etag, - versionId: head.headers["x-amz-version-id"], - }; - } - return yield* Effect.fail(e); - })), + Effect.tap(() => + backend.multipartMetadataStore.remove(params.uploadId!).pipe( + Effect.ignore, + ) + ), ); + return s3Xml.formatCompleteMultipartUpload(result); } return yield* Effect.fail( new Error(`Method POST for key [${key}] not implemented`), ); - }); + }).pipe( + Effect.catchAll((e) => { + return Effect.logError(`postObject error: ${e}`).pipe( + Effect.zipRight(Effect.fail(e)), + ); + }), + ); diff --git a/src/Frontend/Utils.ts b/src/Frontend/Utils.ts index 972743c..f53925c 100644 --- a/src/Frontend/Utils.ts +++ b/src/Frontend/Utils.ts @@ -231,34 +231,40 @@ export function resolveBucket< return yield* resolver.provideForBucket(bucketName, program).pipe( Effect.catchAll((e) => { - if ( - e instanceof NoSuchBucket || - e instanceof NoSuchKey || - e instanceof BucketAlreadyExists || - e instanceof BucketAlreadyOwnedByYou || - e instanceof InternalError || - e instanceof AccessDenied || - e instanceof BucketNotEmpty || - e instanceof NoSuchUpload || - e instanceof InvalidPart || - e instanceof InvalidPartOrder || - e instanceof EntityTooSmall || - e instanceof InvalidRequest || - e instanceof MalformedXML || - e instanceof DeleteObjectsError - ) { - return Effect.succeed(s3Xml.formatError(e, isHead)); - } - return Effect.logError( - `resolveBucket caught unhandled error for bucket ${bucketName}: ${e}`, + return Effect.logInfo( + `resolveBucket caught error for bucket ${bucketName}: ${e}`, ).pipe( - Effect.zipRight( - Effect.fail( - new BadGateway({ - message: e instanceof Error ? e.message : String(e), - }), - ), - ), + Effect.flatMap(() => { + if ( + e instanceof NoSuchBucket || + e instanceof NoSuchKey || + e instanceof BucketAlreadyExists || + e instanceof BucketAlreadyOwnedByYou || + e instanceof InternalError || + e instanceof AccessDenied || + e instanceof BucketNotEmpty || + e instanceof NoSuchUpload || + e instanceof InvalidPart || + e instanceof InvalidPartOrder || + e instanceof EntityTooSmall || + e instanceof InvalidRequest || + e instanceof MalformedXML || + e instanceof DeleteObjectsError + ) { + return Effect.succeed(s3Xml.formatError(e, isHead)); + } + return Effect.logError( + `resolveBucket caught unhandled error for bucket ${bucketName}: ${e}`, + ).pipe( + Effect.zipRight( + Effect.fail( + new BadGateway({ + message: e instanceof Error ? e.message : String(e), + }), + ), + ), + ); + }), ); }), ); diff --git a/src/Http.ts b/src/Http.ts index 3c3c693..6fa2dbc 100644 --- a/src/Http.ts +++ b/src/Http.ts @@ -29,15 +29,10 @@ export const HttpServerHeraldLive = Layer.unwrapEffect( 3000, ); return HttpApiBuilder.serve(HttpMiddleware.logger).pipe( - // provides swagger ui for http api Layer.provide(HttpApiSwagger.layer()), - // provides openapi.json endpoint Layer.provide(HttpApiBuilder.middlewareOpenApi()), - // adds cors support - // FIXME: config support Layer.provide(HttpApiBuilder.middlewareCors()), Layer.provide(HttpHeraldLive), - // log address at startup HttpServer.withLogAddress, Layer.provide(NodeHttpServer.layer(createServer, { port })), Layer.provide(HeraldConfigLive), diff --git a/src/Services/Backend.ts b/src/Services/Backend.ts index 9f3f1f4..30cff6d 100644 --- a/src/Services/Backend.ts +++ b/src/Services/Backend.ts @@ -3,6 +3,7 @@ */ import { Context, type Effect, Schema, type Stream } from "effect"; +import type { KeyValueStore } from "@effect/platform"; export interface BucketInfo { readonly name: string; @@ -289,6 +290,8 @@ export interface BackendService { objects: readonly { key: string; versionId?: string }[], ) => Effect.Effect; + readonly multipartMetadataStore: KeyValueStore.KeyValueStore; + // Multipart Upload readonly createMultipartUpload: ( key: string, @@ -304,6 +307,7 @@ export interface BackendService { key: string, uploadId: string, parts: readonly { etag: string; partNumber: number }[], + metadata: Record, ) => Effect.Effect; readonly abortMultipartUpload: ( key: string, diff --git a/src/Services/BackendKeyValueStore.ts b/src/Services/BackendKeyValueStore.ts new file mode 100644 index 0000000..598e358 --- /dev/null +++ b/src/Services/BackendKeyValueStore.ts @@ -0,0 +1,125 @@ +import { Chunk, Effect, Option, Stream } from "effect"; +import { KeyValueStore } from "@effect/platform"; +import { SystemError } from "@effect/platform/Error"; +import type { BackendService } from "./Backend.ts"; + +/** + * A KeyValueStore that persists its data as objects in a BackendService. + * This is used by backends like Swift that don't natively support S3 multipart metadata + * persistence during the upload lifecycle. + */ +export const makeBackendKeyValueStore = ( + ops: { + getObject: BackendService["getObject"]; + putObject: BackendService["putObject"]; + deleteObject: BackendService["deleteObject"]; + }, + prefix: string, +): KeyValueStore.KeyValueStore => + KeyValueStore.make({ + get: (key) => { + return ops.getObject(`${prefix}${key}`, {}).pipe( + Effect.flatMap((res) => Stream.runCollect(res.stream)), + Effect.map((chunks) => { + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const all = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + all.set(chunk, offset); + offset += chunk.length; + } + return Option.some(new TextDecoder().decode(all)); + }), + Effect.catchTag("NoSuchKey", () => Effect.succeed(Option.none())), + Effect.catchAll((e) => + Effect.fail( + new SystemError({ + module: "KeyValueStore", + method: "get", + reason: "Unknown", + syscall: "getObject", + description: String(e), + cause: e, + }), + ) + ), + ); + }, + getUint8Array: (key) => { + return ops.getObject(`${prefix}${key}`, {}).pipe( + Effect.flatMap((res) => Stream.runCollect(res.stream)), + Effect.map((chunks) => { + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const all = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + all.set(chunk, offset); + offset += chunk.length; + } + return Option.some(all); + }), + Effect.catchTag("NoSuchKey", () => Effect.succeed(Option.none())), + Effect.catchAll((e) => + Effect.fail( + new SystemError({ + module: "KeyValueStore", + method: "getUint8Array", + reason: "Unknown", + syscall: "getObject", + description: String(e), + cause: e, + }), + ) + ), + ); + }, + set: (key, value) => { + const encodedValue = typeof value === "string" + ? new TextEncoder().encode(value) + : value; + return ops.putObject( + `${prefix}${key}`, + Stream.succeed(encodedValue), + { "Content-Type": "application/json" }, + ).pipe( + Effect.asVoid, + Effect.catchAll((e) => + Effect.fail( + new SystemError({ + module: "KeyValueStore", + method: "set", + reason: "Unknown", + syscall: "putObject", + description: String(e), + cause: e, + }), + ) + ), + ); + }, + remove: (key) => + ops.deleteObject(`${prefix}${key}`).pipe( + Effect.catchAll((e) => + Effect.fail( + new SystemError({ + module: "KeyValueStore", + method: "remove", + reason: "Unknown", + syscall: "deleteObject", + description: String(e), + cause: e, + }), + ) + ), + ), + clear: Effect.die("Clear not supported in BackendKeyValueStore"), + size: Effect.die("Size not supported in BackendKeyValueStore"), + }); diff --git a/src/Services/NoopKeyValueStore.ts b/src/Services/NoopKeyValueStore.ts new file mode 100644 index 0000000..2f3ce59 --- /dev/null +++ b/src/Services/NoopKeyValueStore.ts @@ -0,0 +1,12 @@ +import { Effect, Option } from "effect"; +import { KeyValueStore } from "@effect/platform"; + +export const makeNoopKeyValueStore = (): KeyValueStore.KeyValueStore => + KeyValueStore.make({ + get: (_key) => Effect.succeed(Option.none()), + getUint8Array: (_key) => Effect.succeed(Option.none()), + set: (_key, _value) => Effect.void, + remove: (_key) => Effect.void, + clear: Effect.void, + size: Effect.succeed(0), + }); diff --git a/tests/integration/__snapshots__/objects.test.ts.snap b/tests/integration/__snapshots__/objects.test.ts.snap index 63b70f8..c3d1496 100644 --- a/tests/integration/__snapshots__/objects.test.ts.snap +++ b/tests/integration/__snapshots__/objects.test.ts.snap @@ -1,228 +1,13 @@ export const snapshot = {}; -snapshot[`Baseline/objects/put metadata 1`] = ` -{ - headers: { - "accept-ranges": "bytes", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 204, -} -`; - -snapshot[`Proxy/objects/put metadata 1`] = ` -{ - headers: {}, - status: 204, -} -`; - -snapshot[`Swift/objects/put metadata 1`] = ` -{ - headers: {}, - status: 204, -} -`; - -snapshot[`Baseline/objects/get/existing metadata 1`] = ` -{ - headers: { - "accept-ranges": "bytes", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 204, -} -`; - -snapshot[`Proxy/objects/get/existing metadata 1`] = ` -{ - headers: {}, - status: 204, -} -`; - -snapshot[`Swift/objects/get/existing metadata 1`] = ` -{ - headers: {}, - status: 204, -} -`; - -snapshot[`Baseline/objects/get/non-existent metadata 1`] = ` -{ - headers: { - "accept-ranges": "bytes", - "content-length": "359", - "content-type": "application/xml", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 404, -} -`; - -snapshot[`Baseline/objects/get/non-existent body 1`] = ` -' -NoSuchKeyThe specified key does not exist.no-suchtest-objects-bucket/test-objects-bucket/no-suchIDHOST' -`; - -snapshot[`Proxy/objects/get/non-existent metadata 1`] = ` -{ - headers: { - "content-type": "application/xml", - vary: "Accept-Encoding", - }, - status: 404, -} -`; - -snapshot[`Proxy/objects/get/non-existent body 1`] = `'NoSuchKeyThe specified key does not exist.'`; - -snapshot[`Swift/objects/get/non-existent metadata 1`] = ` -{ - headers: { - "content-type": "application/xml", - vary: "Accept-Encoding", - }, - status: 404, -} -`; - -snapshot[`Swift/objects/get/non-existent body 1`] = `'NoSuchKey

Not Found

The resource could not be found.

'`; - -snapshot[`Baseline/objects/head/existing metadata 1`] = ` -{ - headers: { - "accept-ranges": "bytes", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 204, -} -`; - -snapshot[`Proxy/objects/head/existing metadata 1`] = ` -{ - headers: {}, - status: 204, -} -`; - -snapshot[`Swift/objects/head/existing metadata 1`] = ` -{ - headers: {}, - status: 204, -} -`; - -snapshot[`Baseline/objects/head/non-existent metadata 1`] = ` -{ - headers: { - "accept-ranges": "bytes", - "content-length": "0", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-minio-error-code": "NoSuchKey", - "x-minio-error-desc": '"The specified key does not exist."', - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 404, -} -`; - -snapshot[`Proxy/objects/head/non-existent metadata 1`] = ` -{ - headers: {}, - status: 404, -} -`; - -snapshot[`Swift/objects/head/non-existent metadata 1`] = ` -{ - headers: {}, - status: 404, -} -`; - -snapshot[`Baseline/objects/delete/existing metadata 1`] = ` -{ - headers: { - "accept-ranges": "bytes", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 204, -} -`; - -snapshot[`Proxy/objects/delete/existing metadata 1`] = ` -{ - headers: {}, - status: 204, -} -`; - -snapshot[`Swift/objects/delete/existing metadata 1`] = ` -{ - headers: {}, - status: 204, -} -`; - -snapshot[`Baseline/objects/multipart/basic metadata 1`] = ` -{ - headers: { - "accept-ranges": "bytes", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 204, -} -`; - -snapshot[`Proxy/objects/multipart/basic metadata 1`] = ` +snapshot[`Swift/objects/multipart/basic metadata 1`] = ` { headers: {}, status: 204, } `; -snapshot[`Baseline/objects/multipart/abort metadata 1`] = ` -{ - headers: { - "accept-ranges": "bytes", - "content-length": "479", - "content-type": "application/xml", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 404, -} -`; - -snapshot[`Baseline/objects/multipart/abort body 1`] = ` -' -NoSuchUploadThe specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.multipart-abort.txttest-objects-bucket/test-objects-bucket/multipart-abort.txtIDHOST' -`; - -snapshot[`Proxy/objects/multipart/abort metadata 1`] = ` +snapshot[`Swift/objects/multipart/abort metadata 1`] = ` { headers: { "content-type": "application/xml", @@ -232,42 +17,16 @@ snapshot[`Proxy/objects/multipart/abort metadata 1`] = ` } `; -snapshot[`Proxy/objects/multipart/abort body 1`] = `'NoSuchUploadThe specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.'`; +snapshot[`Swift/objects/multipart/abort body 1`] = `'NoSuchUploadThe specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.'`; -snapshot[`Baseline/objects/multipart/list-parts metadata 1`] = ` -{ - headers: { - "accept-ranges": "bytes", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 204, -} -`; - -snapshot[`Proxy/objects/multipart/list-parts metadata 1`] = ` +snapshot[`Swift/objects/multipart/list-parts metadata 1`] = ` { headers: {}, status: 204, } `; -snapshot[`Baseline/objects/multipart/empty metadata 1`] = ` -{ - headers: { - "accept-ranges": "bytes", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 204, -} -`; - -snapshot[`Proxy/objects/multipart/empty metadata 1`] = ` +snapshot[`Swift/objects/multipart/empty metadata 1`] = ` { headers: {}, status: 204, diff --git a/tests/utils.ts b/tests/utils.ts index e5446b5..3442cd3 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -61,8 +61,12 @@ export const makeTestHarness = ( // Start Deno.serve on a random port const server = Deno.serve( { port: 0, onListen: () => {} }, - (req) => { - return webHandler.handler(req); + async (req) => { + try { + return await webHandler.handler(req); + } catch (_e) { + return new Response("Internal Server Error", { status: 500 }); + } }, ); diff --git a/x/swift-s3-tests.ts b/x/swift-s3-tests.ts index 1ca12ec..553f2f3 100644 --- a/x/swift-s3-tests.ts +++ b/x/swift-s3-tests.ts @@ -7,7 +7,7 @@ * configured with an OpenStack Swift backend. */ -import { Config, Effect, Layer, Logger, LogLevel } from "effect"; +import { Config, Effect, Layer, Logger, LogLevel, Stream } from "effect"; import { makeTestHarness } from "../tests/utils.ts"; import type { GlobalConfig } from "../src/Domain/Config.ts"; import * as path from "@std/path"; @@ -22,20 +22,31 @@ const program = Effect.gen(function* () { // Read Swift config from environment const authUrl = yield* Config.string("HERALD_SWIFTTEST_AUTH_URL").pipe( Config.orElse(() => Config.string("HEARLD_SWIFTTEST_AUTH_URL")), - Config.withDefault(""), + Config.orElse(() => Config.string("OS_AUTH_URL")), + Config.withDefault("https://api.pub1.infomaniak.cloud/identity/v3"), ); const region = yield* Config.string("HERALD_SWIFTTEST_OS_REGION_NAME").pipe( Config.orElse(() => Config.string("HEARLD_SWIFTTEST_OS_REGION_NAME")), - Config.withDefault(""), + Config.orElse(() => Config.string("TF_VAR_OS_REGION_NAME")), + Config.orElse(() => Config.string("OS_REGION_NAME")), + Config.withDefault("dc3-a"), ); const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_USERNAME")), + Config.orElse(() => Config.string("OS_USERNAME")), Config.withDefault(""), ); const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PASSWORD")), + Config.orElse(() => Config.string("OS_PASSWORD")), Config.withDefault(""), ); const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") - .pipe(Config.withDefault("")); + .pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PROJECT_NAME")), + Config.orElse(() => Config.string("OS_PROJECT_NAME")), + Config.withDefault(""), + ); if (!authUrl || !username || !password || !projectName) { return yield* Effect.fail( @@ -103,6 +114,33 @@ display_name = alt email = alt@example.com access_key = dummy secret_key = dummy + +[s3 tenant] +user_id = tenant +display_name = tenant +email = tenant@example.com +access_key = dummy +secret_key = dummy +tenant = dummy + +[iam] +email = s3@example.com +user_id = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +access_key = dummy +secret_key = dummy +display_name = youruseridhere + +[iam root] +access_key = dummyroot +secret_key = dummyroot +user_id = RGW11111111111111111 +email = account1@ceph.com + +[iam alt root] +access_key = dummyaltroot +secret_key = dummyaltroot +user_id = RGW22222222222222222 +email = account2@ceph.com `; const confPath = yield* Effect.promise(() => @@ -194,7 +232,7 @@ secret_key = dummy console.log(colors.green(`\n✓ s3-tests completed successfully.`)); }).pipe( Effect.scoped, - Effect.provide(Logger.minimumLogLevel(LogLevel.Info)), + Effect.provide(Logger.minimumLogLevel(LogLevel.Debug)), ); if (import.meta.main) { From c1a1180649beb33d7aba37e53d8b4cc5cb98d7fb Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Thu, 22 Jan 2026 02:52:36 +0300 Subject: [PATCH 02/10] fix: fix issue with list multipart Signed-off-by: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> --- src/Backends/Swift/Backend.ts | 4 +- src/Backends/Swift/Buckets.ts | 30 +- src/Backends/Swift/Objects.ts | 61 ++-- src/Backends/Swift/Utils.ts | 4 + src/Frontend/Objects/Delete.ts | 7 +- src/Frontend/Objects/Post.ts | 39 ++- tests/config.test.ts | 1 + .../__snapshots__/buckets.test.ts.snap | 42 --- .../__snapshots__/objects.test.ts.snap | 274 ++++++++++++++++++ tests/integration/buckets.test.ts | 3 + tests/utils.ts | 7 +- 11 files changed, 369 insertions(+), 103 deletions(-) diff --git a/src/Backends/Swift/Backend.ts b/src/Backends/Swift/Backend.ts index 930a1ad..8063b21 100644 --- a/src/Backends/Swift/Backend.ts +++ b/src/Backends/Swift/Backend.ts @@ -4,7 +4,7 @@ import type { BackendError, BackendService } from "../../Services/Backend.ts"; import type { MaterializedBucket } from "../../Domain/Config.ts"; import { makeBucketOps } from "./Buckets.ts"; import { makeObjectOps } from "./Objects.ts"; -import { getTarget } from "./Utils.ts"; +import { getTarget, MP_META_PREFIX } from "./Utils.ts"; import type { SwiftClient } from "./Client.ts"; import { makeBackendKeyValueStore } from "../../Services/BackendKeyValueStore.ts"; @@ -35,7 +35,7 @@ export const makeSwiftBackend = ( ...baseBackend, multipartMetadataStore: makeBackendKeyValueStore( objectOps, - ".herald/multipart-meta/", + MP_META_PREFIX, ), }; diff --git a/src/Backends/Swift/Buckets.ts b/src/Backends/Swift/Buckets.ts index dde6ea5..cd88726 100644 --- a/src/Backends/Swift/Buckets.ts +++ b/src/Backends/Swift/Buckets.ts @@ -7,7 +7,7 @@ import { type ListObjectsResult, type OwnerInfo, } from "../../Services/Backend.ts"; -import { mapError, type SwiftTarget } from "./Utils.ts"; +import { INTERNAL_PREFIX, mapError, type SwiftTarget } from "./Utils.ts"; export interface SwiftContainer { readonly name: string; @@ -96,20 +96,22 @@ export const makeBucketOps = ( Effect.gen(function* () { const { url, token, container } = target; - // 1. Cleanup .herald/ objects so bucket can be deleted - let marker: string | undefined = undefined; - while (true) { - const heraldObjects: ListObjectsResult = yield* objectOps.listObjects({ - prefix: ".herald/", - marker, - }); - for (const obj of heraldObjects.contents) { - yield* objectOps.deleteObject(obj.key).pipe(Effect.ignore); + // 1. Cleanup .herald/ and .hrld/ objects so bucket can be deleted + for (const prefix of [".herald/", INTERNAL_PREFIX]) { + let marker: string | undefined = undefined; + while (true) { + const objects: ListObjectsResult = yield* objectOps.listObjects({ + prefix, + marker, + }); + for (const obj of objects.contents) { + yield* objectOps.deleteObject(obj.key).pipe(Effect.ignore); + } + if (!objects.isTruncated || !objects.nextMarker) { + break; + } + marker = objects.nextMarker; } - if (!heraldObjects.isTruncated || !heraldObjects.nextMarker) { - break; - } - marker = heraldObjects.nextMarker; } // 2. Delete the bucket diff --git a/src/Backends/Swift/Objects.ts b/src/Backends/Swift/Objects.ts index c2ec6c0..453f5a9 100644 --- a/src/Backends/Swift/Objects.ts +++ b/src/Backends/Swift/Objects.ts @@ -19,7 +19,12 @@ import { type PutObjectResult, type UploadPartResult, } from "../../Services/Backend.ts"; -import { mapError, type SwiftTarget } from "./Utils.ts"; +import { + mapError, + MP_META_PREFIX, + MP_SEGMENTS_PREFIX, + type SwiftTarget, +} from "./Utils.ts"; import { fixHeaderEncoding } from "../../Frontend/Utils.ts"; export interface SwiftObject { @@ -564,8 +569,7 @@ export const makeObjectOps = ( ): Effect.Effect => Effect.gen(function* () { const { url, token, container } = target; - const segmentKey = - `.herald/multipart-segments/${uploadId}/${partNumber}`; + const segmentKey = `${MP_SEGMENTS_PREFIX}${uploadId}/${partNumber}`; const encodedSegmentKey = segmentKey.split("/").map(encodeURIComponent) .join("/"); @@ -625,7 +629,7 @@ export const makeObjectOps = ( let segmentMarker: string | undefined = undefined; while (true) { const segmentsResult: ListObjectsResult = yield* listObjects({ - prefix: `.herald/multipart-segments/${uploadId}/`, + prefix: `${MP_SEGMENTS_PREFIX}${uploadId}/`, marker: segmentMarker, }); for (const c of segmentsResult.contents) { @@ -640,8 +644,7 @@ export const makeObjectOps = ( // 1. Build SLO manifest const manifest = []; for (const p of parts) { - const segmentKey = - `.herald/multipart-segments/${uploadId}/${p.partNumber}`; + const segmentKey = `${MP_SEGMENTS_PREFIX}${uploadId}/${p.partNumber}`; const info = segmentMap.get(segmentKey); if (!info) { return yield* Effect.fail( @@ -714,6 +717,17 @@ export const makeObjectOps = ( ? etagHeader[0] : etagHeader; + // 3. Delete the metadata object + const metaKey = `${MP_META_PREFIX}${key}/${uploadId}`; + const encodedMetaKey = metaKey.split("/").map(encodeURIComponent).join( + "/", + ); + yield* client.execute( + HttpClientRequest.del(`${url}/${encodedMetaKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe(Effect.ignore); + return { location: `${url}/${encodedKey}`, bucket: container, @@ -723,7 +737,7 @@ export const makeObjectOps = ( }), abortMultipartUpload: ( - _key: string, + key: string, uploadId: string, ): Effect.Effect => Effect.gen(function* () { @@ -733,7 +747,7 @@ export const makeObjectOps = ( let marker: string | undefined = undefined; while (true) { const segmentsResult: ListObjectsResult = yield* listObjects({ - prefix: `.herald/multipart-segments/${uploadId}/`, + prefix: `${MP_SEGMENTS_PREFIX}${uploadId}/`, marker, }); @@ -754,7 +768,7 @@ export const makeObjectOps = ( } // 2. Delete the metadata object - const metaKey = `.herald/multipart-meta/${uploadId}`; + const metaKey = `${MP_META_PREFIX}${key}/${uploadId}`; const encodedMetaKey = metaKey.split("/").map(encodeURIComponent).join( "/", ); @@ -775,16 +789,24 @@ export const makeObjectOps = ( }): Effect.Effect => Effect.gen(function* () { const { container } = target; + const prefix = `${MP_META_PREFIX}${args.prefix ?? ""}`; + const marker = args.keyMarker + ? `${MP_META_PREFIX}${args.keyMarker}/${args.uploadIdMarker ?? ""}` + : undefined; + const metaResult = yield* listObjects({ - prefix: ".herald/multipart-meta/", + prefix, + delimiter: args.delimiter, maxKeys: args.maxUploads, - marker: args.uploadIdMarker, + marker, }); const uploads: MultipartUploadInfo[] = metaResult.contents.map((c) => { - const uploadId = c.key.substring(".herald/multipart-meta/".length); + const parts = c.key.substring(MP_META_PREFIX.length).split("/"); + const uploadId = parts.pop()!; + const key = parts.join("/"); return { - key: "unknown", + key, uploadId, owner: { id: "swift", displayName: "Swift User" }, initiator: { id: "swift", displayName: "Swift User" }, @@ -802,7 +824,9 @@ export const makeObjectOps = ( delimiter: args.delimiter, isTruncated: metaResult.isTruncated, uploads, - commonPrefixes: [], + commonPrefixes: metaResult.commonPrefixes.map((cp) => ({ + prefix: cp.prefix.substring(MP_META_PREFIX.length), + })), encodingType: args.encodingType, } satisfies ListMultipartUploadsResult; }), @@ -815,9 +839,12 @@ export const makeObjectOps = ( const { url, token, container } = target; // Check if upload exists by checking for metadata object - const metaKey = `.herald/multipart-meta/${uploadId}`; + const metaKey = `${MP_META_PREFIX}${key}/${uploadId}`; + const encodedMetaKey = metaKey.split("/").map(encodeURIComponent).join( + "/", + ); const metaResponse = yield* client.execute( - HttpClientRequest.head(`${url}/${metaKey}`).pipe( + HttpClientRequest.head(`${url}/${encodedMetaKey}`).pipe( HttpClientRequest.setHeaders({ "X-Auth-Token": token }), ), ).pipe( @@ -835,7 +862,7 @@ export const makeObjectOps = ( } const segmentsResult = yield* listObjects({ - prefix: `.herald/multipart-segments/${uploadId}/`, + prefix: `${MP_SEGMENTS_PREFIX}${uploadId}/`, }); const parts: PartInfo[] = segmentsResult.contents.map((c) => { diff --git a/src/Backends/Swift/Utils.ts b/src/Backends/Swift/Utils.ts index 81e3aa5..c72b0bf 100644 --- a/src/Backends/Swift/Utils.ts +++ b/src/Backends/Swift/Utils.ts @@ -18,6 +18,10 @@ export interface SwiftTarget { readonly url: string; } +export const INTERNAL_PREFIX = ".hrld/"; +export const MP_META_PREFIX = `${INTERNAL_PREFIX}mmp/`; +export const MP_SEGMENTS_PREFIX = `${INTERNAL_PREFIX}msg/`; + export const mapError = ( status: number, message: string, diff --git a/src/Frontend/Objects/Delete.ts b/src/Frontend/Objects/Delete.ts index 054a515..b5e7264 100644 --- a/src/Frontend/Objects/Delete.ts +++ b/src/Frontend/Objects/Delete.ts @@ -12,9 +12,10 @@ export const deleteObject = () => if (params.uploadId) { // Abort Multipart Upload yield* backend.abortMultipartUpload(key, params.uploadId); - yield* backend.multipartMetadataStore.remove(params.uploadId).pipe( - Effect.ignore, - ); + yield* backend.multipartMetadataStore.remove(`${key}/${params.uploadId}`) + .pipe( + Effect.ignore, + ); return HttpServerResponse.empty({ status: 204 }); } diff --git a/src/Frontend/Objects/Post.ts b/src/Frontend/Objects/Post.ts index a5165e9..de80954 100644 --- a/src/Frontend/Objects/Post.ts +++ b/src/Frontend/Objects/Post.ts @@ -2,7 +2,6 @@ import { Effect, Option } from "effect"; import { HttpServerResponse } from "@effect/platform"; import { RequestContext } from "../Utils.ts"; import { S3Xml } from "../../Services/S3Xml.ts"; -import { NoSuchUpload } from "../../Services/Backend.ts"; /** * Handler for POST requests on buckets or objects. @@ -82,7 +81,7 @@ export const postObject = () => } } yield* backend.multipartMetadataStore.set( - result.uploadId, + `${key}/${result.uploadId}`, JSON.stringify(metadata), ).pipe( Effect.tapError((e) => @@ -121,36 +120,31 @@ export const postObject = () => // Retrieve metadata const metadataOpt = yield* backend.multipartMetadataStore.get( - params.uploadId, + `${key}/${params.uploadId}`, ); + let metadata: Record = {}; + if (Option.isNone(metadataOpt)) { + // Check for idempotency const head = yield* backend.headObject(key, {}).pipe( - Effect.orElseFail(() => - new NoSuchUpload({ - uploadId: params.uploadId!, - message: "The specified upload does not exist.", - }) - ), + Effect.option, ); - if (head.etag) { + if (Option.isSome(head) && head.value.etag) { return s3Xml.formatCompleteMultipartUpload({ location: `http://localhost/${bucket}/${key}`, // Approximate bucket, key, - etag: head.etag, + etag: head.value.etag, }); } - return yield* Effect.fail( - new NoSuchUpload({ - uploadId: params.uploadId!, - message: "The specified upload does not exist.", - }), - ); + // If not completed and no metadata, proceed with empty metadata + // Backends like Swift will fail if the upload doesn't exist (no segments) + // Backends like S3 will succeed if S3 says it's okay. + } else { + metadata = JSON.parse(metadataOpt.value); } - const metadata = JSON.parse(metadataOpt.value); - const result = yield* backend.completeMultipartUpload( key, params.uploadId, @@ -158,9 +152,10 @@ export const postObject = () => metadata, ).pipe( Effect.tap(() => - backend.multipartMetadataStore.remove(params.uploadId!).pipe( - Effect.ignore, - ) + backend.multipartMetadataStore.remove(`${key}/${params.uploadId!}`) + .pipe( + Effect.ignore, + ) ), ); diff --git a/tests/config.test.ts b/tests/config.test.ts index eb5981a..9b834d2 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -234,6 +234,7 @@ const cases: TestCase[] = [ backends: { swift_main: { protocol: "swift", + auth_url: "http://keystone.example.com", credentials: { username: "user1", password: "pw1", diff --git a/tests/integration/__snapshots__/buckets.test.ts.snap b/tests/integration/__snapshots__/buckets.test.ts.snap index 76add56..f420d34 100644 --- a/tests/integration/__snapshots__/buckets.test.ts.snap +++ b/tests/integration/__snapshots__/buckets.test.ts.snap @@ -181,45 +181,3 @@ snapshot[`Swift/buckets/head/non-existent metadata 1`] = ` status: 404, } `; - -snapshot[`Baseline/buckets/list metadata 1`] = ` -{ - headers: { - "content-type": "application/xml", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 200, -} -`; - -snapshot[`Baseline/buckets/list body 1`] = ` -' -02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4minioherald-25gnaqqph3oof5kljqtdeu-72026-01-15T00:00:00.000Zherald-74khf7szf4qtrzrth2weoi-2192026-01-15T00:00:00.000Zherald-88ztrfgehycvaw5bh5t625-12026-01-15T00:00:00.000Zherald-almi4r3xt6pj4vmf25mpkc-362026-01-15T00:00:00.000Zherald-b3y7kg3dn3u5q0awin9aj8-72026-01-15T00:00:00.000Zherald-ferrwumx0p2j3tdhrfle4o-642026-01-15T00:00:00.000Zherald-iqm95px2zlxt75mcsx3dms-12026-01-15T00:00:00.000Zherald-l84igcd8jggs3wioh4msk8-12026-01-15T00:00:00.000Zherald-quy3o0n429jznm43dcga5l-312026-01-15T00:00:00.000Zherald-unz0kp56250vjw6umbd0va-12026-01-15T00:00:00.000Zherald-zrs5hcqpud1tn54vd9u700-152026-01-15T00:00:00.000Zherald-zunthialhf5qffc4p9xthk-742026-01-15T00:00:00.000Z' -`; - -snapshot[`Proxy/buckets/list metadata 1`] = ` -{ - headers: { - "content-type": "application/xml", - vary: "Accept-Encoding", - }, - status: 200, -} -`; - -snapshot[`Proxy/buckets/list body 1`] = `'02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4minioherald-25gnaqqph3oof5kljqtdeu-72026-01-15T00:00:00.000Zherald-74khf7szf4qtrzrth2weoi-2192026-01-15T00:00:00.000Zherald-88ztrfgehycvaw5bh5t625-12026-01-15T00:00:00.000Zherald-almi4r3xt6pj4vmf25mpkc-362026-01-15T00:00:00.000Zherald-b3y7kg3dn3u5q0awin9aj8-72026-01-15T00:00:00.000Zherald-ferrwumx0p2j3tdhrfle4o-642026-01-15T00:00:00.000Zherald-iqm95px2zlxt75mcsx3dms-12026-01-15T00:00:00.000Zherald-l84igcd8jggs3wioh4msk8-12026-01-15T00:00:00.000Zherald-quy3o0n429jznm43dcga5l-312026-01-15T00:00:00.000Zherald-unz0kp56250vjw6umbd0va-12026-01-15T00:00:00.000Zherald-zrs5hcqpud1tn54vd9u700-152026-01-15T00:00:00.000Zherald-zunthialhf5qffc4p9xthk-742026-01-15T00:00:00.000Z'`; - -snapshot[`Swift/buckets/list metadata 1`] = ` -{ - headers: { - "content-type": "application/xml", - vary: "Accept-Encoding", - }, - status: 200, -} -`; - -snapshot[`Swift/buckets/list body 1`] = `'swiftSwift User192.168.5.1232026-01-15T00:00:00.000Za2026-01-15T00:00:00.000Zaa2026-01-15T00:00:00.000Zbuilds2026-01-15T00:00:00.000Zfoo-2026-01-15T00:00:00.000Zfoo-.bar2026-01-15T00:00:00.000Zfoo.-bar2026-01-15T00:00:00.000Zfoo..bar2026-01-15T00:00:00.000Zfoo_bar2026-01-15T00:00:00.000Zherald-swift-2w97l75ompcxiypo-12026-01-15T00:00:00.000Zherald-swift-5dfyoor543wddfpb-12026-01-15T00:00:00.000Zherald-swift-5m8pru9nzpno98zp-1572026-01-15T00:00:00.000Zherald-swift-5txup4vs19i8tr6s-292026-01-15T00:00:00.000Zherald-swift-cze1vw7y05q33782-1462026-01-15T00:00:00.000Zherald-swift-fd0oi5radob46p39-12026-01-15T00:00:00.000Zherald-swift-m55m3lqytoaxuxro-132026-01-15T00:00:00.000Zherald-swift-oda2k1hu2ds0wir6-22026-01-15T00:00:00.000Zherald-swift-sy6d1ftl2i7g78jj-12026-01-15T00:00:00.000Zherald-swift-yx0xlhaeebv1g9c7-12026-01-15T00:00:00.000Zherald-task-store-mr-120-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-127-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-130-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-131-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-132-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-137-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-139-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-143-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-144-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-145-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-146-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-147-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-149-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-150-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-151-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-154-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-155-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-157-vivavox2026-01-15T00:00:00.000Zherald-task-store-prd-vivavox2026-01-15T00:00:00.000Zherald-task-store-stg-vivavox2026-01-15T00:00:00.000Ziac-swift2026-01-15T00:00:00.000Zmr-101-vivavox2026-01-15T00:00:00.000Zmr-109-vivavox2026-01-15T00:00:00.000Zmr-111-vivavox2026-01-15T00:00:00.000Zmr-115-vivavox2026-01-15T00:00:00.000Zmr-116-vivavox2026-01-15T00:00:00.000Zmr-120-vivavox2026-01-15T00:00:00.000Zmr-121-vivavox2026-01-15T00:00:00.000Zmr-122-vivavox2026-01-15T00:00:00.000Zmr-124-vivavox2026-01-15T00:00:00.000Zmr-126-vivavox2026-01-15T00:00:00.000Zmr-127-vivavox2026-01-15T00:00:00.000Zmr-130-vivavox2026-01-15T00:00:00.000Zmr-131-vivavox2026-01-15T00:00:00.000Zmr-132-vivavox2026-01-15T00:00:00.000Zmr-137-vivavox2026-01-15T00:00:00.000Zmr-139-vivavox2026-01-15T00:00:00.000Zmr-143-vivavox2026-01-15T00:00:00.000Zmr-144-vivavox2026-01-15T00:00:00.000Zmr-145-vivavox2026-01-15T00:00:00.000Zmr-146-vivavox2026-01-15T00:00:00.000Zmr-147-vivavox2026-01-15T00:00:00.000Zmr-149-vivavox2026-01-15T00:00:00.000Zmr-150-vivavox2026-01-15T00:00:00.000Zmr-151-vivavox2026-01-15T00:00:00.000Zmr-154-vivavox2026-01-15T00:00:00.000Zmr-155-vivavox2026-01-15T00:00:00.000Zmr-157-vivavox2026-01-15T00:00:00.000Zprd-vivavox2026-01-15T00:00:00.000Zstg-vivavox2026-01-15T00:00:00.000Zstg-vivavox+segments2026-01-15T00:00:00.000Ztest-objects-bucket2026-01-15T00:00:00.000Z'`; diff --git a/tests/integration/__snapshots__/objects.test.ts.snap b/tests/integration/__snapshots__/objects.test.ts.snap index c3d1496..e3d5f90 100644 --- a/tests/integration/__snapshots__/objects.test.ts.snap +++ b/tests/integration/__snapshots__/objects.test.ts.snap @@ -1,5 +1,207 @@ export const snapshot = {}; +snapshot[`Baseline/objects/put metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/put metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/put metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/get/existing metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/get/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/get/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/get/non-existent metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "359", + "content-type": "application/xml", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Baseline/objects/get/non-existent body 1`] = ` +' +NoSuchKeyThe specified key does not exist.no-suchtest-objects-bucket/test-objects-bucket/no-suchIDHOST' +`; + +snapshot[`Proxy/objects/get/non-existent metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/objects/get/non-existent body 1`] = `'NoSuchKeyThe specified key does not exist.'`; + +snapshot[`Swift/objects/get/non-existent metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Swift/objects/get/non-existent body 1`] = `'NoSuchKey

Not Found

The resource could not be found.

'`; + +snapshot[`Baseline/objects/head/existing metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/head/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/head/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/head/non-existent metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "0", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-minio-error-code": "NoSuchKey", + "x-minio-error-desc": '"The specified key does not exist."', + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/objects/head/non-existent metadata 1`] = ` +{ + headers: {}, + status: 404, +} +`; + +snapshot[`Swift/objects/head/non-existent metadata 1`] = ` +{ + headers: {}, + status: 404, +} +`; + +snapshot[`Baseline/objects/delete/existing metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/delete/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/delete/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/multipart/basic metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/multipart/basic metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Swift/objects/multipart/basic metadata 1`] = ` { headers: {}, @@ -7,6 +209,38 @@ snapshot[`Swift/objects/multipart/basic metadata 1`] = ` } `; +snapshot[`Baseline/objects/multipart/abort metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "479", + "content-type": "application/xml", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Baseline/objects/multipart/abort body 1`] = ` +' +NoSuchUploadThe specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.multipart-abort.txttest-objects-bucket/test-objects-bucket/multipart-abort.txtIDHOST' +`; + +snapshot[`Proxy/objects/multipart/abort metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/objects/multipart/abort body 1`] = `'NoSuchUploadThe specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.'`; + snapshot[`Swift/objects/multipart/abort metadata 1`] = ` { headers: { @@ -19,6 +253,26 @@ snapshot[`Swift/objects/multipart/abort metadata 1`] = ` snapshot[`Swift/objects/multipart/abort body 1`] = `'NoSuchUploadThe specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.'`; +snapshot[`Baseline/objects/multipart/list-parts metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/multipart/list-parts metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Swift/objects/multipart/list-parts metadata 1`] = ` { headers: {}, @@ -26,6 +280,26 @@ snapshot[`Swift/objects/multipart/list-parts metadata 1`] = ` } `; +snapshot[`Baseline/objects/multipart/empty metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/multipart/empty metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Swift/objects/multipart/empty metadata 1`] = ` { headers: {}, diff --git a/tests/integration/buckets.test.ts b/tests/integration/buckets.test.ts index cd23268..1edd822 100644 --- a/tests/integration/buckets.test.ts +++ b/tests/integration/buckets.test.ts @@ -30,6 +30,7 @@ interface BucketTestSpec { setup?: (client: S3Client) => Promise; teardown?: (client: S3Client) => Promise; expectedErrorCode?: string; + skipSnapshot?: boolean; } const specs: BucketTestSpec[] = [ @@ -88,6 +89,7 @@ const specs: BucketTestSpec[] = [ { name: "buckets/list", fn: (c) => c.send(new ListBucketsCommand({})), + skipSnapshot: true, }, ]; @@ -126,6 +128,7 @@ const cases: ProxyTestCase[] = specs.map((spec) => ({ name: spec.name, config: testConfig, fn: (client: S3Client) => runBucketTest(spec, client), + skipSnapshot: spec.skipSnapshot, })); harness(cases); diff --git a/tests/utils.ts b/tests/utils.ts index 3442cd3..f84bb30 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -276,6 +276,7 @@ export type ProxyTestCase = { ) => Promise | Effect.Effect; ignore?: boolean; only?: boolean; + skipSnapshot?: boolean; }; function baselineRunner(tc: ProxyTestCase, t: Deno.TestContext) { @@ -308,7 +309,7 @@ function baselineRunner(tc: ProxyTestCase, t: Deno.TestContext) { yield* resultEffect; const lastResponse = h.getLastResponse(); - if (lastResponse) { + if (lastResponse && !tc.skipSnapshot) { yield* Effect.tryPromise(() => assertSnapshot(t, { status: lastResponse.status, @@ -370,7 +371,7 @@ function proxyRunner(tc: ProxyTestCase, t: Deno.TestContext) { yield* resultEffect; const lastResponse = h.getLastResponse(); - if (lastResponse) { + if (lastResponse && !tc.skipSnapshot) { yield* Effect.tryPromise(() => assertSnapshot(t, { status: lastResponse.status, @@ -501,7 +502,7 @@ function swiftRunner(tc: ProxyTestCase, t: Deno.TestContext) { yield* resultEffect; const lastResponse = h.getLastResponse(); - if (lastResponse) { + if (lastResponse && !tc.skipSnapshot) { yield* Effect.tryPromise(() => assertSnapshot(t, { status: lastResponse.status, From d5fd5015ffc72b44462c578502563f39d5f0adf4 Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Thu, 22 Jan 2026 03:01:33 +0300 Subject: [PATCH 03/10] fix: better ci legibilty --- .github/workflows/checks.yml | 67 ++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 13c559e..0a82227 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -13,12 +13,61 @@ env: UV_CACHE_DIR: /tmp/.uv-cache jobs: - checks: + setup: runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v16 + + - name: Set up Nix cache + uses: DeterminateSystems/magic-nix-cache-action@v9 + + - name: Run pre-commit hooks via prek + run: nix develop --command prek run --all-files + + - name: Cache Deno + uses: actions/cache@v4 + with: + path: ~/.cache/deno + key: ${{ runner.os }}-deno-${{ hashFiles('deno.lock') }} + restore-keys: | + ${{ runner.os }}-deno- + + - name: Restore uv cache + uses: actions/cache@v5 + with: + path: /tmp/.uv-cache + key: uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }} + restore-keys: | + uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }} + uv-${{ runner.os }} + + - name: Integration tests + run: nix develop --command deno task test + + - name: Minimize uv cache + run: nix develop --command uv cache prune --ci + + s3-compatibility: + needs: setup + runs-on: ubuntu-latest + strategy: + matrix: + backend: [minio, swift] + include: + - backend: swift + skip_if_no_creds: true env: HERALD_SWIFTTEST_OS_USERNAME: ${{ secrets.OPENSTACK_USERNAME }} HERALD_SWIFTTEST_OS_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD }} HERALD_SWIFTTEST_OS_PROJECT_NAME: ${{ secrets.OPENSTACK_PROJECT }} + HERALD_SWIFTTEST_OS_REGION_NAME: dc3-a + HERALD_SWIFTTEST_AUTH_URL: https://api.pub1.infomaniak.cloud/identity/v3 steps: - name: Checkout repository uses: actions/checkout@v4 @@ -31,9 +80,6 @@ jobs: - name: Set up Nix cache uses: DeterminateSystems/magic-nix-cache-action@v9 - - name: Run pre-commit hooks via prek - run: nix develop --command prek run --all-files - - name: Cache Deno uses: actions/cache@v4 with: @@ -55,6 +101,7 @@ jobs: run: nix develop --command deno run --allow-all x/compose-up.ts s3 db - name: Wait for MinIO + if: matrix.backend == 'minio' run: | for i in {1..30}; do if curl -f http://localhost:9000/minio/health/live; then @@ -67,22 +114,14 @@ jobs: echo "MinIO failed to start" exit 1 - - name: Integration tests - run: nix develop --command deno task test - - name: S3 Compatibility (MinIO) + if: matrix.backend == 'minio' run: nix develop --command deno run --allow-all x/s3-tests.ts --backend minio - name: S3 Compatibility (Swift) - if: env.HERALD_SWIFTTEST_OS_USERNAME != '' - env: - HERALD_SWIFTTEST_OS_REGION_NAME: dc3-a - HERALD_SWIFTTEST_AUTH_URL: https://api.pub1.infomaniak.cloud/identity/v3 + if: matrix.backend == 'swift' && env.HERALD_SWIFTTEST_OS_USERNAME != '' run: nix develop --command deno run --allow-all x/s3-tests.ts --backend swift - - name: Minimize uv cache - run: nix develop --command uv cache prune --ci - - name: Dump logs on failure if: failure() run: | From 65713383ed895d80433f253ee006a72efcbad70e Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Thu, 22 Jan 2026 03:27:21 +0300 Subject: [PATCH 04/10] fix: address feedback Signed-off-by: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> --- src/Backends/Swift/Buckets.ts | 8 +++- src/Frontend/Objects/Post.ts | 13 +++++-- src/Frontend/Utils.ts | 11 ++++++ src/Services/BackendKeyValueStore.ts | 57 ++++++++++++++++------------ 4 files changed, 60 insertions(+), 29 deletions(-) diff --git a/src/Backends/Swift/Buckets.ts b/src/Backends/Swift/Buckets.ts index cd88726..554efa3 100644 --- a/src/Backends/Swift/Buckets.ts +++ b/src/Backends/Swift/Buckets.ts @@ -104,13 +104,17 @@ export const makeBucketOps = ( prefix, marker, }); + if (objects.contents.length === 0) { + break; + } for (const obj of objects.contents) { yield* objectOps.deleteObject(obj.key).pipe(Effect.ignore); } - if (!objects.isTruncated || !objects.nextMarker) { + if (!objects.isTruncated) { break; } - marker = objects.nextMarker; + marker = objects.nextMarker ?? + objects.contents[objects.contents.length - 1].key; } } diff --git a/src/Frontend/Objects/Post.ts b/src/Frontend/Objects/Post.ts index de80954..d9c154d 100644 --- a/src/Frontend/Objects/Post.ts +++ b/src/Frontend/Objects/Post.ts @@ -1,6 +1,6 @@ import { Effect, Option } from "effect"; import { HttpServerResponse } from "@effect/platform"; -import { RequestContext } from "../Utils.ts"; +import { deriveBaseUrl, RequestContext } from "../Utils.ts"; import { S3Xml } from "../../Services/S3Xml.ts"; /** @@ -131,8 +131,9 @@ export const postObject = () => Effect.option, ); if (Option.isSome(head) && head.value.etag) { + const baseUrl = deriveBaseUrl(request); return s3Xml.formatCompleteMultipartUpload({ - location: `http://localhost/${bucket}/${key}`, // Approximate + location: `${baseUrl}/${bucket}/${key}`, bucket, key, etag: head.value.etag, @@ -142,7 +143,13 @@ export const postObject = () => // Backends like Swift will fail if the upload doesn't exist (no segments) // Backends like S3 will succeed if S3 says it's okay. } else { - metadata = JSON.parse(metadataOpt.value); + try { + metadata = JSON.parse(metadataOpt.value); + } catch (e) { + yield* Effect.logError( + `Failed to parse multipart metadata for ${key}/${params.uploadId}: ${e}`, + ); + } } const result = yield* backend.completeMultipartUpload( diff --git a/src/Frontend/Utils.ts b/src/Frontend/Utils.ts index f53925c..d00f862 100644 --- a/src/Frontend/Utils.ts +++ b/src/Frontend/Utils.ts @@ -45,6 +45,17 @@ export function fixHeaderEncoding(value: string): string { ); } +/** + * Derives the base URL for the S3 response, using the Host header. + */ +export function deriveBaseUrl( + request: HttpServerRequest.HttpServerRequest, +): string { + const host = request.headers["host"] || "localhost"; + const protocol = request.url.startsWith("https") ? "https" : "http"; + return `${protocol}://${host}`; +} + /** * Extracts the object key from the request URL, given the bucket name. */ diff --git a/src/Services/BackendKeyValueStore.ts b/src/Services/BackendKeyValueStore.ts index 598e358..53f022b 100644 --- a/src/Services/BackendKeyValueStore.ts +++ b/src/Services/BackendKeyValueStore.ts @@ -3,6 +3,21 @@ import { KeyValueStore } from "@effect/platform"; import { SystemError } from "@effect/platform/Error"; import type { BackendService } from "./Backend.ts"; +const collectChunks = (chunks: Chunk.Chunk) => { + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const all = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + all.set(chunk, offset); + offset += chunk.length; + } + return all; +}; + /** * A KeyValueStore that persists its data as objects in a BackendService. * This is used by backends like Swift that don't natively support S3 multipart metadata @@ -21,17 +36,7 @@ export const makeBackendKeyValueStore = ( return ops.getObject(`${prefix}${key}`, {}).pipe( Effect.flatMap((res) => Stream.runCollect(res.stream)), Effect.map((chunks) => { - const totalLength = Chunk.reduce( - chunks, - 0, - (acc, chunk) => acc + chunk.length, - ); - const all = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of chunks) { - all.set(chunk, offset); - offset += chunk.length; - } + const all = collectChunks(chunks); return Option.some(new TextDecoder().decode(all)); }), Effect.catchTag("NoSuchKey", () => Effect.succeed(Option.none())), @@ -53,17 +58,7 @@ export const makeBackendKeyValueStore = ( return ops.getObject(`${prefix}${key}`, {}).pipe( Effect.flatMap((res) => Stream.runCollect(res.stream)), Effect.map((chunks) => { - const totalLength = Chunk.reduce( - chunks, - 0, - (acc, chunk) => acc + chunk.length, - ); - const all = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of chunks) { - all.set(chunk, offset); - offset += chunk.length; - } + const all = collectChunks(chunks); return Option.some(all); }), Effect.catchTag("NoSuchKey", () => Effect.succeed(Option.none())), @@ -120,6 +115,20 @@ export const makeBackendKeyValueStore = ( ) ), ), - clear: Effect.die("Clear not supported in BackendKeyValueStore"), - size: Effect.die("Size not supported in BackendKeyValueStore"), + clear: Effect.fail( + new SystemError({ + module: "KeyValueStore", + method: "clear", + reason: "Unknown", + description: "Clear not supported in BackendKeyValueStore", + }), + ), + size: Effect.fail( + new SystemError({ + module: "KeyValueStore", + method: "size", + reason: "Unknown", + description: "Size not supported in BackendKeyValueStore", + }), + ), }); From de81e974f6abd1f3af287040b14057b86c8471ea Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Thu, 22 Jan 2026 03:47:10 +0300 Subject: [PATCH 05/10] feat: benchmarks Signed-off-by: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> --- .github/workflows/checks.yml | 3 + benchmarks/buckets.bench.ts | 83 ++++++++ benchmarks/objects.bench.ts | 381 +++++++++++++++++++++++++++++++++++ benchmarks/utils.ts | 346 +++++++++++++++++++++++++++++++ 4 files changed, 813 insertions(+) create mode 100644 benchmarks/buckets.bench.ts create mode 100644 benchmarks/objects.bench.ts create mode 100644 benchmarks/utils.ts diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 0a82227..f1b3345 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -50,6 +50,9 @@ jobs: - name: Integration tests run: nix develop --command deno task test + - name: Benchmarks + run: nix develop --command deno bench --allow-all benchmarks/ + - name: Minimize uv cache run: nix develop --command uv cache prune --ci diff --git a/benchmarks/buckets.bench.ts b/benchmarks/buckets.bench.ts new file mode 100644 index 0000000..e4dca9f --- /dev/null +++ b/benchmarks/buckets.bench.ts @@ -0,0 +1,83 @@ +import { + CreateBucketCommand, + DeleteBucketCommand, + HeadBucketCommand, + ListBucketsCommand, +} from "@aws-sdk/client-s3"; +import { type BenchmarkCase, benchmarkHarness } from "./utils.ts"; +import type { GlobalConfig } from "../src/Domain/Config.ts"; + +const benchConfig: GlobalConfig = { + backends: { + minio: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + buckets: "*", + }, + }, +}; + +const BUCKET_PREFIX = "bench-bucket-"; + +const cases: BenchmarkCase[] = [ + { + name: "create/new", + group: "buckets", + config: benchConfig, + fn: async (client, b) => { + const bucketName = `${BUCKET_PREFIX}${ + Math.random().toString(36).substring(7) + }`; + b.start(); + await client.send(new CreateBucketCommand({ Bucket: bucketName })); + b.end(); + // Cleanup after measurement + await client.send(new DeleteBucketCommand({ Bucket: bucketName })).catch( + () => {}, + ); + }, + }, + { + name: "list/all", + group: "buckets", + config: benchConfig, + fn: async (client, b) => { + b.start(); + await client.send(new ListBucketsCommand({})); + b.end(); + }, + }, + { + name: "head/existing", + group: "buckets", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: "head-bucket" })) + .catch(() => {}); + }, + fn: async (client, b) => { + b.start(); + await client.send(new HeadBucketCommand({ Bucket: "head-bucket" })); + b.end(); + }, + }, + { + name: "delete/existing", + group: "buckets", + config: benchConfig, + fn: async (client, b) => { + const bucketName = `delete-${Math.random().toString(36).substring(7)}`; + await client.send(new CreateBucketCommand({ Bucket: bucketName })); + b.start(); + await client.send(new DeleteBucketCommand({ Bucket: bucketName })); + b.end(); + }, + }, +]; + +benchmarkHarness(cases); diff --git a/benchmarks/objects.bench.ts b/benchmarks/objects.bench.ts new file mode 100644 index 0000000..10ef4ed --- /dev/null +++ b/benchmarks/objects.bench.ts @@ -0,0 +1,381 @@ +import { + CompleteMultipartUploadCommand, + CreateBucketCommand, + CreateMultipartUploadCommand, + DeleteObjectCommand, + GetObjectCommand, + HeadObjectCommand, + PutObjectCommand, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import { type BenchmarkCase, benchmarkHarness } from "./utils.ts"; +import type { GlobalConfig } from "../src/Domain/Config.ts"; +import { Effect, Stream } from "effect"; +import { HttpClientRequest } from "@effect/platform"; + +const benchConfig: GlobalConfig = { + backends: { + minio: { + protocol: "s3", + endpoint: "http://localhost:9000", + region: "us-east-1", + credentials: { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }, + buckets: "*", + }, + }, +}; + +const BUCKET = "bench-bucket-objects"; +const DATA_1KB = new Uint8Array(1024).fill(97); +const DATA_1MB = new Uint8Array(1024 * 1024).fill(97); +const DATA_10MB = new Uint8Array(10 * 1024 * 1024).fill(97); + +const cases: BenchmarkCase[] = [ + // --- PutObject --- + { + name: "put/1kb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + b.start(); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "1kb.txt", + Body: DATA_1KB, + }), + ); + b.end(); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.put(`${url}/1kb.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + HttpClientRequest.bodyUint8Array(DATA_1KB), + ); + const response = await Effect.runPromise(client.execute(request)); + await response.text; // Ensure body is consumed + b.end(); + }, + }, + { + name: "put/1mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + b.start(); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "1mb.txt", + Body: DATA_1MB, + }), + ); + b.end(); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.put(`${url}/1mb.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + HttpClientRequest.bodyUint8Array(DATA_1MB), + ); + const response = await Effect.runPromise(client.execute(request)); + await response.text; + b.end(); + }, + }, + { + name: "put/10mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + b.start(); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "10mb.txt", + Body: DATA_10MB, + }), + ); + b.end(); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.put(`${url}/10mb.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + HttpClientRequest.bodyUint8Array(DATA_10MB), + ); + const response = await Effect.runPromise(client.execute(request)); + await response.text; + b.end(); + }, + }, + + // --- GetObject --- + { + name: "get/1kb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "get-1kb.txt", + Body: DATA_1KB, + }), + ); + }, + fn: async (client, b) => { + b.start(); + const res = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: "get-1kb.txt" }), + ); + await res.Body?.transformToByteArray(); + b.end(); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.get(`${url}/get-1kb.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + const response = await Effect.runPromise(client.execute(request)); + await Stream.runDrain(response.stream); + b.end(); + }, + }, + { + name: "get/1mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "get-1mb.txt", + Body: DATA_1MB, + }), + ); + }, + fn: async (client, b) => { + b.start(); + const res = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: "get-1mb.txt" }), + ); + await res.Body?.transformToByteArray(); + b.end(); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.get(`${url}/get-1mb.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + const response = await Effect.runPromise(client.execute(request)); + await Stream.runDrain(response.stream); + b.end(); + }, + }, + { + name: "get/10mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "get-10mb.txt", + Body: DATA_10MB, + }), + ); + }, + fn: async (client, b) => { + b.start(); + const res = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: "get-10mb.txt" }), + ); + await res.Body?.transformToByteArray(); + b.end(); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.get(`${url}/get-10mb.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + const response = await Effect.runPromise(client.execute(request)); + await Stream.runDrain(response.stream); + b.end(); + }, + }, + + // --- HeadObject --- + { + name: "head/existing", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "head.txt", + Body: DATA_1KB, + }), + ); + }, + fn: async (client, b) => { + b.start(); + await client.send( + new HeadObjectCommand({ Bucket: BUCKET, Key: "head.txt" }), + ); + b.end(); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.head(`${url}/head.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(request)); + b.end(); + }, + }, + + // --- DeleteObject --- + { + name: "delete/existing", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "delete.txt", + Body: DATA_1KB, + }), + ); + + b.start(); + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "delete.txt" }), + ); + b.end(); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + // Pre-upload for delete + const putReq = HttpClientRequest.put(`${url}/delete-direct.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + HttpClientRequest.bodyUint8Array(DATA_1KB), + ); + await Effect.runPromise(client.execute(putReq)); + + b.start(); + const request = HttpClientRequest.del(`${url}/delete-direct.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(request)); + b.end(); + }, + }, + + // --- Multipart Upload --- + { + name: "multipart/upload", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + const key = "multipart.txt"; + const partSize = 5 * 1024 * 1024 + 1; + const body1 = new Uint8Array(partSize).fill(97); + const body2 = new Uint8Array(10).fill(98); + + b.start(); + const { UploadId } = await client.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + const { ETag: etag1 } = await client.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: body1, + }), + ); + const { ETag: etag2 } = await client.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 2, + Body: body2, + }), + ); + await client.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { + Parts: [{ ETag: etag1, PartNumber: 1 }, { + ETag: etag2, + PartNumber: 2, + }], + }, + }), + ); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "multipart.txt" }), + ).catch(() => {}); + }, + }, +]; + +benchmarkHarness(cases); diff --git a/benchmarks/utils.ts b/benchmarks/utils.ts new file mode 100644 index 0000000..614040b --- /dev/null +++ b/benchmarks/utils.ts @@ -0,0 +1,346 @@ +import { S3Client } from "@aws-sdk/client-s3"; +import { Config, Effect, Layer, Logger, LogLevel, Option, Scope } from "effect"; +import { HttpHeraldLive } from "../src/Http.ts"; +import { HeraldConfig } from "../src/Config/Layer.ts"; +import { lookupBucket } from "../src/Domain/Config.ts"; +import { BackendResolverLive } from "../src/Services/BackendResolver.ts"; +import { S3ClientLive } from "../src/Backends/S3/Client.ts"; +import { SwiftClient, SwiftClientLive } from "../src/Backends/Swift/Client.ts"; +import { S3XmlLive } from "../src/Services/S3Xml.ts"; +import { HttpApiBuilder, HttpServer } from "@effect/platform"; +import { FetchHttpClient, HttpClient } from "@effect/platform"; +import type { GlobalConfig } from "../src/Domain/Config.ts"; + +export type BenchmarkCase = { + name: string; + config: GlobalConfig; + fn: (client: S3Client, b: Deno.BenchContext) => Promise; + // For direct comparisons that don't use S3 SDK + directSwiftFn?: ( + target: { url: string; token: string; container: string }, + client: HttpClient.HttpClient, + b: Deno.BenchContext, + ) => Promise; + setup?: (client: S3Client) => Promise; + teardown?: (client: S3Client) => Promise; + group?: string; + baseline?: boolean; + ignore?: boolean; + only?: boolean; +}; + +export const getSwiftConfig = () => + Effect.gen(function* () { + const authUrl = yield* Config.string("HEARLD_SWIFTTEST_AUTH_URL").pipe( + Config.orElse(() => Config.string("HERALD_SWIFTTEST_AUTH_URL")), + Config.orElse(() => Config.string("OS_AUTH_URL")), + Config.withDefault("https://api.pub1.infomaniak.cloud/identity/v3"), + Config.option, + ); + + const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_USERNAME")), + Config.orElse(() => Config.string("OS_USERNAME")), + Config.option, + ); + const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PASSWORD")), + Config.orElse(() => Config.string("OS_PASSWORD")), + Config.option, + ); + const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") + .pipe( + Config.orElse(() => Config.string("TF_VAR_OS_PROJECT_NAME")), + Config.orElse(() => Config.string("OS_PROJECT_NAME")), + Config.option, + ); + const region = yield* Config.string("HEARLD_SWIFTTEST_OS_REGION_NAME").pipe( + Config.orElse(() => Config.string("HERALD_SWIFTTEST_OS_REGION_NAME")), + Config.orElse(() => Config.string("TF_VAR_OS_REGION_NAME")), + Config.orElse(() => Config.string("OS_REGION_NAME")), + Config.withDefault("dc3-a"), + Config.option, + ); + + if ( + Option.isNone(username) || Option.isNone(password) || + Option.isNone(projectName) || Option.isNone(authUrl) + ) { + return Option.none(); + } + + const config: GlobalConfig = { + backends: { + swift: { + protocol: "swift", + auth_url: authUrl.value, + region: Option.getOrUndefined(region), + credentials: { + username: username.value, + password: password.value, + project_name: projectName.value, + user_domain_name: "Default", + project_domain_name: "Default", + }, + buckets: "*", + }, + }, + }; + return Option.some(config); + }); + +export interface BenchHarness { + proxyUrl: string; + minioUrl: string; + directClient: S3Client; + proxyClient: S3Client; + // Raw swift target for direct comparisons + swiftTarget?: { url: string; token: string; container: string }; + httpClient?: HttpClient.HttpClient; +} + +export const makeBenchHarness = ( + config: GlobalConfig, +): Effect.Effect => + Effect.gen(function* () { + const HeraldConfigLive = Layer.succeed(HeraldConfig, { + raw: config, + lookupBucket: (name: string) => lookupBucket(config, name), + }); + + const ApiWithRequirements = HttpHeraldLive.pipe( + Layer.provide(BackendResolverLive), + Layer.provide(S3ClientLive), + Layer.provide(SwiftClientLive), + Layer.provide(S3XmlLive), + Layer.provide(HeraldConfigLive), + Layer.provide(FetchHttpClient.layer), + Layer.provideMerge(HttpServer.layerContext), + Layer.provideMerge(Logger.minimumLogLevel(LogLevel.None)), + ); + + const webHandler = HttpApiBuilder.toWebHandler(ApiWithRequirements); + + const server = Deno.serve( + { port: 0, onListen: () => {} }, + async (req) => { + try { + return await webHandler.handler(req); + } catch (_e) { + return new Response("Internal Server Error", { status: 500 }); + } + }, + ); + + yield* Effect.addFinalizer(() => + Effect.tryPromise(() => server.shutdown()).pipe(Effect.orDie) + ); + yield* Effect.addFinalizer(() => + Effect.tryPromise(() => webHandler.dispose()).pipe(Effect.orDie) + ); + + const proxyUrl = `http://localhost:${server.addr.port}`; + const minioUrl = "http://localhost:9000"; + const credentials = { + accessKeyId: "minioadmin", + secretAccessKey: "minioadmin", + }; + + const directClient = new S3Client({ + endpoint: minioUrl, + region: "us-east-1", + credentials, + forcePathStyle: true, + }); + + const proxyClient = new S3Client({ + endpoint: proxyUrl, + region: "us-east-1", + credentials, + forcePathStyle: true, + }); + + let swiftTarget: BenchHarness["swiftTarget"] = undefined; + let httpClient: HttpClient.HttpClient | undefined = undefined; + + // If swift is configured, get a token for direct benchmarks + const swiftBackendId = Object.keys(config.backends).find((k) => + config.backends[k].protocol === "swift" + ); + if (swiftBackendId) { + const swiftClient = yield* SwiftClient; + const authMeta = yield* swiftClient.getAuthMeta({ + backend_id: swiftBackendId, + }); + swiftTarget = { + url: authMeta.storageUrl, + token: authMeta.token, + container: "bench-bucket", // Fixed for bench + }; + httpClient = yield* HttpClient.HttpClient; + } + + return { + proxyUrl, + minioUrl, + directClient, + proxyClient, + swiftTarget, + httpClient, + }; + }).pipe( + // We need to provide the requirements for SwiftClient and HttpClient + Effect.provide(SwiftClientLive), + Effect.provide(FetchHttpClient.layer), + Effect.provide( + Layer.succeed(HeraldConfig, { + raw: config, + lookupBucket: (name: string) => lookupBucket(config, name), + }), + ), + ); + +// Global state for harnesses to avoid iterative restarts +let minioHarness: BenchHarness | null = null; +let swiftHarness: BenchHarness | null = null; +let globalScope: Scope.Scope | null = null; + +// Check swift config once at the beginning +const swiftConfigOpt = await Effect.runPromise(getSwiftConfig()); + +async function ensureHarnesses(bc: BenchmarkCase) { + if (globalScope) return; + + globalScope = Effect.runSync(Scope.make()); + + minioHarness = await Effect.runPromise( + makeBenchHarness(bc.config).pipe( + Effect.provideService(Scope.Scope, globalScope), + ), + ); + + if (Option.isSome(swiftConfigOpt)) { + swiftHarness = await Effect.runPromise( + makeBenchHarness(swiftConfigOpt.value).pipe( + Effect.provideService(Scope.Scope, globalScope), + ), + ); + } +} + +export function benchmarkHarness(cases: BenchmarkCase[]) { + for (const bc of cases) { + const operationName = `${bc.group ? `${bc.group}/` : ""}${bc.name}`; + + // 1. Baseline (Direct Minio) + Deno.bench({ + name: "Baseline", + group: operationName, + baseline: true, + ignore: bc.ignore, + only: bc.only, + fn: async (b) => { + await ensureHarnesses(bc); + const client = minioHarness!.directClient; + + try { + if (bc.setup) await bc.setup(client); + } catch (e) { + throw new Error(`Setup failed for ${operationName} (Baseline): ${e}`); + } + + await bc.fn(client, b); + + if (bc.teardown) { + await bc.teardown(client).catch(() => {}); + } + }, + }); + + // 2. Proxy (Herald + Minio) + Deno.bench({ + name: "Proxy", + group: operationName, + ignore: bc.ignore, + only: bc.only, + fn: async (b) => { + await ensureHarnesses(bc); + const client = minioHarness!.proxyClient; + + try { + if (bc.setup) await bc.setup(client); + } catch (e) { + throw new Error(`Setup failed for ${operationName} (Proxy): ${e}`); + } + + await bc.fn(client, b); + + if (bc.teardown) { + await bc.teardown(client).catch(() => {}); + } + }, + }); + + // 3. Swift Proxy (Herald + Swift) + Deno.bench({ + name: "Swift-Proxy", + group: operationName, + ignore: bc.ignore || Option.isNone(swiftConfigOpt), + only: bc.only, + fn: async (b) => { + await ensureHarnesses(bc); + if (!swiftHarness) return; + const client = swiftHarness.proxyClient; + + try { + if (bc.setup) await bc.setup(client); + } catch (e) { + throw new Error( + `Setup failed for ${operationName} (Swift-Proxy): ${e}`, + ); + } + + await bc.fn(client, b); + + if (bc.teardown) { + await bc.teardown(client).catch(() => {}); + } + }, + }); + + // 4. Swift Direct (Raw Swift API) + if (bc.directSwiftFn) { + Deno.bench({ + name: "Swift-Direct", + group: operationName, + ignore: bc.ignore || Option.isNone(swiftConfigOpt), + only: bc.only, + fn: async (b) => { + await ensureHarnesses(bc); + if ( + !swiftHarness || !swiftHarness.swiftTarget || + !swiftHarness.httpClient + ) return; + + try { + if (bc.setup) await bc.setup(swiftHarness.proxyClient); + } catch (e) { + throw new Error( + `Setup failed for ${operationName} (Swift-Direct): ${e}`, + ); + } + + await bc.directSwiftFn!( + swiftHarness.swiftTarget, + swiftHarness.httpClient, + b, + ); + + if (bc.teardown) { + await bc.teardown(swiftHarness.proxyClient).catch(() => {}); + } + }, + }); + } + } +} From ede971636807df5175fea8dfb8fa5b4e9d025c42 Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Thu, 22 Jan 2026 04:27:56 +0300 Subject: [PATCH 06/10] wip: bad worklfow Signed-off-by: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> --- .github/workflows/checks.yml | 75 ++++++++++-------------------------- 1 file changed, 20 insertions(+), 55 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f1b3345..3bba77f 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -13,64 +13,15 @@ env: UV_CACHE_DIR: /tmp/.uv-cache jobs: - setup: + checks: runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Install Nix - uses: DeterminateSystems/nix-installer-action@v16 - - - name: Set up Nix cache - uses: DeterminateSystems/magic-nix-cache-action@v9 - - - name: Run pre-commit hooks via prek - run: nix develop --command prek run --all-files - - - name: Cache Deno - uses: actions/cache@v4 - with: - path: ~/.cache/deno - key: ${{ runner.os }}-deno-${{ hashFiles('deno.lock') }} - restore-keys: | - ${{ runner.os }}-deno- - - - name: Restore uv cache - uses: actions/cache@v5 - with: - path: /tmp/.uv-cache - key: uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }} - restore-keys: | - uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }} - uv-${{ runner.os }} - - - name: Integration tests - run: nix develop --command deno task test - - - name: Benchmarks - run: nix develop --command deno bench --allow-all benchmarks/ - - - name: Minimize uv cache - run: nix develop --command uv cache prune --ci - - s3-compatibility: - needs: setup - runs-on: ubuntu-latest - strategy: - matrix: - backend: [minio, swift] - include: - - backend: swift - skip_if_no_creds: true env: HERALD_SWIFTTEST_OS_USERNAME: ${{ secrets.OPENSTACK_USERNAME }} HERALD_SWIFTTEST_OS_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD }} HERALD_SWIFTTEST_OS_PROJECT_NAME: ${{ secrets.OPENSTACK_PROJECT }} - HERALD_SWIFTTEST_OS_REGION_NAME: dc3-a - HERALD_SWIFTTEST_AUTH_URL: https://api.pub1.infomaniak.cloud/identity/v3 + strategy: + matrix: + backend: [minio, swift] steps: - name: Checkout repository uses: actions/checkout@v4 @@ -81,7 +32,10 @@ jobs: uses: DeterminateSystems/nix-installer-action@v16 - name: Set up Nix cache - uses: DeterminateSystems/magic-nix-cache-action@v9 + uses: DeterminateSystems/flakehub-cache-action@v3 + + - name: Run pre-commit hooks via prek + run: nix develop --command prek run --all-files - name: Cache Deno uses: actions/cache@v4 @@ -104,7 +58,6 @@ jobs: run: nix develop --command deno run --allow-all x/compose-up.ts s3 db - name: Wait for MinIO - if: matrix.backend == 'minio' run: | for i in {1..30}; do if curl -f http://localhost:9000/minio/health/live; then @@ -117,14 +70,26 @@ jobs: echo "MinIO failed to start" exit 1 + - name: Integration tests + run: nix develop --command deno task test + + - name: Benchmarks + run: nix develop --command deno bench --allow-all benchmarks/ + - name: S3 Compatibility (MinIO) if: matrix.backend == 'minio' run: nix develop --command deno run --allow-all x/s3-tests.ts --backend minio - name: S3 Compatibility (Swift) if: matrix.backend == 'swift' && env.HERALD_SWIFTTEST_OS_USERNAME != '' + env: + HERALD_SWIFTTEST_OS_REGION_NAME: dc3-a + HERALD_SWIFTTEST_AUTH_URL: https://api.pub1.infomaniak.cloud/identity/v3 run: nix develop --command deno run --allow-all x/s3-tests.ts --backend swift + - name: Minimize uv cache + run: nix develop --command uv cache prune --ci + - name: Dump logs on failure if: failure() run: | From a90e522888cda1b91452230599f4fc6e973e35fb Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Thu, 22 Jan 2026 04:32:41 +0300 Subject: [PATCH 07/10] fix: no matrices Signed-off-by: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> --- .github/workflows/checks.yml | 51 ++++++++++++++++++++++++++++-------- x/s3-tests.ts | 3 ++- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 3bba77f..f205f27 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -8,6 +8,10 @@ on: types: [opened, synchronize, reopened, ready_for_review] workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + env: DOCKER_CMD: docker UV_CACHE_DIR: /tmp/.uv-cache @@ -19,9 +23,6 @@ jobs: HERALD_SWIFTTEST_OS_USERNAME: ${{ secrets.OPENSTACK_USERNAME }} HERALD_SWIFTTEST_OS_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD }} HERALD_SWIFTTEST_OS_PROJECT_NAME: ${{ secrets.OPENSTACK_PROJECT }} - strategy: - matrix: - backend: [minio, swift] steps: - name: Checkout repository uses: actions/checkout@v4 @@ -76,16 +77,42 @@ jobs: - name: Benchmarks run: nix develop --command deno bench --allow-all benchmarks/ - - name: S3 Compatibility (MinIO) - if: matrix.backend == 'minio' - run: nix develop --command deno run --allow-all x/s3-tests.ts --backend minio - - - name: S3 Compatibility (Swift) - if: matrix.backend == 'swift' && env.HERALD_SWIFTTEST_OS_USERNAME != '' + - name: S3 Compatibility env: HERALD_SWIFTTEST_OS_REGION_NAME: dc3-a HERALD_SWIFTTEST_AUTH_URL: https://api.pub1.infomaniak.cloud/identity/v3 - run: nix develop --command deno run --allow-all x/s3-tests.ts --backend swift + run: | + # Run MinIO tests in background + nix develop --command deno run --allow-all x/s3-tests.ts --backend minio & + MINIO_PID=$! + + # Run Swift tests in background if credentials exist + SWIFT_PID="" + if [ -n "$HERALD_SWIFTTEST_OS_USERNAME" ]; then + nix develop --command deno run --allow-all x/s3-tests.ts --backend swift & + SWIFT_PID=$! + else + echo "Swift credentials missing, skipping Swift compatibility tests." + fi + + # Wait for both and capture exit codes + MINIO_EXIT=0 + if ! wait $MINIO_PID; then + MINIO_EXIT=$? + fi + + SWIFT_EXIT=0 + if [ -n "$SWIFT_PID" ]; then + if ! wait $SWIFT_PID; then + SWIFT_EXIT=$? + fi + fi + + # Exit with error if either failed + if [ $MINIO_EXIT -ne 0 ] || [ $SWIFT_EXIT -ne 0 ]; then + echo "One or more compatibility tests failed (MinIO: $MINIO_EXIT, Swift: $SWIFT_EXIT)" + exit 1 + fi - name: Minimize uv cache run: nix develop --command uv cache prune --ci @@ -93,8 +120,10 @@ jobs: - name: Dump logs on failure if: failure() run: | - echo "--- s3-tests/s3-tests.log ---" + echo "--- s3-tests/s3-tests.log (MinIO) ---" cat s3-tests/s3-tests.log || true + echo "--- s3-tests/s3-tests-swift.log (Swift) ---" + cat s3-tests/s3-tests-swift.log || true echo "--- s3-tests/herald-proxy.log ---" cat s3-tests/herald-proxy.log || true echo "--- s3-tests/herald-proxy-swift.log ---" diff --git a/x/s3-tests.ts b/x/s3-tests.ts index d4e958e..45ac7b9 100755 --- a/x/s3-tests.ts +++ b/x/s3-tests.ts @@ -267,7 +267,8 @@ email = iam_alt_root@example.com ); yield* Effect.promise(() => Deno.writeTextFile(confPath, confContent)); - const logPath = path.join(s3TestsDir, "s3-tests.log"); + const logName = backend === "swift" ? "s3-tests-swift.log" : "s3-tests.log"; + const logPath = path.join(s3TestsDir, logName); console.log(`s3-tests directory: ${colors.gray(s3TestsDir)}`); console.log(`Log file: ${colors.gray(logPath)}`); From 27853709169b0d3889409462361bf19cbc9ce59e Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Thu, 22 Jan 2026 07:07:18 +0300 Subject: [PATCH 08/10] fix: benchmarked speed issues Signed-off-by: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> --- .github/workflows/checks.yml | 60 ++++----- CONTRIBUTING.md | 102 +++++++------- README.md | 3 + TODO.md | 2 +- benchmarks/buckets.bench.ts | 57 ++++++++ benchmarks/objects.bench.ts | 232 +++++++++++++++++++++++++++----- benchmarks/utils.ts | 37 +++-- src/Backends/S3/Objects.ts | 46 +++---- src/Backends/Swift/Buckets.ts | 47 ++++--- src/Backends/Swift/Client.ts | 48 +++++++ src/Backends/Swift/Objects.ts | 195 +++++++++++++++++---------- src/Frontend/Objects/Get.ts | 9 ++ src/Frontend/Objects/Put.ts | 1 + src/Services/Backend.ts | 2 + src/main.ts | 4 + tests/utils.ts | 8 +- tools/compose.yml | 6 + x/s3-tests.ts | 12 +- x/swift-debug.ts | 42 ------ x/swift-s3-tests.ts | 245 ---------------------------------- 20 files changed, 614 insertions(+), 544 deletions(-) delete mode 100644 x/swift-debug.ts delete mode 100644 x/swift-s3-tests.ts diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f205f27..9908035 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -24,21 +24,18 @@ jobs: HERALD_SWIFTTEST_OS_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD }} HERALD_SWIFTTEST_OS_PROJECT_NAME: ${{ secrets.OPENSTACK_PROJECT }} steps: - - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: submodules: recursive - - name: Install Nix - uses: DeterminateSystems/nix-installer-action@v16 + - uses: DeterminateSystems/nix-installer-action@v16 - - name: Set up Nix cache - uses: DeterminateSystems/flakehub-cache-action@v3 + - uses: DeterminateSystems/flakehub-cache-action@v3 - - name: Run pre-commit hooks via prek + - name: pre-commit hooks run: nix develop --command prek run --all-files - - name: Cache Deno + - name: deno cache uses: actions/cache@v4 with: path: ~/.cache/deno @@ -46,7 +43,7 @@ jobs: restore-keys: | ${{ runner.os }}-deno- - - name: Restore uv cache + - name: uv cache uses: actions/cache@v5 with: path: /tmp/.uv-cache @@ -55,45 +52,46 @@ jobs: uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }} uv-${{ runner.os }} - - name: Start services - run: nix develop --command deno run --allow-all x/compose-up.ts s3 db + - name: start container + run: nix develop --command deno run --allow-all x/compose-up.ts s3 swift db - - name: Wait for MinIO + - name: wait for services run: | + echo "Waiting for MinIO..." for i in {1..30}; do - if curl -f http://localhost:9000/minio/health/live; then + if curl -sf http://localhost:9000/minio/health/live; then echo "MinIO is ready" + break + fi + sleep 2 + done || (echo "MinIO failed to start" && exit 1) + + echo "Waiting for SAIO..." + for i in {1..60}; do + if curl -sf http://localhost:8080/auth/v1.0; then + echo "SAIO is ready" exit 0 fi - echo "Waiting for MinIO..." sleep 2 done - echo "MinIO failed to start" + echo "SAIO failed to start" exit 1 - - name: Integration tests + - name: integration tests run: nix develop --command deno task test - - name: Benchmarks + - name: benchmarks run: nix develop --command deno bench --allow-all benchmarks/ - - name: S3 Compatibility - env: - HERALD_SWIFTTEST_OS_REGION_NAME: dc3-a - HERALD_SWIFTTEST_AUTH_URL: https://api.pub1.infomaniak.cloud/identity/v3 + - name: s3-tests run: | # Run MinIO tests in background nix develop --command deno run --allow-all x/s3-tests.ts --backend minio & MINIO_PID=$! - # Run Swift tests in background if credentials exist - SWIFT_PID="" - if [ -n "$HERALD_SWIFTTEST_OS_USERNAME" ]; then - nix develop --command deno run --allow-all x/s3-tests.ts --backend swift & - SWIFT_PID=$! - else - echo "Swift credentials missing, skipping Swift compatibility tests." - fi + # Run Swift tests in background against SAIO + nix develop --command deno run --allow-all x/s3-tests.ts --backend swift & + SWIFT_PID=$! # Wait for both and capture exit codes MINIO_EXIT=0 @@ -114,10 +112,10 @@ jobs: exit 1 fi - - name: Minimize uv cache + - name: prune uv cache run: nix develop --command uv cache prune --ci - - name: Dump logs on failure + - name: failure logs if: failure() run: | echo "--- s3-tests/s3-tests.log (MinIO) ---" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c73b2f9..703051e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,84 +1,84 @@ # Contributing +## Starting Services + +You can start the containers used for development using the provided scripts: + +```bash +# Start MinIO and Redis +deno run --allow-all x/compose-up.ts s3 db + +# Start Swift (SAIO) +deno run --allow-all x/compose-up.ts swift +``` + +## Running Tests + +```bash +# Run all tests +deno task test + +# Run Swift integration tests specifically +deno task test --filter "Swift/" +``` + +## Benchmarking + +```bash +deno bench --allow-all benchmarks/ +``` + ## Repo Map - `src/Domain`: Core logic and data models. Contains Effect Schemas for global - configuration and logic for bucket matching. + configuration and logic for backend resolution/matching. - `src/Config`: Application configuration loading. Defines the HeraldConfig service layer. - `src/Services`: Shared service abstractions and implementations. - - `src/Services/Backend.ts`: Generic storage backend interface with structured request/response types and domain-specific error types. - - `src/Services/BackendResolver.ts`: Logic for dynamically providing the correct backend based on request context. - - `src/Services/S3Xml.ts`: S3-compatible XML response formatting for errors, bucket listings, and object listings. + - `src/Services/BackendKeyValueStore.ts`: Abstraction for backend-specific + key-value storage. -- `src/Backends/S3`: S3 protocol implementation. - - - `src/Backends/S3/Backend.ts`: S3-specific implementation of the - BackendService using AWS SDK, handling MinIO metadata stripping and encoding - normalization. - - - `src/Backends/S3/Client.ts`: Low-level AWS SDK S3 client management and - credential resolution. - - - `src/Backends/S3/Signer.ts`: AWS Signature Version 4 implementation for - request signing. +- `src/Backends`: Specific storage backend implementations. + - `src/Backends/S3`: S3 protocol implementation using AWS SDK. + - `src/Backends/Swift`: OpenStack Swift protocol implementation. - `src/Frontend`: HTTP ingress layer. - - `src/Frontend/Api.ts`: HttpApi definition for the S3 compatibility layer. - - `src/Frontend/Http.ts`: Main HTTP server setup and endpoint group registrations. - - - `src/Frontend/Utils.ts`: Shared frontend helpers for backend resolution and - S3-compliant error mapping. - - - `src/Frontend/Buckets/`: Handlers for bucket-level S3 operations (Create, - Delete, List, Head). - - - `src/Frontend/Objects/`: Handlers for object-level S3 operations (Get, Put, - Delete, Head, List, Multi-Object Delete). - + - `src/Frontend/Buckets/`: Handlers for bucket-level S3 operations. + - `src/Frontend/Objects/`: Handlers for object-level S3 operations, including + Multipart Upload (via `Post.ts`). - `src/Frontend/Health/`: Handlers for system health monitoring. -- `tests/`: Test suite. +- `src/Logging` & `src/Tracing.ts`: Diagnostic observability layers. +- `tests/`: Test suite. - `tests/integration/`: End-to-end tests comparing Herald proxy behavior against a MinIO baseline using snapshots. + - `tests/config.test.ts`: Unit tests for configuration and backend resolution. + - `tests/utils.ts`: Shared test harness and snapshot normalization logic. - - `tests/config.test.ts`: Unit tests for configuration inheritance, glob - matching, and backend resolution. - - - `tests/utils.ts`: Shared test harness, Effect-based assertions, and snapshot - normalization logic. +- `benchmarks/`: Performance testing suite for evaluating proxy overhead and + streaming efficiency. - `x/`: CLI utilities and development scripts. - - - `x/s3-tests.ts`: Orchestration script for running the ceph `s3-tests` suite - against the proxy. - - - `x/snapdiff.ts`: Tool for comparing Herald proxy snapshots against baseline + - `x/dev.ts`: Main development entry point for running the proxy locally. + - `x/s3-tests.ts`: Orchestration for running the ceph `s3-tests` suite. + - `x/snapdiff.ts`: Tool for comparing proxy snapshots against baseline responses. + - `x/compose-up.ts` & `x/compose-down.ts`: Helpers for managing local Docker + dependencies. - - `x/swift-s3-tests.ts`: Orchestration script for running the ceph `s3-tests` - suite against the proxy with a Swift backend. Requires `infisical` for - secrets. - - ```bash - infisical run -- deno task test x/swift-s3-tests.ts - ``` - - - `x/utils.ts`: Shell scripting utilities powered by `dax`. - -- `tools/`: Infrastructure and development tools. +- `chart/`: Helm charts for Kubernetes deployment. - - `tools/compose.yml`: Docker configuration for local development services - (MinIO, Redis). +- `tools/`: Infrastructure and development tools (Docker Compose, + Containerfiles). diff --git a/README.md b/README.md index cf44782..a0b95d7 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,9 @@ backends: # Glob pattern support within the map "test-*": region: us-east-1 + minio2: + # simple config for matching glob buckets + buckets: "my-*" ``` ### Routing Logic diff --git a/TODO.md b/TODO.md index 348c0b4..822a9a9 100644 --- a/TODO.md +++ b/TODO.md @@ -52,7 +52,7 @@ implementation. `CompleteMultipartUpload`, `AbortMultipartUpload`, and `ListParts`. _(Focus tests: `test_multipart_upload`, `test_multipart_upload_empty`, `test_abort_multipart_upload`)_ - - [ ] **Swift Multipart Upload**: Implement S3 multipart mapping to Swift SLO. + - [x] **Swift Multipart Upload**: Implement S3 multipart mapping to Swift SLO. - [ ] **GetObject Attributes**: Implementation of `GET /bucket/key?attributes`. _(Focus tests: `test_get_object_attributes`)_ - [ ] **HeadObject Consistency**: Fix `404 Not Found` errors on existing objects diff --git a/benchmarks/buckets.bench.ts b/benchmarks/buckets.bench.ts index e4dca9f..09d1252 100644 --- a/benchmarks/buckets.bench.ts +++ b/benchmarks/buckets.bench.ts @@ -6,6 +6,8 @@ import { } from "@aws-sdk/client-s3"; import { type BenchmarkCase, benchmarkHarness } from "./utils.ts"; import type { GlobalConfig } from "../src/Domain/Config.ts"; +import { Effect } from "effect"; +import { HttpClientRequest } from "@effect/platform"; const benchConfig: GlobalConfig = { backends: { @@ -41,6 +43,23 @@ const cases: BenchmarkCase[] = [ () => {}, ); }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + const bucketName = `${BUCKET_PREFIX}${ + Math.random().toString(36).substring(7) + }`; + b.start(); + const request = HttpClientRequest.put(`${url}/${bucketName}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(request)); + b.end(); + // Cleanup + const delReq = HttpClientRequest.del(`${url}/${bucketName}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(delReq)).catch(() => {}); + }, }, { name: "list/all", @@ -51,6 +70,15 @@ const cases: BenchmarkCase[] = [ await client.send(new ListBucketsCommand({})); b.end(); }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.get(`${url}?format=json`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(request)); + b.end(); + }, }, { name: "head/existing", @@ -65,6 +93,19 @@ const cases: BenchmarkCase[] = [ await client.send(new HeadBucketCommand({ Bucket: "head-bucket" })); b.end(); }, + teardown: async (client) => { + await client.send(new DeleteBucketCommand({ Bucket: "head-bucket" })) + .catch(() => {}); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.head(`${url}/head-bucket`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(request)); + b.end(); + }, }, { name: "delete/existing", @@ -77,6 +118,22 @@ const cases: BenchmarkCase[] = [ await client.send(new DeleteBucketCommand({ Bucket: bucketName })); b.end(); }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + const bucketName = `delete-${Math.random().toString(36).substring(7)}`; + // Setup + const putReq = HttpClientRequest.put(`${url}/${bucketName}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(putReq)); + + b.start(); + const request = HttpClientRequest.del(`${url}/${bucketName}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + await Effect.runPromise(client.execute(request)); + b.end(); + }, }, ]; diff --git a/benchmarks/objects.bench.ts b/benchmarks/objects.bench.ts index 10ef4ed..096e1f7 100644 --- a/benchmarks/objects.bench.ts +++ b/benchmarks/objects.bench.ts @@ -33,6 +33,9 @@ const DATA_1KB = new Uint8Array(1024).fill(97); const DATA_1MB = new Uint8Array(1024 * 1024).fill(97); const DATA_10MB = new Uint8Array(10 * 1024 * 1024).fill(97); +const getLargeData = (sizeMb: number) => + new Uint8Array(sizeMb * 1024 * 1024).fill(97); + const cases: BenchmarkCase[] = [ // --- PutObject --- { @@ -55,10 +58,15 @@ const cases: BenchmarkCase[] = [ ); b.end(); }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "1kb.txt" }), + ).catch(() => {}); + }, directSwiftFn: async (target, client, b) => { const { url, token } = target; b.start(); - const request = HttpClientRequest.put(`${url}/1kb.txt`).pipe( + const request = HttpClientRequest.put(`${url}/${BUCKET}/1kb.txt`).pipe( HttpClientRequest.setHeaders({ "X-Auth-Token": token }), HttpClientRequest.bodyUint8Array(DATA_1KB), ); @@ -87,10 +95,15 @@ const cases: BenchmarkCase[] = [ ); b.end(); }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "1mb.txt" }), + ).catch(() => {}); + }, directSwiftFn: async (target, client, b) => { const { url, token } = target; b.start(); - const request = HttpClientRequest.put(`${url}/1mb.txt`).pipe( + const request = HttpClientRequest.put(`${url}/${BUCKET}/1mb.txt`).pipe( HttpClientRequest.setHeaders({ "X-Auth-Token": token }), HttpClientRequest.bodyUint8Array(DATA_1MB), ); @@ -119,10 +132,15 @@ const cases: BenchmarkCase[] = [ ); b.end(); }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "10mb.txt" }), + ).catch(() => {}); + }, directSwiftFn: async (target, client, b) => { const { url, token } = target; b.start(); - const request = HttpClientRequest.put(`${url}/10mb.txt`).pipe( + const request = HttpClientRequest.put(`${url}/${BUCKET}/10mb.txt`).pipe( HttpClientRequest.setHeaders({ "X-Auth-Token": token }), HttpClientRequest.bodyUint8Array(DATA_10MB), ); @@ -131,8 +149,46 @@ const cases: BenchmarkCase[] = [ b.end(); }, }, - - // --- GetObject --- + { + name: "put/100mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + const data = getLargeData(100); + b.start(); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "100mb.txt", + Body: data, + }), + ); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "100mb.txt" }), + ).catch(() => {}); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + const data = getLargeData(100); + b.start(); + const request = HttpClientRequest.put(`${url}/${BUCKET}/100mb.txt`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + HttpClientRequest.bodyUint8Array(data), + ); + const response = await Effect.runPromise(client.execute(request)); + await response.text; + b.end(); + }, + }, + // --- HeadObject --- { name: "get/1kb", group: "objects", @@ -157,14 +213,20 @@ const cases: BenchmarkCase[] = [ await res.Body?.transformToByteArray(); b.end(); }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "get-1kb.txt" }), + ).catch(() => {}); + }, directSwiftFn: async (target, client, b) => { const { url, token } = target; b.start(); - const request = HttpClientRequest.get(`${url}/get-1kb.txt`).pipe( - HttpClientRequest.setHeaders({ "X-Auth-Token": token }), - ); + const request = HttpClientRequest.get(`${url}/${BUCKET}/get-1kb.txt`) + .pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); const response = await Effect.runPromise(client.execute(request)); - await Stream.runDrain(response.stream); + await Effect.runPromise(Stream.runDrain(response.stream)); b.end(); }, }, @@ -192,14 +254,20 @@ const cases: BenchmarkCase[] = [ await res.Body?.transformToByteArray(); b.end(); }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "get-1mb.txt" }), + ).catch(() => {}); + }, directSwiftFn: async (target, client, b) => { const { url, token } = target; b.start(); - const request = HttpClientRequest.get(`${url}/get-1mb.txt`).pipe( - HttpClientRequest.setHeaders({ "X-Auth-Token": token }), - ); + const request = HttpClientRequest.get(`${url}/${BUCKET}/get-1mb.txt`) + .pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); const response = await Effect.runPromise(client.execute(request)); - await Stream.runDrain(response.stream); + await Effect.runPromise(Stream.runDrain(response.stream)); b.end(); }, }, @@ -227,14 +295,61 @@ const cases: BenchmarkCase[] = [ await res.Body?.transformToByteArray(); b.end(); }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "get-10mb.txt" }), + ).catch(() => {}); + }, directSwiftFn: async (target, client, b) => { const { url, token } = target; b.start(); - const request = HttpClientRequest.get(`${url}/get-10mb.txt`).pipe( - HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + const request = HttpClientRequest.get(`${url}/${BUCKET}/get-10mb.txt`) + .pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); + const response = await Effect.runPromise(client.execute(request)); + await Effect.runPromise(Stream.runDrain(response.stream)); + b.end(); + }, + }, + { + name: "get/100mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + await client.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: "get-100mb.txt", + Body: getLargeData(100), + }), + ); + }, + fn: async (client, b) => { + b.start(); + const res = await client.send( + new GetObjectCommand({ Bucket: BUCKET, Key: "get-100mb.txt" }), ); + await res.Body?.transformToByteArray(); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "get-100mb.txt" }), + ).catch(() => {}); + }, + directSwiftFn: async (target, client, b) => { + const { url, token } = target; + b.start(); + const request = HttpClientRequest.get(`${url}/${BUCKET}/get-100mb.txt`) + .pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ); const response = await Effect.runPromise(client.execute(request)); - await Stream.runDrain(response.stream); + await Effect.runPromise(Stream.runDrain(response.stream)); b.end(); }, }, @@ -266,7 +381,7 @@ const cases: BenchmarkCase[] = [ directSwiftFn: async (target, client, b) => { const { url, token } = target; b.start(); - const request = HttpClientRequest.head(`${url}/head.txt`).pipe( + const request = HttpClientRequest.head(`${url}/${BUCKET}/head.txt`).pipe( HttpClientRequest.setHeaders({ "X-Auth-Token": token }), ); await Effect.runPromise(client.execute(request)); @@ -302,14 +417,18 @@ const cases: BenchmarkCase[] = [ directSwiftFn: async (target, client, b) => { const { url, token } = target; // Pre-upload for delete - const putReq = HttpClientRequest.put(`${url}/delete-direct.txt`).pipe( + const putReq = HttpClientRequest.put( + `${url}/${BUCKET}/delete-direct.txt`, + ).pipe( HttpClientRequest.setHeaders({ "X-Auth-Token": token }), HttpClientRequest.bodyUint8Array(DATA_1KB), ); await Effect.runPromise(client.execute(putReq)); b.start(); - const request = HttpClientRequest.del(`${url}/delete-direct.txt`).pipe( + const request = HttpClientRequest.del( + `${url}/${BUCKET}/delete-direct.txt`, + ).pipe( HttpClientRequest.setHeaders({ "X-Auth-Token": token }), ); await Effect.runPromise(client.execute(request)); @@ -319,7 +438,7 @@ const cases: BenchmarkCase[] = [ // --- Multipart Upload --- { - name: "multipart/upload", + name: "multipart/10mb", group: "objects", config: benchConfig, setup: async (client) => { @@ -328,10 +447,9 @@ const cases: BenchmarkCase[] = [ ); }, fn: async (client, b) => { - const key = "multipart.txt"; - const partSize = 5 * 1024 * 1024 + 1; - const body1 = new Uint8Array(partSize).fill(97); - const body2 = new Uint8Array(10).fill(98); + const key = "multipart-10mb.txt"; + const partSize = 5 * 1024 * 1024; + const body = new Uint8Array(partSize).fill(97); b.start(); const { UploadId } = await client.send( @@ -343,7 +461,7 @@ const cases: BenchmarkCase[] = [ Key: key, UploadId, PartNumber: 1, - Body: body1, + Body: body, }), ); const { ETag: etag2 } = await client.send( @@ -352,7 +470,7 @@ const cases: BenchmarkCase[] = [ Key: key, UploadId, PartNumber: 2, - Body: body2, + Body: body, }), ); await client.send( @@ -361,10 +479,62 @@ const cases: BenchmarkCase[] = [ Key: key, UploadId, MultipartUpload: { - Parts: [{ ETag: etag1, PartNumber: 1 }, { - ETag: etag2, - PartNumber: 2, - }], + Parts: [ + { ETag: etag1, PartNumber: 1 }, + { ETag: etag2, PartNumber: 2 }, + ], + }, + }), + ); + b.end(); + }, + teardown: async (client) => { + await client.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "multipart-10mb.txt" }), + ).catch(() => {}); + }, + }, + { + name: "multipart/100mb", + group: "objects", + config: benchConfig, + setup: async (client) => { + await client.send(new CreateBucketCommand({ Bucket: BUCKET })).catch( + () => {}, + ); + }, + fn: async (client, b) => { + const key = "multipart-100mb.txt"; + const partSize = 10 * 1024 * 1024; + const body = new Uint8Array(partSize).fill(97); + + b.start(); + const { UploadId } = await client.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + + const parts = await Promise.all( + Array.from({ length: 10 }, (_, i) => i + 1).map(async (i) => { + const { ETag } = await client.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: i, + Body: body, + }), + ); + return { ETag, PartNumber: i }; + }), + ); + + await client.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { + Parts: parts, }, }), ); @@ -372,7 +542,7 @@ const cases: BenchmarkCase[] = [ }, teardown: async (client) => { await client.send( - new DeleteObjectCommand({ Bucket: BUCKET, Key: "multipart.txt" }), + new DeleteObjectCommand({ Bucket: BUCKET, Key: "multipart-100mb.txt" }), ).catch(() => {}); }, }, diff --git a/benchmarks/utils.ts b/benchmarks/utils.ts index 614040b..deeda7b 100644 --- a/benchmarks/utils.ts +++ b/benchmarks/utils.ts @@ -34,18 +34,20 @@ export const getSwiftConfig = () => const authUrl = yield* Config.string("HEARLD_SWIFTTEST_AUTH_URL").pipe( Config.orElse(() => Config.string("HERALD_SWIFTTEST_AUTH_URL")), Config.orElse(() => Config.string("OS_AUTH_URL")), - Config.withDefault("https://api.pub1.infomaniak.cloud/identity/v3"), + Config.withDefault("http://localhost:8080/auth/v1.0"), Config.option, ); const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( Config.orElse(() => Config.string("TF_VAR_OS_USERNAME")), Config.orElse(() => Config.string("OS_USERNAME")), + Config.withDefault("test:tester"), Config.option, ); const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( Config.orElse(() => Config.string("TF_VAR_OS_PASSWORD")), Config.orElse(() => Config.string("OS_PASSWORD")), + Config.withDefault("testing"), Config.option, ); const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") @@ -64,7 +66,7 @@ export const getSwiftConfig = () => if ( Option.isNone(username) || Option.isNone(password) || - Option.isNone(projectName) || Option.isNone(authUrl) + Option.isNone(authUrl) ) { return Option.none(); } @@ -78,7 +80,7 @@ export const getSwiftConfig = () => credentials: { username: username.value, password: password.value, - project_name: projectName.value, + project_name: Option.getOrUndefined(projectName), user_domain_name: "Default", project_domain_name: "Default", }, @@ -115,6 +117,10 @@ export const makeBenchHarness = ( Layer.provide(S3XmlLive), Layer.provide(HeraldConfigLive), Layer.provide(FetchHttpClient.layer), + Layer.provide(Layer.succeed(FetchHttpClient.RequestInit, { + // @ts-ignore: duplex is required for streaming body in fetch + duplex: "half", + })), Layer.provideMerge(HttpServer.layerContext), Layer.provideMerge(Logger.minimumLogLevel(LogLevel.None)), ); @@ -192,6 +198,10 @@ export const makeBenchHarness = ( // We need to provide the requirements for SwiftClient and HttpClient Effect.provide(SwiftClientLive), Effect.provide(FetchHttpClient.layer), + Effect.provide(Layer.succeed(FetchHttpClient.RequestInit, { + // @ts-ignore: duplex is required for streaming body in fetch + duplex: "half", + })), Effect.provide( Layer.succeed(HeraldConfig, { raw: config, @@ -231,12 +241,13 @@ async function ensureHarnesses(bc: BenchmarkCase) { export function benchmarkHarness(cases: BenchmarkCase[]) { for (const bc of cases) { const operationName = `${bc.group ? `${bc.group}/` : ""}${bc.name}`; + const s3Group = `${operationName} (S3)`; + const swiftGroup = `${operationName} (Swift)`; // 1. Baseline (Direct Minio) Deno.bench({ - name: "Baseline", - group: operationName, - baseline: true, + name: `Minio-Direct`, + group: s3Group, ignore: bc.ignore, only: bc.only, fn: async (b) => { @@ -259,8 +270,9 @@ export function benchmarkHarness(cases: BenchmarkCase[]) { // 2. Proxy (Herald + Minio) Deno.bench({ - name: "Proxy", - group: operationName, + name: `Herald-Proxy`, + baseline: true, + group: s3Group, ignore: bc.ignore, only: bc.only, fn: async (b) => { @@ -283,8 +295,9 @@ export function benchmarkHarness(cases: BenchmarkCase[]) { // 3. Swift Proxy (Herald + Swift) Deno.bench({ - name: "Swift-Proxy", - group: operationName, + name: `Swift-Proxy`, + group: swiftGroup, + baseline: true, ignore: bc.ignore || Option.isNone(swiftConfigOpt), only: bc.only, fn: async (b) => { @@ -311,8 +324,8 @@ export function benchmarkHarness(cases: BenchmarkCase[]) { // 4. Swift Direct (Raw Swift API) if (bc.directSwiftFn) { Deno.bench({ - name: "Swift-Direct", - group: operationName, + name: `Swift-Direct`, + group: swiftGroup, ignore: bc.ignore || Option.isNone(swiftConfigOpt), only: bc.only, fn: async (b) => { diff --git a/src/Backends/S3/Objects.ts b/src/Backends/S3/Objects.ts index 62761ba..481825c 100644 --- a/src/Backends/S3/Objects.ts +++ b/src/Backends/S3/Objects.ts @@ -276,10 +276,12 @@ export const makeObjectOps = (target: S3Target) => ({ return body as ReadableStream; }; - const stream = Stream.fromReadableStream( - getWebStream, - (e) => new Error(String(e)), - ); + const webStream = getWebStream(); + const stream: Stream.Stream = Stream + .fromReadableStream( + () => webStream, + (e) => new Error(String(e)), + ); const metadata: Record = {}; if (result.Metadata) { @@ -314,31 +316,16 @@ export const makeObjectOps = (target: S3Target) => ({ s3Headers[`x-amz-meta-${k}`] = v; } - return yield* Stream.runCollect(stream).pipe( - Effect.mapError((e) => new InternalError({ message: String(e) })), - Effect.map((chunks) => { - const totalLength = Chunk.reduce( - chunks, - 0, - (acc, chunk) => acc + chunk.length, - ); - const all = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of chunks) { - all.set(chunk, offset); - offset += chunk.length; - } - return { - stream: Stream.succeed(all), - contentType: result.ContentType, - contentLength: all.length, - etag: result.ETag, - lastModified: result.LastModified, - metadata, - headers: s3Headers, - } satisfies ObjectResponse; - }), - ); + return { + stream, + nativeStream: webStream, + contentType: result.ContentType, + contentLength: result.ContentLength, + etag: result.ETag, + lastModified: result.LastModified, + metadata, + headers: s3Headers, + } satisfies ObjectResponse; }), headObject: ( @@ -551,6 +538,7 @@ export const makeObjectOps = (target: S3Target) => ({ uploadId: string, partNumber: number, bodyStream: Stream.Stream, + _headers: Record, ) => Effect.gen(function* () { const { client, bucketName } = target; diff --git a/src/Backends/Swift/Buckets.ts b/src/Backends/Swift/Buckets.ts index 554efa3..a4cf408 100644 --- a/src/Backends/Swift/Buckets.ts +++ b/src/Backends/Swift/Buckets.ts @@ -97,26 +97,33 @@ export const makeBucketOps = ( const { url, token, container } = target; // 1. Cleanup .herald/ and .hrld/ objects so bucket can be deleted - for (const prefix of [".herald/", INTERNAL_PREFIX]) { - let marker: string | undefined = undefined; - while (true) { - const objects: ListObjectsResult = yield* objectOps.listObjects({ - prefix, - marker, - }); - if (objects.contents.length === 0) { - break; - } - for (const obj of objects.contents) { - yield* objectOps.deleteObject(obj.key).pipe(Effect.ignore); - } - if (!objects.isTruncated) { - break; - } - marker = objects.nextMarker ?? - objects.contents[objects.contents.length - 1].key; - } - } + yield* Effect.all( + [".herald/", INTERNAL_PREFIX].map((prefix) => + Effect.gen(function* () { + let marker: string | undefined = undefined; + while (true) { + const objects: ListObjectsResult = yield* objectOps.listObjects({ + prefix, + marker, + }); + if (objects.contents.length === 0) { + break; + } + yield* Effect.all( + objects.contents.map((obj) => + objectOps.deleteObject(obj.key).pipe(Effect.ignore) + ), + { concurrency: 10 }, + ); + if (!objects.isTruncated || !objects.nextMarker) { + break; + } + marker = objects.nextMarker; + } + }) + ), + { concurrency: 2 }, + ); // 2. Delete the bucket const response = yield* client.execute( diff --git a/src/Backends/Swift/Client.ts b/src/Backends/Swift/Client.ts index 0feda08..c3f3df0 100644 --- a/src/Backends/Swift/Client.ts +++ b/src/Backends/Swift/Client.ts @@ -61,6 +61,54 @@ export const SwiftClientLive = Layer.effect( project_domain_name = "Default", } = credentials; + const isV1 = auth_url.endsWith("/v1.0") || !project_name; + + if (isV1) { + return Effect.gen(function* () { + const request = HttpClientRequest.get(auth_url).pipe( + HttpClientRequest.setHeaders({ + "X-Auth-User": username || "", + "X-Auth-Key": password || "", + }), + ); + const response = yield* client.execute(request).pipe( + Effect.mapError((e) => new Error(String(e))), + ); + + if (response.status < 200 || response.status >= 300) { + const msg = yield* response.text.pipe( + Effect.orElseSucceed(() => "Unknown error"), + ); + return yield* Effect.fail( + new Error(`Failed to authenticate with Swift v1.0: ${msg}`), + ); + } + + const token = response.headers["x-auth-token"]; + const storageUrl = response.headers["x-storage-url"]; + + const tokenStr = Array.isArray(token) ? token[0] : (token || ""); + const storageUrlStr = Array.isArray(storageUrl) + ? storageUrl[0] + : (storageUrl || ""); + + if (!tokenStr || !storageUrlStr) { + return yield* Effect.fail( + new Error( + "X-Auth-Token or X-Storage-Url header missing from Swift v1.0 response", + ), + ); + } + + return { + token: tokenStr, + storageUrl: storageUrlStr, + }; + }).pipe( + Effect.mapError((e) => e instanceof Error ? e : new Error(String(e))), + ); + } + const requestBody = { auth: { identity: { diff --git a/src/Backends/Swift/Objects.ts b/src/Backends/Swift/Objects.ts index 453f5a9..cd02f10 100644 --- a/src/Backends/Swift/Objects.ts +++ b/src/Backends/Swift/Objects.ts @@ -1,4 +1,4 @@ -import { Effect, Option, type Stream } from "effect"; +import { Effect, Option, Schedule, type Stream } from "effect"; import { type HttpClient, HttpClientRequest } from "@effect/platform"; import { type BackendError, @@ -270,8 +270,16 @@ export const makeObjectOps = ( ? lastModifiedHeader[0] : lastModifiedHeader; + // Try to get the native stream to avoid Effect <-> WebStream conversion overhead + const nativeStream = + (response as unknown as { source?: unknown }).source instanceof + Response + ? (response as unknown as { source: Response }).source.body + : undefined; + return { stream: response.stream, + nativeStream: nativeStream || undefined, contentType: (Array.isArray(response.headers["content-type"]) ? response.headers["content-type"][0] : response.headers["content-type"]) || undefined, @@ -497,52 +505,69 @@ export const makeObjectOps = ( deleteObjects: (objects: readonly { key: string; versionId?: string }[]) => Effect.gen(function* () { const { url, token, container } = target; - const deleted: string[] = []; - const errors: { key: string; code: string; message: string }[] = []; - for (const obj of objects) { - const encodedKey = obj.key.split("/").map(encodeURIComponent).join( - "/", - ); - let response = yield* client.execute( - HttpClientRequest.del(`${url}/${encodedKey}`).pipe( - HttpClientRequest.setHeaders({ - "X-Auth-Token": token, - "X-Static-Large-Object": "true", - }), - HttpClientRequest.setUrlParams({ - "multipart-manifest": "delete", - }), - ), - ).pipe( - Effect.mapError((e) => mapError(500, String(e), container)), - ); + const results = yield* Effect.all( + objects.map((obj) => + Effect.gen(function* () { + const encodedKey = obj.key.split("/").map(encodeURIComponent) + .join( + "/", + ); + let response = yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ + "X-Auth-Token": token, + "X-Static-Large-Object": "true", + }), + HttpClientRequest.setUrlParams({ + "multipart-manifest": "delete", + }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status === 400) { + // Not an SLO, try regular delete + response = yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + } + + if ( + (response.status >= 200 && response.status < 300) || + response.status === 204 || response.status === 404 + ) { + return { key: obj.key, error: null }; + } else { + const errorBody = yield* response.text.pipe( + Effect.orElseSucceed(() => "Unknown error"), + ); + return { + key: obj.key, + error: { + code: String(response.status), + message: errorBody, + }, + }; + } + }) + ), + { concurrency: 10 }, + ); - if (response.status === 400) { - // Not an SLO, try regular delete - response = yield* client.execute( - HttpClientRequest.del(`${url}/${encodedKey}`).pipe( - HttpClientRequest.setHeaders({ "X-Auth-Token": token }), - ), - ).pipe( - Effect.mapError((e) => mapError(500, String(e), container)), - ); - } + const deleted: string[] = []; + const errors: { key: string; code: string; message: string }[] = []; - if ( - (response.status >= 200 && response.status < 300) || - response.status === 204 || response.status === 404 - ) { - deleted.push(obj.key); + for (const res of results) { + if (res.error) { + errors.push({ key: res.key, ...res.error }); } else { - const errorBody = yield* response.text.pipe( - Effect.orElseSucceed(() => "Unknown error"), - ); - errors.push({ - key: obj.key, - code: String(response.status), - message: errorBody, - }); + deleted.push(res.key); } } @@ -566,6 +591,7 @@ export const makeObjectOps = ( uploadId: string, partNumber: number, body: Stream.Stream, + _headers: Record, ): Effect.Effect => Effect.gen(function* () { const { url, token, container } = target; @@ -626,35 +652,53 @@ export const makeObjectOps = ( // Fetch segment info to get sizes const segmentMap = new Map(); - let segmentMarker: string | undefined = undefined; - while (true) { - const segmentsResult: ListObjectsResult = yield* listObjects({ - prefix: `${MP_SEGMENTS_PREFIX}${uploadId}/`, - marker: segmentMarker, - }); - for (const c of segmentsResult.contents) { - segmentMap.set(c.key, c); + const buildSegmentMap = Effect.gen(function* () { + segmentMap.clear(); + let segmentMarker: string | undefined = undefined; + while (true) { + const segmentsResult: ListObjectsResult = yield* listObjects({ + prefix: `${MP_SEGMENTS_PREFIX}${uploadId}/`, + marker: segmentMarker, + }); + for (const c of segmentsResult.contents) { + segmentMap.set(c.key, c); + } + if (!segmentsResult.isTruncated || !segmentsResult.nextMarker) { + break; + } + segmentMarker = segmentsResult.nextMarker; } - if (!segmentsResult.isTruncated || !segmentsResult.nextMarker) { - break; + + // Verify all parts are present + for (const p of parts) { + const segmentKey = + `${MP_SEGMENTS_PREFIX}${uploadId}/${p.partNumber}`; + if (!segmentMap.has(segmentKey)) { + return yield* Effect.fail( + new NoSuchUpload({ + uploadId, + message: `Part ${p.partNumber} not found in segment listing`, + }), + ); + } } - segmentMarker = segmentsResult.nextMarker; - } + }); + + // Retry with exponential backoff for eventual consistency + yield* buildSegmentMap.pipe( + Effect.retry({ + while: (e) => e instanceof NoSuchUpload, + schedule: Schedule.exponential("100 millis").pipe( + Schedule.compose(Schedule.recurs(4)), + ), + }), + ); // 1. Build SLO manifest const manifest = []; for (const p of parts) { const segmentKey = `${MP_SEGMENTS_PREFIX}${uploadId}/${p.partNumber}`; - const info = segmentMap.get(segmentKey); - if (!info) { - return yield* Effect.fail( - new NoSuchUpload({ - uploadId, - message: - `Part ${p.partNumber} not found. The upload might have already been completed or aborted.`, - }), - ); - } + const info = segmentMap.get(segmentKey)!; manifest.push({ path: `/${container}/${segmentKey}`, etag: p.etag.replace(/"/g, ""), @@ -751,15 +795,18 @@ export const makeObjectOps = ( marker, }); - for (const content of segmentsResult.contents) { - const encodedKey = content.key.split("/").map(encodeURIComponent) - .join("/"); - yield* client.execute( - HttpClientRequest.del(`${url}/${encodedKey}`).pipe( - HttpClientRequest.setHeaders({ "X-Auth-Token": token }), - ), - ).pipe(Effect.ignore); - } + yield* Effect.all( + segmentsResult.contents.map((content) => { + const encodedKey = content.key.split("/").map(encodeURIComponent) + .join("/"); + return client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe(Effect.ignore); + }), + { concurrency: 10 }, + ); if (!segmentsResult.isTruncated || !segmentsResult.nextMarker) { break; diff --git a/src/Frontend/Objects/Get.ts b/src/Frontend/Objects/Get.ts index c223fec..193c438 100644 --- a/src/Frontend/Objects/Get.ts +++ b/src/Frontend/Objects/Get.ts @@ -27,6 +27,15 @@ export const getObject = () => const status = (request.headers["range"] || request.headers["Range"]) ? 206 : 200; + + if (result.nativeStream) { + return HttpServerResponse.raw(result.nativeStream, { + status, + headers: result.headers, + contentType: result.contentType, + }); + } + return HttpServerResponse.stream(result.stream, { status, headers: result.headers, diff --git a/src/Frontend/Objects/Put.ts b/src/Frontend/Objects/Put.ts index 08421cd..c6bbe2f 100644 --- a/src/Frontend/Objects/Put.ts +++ b/src/Frontend/Objects/Put.ts @@ -16,6 +16,7 @@ export const putObject = () => params.uploadId, params.partNumber, request.stream, + request.headers, ); return HttpServerResponse.empty({ status: 200, diff --git a/src/Services/Backend.ts b/src/Services/Backend.ts index 30cff6d..5ad2e51 100644 --- a/src/Services/Backend.ts +++ b/src/Services/Backend.ts @@ -51,6 +51,7 @@ export interface ListObjectsResult { export interface ObjectResponse { readonly stream: Stream.Stream; + readonly nativeStream?: ReadableStream; readonly contentType?: string; readonly contentLength?: number; readonly etag?: string; @@ -302,6 +303,7 @@ export interface BackendService { uploadId: string, partNumber: number, body: Stream.Stream, + headers: Record, ) => Effect.Effect; readonly completeMultipartUpload: ( key: string, diff --git a/src/main.ts b/src/main.ts index 56ccc3b..0d5f32f 100644 --- a/src/main.ts +++ b/src/main.ts @@ -11,6 +11,10 @@ HttpServerHeraldLive.pipe( // provider an HttpClient impl based on `fetch` // used to talk the the swift impl Layer.provide(FetchHttpClient.layer), + Layer.provide(Layer.succeed(FetchHttpClient.RequestInit, { + // @ts-ignore: duplex is required for streaming body in fetch + duplex: "half", + })), // run layer until interrupted Layer.launch, // add support for Cli goodies like diff --git a/tests/utils.ts b/tests/utils.ts index f84bb30..129a8cf 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -408,18 +408,20 @@ const getSwiftConfig = () => const authUrl = yield* Config.string("HEARLD_SWIFTTEST_AUTH_URL").pipe( Config.orElse(() => Config.string("HERALD_SWIFTTEST_AUTH_URL")), Config.orElse(() => Config.string("OS_AUTH_URL")), - Config.withDefault("https://api.pub1.infomaniak.cloud/identity/v3"), + Config.withDefault("http://localhost:8080/auth/v1.0"), Config.option, ); const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( Config.orElse(() => Config.string("TF_VAR_OS_USERNAME")), Config.orElse(() => Config.string("OS_USERNAME")), + Config.withDefault("test:tester"), Config.option, ); const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( Config.orElse(() => Config.string("TF_VAR_OS_PASSWORD")), Config.orElse(() => Config.string("OS_PASSWORD")), + Config.withDefault("testing"), Config.option, ); const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") @@ -438,7 +440,7 @@ const getSwiftConfig = () => if ( Option.isNone(username) || Option.isNone(password) || - Option.isNone(projectName) || Option.isNone(authUrl) + Option.isNone(authUrl) ) { return Option.none(); } @@ -452,7 +454,7 @@ const getSwiftConfig = () => credentials: { username: username.value, password: password.value, - project_name: projectName.value, + project_name: Option.getOrUndefined(projectName), user_domain_name: "Default", project_domain_name: "Default", }, diff --git a/tools/compose.yml b/tools/compose.yml index 0067818..8abebe0 100644 --- a/tools/compose.yml +++ b/tools/compose.yml @@ -33,6 +33,12 @@ services: volumes: - miniodata:/data + saio: + profiles: ["swift"] + image: docker.io/openstackswift/saio:latest + ports: + - "8080:8080" + volumes: redisdata: miniodata: diff --git a/x/s3-tests.ts b/x/s3-tests.ts index 45ac7b9..75c0772 100755 --- a/x/s3-tests.ts +++ b/x/s3-tests.ts @@ -13,7 +13,7 @@ * ./x/s3-tests.ts [pytest-args] [--backend ] [--no-abort] * * Environment Variables: - * S3TEST_TAGS: Custom pytest marks (default: not fails_on_s3proxy and ...) + * S3TEST_TAGS: Custom pytest marks (default: not buckets and ...) * S3TEST_PYTEST_ARGS: Additional pytest arguments * S3TEST_NO_ABORT: Set to "true" to disable abort-on-error * HERALD_LOG_LEVEL: Set to "DEBUG" for verbose proxy logging @@ -33,7 +33,7 @@ import { makeTestHarness } from "../tests/utils.ts"; import { GlobalConfig } from "../src/Domain/Config.ts"; const DEFAULT_TAGS = - "not fails_on_s3proxy and not appendobject and not bucket_policy and not copy and not cors and not encryption and not fails_strict_rfc2616 and not iam_tenant and not lifecycle and not object_lock and not policy and not policy_status and not s3select and not s3website and not sse_s3 and not tagging and not test_of_sts and not user_policy and not versioning and not webidentity_test"; + "not appendobject and not bucket_policy and not copy and not cors and not encryption and not fails_strict_rfc2616 and not iam_tenant and not lifecycle and not object_lock and not policy and not policy_status and not s3select and not s3website and not sse_s3 and not tagging and not test_of_sts and not user_policy and not versioning and not webidentity_test"; function getMinioConfig(): GlobalConfig { return { @@ -57,18 +57,20 @@ const getSwiftConfig = () => const authUrl = yield* Config.string("HEARLD_SWIFTTEST_AUTH_URL").pipe( Config.orElse(() => Config.string("HERALD_SWIFTTEST_AUTH_URL")), Config.orElse(() => Config.string("OS_AUTH_URL")), - Config.withDefault("https://api.pub1.infomaniak.cloud/identity/v3"), + Config.withDefault("http://localhost:8080/auth/v1.0"), Config.option, ); const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( Config.orElse(() => Config.string("TF_VAR_OS_USERNAME")), Config.orElse(() => Config.string("OS_USERNAME")), + Config.withDefault("test:tester"), Config.option, ); const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( Config.orElse(() => Config.string("TF_VAR_OS_PASSWORD")), Config.orElse(() => Config.string("OS_PASSWORD")), + Config.withDefault("testing"), Config.option, ); const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") @@ -87,7 +89,7 @@ const getSwiftConfig = () => if ( Option.isNone(username) || Option.isNone(password) || - Option.isNone(projectName) || Option.isNone(authUrl) + Option.isNone(authUrl) ) { return Option.none(); } @@ -101,7 +103,7 @@ const getSwiftConfig = () => credentials: { username: username.value, password: password.value, - project_name: projectName.value, + project_name: Option.getOrUndefined(projectName), user_domain_name: "Default", project_domain_name: "Default", }, diff --git a/x/swift-debug.ts b/x/swift-debug.ts deleted file mode 100644 index 76e888c..0000000 --- a/x/swift-debug.ts +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env -S deno run --allow-all -import { Effect, Logger, LogLevel } from "effect"; -import { SwiftClient, SwiftClientLive } from "../src/Backends/Swift/Client.ts"; -import { HeraldConfigLive } from "../src/Config/Layer.ts"; -import { makeSwiftBackend } from "../src/Backends/Swift/Backend.ts"; -import { FetchHttpClient } from "@effect/platform"; - -const program = Effect.gen(function* () { - console.log("Checking Swift connection..."); - - // We'll use the 'default' backend which should be configured via HERALD_ env vars - const backendId = "default"; - - const swiftClient = yield* SwiftClient; - const auth = yield* swiftClient.getAuthMeta({ backend_id: backendId }); - - console.log("Auth successful!"); - console.log(`Storage URL: ${auth.storageUrl}`); - console.log(`Token: ${auth.token.substring(0, 10)}...`); - - const backend = yield* makeSwiftBackend({ backend_id: backendId }); - const { buckets } = yield* backend.listBuckets(); - - console.log(`Found ${buckets.length} buckets:`); - for (const b of buckets) { - console.log(` - ${b.name} (created: ${b.creationDate})`); - } -}).pipe( - Effect.provide(SwiftClientLive), - Effect.provide(HeraldConfigLive), - Effect.provide(FetchHttpClient.layer), - Effect.provide(Logger.minimumLogLevel(LogLevel.Debug)), -); - -if (import.meta.main) { - Effect.runPromiseExit(program).then((exit) => { - if (exit._tag === "Failure") { - console.error("Program failed:", exit.cause); - Deno.exit(1); - } - }); -} diff --git a/x/swift-s3-tests.ts b/x/swift-s3-tests.ts deleted file mode 100644 index 553f2f3..0000000 --- a/x/swift-s3-tests.ts +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env -S deno run --allow-all - -/** - * Herald Swift Compatibility Test Runner - * - * This script runs the Ceph s3-tests suite against a Herald proxy instance - * configured with an OpenStack Swift backend. - */ - -import { Config, Effect, Layer, Logger, LogLevel, Stream } from "effect"; -import { makeTestHarness } from "../tests/utils.ts"; -import type { GlobalConfig } from "../src/Domain/Config.ts"; -import * as path from "@std/path"; -import { $ } from "./utils.ts"; -import * as colors from "@std/fmt/colors"; - -const program = Effect.gen(function* () { - const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); - const s3TestsDir = path.resolve(__dirname, "../s3-tests"); - const proxyLogPath = path.join(s3TestsDir, "herald-proxy-swift.log"); - - // Read Swift config from environment - const authUrl = yield* Config.string("HERALD_SWIFTTEST_AUTH_URL").pipe( - Config.orElse(() => Config.string("HEARLD_SWIFTTEST_AUTH_URL")), - Config.orElse(() => Config.string("OS_AUTH_URL")), - Config.withDefault("https://api.pub1.infomaniak.cloud/identity/v3"), - ); - const region = yield* Config.string("HERALD_SWIFTTEST_OS_REGION_NAME").pipe( - Config.orElse(() => Config.string("HEARLD_SWIFTTEST_OS_REGION_NAME")), - Config.orElse(() => Config.string("TF_VAR_OS_REGION_NAME")), - Config.orElse(() => Config.string("OS_REGION_NAME")), - Config.withDefault("dc3-a"), - ); - const username = yield* Config.string("HERALD_SWIFTTEST_OS_USERNAME").pipe( - Config.orElse(() => Config.string("TF_VAR_OS_USERNAME")), - Config.orElse(() => Config.string("OS_USERNAME")), - Config.withDefault(""), - ); - const password = yield* Config.string("HERALD_SWIFTTEST_OS_PASSWORD").pipe( - Config.orElse(() => Config.string("TF_VAR_OS_PASSWORD")), - Config.orElse(() => Config.string("OS_PASSWORD")), - Config.withDefault(""), - ); - const projectName = yield* Config.string("HERALD_SWIFTTEST_OS_PROJECT_NAME") - .pipe( - Config.orElse(() => Config.string("TF_VAR_OS_PROJECT_NAME")), - Config.orElse(() => Config.string("OS_PROJECT_NAME")), - Config.withDefault(""), - ); - - if (!authUrl || !username || !password || !projectName) { - return yield* Effect.fail( - new Error( - "Swift environment variables (HERALD_SWIFTTEST_...) are missing. Run with infisical.", - ), - ); - } - - const swiftConfig: GlobalConfig = { - backends: { - swift: { - protocol: "swift", - auth_url: authUrl, - region: region || undefined, - credentials: { - username, - password, - project_name: projectName, - user_domain_name: "Default", - project_domain_name: "Default", - }, - buckets: "*", - }, - }, - }; - - // Create a file logger for the proxy - const proxyLogFile = yield* Effect.tryPromise(() => - Deno.open(proxyLogPath, { write: true, create: true, truncate: true }) - ); - - yield* Effect.addFinalizer(() => - Effect.tryPromise({ - try: () => Promise.resolve(proxyLogFile.close()), - catch: (e) => new Error(`Failed to close proxy log file: ${e}`), - }).pipe(Effect.orDie) - ); - - // Provide the test harness - const h = yield* makeTestHarness(swiftConfig); - const port = new URL(h.proxyUrl).port; - - console.log(`Starting Herald (Swift backend) on port ${colors.cyan(port)}`); - console.log(`Proxy logs: ${colors.gray(proxyLogPath)}`); - - const confContent = `[DEFAULT] -host = 127.0.0.1 -port = ${port} -is_secure = no - -[fixtures] -bucket prefix = herald-swift-{random}- - -[s3 main] -user_id = main -display_name = main -email = main@example.com -access_key = dummy -secret_key = dummy - -[s3 alt] -user_id = alt -display_name = alt -email = alt@example.com -access_key = dummy -secret_key = dummy - -[s3 tenant] -user_id = tenant -display_name = tenant -email = tenant@example.com -access_key = dummy -secret_key = dummy -tenant = dummy - -[iam] -email = s3@example.com -user_id = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef -access_key = dummy -secret_key = dummy -display_name = youruseridhere - -[iam root] -access_key = dummyroot -secret_key = dummyroot -user_id = RGW11111111111111111 -email = account1@ceph.com - -[iam alt root] -access_key = dummyaltroot -secret_key = dummyaltroot -user_id = RGW22222222222222222 -email = account2@ceph.com -`; - - const confPath = yield* Effect.promise(() => - Deno.makeTempFile({ suffix: ".conf" }) - ); - yield* Effect.promise(() => Deno.writeTextFile(confPath, confContent)); - yield* Effect.addFinalizer(() => - Effect.promise(() => - Deno.remove(confPath).catch((e) => { - console.error(`Failed to remove conf file ${confPath}: ${e}`); - }) - ) - ); - - const logPath = path.join(s3TestsDir, "s3-tests-swift.log"); - const junitXmlPath = path.join(s3TestsDir, "junit-swift.xml"); - - const rawArgs = $.argv; - const noAbort = rawArgs.includes("--no-abort"); - const pytestArgsFromCli = rawArgs.filter((arg) => arg !== "--no-abort"); - - const cmdArgs: string[] = [ - "run", - "pytest", - "-v", - "--tb=short", - `--junit-xml=${junitXmlPath}`, - ...pytestArgsFromCli, - ]; - - // If no specific test path, default to test_s3.py - if ( - !pytestArgsFromCli.some((arg) => - arg.includes("s3tests/") || arg.endsWith(".py") - ) - ) { - cmdArgs.push("s3tests/functional/test_s3.py"); - } - - console.log(`Running s3-tests against Herald (Swift)...`); - - const logFile = yield* Effect.tryPromise(() => - Deno.open(logPath, { write: true, create: true, truncate: true }) - ); - yield* Effect.addFinalizer(() => - Effect.promise(() => Promise.resolve(logFile.close())) - ); - - const result = yield* Effect.tryPromise({ - try: async () => { - const child = $`uv ${cmdArgs}` - .cwd(s3TestsDir) - .env({ - S3TEST_CONF: confPath, - UV_PYTHON: "3.11", - PYTHONUNBUFFERED: "1", - }) - .noThrow() - .stdout("piped") - .stderr("piped") - .spawn(); - - const decoder = new TextDecoder(); - async function streamToLog(stream: ReadableStream) { - const reader = stream.getReader(); - while (true) { - const { done, value } = await reader.read(); - if (done) break; - await logFile.write(value); - Deno.stdout.writeSync(value); // Echo to console for now - } - } - - const [procResult] = await Promise.all([ - child, - streamToLog(child.stdout()), - streamToLog(child.stderr()), - ]); - - return procResult; - }, - catch: (e) => new Error(`Failed to run pytest: ${e}`), - }); - - if (result.code !== 0) { - yield* Effect.fail(new Error(`s3-tests failed with code ${result.code}`)); - } - - console.log(colors.green(`\n✓ s3-tests completed successfully.`)); -}).pipe( - Effect.scoped, - Effect.provide(Logger.minimumLogLevel(LogLevel.Debug)), -); - -if (import.meta.main) { - Effect.runPromiseExit(program).then((exit) => { - if (exit._tag === "Failure") { - console.error(colors.red(`Error: ${exit.cause}`)); - Deno.exit(1); - } - }); -} From dd3f9e75a4e124272af28f0279189d7bad0c0b6d Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Thu, 22 Jan 2026 07:37:46 +0300 Subject: [PATCH 09/10] fix: bad check for SAIO start --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 9908035..b2bdc8b 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -68,7 +68,7 @@ jobs: echo "Waiting for SAIO..." for i in {1..60}; do - if curl -sf http://localhost:8080/auth/v1.0; then + if curl -sf http://localhost:8080/healthcheck; then echo "SAIO is ready" exit 0 fi From 61346237baf0aefcf782a1fd1b2398207a4df267 Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Thu, 22 Jan 2026 09:03:21 +0300 Subject: [PATCH 10/10] feat: cors (#83) --- .github/workflows/checks.yml | 4 - README.md | 79 ++++++++++++- benchmarks/utils.ts | 6 +- src/Config/Layer.ts | 75 +++++++++++- src/Domain/Config.ts | 85 ++++++++++++++ src/Frontend/Cors.ts | 126 ++++++++++++++++++++ src/Http.ts | 7 +- tests/config.test.ts | 38 ++++++ tests/cors.test.ts | 219 +++++++++++++++++++++++++++++++++++ tests/utils.ts | 6 +- x/s3-tests.ts | 6 +- 11 files changed, 626 insertions(+), 25 deletions(-) create mode 100644 src/Frontend/Cors.ts create mode 100644 tests/cors.test.ts diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b2bdc8b..639adca 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,10 +19,6 @@ env: jobs: checks: runs-on: ubuntu-latest - env: - HERALD_SWIFTTEST_OS_USERNAME: ${{ secrets.OPENSTACK_USERNAME }} - HERALD_SWIFTTEST_OS_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD }} - HERALD_SWIFTTEST_OS_PROJECT_NAME: ${{ secrets.OPENSTACK_PROJECT }} steps: - uses: actions/checkout@v4 with: diff --git a/README.md b/README.md index a0b95d7..00927ee 100644 --- a/README.md +++ b/README.md @@ -68,9 +68,78 @@ backends: # Glob pattern support within the map "test-*": region: us-east-1 - minio2: - # simple config for matching glob buckets - buckets: "my-*" + + # Example Swift backend + swift-storage: + protocol: swift + auth_url: http://keystone.example.com/v3 + region: RegionOne + # Optional: override the Swift container name for all buckets in this backend + # container: my-fixed-container + credentials: + username: my-user + password: my-password + project_name: my-project + user_domain_name: Default + project_domain_name: Default + # Route all archive buckets to Swift + buckets: "archive-*" + +cors: + # Global CORS defaults + allowedOrigins: ["*"] + allowedMethods: ["GET", "PUT", "POST", "DELETE", "HEAD", "OPTIONS"] + allowedHeaders: ["*"] + exposedHeaders: ["*"] + maxAge: 3600 + credentials: false +``` + +### CORS Configuration + +Herald supports fine-grained CORS control at three levels with the following +precedence: **Bucket > Backend > Global**. + +- **Global**: Defined at the root of the config file under `cors`. +- **Backend**: Defined within a backend block under `cors`. Overrides global + settings. +- **Bucket**: Defined within a bucket definition under `cors`. Overrides both + backend and global settings. + +#### Default Behavior + +If no CORS configuration is provided at any level, **CORS is disabled** and +Herald will not add any CORS-related headers to responses. Preflight `OPTIONS` +requests will be passed through to the backend. + +If you enable CORS by providing configuration at any level, the following +defaults are applied for any omitted fields: + +| Field | Default Value | Description | +| ---------------- | --------------------------------------- | ------------------------------------------------------ | +| `maxAge` | `3600` | Max age in seconds for preflight results | +| `allowedMethods` | `GET, PUT, POST, DELETE, HEAD, OPTIONS` | Allowed HTTP methods | +| `allowedHeaders` | (Mirrors request) | Defaults to mirroring `Access-Control-Request-Headers` | +| `credentials` | `false` | Whether to allow credentials | +| `allowedOrigins` | (None) | Headers only added if `Origin` matches an entry | + +Example with overrides: + +```yaml +cors: # Global defaults + allowedOrigins: ["*"] + credentials: false + +backends: + prod: + protocol: s3 + cors: # Backend-level override + allowedOrigins: ["https://app.example.com"] + credentials: true + buckets: + assets: + cors: # Bucket-level override + allowedOrigins: ["https://cdn.example.com"] ``` ### Routing Logic @@ -81,5 +150,5 @@ resolves the backend using the following priority: 1. **Direct match**: Looks for `my-bucket` in all backends' `buckets` maps. 2. **Glob match (map)**: Looks for glob patterns (like `test-*`) in all backends' `buckets` maps. -3. **Glob match (string)**: If a backend has `buckets: "..."`, it checks if the - bucket name matches that pattern. +3. **Glob match (string)**: If a backend has `buckets: "string-*"`, it checks if + the bucket name matches that pattern. diff --git a/benchmarks/utils.ts b/benchmarks/utils.ts index deeda7b..27e4537 100644 --- a/benchmarks/utils.ts +++ b/benchmarks/utils.ts @@ -31,8 +31,7 @@ export type BenchmarkCase = { export const getSwiftConfig = () => Effect.gen(function* () { - const authUrl = yield* Config.string("HEARLD_SWIFTTEST_AUTH_URL").pipe( - Config.orElse(() => Config.string("HERALD_SWIFTTEST_AUTH_URL")), + const authUrl = yield* Config.string("HERALD_SWIFTTEST_AUTH_URL").pipe( Config.orElse(() => Config.string("OS_AUTH_URL")), Config.withDefault("http://localhost:8080/auth/v1.0"), Config.option, @@ -56,8 +55,7 @@ export const getSwiftConfig = () => Config.orElse(() => Config.string("OS_PROJECT_NAME")), Config.option, ); - const region = yield* Config.string("HEARLD_SWIFTTEST_OS_REGION_NAME").pipe( - Config.orElse(() => Config.string("HERALD_SWIFTTEST_OS_REGION_NAME")), + const region = yield* Config.string("HERALD_SWIFTTEST_OS_REGION_NAME").pipe( Config.orElse(() => Config.string("TF_VAR_OS_REGION_NAME")), Config.orElse(() => Config.string("OS_REGION_NAME")), Config.withDefault("dc3-a"), diff --git a/src/Config/Layer.ts b/src/Config/Layer.ts index 8efd2d0..343fa12 100644 --- a/src/Config/Layer.ts +++ b/src/Config/Layer.ts @@ -1,6 +1,7 @@ import { Config, Context, Effect, Layer, type Option, Schema } from "effect"; import { parse } from "@std/yaml"; import { + type BackendConfig, GlobalConfig, lookupBucket, type MaterializedBucket, @@ -55,6 +56,12 @@ export function parseConfig( "PROJECT_NAME", "USER_DOMAIN_NAME", "PROJECT_DOMAIN_NAME", + "CORS_ALLOWED_ORIGINS", + "CORS_ALLOWED_METHODS", + "CORS_ALLOWED_HEADERS", + "CORS_EXPOSED_HEADERS", + "CORS_MAX_AGE", + "CORS_CREDENTIALS", ]; for (const [key, value] of Object.entries(env)) { @@ -93,11 +100,67 @@ export function parseConfig( backend.credentials = {} as Record; } (backend.credentials as Record)[configKey] = value; + } else if (configKey.startsWith("cors_")) { + if (!backend.cors) { + backend.cors = {} as Record; + } + const corsKey = configKey.substring(5); + const camelCorsKey = corsKey.replace( + /_([a-z])/g, + (_, g) => g.toUpperCase(), + ); + + if ( + camelCorsKey === "allowedOrigins" || + camelCorsKey === "allowedMethods" || + camelCorsKey === "allowedHeaders" || camelCorsKey === "exposedHeaders" + ) { + (backend.cors as Record)[camelCorsKey] = value.split( + ",", + ).map((s) => s.trim()); + } else if (camelCorsKey === "maxAge") { + const parsed = parseInt(value, 10); + if (Number.isInteger(parsed) && Number.isFinite(parsed)) { + (backend.cors as Record)[camelCorsKey] = parsed; + } + } else if (camelCorsKey === "credentials") { + (backend.cors as Record)[camelCorsKey] = + value.toLowerCase() === "true"; + } } else { backend[configKey] = value; } } + // Handle global CORS from env + const globalCors: Record = (yamlConfig && + typeof yamlConfig === "object" && "cors" in yamlConfig) + ? { ...(yamlConfig as { cors: Record }).cors } + : {}; + + for (const [key, value] of Object.entries(env)) { + if (!key.startsWith("HERALD_CORS_")) continue; + const corsKey = key.substring(12).toLowerCase(); + const camelCorsKey = corsKey.replace( + /_([a-z])/g, + (_, g) => g.toUpperCase(), + ); + + if ( + camelCorsKey === "allowedOrigins" || camelCorsKey === "allowedMethods" || + camelCorsKey === "allowedHeaders" || camelCorsKey === "exposedHeaders" + ) { + globalCors[camelCorsKey] = value.split(",").map((s) => s.trim()); + } else if (camelCorsKey === "maxAge") { + const parsed = parseInt(value, 10); + if (Number.isInteger(parsed) && Number.isFinite(parsed)) { + globalCors[camelCorsKey] = parsed; + } + } else if (camelCorsKey === "credentials") { + globalCors[camelCorsKey] = value.toLowerCase() === "true"; + } + } + // Default backend fallback if no backends defined at all if (Object.keys(backends).length === 0) { backends["default"] = { @@ -106,7 +169,17 @@ export function parseConfig( }; } - return Schema.decodeUnknownSync(GlobalConfig)({ backends }); + const validatedBackends: Record = {}; + for (const [id, b] of Object.entries(backends)) { + if (b.protocol === "s3" || b.protocol === "swift") { + validatedBackends[id] = b as BackendConfig; + } + } + + return Schema.decodeUnknownSync(GlobalConfig)({ + backends: validatedBackends, + cors: Object.keys(globalCors).length > 0 ? globalCors : undefined, + }); } export const HeraldConfigLive = Layer.effect( diff --git a/src/Domain/Config.ts b/src/Domain/Config.ts index 77f32a0..c1edc62 100644 --- a/src/Domain/Config.ts +++ b/src/Domain/Config.ts @@ -15,10 +15,29 @@ export const SwiftCredentials = Schema.Struct({ export const Credentials = Schema.Union(S3Credentials, SwiftCredentials); +export const CorsConfig = Schema.Struct({ + allowedOrigins: Schema.optional(Schema.Array(Schema.String)), + allowedMethods: Schema.optional(Schema.Array(Schema.String)), + allowedHeaders: Schema.optional(Schema.Array(Schema.String)), + exposedHeaders: Schema.optional(Schema.Array(Schema.String)), + maxAge: Schema.optional(Schema.Number), + credentials: Schema.optional(Schema.Boolean), +}).pipe( + Schema.filter((c) => { + if (c.allowedOrigins?.includes("*") && c.credentials) { + return "CORS configuration cannot have allowedOrigins: ['*'] when credentials: true"; + } + return true; + }), +); + +export type CorsConfig = Schema.Schema.Type; + export const BucketOverride = Schema.Struct({ endpoint: Schema.optional(Schema.String), bucket_name: Schema.optional(Schema.String), region: Schema.optional(Schema.String), + cors: Schema.optional(CorsConfig), }); export type BucketOverride = Schema.Schema.Type; @@ -37,6 +56,7 @@ export const S3Config = Schema.Struct({ region: Schema.optional(Schema.String), credentials: Schema.optional(S3Credentials), buckets: BucketsConfig, + cors: Schema.optional(CorsConfig), }); export const SwiftConfig = Schema.Struct({ @@ -46,6 +66,7 @@ export const SwiftConfig = Schema.Struct({ container: Schema.optional(Schema.String), credentials: Schema.optional(SwiftCredentials), buckets: BucketsConfig, + cors: Schema.optional(CorsConfig), }); export const BackendConfig = Schema.Union(S3Config, SwiftConfig); @@ -54,6 +75,7 @@ export type BackendConfig = Schema.Schema.Type; export const GlobalConfig = Schema.Struct({ backends: Schema.Record({ key: Schema.String, value: BackendConfig }), + cors: Schema.optional(CorsConfig), }); export type GlobalConfig = Schema.Schema.Type; @@ -165,3 +187,66 @@ export const lookupBucket = ( return Option.none(); }; + +export const resolveCorsConfig = ( + config: GlobalConfig, + bucketName: string, +): CorsConfig | undefined => { + // 1. Find the backend and bucket override + let bucketCors: CorsConfig | undefined; + let backendCors: CorsConfig | undefined; + + for (const backend of Object.values(config.backends)) { + const buckets = backend.buckets; + if (buckets && typeof buckets !== "string" && buckets[bucketName]) { + bucketCors = buckets[bucketName].cors; + backendCors = backend.cors; + break; + } + } + + // If not found by direct hit, try glob match (similar to lookupBucket) + if (!bucketCors && !backendCors) { + for (const backend of Object.values(config.backends)) { + const buckets = backend.buckets; + if (buckets && typeof buckets !== "string") { + let foundMatch = false; + for (const [key, override] of Object.entries(buckets)) { + if (globToRegex(key).test(bucketName)) { + bucketCors = (override as BucketOverride).cors; + backendCors = backend.cors; + foundMatch = true; + break; + } + } + if (foundMatch) break; + } + } + } + + // If still not found, check if it's a general backend match + if (!bucketCors && !backendCors) { + for (const backend of Object.values(config.backends)) { + const buckets = backend.buckets; + if ( + typeof buckets === "string" && globToRegex(buckets).test(bucketName) + ) { + backendCors = backend.cors; + break; + } + } + } + + const globalCors = config.cors; + + if (!bucketCors && !backendCors && !globalCors) { + return undefined; + } + + // Merge with precedence: bucket > backend > global + return { + ...globalCors, + ...backendCors, + ...bucketCors, + }; +}; diff --git a/src/Frontend/Cors.ts b/src/Frontend/Cors.ts new file mode 100644 index 0000000..c4f94c9 --- /dev/null +++ b/src/Frontend/Cors.ts @@ -0,0 +1,126 @@ +import { + HttpMiddleware, + HttpServerRequest, + HttpServerResponse, +} from "@effect/platform"; +import { Effect } from "effect"; +import { HeraldConfig } from "../Config/Layer.ts"; +import { resolveCorsConfig } from "../Domain/Config.ts"; + +/** + * Extracts the bucket name from the request URL path. + * Assumes path format like /:bucket or /:bucket/* + */ +function extractBucketFromPath(url: string): string | undefined { + try { + const path = new URL(url, "http://localhost").pathname; + const parts = path.split("/").filter((p) => p.length > 0); + return parts.length > 0 ? parts[0] : undefined; + } catch { + return undefined; + } +} + +/** + * Adds CORS headers to a response based on the provided config. + */ +function addCorsHeaders( + response: HttpServerResponse.HttpServerResponse, + cors: NonNullable>, + request: HttpServerRequest.HttpServerRequest, +): HttpServerResponse.HttpServerResponse { + const origin = request.headers["origin"]; + const headers = { ...response.headers }; + + if (cors.allowedOrigins) { + if (cors.allowedOrigins.includes("*")) { + headers["access-control-allow-origin"] = "*"; + } else if (origin && cors.allowedOrigins.includes(origin)) { + headers["access-control-allow-origin"] = origin; + headers["vary"] = headers["vary"] + ? `${headers["vary"]}, Origin` + : "Origin"; + } + } + + if (cors.credentials) { + headers["access-control-allow-credentials"] = "true"; + } + + if (cors.exposedHeaders) { + headers["access-control-expose-headers"] = cors.exposedHeaders.join(", "); + } + + return HttpServerResponse.setHeaders(response, headers); +} + +/** + * Creates a 204 No Content response for OPTIONS preflight requests. + */ +function makePreflightResponse( + cors: NonNullable>, + request: HttpServerRequest.HttpServerRequest, +): HttpServerResponse.HttpServerResponse { + const origin = request.headers["origin"]; + const headers: Record = { + "access-control-max-age": String(cors.maxAge ?? 3600), + }; + + if (cors.allowedOrigins) { + if (cors.allowedOrigins.includes("*")) { + headers["access-control-allow-origin"] = "*"; + } else if (origin && cors.allowedOrigins.includes(origin)) { + headers["access-control-allow-origin"] = origin; + headers["vary"] = "Origin"; + } + } + + if (cors.credentials) { + headers["access-control-allow-credentials"] = "true"; + } + + if (cors.allowedMethods) { + headers["access-control-allow-methods"] = cors.allowedMethods.join(", "); + } else { + // Default to common S3 methods if not specified + headers["access-control-allow-methods"] = + "GET, PUT, POST, DELETE, HEAD, OPTIONS"; + } + + if (cors.allowedHeaders) { + headers["access-control-allow-headers"] = cors.allowedHeaders.join(", "); + } else { + const requestedHeaders = request.headers["access-control-request-headers"]; + if (requestedHeaders) { + headers["access-control-allow-headers"] = requestedHeaders; + } + } + + return HttpServerResponse.empty({ status: 204, headers }); +} + +/** + * Custom CORS middleware that resolves configuration per-request based on the bucket. + */ +export const corsMiddleware = HttpMiddleware.make((app) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const config = yield* HeraldConfig; + + const bucket = extractBucketFromPath(request.url); + const corsConfig = bucket + ? resolveCorsConfig(config.raw, bucket) + : config.raw.cors; + + if (!corsConfig) { + return yield* app; + } + + if (request.method === "OPTIONS") { + return makePreflightResponse(corsConfig, request); + } + + const response = yield* app; + return addCorsHeaders(response, corsConfig, request); + }) +); diff --git a/src/Http.ts b/src/Http.ts index 6fa2dbc..2765ce0 100644 --- a/src/Http.ts +++ b/src/Http.ts @@ -5,7 +5,7 @@ import { HttpServer, } from "@effect/platform"; import { NodeHttpServer } from "@effect/platform-node"; -import { Config, Effect, Layer } from "effect"; +import { Config, Effect, flow, Layer } from "effect"; // deno-lint-ignore no-external-import import { createServer } from "node:http"; @@ -16,6 +16,7 @@ import { HeraldConfigLive } from "./Config/Layer.ts"; import { HttpHealthLive } from "./Frontend/Health/Http.ts"; import { HttpS3Live } from "./Frontend/Http.ts"; import { HttpHeraldApi } from "./Api.ts"; +import { corsMiddleware } from "./Frontend/Cors.ts"; export const HttpHeraldLive = HttpApiBuilder.api(HttpHeraldApi).pipe( Layer.provide(HttpHealthLive), @@ -28,10 +29,10 @@ export const HttpServerHeraldLive = Layer.unwrapEffect( Config.integer("PORT"), 3000, ); - return HttpApiBuilder.serve(HttpMiddleware.logger).pipe( + const middleware = flow(corsMiddleware, HttpMiddleware.logger); + return HttpApiBuilder.serve(middleware).pipe( Layer.provide(HttpApiSwagger.layer()), Layer.provide(HttpApiBuilder.middlewareOpenApi()), - Layer.provide(HttpApiBuilder.middlewareCors()), Layer.provide(HttpHeraldLive), HttpServer.withLogAddress, Layer.provide(NodeHttpServer.layer(createServer, { port })), diff --git a/tests/config.test.ts b/tests/config.test.ts index 9b834d2..442f615 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -250,6 +250,44 @@ const cases: TestCase[] = [ }, }, }, + { + id: "priority_full_hierarchy", + name: "full priority hierarchy (direct > map-glob > string-glob)", + input: { + backends: { + string_glob: { + protocol: "s3", + endpoint: "http://string-glob.com", + buckets: "logs-*", + }, + map_glob: { + protocol: "s3", + endpoint: "http://map-glob.com", + buckets: { + "logs-2025-*": {}, + }, + }, + direct: { + protocol: "s3", + endpoint: "http://direct.com", + buckets: { + "logs-2025-01": {}, + }, + }, + }, + }, + expectedBuckets: { + "logs-2025-01": { backend_id: "direct", endpoint: "http://direct.com" }, + "logs-2025-02": { + backend_id: "map_glob", + endpoint: "http://map-glob.com", + }, + "logs-2024-12": { + backend_id: "string_glob", + endpoint: "http://string-glob.com", + }, + }, + }, ]; for (const tc of cases) { diff --git a/tests/cors.test.ts b/tests/cors.test.ts new file mode 100644 index 0000000..db11f8a --- /dev/null +++ b/tests/cors.test.ts @@ -0,0 +1,219 @@ +import { Effect, Option, Schema } from "effect"; +import { assertEquals, testEffect } from "./utils.ts"; +import { GlobalConfig, resolveCorsConfig } from "../src/Domain/Config.ts"; +import { parseConfig } from "../src/Config/Layer.ts"; +import { corsMiddleware } from "../src/Frontend/Cors.ts"; +import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; +import { HeraldConfig } from "../src/Config/Layer.ts"; + +function makeMockRequest( + url: string, + init: RequestInit, +): HttpServerRequest.HttpServerRequest { + const req = new Request(url, init); + return { + method: req.method, + url: req.url, + headers: Object.fromEntries(req.headers.entries()), + remoteAddress: Option.none(), + } as unknown as HttpServerRequest.HttpServerRequest; +} + +testEffect("cors/resolveCorsConfig/inheritance", () => + Effect.gen(function* () { + yield* Effect.void; + const configInput = { + cors: { + allowedOrigins: ["https://global.com"], + credentials: false, + }, + backends: { + s3_main: { + protocol: "s3", + cors: { + allowedOrigins: ["https://backend.com"], + maxAge: 3600, + }, + buckets: { + bucket_with_cors: { + cors: { + allowedOrigins: ["https://bucket.com"], + credentials: true, + }, + }, + bucket_no_cors: {}, + }, + }, + other: { + protocol: "s3", + buckets: "*", + }, + }, + }; + + const config = Schema.decodeUnknownSync(GlobalConfig)(configInput); + + // 1. Bucket level override + const cors1 = resolveCorsConfig(config, "bucket_with_cors"); + assertEquals(cors1?.allowedOrigins, ["https://bucket.com"]); + assertEquals(cors1?.credentials, true); + assertEquals(cors1?.maxAge, 3600); // Inherited from backend + + // 2. Backend level override + const cors2 = resolveCorsConfig(config, "bucket_no_cors"); + assertEquals(cors2?.allowedOrigins, ["https://backend.com"]); + assertEquals(cors2?.credentials, false); // Inherited from global + assertEquals(cors2?.maxAge, 3600); + + // 3. Global level + const cors3 = resolveCorsConfig(config, "any-other-bucket"); + assertEquals(cors3?.allowedOrigins, ["https://global.com"]); + assertEquals(cors3?.credentials, false); + assertEquals(cors3?.maxAge, undefined); + })); + +testEffect("cors/parseConfig/env_vars", () => + Effect.gen(function* () { + yield* Effect.void; + const env = { + HERALD_CORS_ALLOWED_ORIGINS: "https://global.com, https://other.com", + HERALD_CORS_CREDENTIALS: "true", + HERALD_PROD_PROTOCOL: "s3", + HERALD_PROD_BUCKETS: "*", + HERALD_PROD_CORS_ALLOWED_ORIGINS: "https://s3.com", + HERALD_PROD_CORS_MAX_AGE: "7200", + }; + const config = parseConfig({ backends: {} }, env); + + assertEquals(config.cors?.allowedOrigins, [ + "https://global.com", + "https://other.com", + ]); + assertEquals(config.cors?.credentials, true); + + const prodBackend = config.backends.prod; + assertEquals(prodBackend.protocol, "s3"); + assertEquals(prodBackend.cors?.allowedOrigins, ["https://s3.com"]); + assertEquals(prodBackend.cors?.maxAge, 7200); + })); + +testEffect("cors/parseConfig/yaml_merge", () => + Effect.gen(function* () { + yield* Effect.void; + const yaml = { + cors: { + allowedOrigins: ["https://yaml.com"], + maxAge: 100, + }, + backends: { + s3: { + protocol: "s3", + buckets: "*", + cors: { + allowedMethods: ["GET"], + }, + }, + }, + }; + const env = { + HERALD_CORS_MAX_AGE: "200", + HERALD_S3_CORS_ALLOWED_METHODS: "POST, PUT", + }; + const config = parseConfig(yaml, env); + + assertEquals(config.cors?.allowedOrigins, ["https://yaml.com"]); + assertEquals(config.cors?.maxAge, 200); // Env overrides YAML + + const s3Backend = config.backends.s3; + assertEquals(s3Backend.cors?.allowedMethods, ["POST", "PUT"]); // Env overrides YAML + })); + +testEffect("cors/middleware/preflight", () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3: { + protocol: "s3", + buckets: "*", + cors: { + allowedOrigins: ["https://example.com"], + allowedMethods: ["GET", "PUT"], + credentials: true, + }, + }, + }, + }; + + const heraldConfig = { + raw: config, + lookupBucket: () => Option.none(), + }; + + const request = makeMockRequest("http://localhost/s3/obj", { + method: "OPTIONS", + headers: { + "origin": "https://example.com", + "access-control-request-method": "PUT", + }, + }); + + const middleware = corsMiddleware( + Effect.fail(new Error("Should not reach handler")), + ); + + const response = yield* middleware.pipe( + // deno-lint-ignore no-explicit-any + Effect.provideService(HeraldConfig, heraldConfig as any), + Effect.provideService(HttpServerRequest.HttpServerRequest, request), + ); + + assertEquals(response.status, 204); + assertEquals( + response.headers["access-control-allow-origin"], + "https://example.com", + ); + assertEquals(response.headers["access-control-allow-methods"], "GET, PUT"); + assertEquals(response.headers["access-control-allow-credentials"], "true"); + })); + +testEffect("cors/middleware/headers", () => + Effect.gen(function* () { + const config: GlobalConfig = { + backends: { + s3: { + protocol: "s3", + buckets: "*", + cors: { + allowedOrigins: ["*"], + exposedHeaders: ["x-amz-meta-custom"], + }, + }, + }, + }; + + const heraldConfig = { + raw: config, + lookupBucket: () => Option.none(), + }; + + const request = makeMockRequest("http://localhost/s3/obj", { + method: "GET", + headers: { "origin": "https://any.com" }, + }); + + const handler = Effect.succeed(HttpServerResponse.empty({ status: 200 })); + const middleware = corsMiddleware(handler); + + const response = yield* middleware.pipe( + // deno-lint-ignore no-explicit-any + Effect.provideService(HeraldConfig, heraldConfig as any), + Effect.provideService(HttpServerRequest.HttpServerRequest, request), + ); + + assertEquals(response.status, 200); + assertEquals(response.headers["access-control-allow-origin"], "*"); + assertEquals( + response.headers["access-control-expose-headers"], + "x-amz-meta-custom", + ); + })); diff --git a/tests/utils.ts b/tests/utils.ts index 129a8cf..1847e02 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -405,8 +405,7 @@ function proxyRunner(tc: ProxyTestCase, t: Deno.TestContext) { const getSwiftConfig = () => Effect.gen(function* () { - const authUrl = yield* Config.string("HEARLD_SWIFTTEST_AUTH_URL").pipe( - Config.orElse(() => Config.string("HERALD_SWIFTTEST_AUTH_URL")), + const authUrl = yield* Config.string("HERALD_SWIFTTEST_AUTH_URL").pipe( Config.orElse(() => Config.string("OS_AUTH_URL")), Config.withDefault("http://localhost:8080/auth/v1.0"), Config.option, @@ -430,8 +429,7 @@ const getSwiftConfig = () => Config.orElse(() => Config.string("OS_PROJECT_NAME")), Config.option, ); - const region = yield* Config.string("HEARLD_SWIFTTEST_OS_REGION_NAME").pipe( - Config.orElse(() => Config.string("HERALD_SWIFTTEST_OS_REGION_NAME")), + const region = yield* Config.string("HERALD_SWIFTTEST_OS_REGION_NAME").pipe( Config.orElse(() => Config.string("TF_VAR_OS_REGION_NAME")), Config.orElse(() => Config.string("OS_REGION_NAME")), Config.withDefault("dc3-a"), diff --git a/x/s3-tests.ts b/x/s3-tests.ts index 75c0772..dca1405 100755 --- a/x/s3-tests.ts +++ b/x/s3-tests.ts @@ -54,8 +54,7 @@ function getMinioConfig(): GlobalConfig { const getSwiftConfig = () => Effect.gen(function* () { - const authUrl = yield* Config.string("HEARLD_SWIFTTEST_AUTH_URL").pipe( - Config.orElse(() => Config.string("HERALD_SWIFTTEST_AUTH_URL")), + const authUrl = yield* Config.string("HERALD_SWIFTTEST_AUTH_URL").pipe( Config.orElse(() => Config.string("OS_AUTH_URL")), Config.withDefault("http://localhost:8080/auth/v1.0"), Config.option, @@ -79,8 +78,7 @@ const getSwiftConfig = () => Config.orElse(() => Config.string("OS_PROJECT_NAME")), Config.option, ); - const region = yield* Config.string("HEARLD_SWIFTTEST_OS_REGION_NAME").pipe( - Config.orElse(() => Config.string("HERALD_SWIFTTEST_OS_REGION_NAME")), + const region = yield* Config.string("HERALD_SWIFTTEST_OS_REGION_NAME").pipe( Config.orElse(() => Config.string("TF_VAR_OS_REGION_NAME")), Config.orElse(() => Config.string("OS_REGION_NAME")), Config.withDefault("dc3-a"),