From 53b8ad36943a15f42bc10c6cfbf50eaa0ac28576 Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Tue, 20 Jan 2026 03:46:02 +0300 Subject: [PATCH 1/2] feat: multipart support for s3 backend --- src/Backends/S3/Backend.ts | 356 +++++++++++++++++- src/Backends/Swift/Backend.ts | 66 +++- src/Frontend/Objects/Delete.ts | 9 + src/Frontend/Objects/Get.ts | 24 +- src/Frontend/Objects/Head.ts | 9 +- src/Frontend/Objects/List.ts | 14 + src/Frontend/Objects/Post.ts | 77 ++++ src/Frontend/Objects/Put.ts | 20 +- src/Frontend/Utils.ts | 25 +- src/Services/Backend.ts | 135 ++++++- src/Services/S3Xml.ts | 113 ++++++ .../__snapshots__/objects.test.ts.snap | 125 +----- tests/integration/objects.test.ts | 179 +++++++++ x/s3-tests.ts | 3 - 14 files changed, 1014 insertions(+), 141 deletions(-) diff --git a/src/Backends/S3/Backend.ts b/src/Backends/S3/Backend.ts index e888527..3aa7309 100644 --- a/src/Backends/S3/Backend.ts +++ b/src/Backends/S3/Backend.ts @@ -1,6 +1,9 @@ import { Chunk, Effect, Option, Stream } from "effect"; import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, CreateBucketCommand, + CreateMultipartUploadCommand, DeleteBucketCommand, DeleteObjectCommand, DeleteObjectsCommand, @@ -9,12 +12,15 @@ import { HeadObjectCommand, ListBucketsCommand, type ListBucketsCommandOutput, + ListMultipartUploadsCommand, ListObjectsCommand, type ListObjectsCommandOutput, ListObjectsV2Command, type ListObjectsV2CommandOutput, ListObjectVersionsCommand, + ListPartsCommand, PutObjectCommand, + UploadPartCommand, } from "@aws-sdk/client-s3"; import type { MaterializedBucket } from "../../Domain/Config.ts"; import { AppConfig } from "../../Config/Layer.ts"; @@ -28,10 +34,16 @@ import { BucketNotEmpty, type CommonPrefix, type DeleteObjectsResult, + EntityTooSmall, InternalError, + InvalidPart, + InvalidPartOrder, + InvalidRequest, type ListObjectsResult, + MalformedXML, NoSuchBucket, NoSuchKey, + NoSuchUpload, type ObjectInfo, } from "../../Services/Backend.ts"; import { S3Client } from "./Client.ts"; @@ -70,6 +82,25 @@ function mapS3Error(e: unknown, bucketName?: string): BackendError { key: "unknown", message: message, }); + case "NoSuchUpload": + return new NoSuchUpload({ + uploadId: "unknown", + message: message, + }); + case "InvalidPart": + case "InvalidPartNumber": + return new InvalidPart({ message }); + case "InvalidPartOrder": + return new InvalidPartOrder({ message }); + case "EntityTooSmall": + return new EntityTooSmall({ message }); + case "InvalidRequest": + if (message.includes("at least one part")) { + return new MalformedXML({ message }); + } + return new InvalidRequest({ message }); + case "MalformedXML": + return new MalformedXML({ message }); case "BucketAlreadyExists": return new BucketAlreadyExists({ bucketName: bucket, message }); case "BucketAlreadyOwnedByYou": @@ -384,7 +415,7 @@ export const makeS3Backend = ( })), ), - getObject: (key) => + getObject: (key, headers) => s3Service.getClient(targetBucket).pipe( Effect.mapError((e) => mapS3Error(e, targetBucket.name)), Effect.flatMap((client) => @@ -394,6 +425,34 @@ export const makeS3Backend = ( new GetObjectCommand({ Bucket: targetBucket.bucket_name, Key: key, + Range: (headers["range"] || headers["Range"]) as string, + PartNumber: (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) + ? parseInt( + (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) as string, + ) + : undefined, + IfMatch: + (headers["if-match"] || headers["If-Match"]) as string, + IfNoneMatch: (headers["if-none-match"] || + headers["If-None-Match"]) as string, + IfModifiedSince: (headers["if-modified-since"] || + headers["If-Modified-Since"]) + ? new Date( + (headers["if-modified-since"] || + headers["If-Modified-Since"]) as string, + ) + : undefined, + IfUnmodifiedSince: (headers["if-unmodified-since"] || + headers["If-Unmodified-Since"]) + ? new Date( + (headers["if-unmodified-since"] || + headers["If-Unmodified-Since"]) as string, + ) + : undefined, }), ), catch: (e) => mapS3Error(e, targetBucket.bucket_name), @@ -446,7 +505,16 @@ export const makeS3Backend = ( if (result.ContentType) { headers["content-type"] = result.ContentType; } + if (result.ContentLength !== undefined) { + headers["content-length"] = String(result.ContentLength); + } if (result.ETag) headers["etag"] = result.ETag; + if (result.PartsCount !== undefined) { + headers["x-amz-mp-parts-count"] = String(result.PartsCount); + } + if (result.VersionId) { + headers["x-amz-version-id"] = result.VersionId; + } if (result.LastModified) { headers["last-modified"] = result.LastModified.toUTCString(); } @@ -487,21 +555,28 @@ export const makeS3Backend = ( }), ), - headObject: (key) => + headObject: (key, headers) => s3Service.getClient(targetBucket).pipe( Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new HeadObjectCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - }), - ), + Effect.flatMap((client) => { + const commandInput = { + Bucket: targetBucket.bucket_name, + Key: key, + PartNumber: (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) + ? parseInt( + (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) as string, + ) + : undefined, + }; + return Effect.tryPromise({ + try: () => client.send(new HeadObjectCommand(commandInput)), catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), + }); + }), Effect.map((result) => { const metadata: Record = {}; if (result.Metadata) { @@ -522,6 +597,12 @@ export const makeS3Backend = ( headers["content-length"] = String(result.ContentLength); } if (result.ETag) headers["etag"] = result.ETag; + if (result.PartsCount !== undefined) { + headers["x-amz-mp-parts-count"] = String(result.PartsCount); + } + if (result.VersionId) { + headers["x-amz-version-id"] = result.VersionId; + } if (result.LastModified) { headers["last-modified"] = result .LastModified.toUTCString(); @@ -651,6 +732,255 @@ export const makeS3Backend = ( })), })), ), + + createMultipartUpload: (key, headers) => + s3Service.getClient(targetBucket).pipe( + Effect.mapError((e) => mapS3Error(e, targetBucket.name)), + Effect.flatMap((client) => { + const metadata: Record = {}; + for (const [k, v] of Object.entries(headers)) { + if (k.toLowerCase().startsWith("x-amz-meta-")) { + const metaKey = k.substring("x-amz-meta-".length); + metadata[metaKey] = String(v); + } + } + const contentType = headers["content-type"]; + + return Effect.tryPromise({ + try: () => + client.send( + new CreateMultipartUploadCommand({ + Bucket: targetBucket.bucket_name, + Key: key, + Metadata: metadata, + ContentType: contentType + ? String(contentType) + : undefined, + }), + ), + catch: (e) => mapS3Error(e, targetBucket.bucket_name), + }); + }), + Effect.flatMap((result) => { + if (!result.UploadId) { + return Effect.fail( + new InternalError({ + message: "S3 returned empty UploadId", + }), + ); + } + return Effect.succeed({ uploadId: result.UploadId }); + }), + ), + + uploadPart: (key, uploadId, partNumber, bodyStream) => + s3Service.getClient(targetBucket).pipe( + Effect.mapError((e) => mapS3Error(e, targetBucket.name)), + Effect.flatMap((client) => + Stream.runCollect(bodyStream).pipe( + Effect.mapError((e) => + new InternalError({ message: String(e) }) + ), + Effect.flatMap((chunks) => { + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const body = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.length; + } + + return Effect.tryPromise({ + try: () => + client.send( + new UploadPartCommand({ + Bucket: targetBucket.bucket_name, + Key: key, + UploadId: uploadId, + PartNumber: partNumber, + Body: body, + }), + ), + catch: (e) => mapS3Error(e, targetBucket.bucket_name), + }); + }), + ) + ), + Effect.flatMap((result) => { + if (!result.ETag) { + return Effect.fail( + new InternalError({ + message: "S3 returned empty ETag for UploadPart", + }), + ); + } + return Effect.succeed({ etag: result.ETag }); + }), + ), + + completeMultipartUpload: (key, uploadId, parts) => + s3Service.getClient(targetBucket).pipe( + Effect.mapError((e) => mapS3Error(e, targetBucket.name)), + Effect.flatMap((client) => + Effect.tryPromise({ + try: () => + client.send( + new CompleteMultipartUploadCommand({ + Bucket: targetBucket.bucket_name, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: parts.map((p) => ({ + ETag: p.etag, + PartNumber: p.partNumber, + })), + }, + }), + ), + catch: (e) => mapS3Error(e, targetBucket.bucket_name), + }) + ), + Effect.flatMap((result) => { + if ( + !result.Location || !result.Bucket || !result.Key || + !result.ETag + ) { + return Effect.fail( + new InternalError({ + message: + "S3 returned incomplete CompleteMultipartUploadResult", + }), + ); + } + return Effect.succeed({ + location: result.Location, + bucket: result.Bucket, + key: result.Key, + etag: result.ETag, + versionId: result.VersionId, + }); + }), + ), + + abortMultipartUpload: (key, uploadId) => + s3Service.getClient(targetBucket).pipe( + Effect.mapError((e) => mapS3Error(e, targetBucket.name)), + Effect.flatMap((client) => + Effect.tryPromise({ + try: () => + client.send( + new AbortMultipartUploadCommand({ + Bucket: targetBucket.bucket_name, + Key: key, + UploadId: uploadId, + }), + ), + catch: (e) => mapS3Error(e, targetBucket.bucket_name), + }) + ), + Effect.map(() => undefined), + ), + + listMultipartUploads: (args) => + s3Service.getClient(targetBucket).pipe( + Effect.mapError((e) => mapS3Error(e, targetBucket.name)), + Effect.flatMap((client) => + Effect.tryPromise({ + try: () => + client.send( + new ListMultipartUploadsCommand({ + Bucket: targetBucket.bucket_name, + Prefix: args.prefix, + Delimiter: args.delimiter, + KeyMarker: args.keyMarker, + UploadIdMarker: args.uploadIdMarker, + MaxUploads: args.maxUploads, + EncodingType: args.encodingType as "url" | undefined, + }), + ), + catch: (e) => mapS3Error(e, targetBucket.bucket_name), + }) + ), + Effect.map((result) => ({ + bucket: result.Bucket ?? targetBucket.bucket_name, + prefix: result.Prefix, + keyMarker: result.KeyMarker, + uploadIdMarker: result.UploadIdMarker, + nextKeyMarker: result.NextKeyMarker, + nextUploadIdMarker: result.NextUploadIdMarker, + maxUploads: result.MaxUploads ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: result.EncodingType as string, + uploads: (result.Uploads ?? []).map((u) => ({ + key: u.Key ?? "", + uploadId: u.UploadId ?? "", + owner: { + id: u.Owner?.ID ?? "", + displayName: u.Owner?.DisplayName ?? "", + }, + initiator: { + id: u.Initiator?.ID ?? "", + displayName: u.Initiator?.DisplayName ?? "", + }, + storageClass: u.StorageClass ?? "STANDARD", + initiated: u.Initiated ?? new Date(), + })), + commonPrefixes: (result.CommonPrefixes ?? []).map((cp) => ({ + prefix: cp.Prefix ?? "", + })), + })), + ), + + listParts: (key, uploadId) => + s3Service.getClient(targetBucket).pipe( + Effect.mapError((e) => mapS3Error(e, targetBucket.name)), + Effect.flatMap((client) => + Effect.tryPromise({ + try: () => + client.send( + new ListPartsCommand({ + Bucket: targetBucket.bucket_name, + Key: key, + UploadId: uploadId, + }), + ), + catch: (e) => mapS3Error(e, targetBucket.bucket_name), + }) + ), + Effect.map((result) => ({ + bucket: result.Bucket ?? targetBucket.bucket_name, + key: result.Key ?? key, + uploadId: result.UploadId ?? uploadId, + owner: { + id: result.Owner?.ID ?? "", + displayName: result.Owner?.DisplayName ?? "", + }, + initiator: { + id: result.Initiator?.ID ?? "", + displayName: result.Initiator?.DisplayName ?? "", + }, + storageClass: result.StorageClass ?? "STANDARD", + partNumberMarker: result.PartNumberMarker + ? parseInt(String(result.PartNumberMarker)) + : 0, + nextPartNumberMarker: result.NextPartNumberMarker + ? parseInt(String(result.NextPartNumberMarker)) + : 0, + maxParts: result.MaxParts ?? 1000, + isTruncated: result.IsTruncated ?? false, + parts: (result.Parts ?? []).map((p) => ({ + partNumber: p.PartNumber ?? 0, + lastModified: p.LastModified ?? new Date(), + etag: p.ETag ?? "", + size: p.Size ?? 0, + })), + })), + ), }; return service; diff --git a/src/Backends/Swift/Backend.ts b/src/Backends/Swift/Backend.ts index 32079da..8689888 100644 --- a/src/Backends/Swift/Backend.ts +++ b/src/Backends/Swift/Backend.ts @@ -330,14 +330,52 @@ export const makeSwiftBackend = ( }; }), - getObject: (key: string) => + getObject: ( + key: string, + headers: Record, + ) => Effect.gen(function* () { const { url, token, container } = yield* getTarget(); const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const swiftHeaders: Record = { + "X-Auth-Token": token, + }; + if (headers["range"] || headers["Range"]) { + swiftHeaders["Range"] = String( + headers["range"] || headers["Range"], + ); + } + if (headers["if-match"] || headers["If-Match"]) { + swiftHeaders["If-Match"] = String( + headers["if-match"] || + headers["If-Match"], + ); + } + if (headers["if-none-match"] || headers["If-None-Match"]) { + swiftHeaders["If-None-Match"] = String( + headers["if-none-match"] || + headers["If-None-Match"], + ); + } + if (headers["if-modified-since"] || headers["If-Modified-Since"]) { + swiftHeaders["If-Modified-Since"] = String( + headers["if-modified-since"] || + headers["If-Modified-Since"], + ); + } + if ( + headers["if-unmodified-since"] || headers["If-Unmodified-Since"] + ) { + swiftHeaders["If-Unmodified-Since"] = String( + headers["if-unmodified-since"] || + headers["If-Unmodified-Since"], + ); + } + const response = yield* Effect.tryPromise({ try: () => fetch(`${url}/${encodedKey}`, { - headers: { "X-Auth-Token": token }, + headers: swiftHeaders, }), catch: (e) => new InternalError({ message: String(e) }), }); @@ -396,15 +434,22 @@ export const makeSwiftBackend = ( } satisfies ObjectResponse; }), - headObject: (key: string) => + headObject: ( + key: string, + _headers: Record, + ) => Effect.gen(function* () { const { url, token, container } = yield* getTarget(); const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const swiftHeaders: Record = { + "X-Auth-Token": token, + }; + // ... handle headers if needed const response = yield* Effect.tryPromise({ try: () => fetch(`${url}/${encodedKey}`, { method: "HEAD", - headers: { "X-Auth-Token": token }, + headers: swiftHeaders, }), catch: (e) => new InternalError({ message: String(e) }), }); @@ -588,5 +633,18 @@ export const makeSwiftBackend = ( return { deleted, errors } satisfies DeleteObjectsResult; }), + + createMultipartUpload: (_key, _headers) => + Effect.fail(new InternalError({ message: "Not implemented" })), + uploadPart: (_key, _uploadId, _partNumber, _body) => + Effect.fail(new InternalError({ message: "Not implemented" })), + completeMultipartUpload: (_key, _uploadId, _parts) => + Effect.fail(new InternalError({ message: "Not implemented" })), + abortMultipartUpload: (_key, _uploadId) => + Effect.fail(new InternalError({ message: "Not implemented" })), + listMultipartUploads: (_args) => + Effect.fail(new InternalError({ message: "Not implemented" })), + listParts: (_key, _uploadId) => + Effect.fail(new InternalError({ message: "Not implemented" })), }; }); diff --git a/src/Frontend/Objects/Delete.ts b/src/Frontend/Objects/Delete.ts index f2c9270..daa07ac 100644 --- a/src/Frontend/Objects/Delete.ts +++ b/src/Frontend/Objects/Delete.ts @@ -12,6 +12,15 @@ export const deleteObject = ( Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest; const key = extractKey(request.url, bucket); + const url = new URL(request.url, "http://localhost"); + const searchParams = url.searchParams; + + if (searchParams.has("uploadId")) { + // Abort Multipart Upload + const uploadId = searchParams.get("uploadId")!; + yield* backend.abortMultipartUpload(key, uploadId); + return HttpServerResponse.empty({ status: 204 }); + } yield* backend.deleteObject(key); return HttpServerResponse.empty({ status: 204 }); diff --git a/src/Frontend/Objects/Get.ts b/src/Frontend/Objects/Get.ts index fff8504..65e0a28 100644 --- a/src/Frontend/Objects/Get.ts +++ b/src/Frontend/Objects/Get.ts @@ -1,19 +1,39 @@ import { Effect } from "effect"; import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; import { extractKey, resolveBucket } from "../Utils.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; /** * Handler for GetObject (GET /:bucket/*) + * Also handles ListParts (?uploadId=...). */ export const getObject = ({ path: { bucket } }: { path: { bucket: string } }) => resolveBucket(bucket, (backend) => Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest; + const s3Xml = yield* S3Xml; const key = extractKey(request.url, bucket); + const url = new URL(request.url, "http://localhost"); + const searchParams = url.searchParams; - const result = yield* backend.getObject(key); + if (searchParams.has("uploadId")) { + // List Parts + const uploadId = searchParams.get("uploadId")!; + const result = yield* backend.listParts(key, uploadId); + return s3Xml.formatListParts(result); + } + + const combinedHeaders = { ...request.headers }; + if (searchParams.has("partNumber")) { + combinedHeaders["x-amz-part-number"] = searchParams.get("partNumber")!; + } + + const result = yield* backend.getObject(key, combinedHeaders); + const status = (request.headers["range"] || request.headers["Range"]) + ? 206 + : 200; return HttpServerResponse.stream(result.stream, { - status: 200, + status, headers: result.headers, contentType: result.contentType, }); diff --git a/src/Frontend/Objects/Head.ts b/src/Frontend/Objects/Head.ts index b91a0a2..483d854 100644 --- a/src/Frontend/Objects/Head.ts +++ b/src/Frontend/Objects/Head.ts @@ -12,8 +12,15 @@ export const headObject = ( Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest; const key = extractKey(request.url, bucket); + const url = new URL(request.url, "http://localhost"); + const combinedHeaders = { ...request.headers }; + if (url.searchParams.has("partNumber")) { + combinedHeaders["x-amz-part-number"] = url.searchParams.get( + "partNumber", + )!; + } - const result = yield* backend.headObject(key); + const result = yield* backend.headObject(key, combinedHeaders); return HttpServerResponse.empty({ status: 200, headers: result.headers, diff --git a/src/Frontend/Objects/List.ts b/src/Frontend/Objects/List.ts index 3552e2e..f92cdc1 100644 --- a/src/Frontend/Objects/List.ts +++ b/src/Frontend/Objects/List.ts @@ -30,6 +30,20 @@ export const listObjects = ( return s3Xml.formatListVersions(result); } + if (searchParams.has("uploads")) { + const result = yield* backend.listMultipartUploads({ + prefix: searchParams.get("prefix") ?? undefined, + delimiter: searchParams.get("delimiter") ?? undefined, + keyMarker: searchParams.get("key-marker") ?? undefined, + uploadIdMarker: searchParams.get("upload-id-marker") ?? undefined, + maxUploads: searchParams.has("max-uploads") + ? parseInt(searchParams.get("max-uploads")!) + : undefined, + encodingType: searchParams.get("encoding-type") ?? undefined, + }); + return s3Xml.formatListMultipartUploads(result); + } + const result = yield* backend.listObjects({ prefix: searchParams.get("prefix") ?? undefined, delimiter: searchParams.get("delimiter") ?? undefined, diff --git a/src/Frontend/Objects/Post.ts b/src/Frontend/Objects/Post.ts index c61c6ba..7287c3a 100644 --- a/src/Frontend/Objects/Post.ts +++ b/src/Frontend/Objects/Post.ts @@ -1,10 +1,12 @@ import { Effect, Option, Stream } from "effect"; import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; import { extractKey, resolveBucket } from "../Utils.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; /** * Handler for POST requests on buckets or objects. * Primarily used for Multi-Object Delete (POST /:bucket?delete). + * Also handles InitiateMultipartUpload (?uploads) and CompleteMultipartUpload (?uploadId=...). */ export const postObject = ( { path: { bucket } }: { path: { bucket: string } }, @@ -12,11 +14,13 @@ export const postObject = ( resolveBucket(bucket, (backend) => Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest; + const s3Xml = yield* S3Xml; const url = new URL(request.url, "http://localhost"); const searchParams = url.searchParams; const key = extractKey(request.url, bucket); if (searchParams.has("delete")) { + // ... (Multi-Object Delete logic) // Multi-Object Delete const bodyChunks = yield* Stream.runCollect(request.stream); let totalLength = 0; @@ -76,6 +80,79 @@ export const postObject = ( }); } + if (searchParams.has("uploads")) { + // Initiate Multipart Upload + const result = yield* backend.createMultipartUpload( + key, + request.headers, + ); + return s3Xml.formatInitiateMultipartUpload( + bucket, + key, + result.uploadId, + ); + } + + if (searchParams.has("uploadId")) { + // Complete Multipart Upload + const uploadId = searchParams.get("uploadId")!; + const bodyChunks = yield* Stream.runCollect(request.stream); + let totalLength = 0; + for (const chunk of Array.from(bodyChunks)) { + totalLength += chunk.length; + } + const bodyBytes = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of Array.from(bodyChunks)) { + bodyBytes.set(chunk, offset); + offset += chunk.length; + } + const bodyText = new TextDecoder().decode(bodyBytes); + + const parts: { etag: string; partNumber: number }[] = []; + const partMatches = Array.from( + bodyText.matchAll(/(.*?)<\/Part>/gs), + ); + for (const match of partMatches) { + const content = match[1]; + const partNumberMatch = content.match( + /(.*?)<\/PartNumber>/, + ); + const etagMatch = content.match(/(.*?)<\/ETag>/); + if (partNumberMatch && etagMatch) { + parts.push({ + partNumber: parseInt(partNumberMatch[1]), + etag: etagMatch[1].replace(/"/g, '"'), + }); + } + } + + const result = yield* backend.completeMultipartUpload( + key, + uploadId, + parts, + ).pipe( + Effect.catchTag("NoSuchUpload", (e) => + Effect.gen(function* () { + // Idempotency: check if object already exists + const head = yield* backend.headObject(key, {}).pipe( + Effect.orElseFail(() => e), + ); + if (head.etag) { + return { + location: `http://localhost/${bucket}/${key}`, // Approximate + bucket, + key, + etag: head.etag, + versionId: head.headers["x-amz-version-id"], + }; + } + return yield* Effect.fail(e); + })), + ); + return s3Xml.formatCompleteMultipartUpload(result); + } + return yield* Effect.fail( new Error(`Method POST for key [${key}] not implemented`), ); diff --git a/src/Frontend/Objects/Put.ts b/src/Frontend/Objects/Put.ts index 62894cc..4b9638b 100644 --- a/src/Frontend/Objects/Put.ts +++ b/src/Frontend/Objects/Put.ts @@ -10,6 +10,24 @@ export const putObject = ({ path: { bucket } }: { path: { bucket: string } }) => Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest; const key = extractKey(request.url, bucket); + const url = new URL(request.url, "http://localhost"); + const searchParams = url.searchParams; + + if (searchParams.has("partNumber") && searchParams.has("uploadId")) { + // Upload Part + const partNumber = parseInt(searchParams.get("partNumber")!); + const uploadId = searchParams.get("uploadId")!; + const result = yield* backend.uploadPart( + key, + uploadId, + partNumber, + request.stream, + ); + return HttpServerResponse.empty({ + status: 200, + headers: { ETag: result.etag }, + }); + } const result = yield* backend.putObject( key, @@ -17,7 +35,7 @@ export const putObject = ({ path: { bucket } }: { path: { bucket: string } }) => request.headers, ); const headers: Record = {}; - if (result.etag) headers["etag"] = result.etag; + if (result.etag) headers["ETag"] = result.etag; if (result.versionId) headers["x-amz-version-id"] = result.versionId; return HttpServerResponse.empty({ diff --git a/src/Frontend/Utils.ts b/src/Frontend/Utils.ts index 9e08b74..58733d4 100644 --- a/src/Frontend/Utils.ts +++ b/src/Frontend/Utils.ts @@ -7,9 +7,16 @@ import { BucketAlreadyExists, BucketAlreadyOwnedByYou, BucketNotEmpty, + DeleteObjectsError, + EntityTooSmall, InternalError, + InvalidPart, + InvalidPartOrder, + InvalidRequest, + MalformedXML, NoSuchBucket, NoSuchKey, + NoSuchUpload, } from "../Services/Backend.ts"; import { HttpServerRequest, type HttpServerResponse } from "@effect/platform"; import type { AppConfig } from "../Config/Layer.ts"; @@ -120,7 +127,14 @@ export function resolveBucket< e instanceof BucketAlreadyOwnedByYou || e instanceof InternalError || e instanceof AccessDenied || - e instanceof BucketNotEmpty + e instanceof BucketNotEmpty || + e instanceof NoSuchUpload || + e instanceof InvalidPart || + e instanceof InvalidPartOrder || + e instanceof EntityTooSmall || + e instanceof InvalidRequest || + e instanceof MalformedXML || + e instanceof DeleteObjectsError ) { return Effect.succeed(s3Xml.formatError(e, isHead)); } @@ -186,7 +200,14 @@ export function resolveBackend< e instanceof BucketAlreadyOwnedByYou || e instanceof InternalError || e instanceof AccessDenied || - e instanceof BucketNotEmpty + e instanceof BucketNotEmpty || + e instanceof NoSuchUpload || + e instanceof InvalidPart || + e instanceof InvalidPartOrder || + e instanceof EntityTooSmall || + e instanceof InvalidRequest || + e instanceof MalformedXML || + e instanceof DeleteObjectsError ) { return Effect.succeed(s3Xml.formatError(e, isHead)); } diff --git a/src/Services/Backend.ts b/src/Services/Backend.ts index 8da9574..9a6f5a5 100644 --- a/src/Services/Backend.ts +++ b/src/Services/Backend.ts @@ -68,6 +68,67 @@ export interface PutObjectResult { readonly versionId?: string; } +export interface MultipartUploadResult { + readonly uploadId: string; +} + +export interface UploadPartResult { + readonly etag: string; +} + +export interface CompleteMultipartUploadResult { + readonly location: string; + readonly bucket: string; + readonly key: string; + readonly etag: string; + readonly versionId?: string; +} + +export interface PartInfo { + readonly partNumber: number; + readonly lastModified: Date; + readonly etag: string; + readonly size: number; +} + +export interface ListPartsResult { + readonly bucket: string; + readonly key: string; + readonly uploadId: string; + readonly owner: OwnerInfo; + readonly initiator: OwnerInfo; + readonly storageClass: string; + readonly partNumberMarker: number; + readonly nextPartNumberMarker: number; + readonly maxParts: number; + readonly isTruncated: boolean; + readonly parts: readonly PartInfo[]; +} + +export interface MultipartUploadInfo { + readonly key: string; + readonly uploadId: string; + readonly owner: OwnerInfo; + readonly initiator: OwnerInfo; + readonly storageClass: string; + readonly initiated: Date; +} + +export interface ListMultipartUploadsResult { + readonly bucket: string; + readonly prefix?: string; + readonly keyMarker?: string; + readonly uploadIdMarker?: string; + readonly nextKeyMarker?: string; + readonly nextUploadIdMarker?: string; + readonly maxUploads: number; + readonly delimiter?: string; + readonly isTruncated: boolean; + readonly uploads: readonly MultipartUploadInfo[]; + readonly commonPrefixes: readonly CommonPrefix[]; + readonly encodingType?: string; +} + export class NoSuchBucket extends Schema.TaggedError()("NoSuchBucket", { bucketName: Schema.String, @@ -111,6 +172,37 @@ export class BucketNotEmpty message: Schema.String, }) {} +export class NoSuchUpload + extends Schema.TaggedError()("NoSuchUpload", { + uploadId: Schema.String, + message: Schema.String, + }) {} + +export class InvalidPart + extends Schema.TaggedError()("InvalidPart", { + message: Schema.String, + }) {} + +export class InvalidPartOrder + extends Schema.TaggedError()("InvalidPartOrder", { + message: Schema.String, + }) {} + +export class EntityTooSmall + extends Schema.TaggedError()("EntityTooSmall", { + message: Schema.String, + }) {} + +export class InvalidRequest + extends Schema.TaggedError()("InvalidRequest", { + message: Schema.String, + }) {} + +export class MalformedXML + extends Schema.TaggedError()("MalformedXML", { + message: Schema.String, + }) {} + export interface DeleteError { readonly key: string; readonly code: string; @@ -141,7 +233,13 @@ export type BackendError = | AccessDenied | NoSuchKey | BucketNotEmpty - | DeleteObjectsError; + | DeleteObjectsError + | NoSuchUpload + | InvalidPart + | InvalidPartOrder + | EntityTooSmall + | InvalidRequest + | MalformedXML; export interface BackendService { readonly listBuckets: () => Effect.Effect< @@ -171,9 +269,11 @@ export interface BackendService { }) => Effect.Effect; readonly getObject: ( key: string, + headers: Record, ) => Effect.Effect; readonly headObject: ( key: string, + headers: Record, ) => Effect.Effect; readonly putObject: ( key: string, @@ -184,6 +284,39 @@ export interface BackendService { readonly deleteObjects: ( objects: readonly { key: string; versionId?: string }[], ) => Effect.Effect; + + // Multipart Upload + readonly createMultipartUpload: ( + key: string, + headers: Record, + ) => Effect.Effect; + readonly uploadPart: ( + key: string, + uploadId: string, + partNumber: number, + body: Stream.Stream, + ) => Effect.Effect; + readonly completeMultipartUpload: ( + key: string, + uploadId: string, + parts: readonly { etag: string; partNumber: number }[], + ) => Effect.Effect; + readonly abortMultipartUpload: ( + key: string, + uploadId: string, + ) => Effect.Effect; + readonly listMultipartUploads: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + uploadIdMarker?: string; + maxUploads?: number; + encodingType?: string; + }) => Effect.Effect; + readonly listParts: ( + key: string, + uploadId: string, + ) => Effect.Effect; } /** diff --git a/src/Services/S3Xml.ts b/src/Services/S3Xml.ts index 444cfb5..991c657 100644 --- a/src/Services/S3Xml.ts +++ b/src/Services/S3Xml.ts @@ -6,10 +6,18 @@ import { BucketAlreadyOwnedByYou, type BucketInfo, BucketNotEmpty, + EntityTooSmall, InternalError, + InvalidPart, + InvalidPartOrder, + InvalidRequest, + type ListMultipartUploadsResult, type ListObjectsResult, + type ListPartsResult, + MalformedXML, NoSuchBucket, NoSuchKey, + NoSuchUpload, type OwnerInfo, } from "./Backend.ts"; @@ -30,6 +38,25 @@ export class S3Xml extends Context.Tag("S3Xml")< readonly formatListVersions: ( result: ListObjectsResult, ) => HttpServerResponse.HttpServerResponse; + readonly formatListMultipartUploads: ( + result: ListMultipartUploadsResult, + ) => HttpServerResponse.HttpServerResponse; + readonly formatInitiateMultipartUpload: ( + bucket: string, + key: string, + uploadId: string, + ) => HttpServerResponse.HttpServerResponse; + readonly formatCompleteMultipartUpload: ( + result: { + location: string; + bucket: string; + key: string; + etag: string; + }, + ) => HttpServerResponse.HttpServerResponse; + readonly formatListParts: ( + result: ListPartsResult, + ) => HttpServerResponse.HttpServerResponse; } >() {} @@ -66,6 +93,30 @@ export const S3XmlLive = Layer.succeed( code = "BucketNotEmpty"; message = e.message; status = 409; + } else if (e instanceof NoSuchUpload) { + code = "NoSuchUpload"; + message = e.message; + status = 404; + } else if (e instanceof InvalidPart) { + code = "InvalidPart"; + message = e.message; + status = 400; + } else if (e instanceof InvalidPartOrder) { + code = "InvalidPartOrder"; + message = e.message; + status = 400; + } else if (e instanceof EntityTooSmall) { + code = "EntityTooSmall"; + message = e.message; + status = 400; + } else if (e instanceof InvalidRequest) { + code = "InvalidRequest"; + message = e.message; + status = 400; + } else if (e instanceof MalformedXML) { + code = "MalformedXML"; + message = e.message; + status = 400; } else if (e instanceof InternalError) { code = "InternalError"; message = e.message; @@ -260,5 +311,67 @@ export const S3XmlLive = Layer.succeed( }, }); }, + + formatListMultipartUploads: (result) => { + const uploadsXml = result.uploads.map((u) => + `${u.key}${u.uploadId}${u.initiator.id}${u.initiator.displayName}${u.owner.id}${u.owner.displayName}${u.storageClass}${u.initiated.toISOString()}` + ).join(""); + + const commonPrefixesXml = result.commonPrefixes.map((cp) => + `${cp.prefix}` + ).join(""); + + const xml = + `${result.bucket}${ + result.keyMarker ?? "" + }${ + result.uploadIdMarker ?? "" + }${ + result.nextKeyMarker ?? "" + }${ + result.nextUploadIdMarker ?? "" + }${result.maxUploads}${result.isTruncated}${uploadsXml}${commonPrefixesXml}`; + + return HttpServerResponse.text(xml, { + headers: { "Content-Type": "application/xml" }, + }); + }, + + formatInitiateMultipartUpload: (bucket, key, uploadId) => { + const xml = + `${bucket}${key}${uploadId}`; + + return HttpServerResponse.text(xml, { + headers: { + "Content-Type": "application/xml", + }, + }); + }, + + formatCompleteMultipartUpload: (result) => { + const xml = + `${result.location}${result.bucket}${result.key}${result.etag}`; + + return HttpServerResponse.text(xml, { + headers: { + "Content-Type": "application/xml", + }, + }); + }, + + formatListParts: (result) => { + const partsXml = result.parts.map((p) => + `${p.partNumber}${p.lastModified.toISOString()}${p.etag}${p.size}` + ).join(""); + + const xml = + `${result.bucket}${result.key}${result.uploadId}${result.initiator.id}${result.initiator.displayName}${result.owner.id}${result.owner.displayName}${result.storageClass}${result.partNumberMarker}${result.nextPartNumberMarker}${result.maxParts}${result.isTruncated}${partsXml}`; + + return HttpServerResponse.text(xml, { + headers: { + "Content-Type": "application/xml", + }, + }); + }, }), ); diff --git a/tests/integration/__snapshots__/objects.test.ts.snap b/tests/integration/__snapshots__/objects.test.ts.snap index ba27552..798fa82 100644 --- a/tests/integration/__snapshots__/objects.test.ts.snap +++ b/tests/integration/__snapshots__/objects.test.ts.snap @@ -1,6 +1,6 @@ export const snapshot = {}; -snapshot[`Baseline/objects/put metadata 1`] = ` +snapshot[`Baseline/objects/multipart/basic metadata 1`] = ` { headers: { "accept-ranges": "bytes", @@ -13,52 +13,18 @@ snapshot[`Baseline/objects/put metadata 1`] = ` } `; -snapshot[`Proxy/objects/put metadata 1`] = ` +snapshot[`Proxy/objects/multipart/basic metadata 1`] = ` { headers: {}, status: 204, } `; -snapshot[`Swift/objects/put metadata 1`] = ` -{ - headers: {}, - status: 204, -} -`; - -snapshot[`Baseline/objects/get/existing metadata 1`] = ` -{ - headers: { - "accept-ranges": "bytes", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 204, -} -`; - -snapshot[`Proxy/objects/get/existing metadata 1`] = ` -{ - headers: {}, - status: 204, -} -`; - -snapshot[`Swift/objects/get/existing metadata 1`] = ` -{ - headers: {}, - status: 204, -} -`; - -snapshot[`Baseline/objects/get/non-existent metadata 1`] = ` +snapshot[`Baseline/objects/multipart/abort metadata 1`] = ` { headers: { "accept-ranges": "bytes", - "content-length": "359", + "content-length": "479", "content-type": "application/xml", "strict-transport-security": "max-age=31536000; includeSubDomains", "x-content-type-options": "nosniff", @@ -69,24 +35,12 @@ snapshot[`Baseline/objects/get/non-existent metadata 1`] = ` } `; -snapshot[`Baseline/objects/get/non-existent body 1`] = ` +snapshot[`Baseline/objects/multipart/abort body 1`] = ` ' -NoSuchKeyThe specified key does not exist.no-suchtest-objects-bucket/test-objects-bucket/no-suchIDHOST' -`; - -snapshot[`Proxy/objects/get/non-existent metadata 1`] = ` -{ - headers: { - "content-type": "application/xml", - vary: "Accept-Encoding", - }, - status: 404, -} +NoSuchUploadThe specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.multipart-abort.txttest-objects-bucket/test-objects-bucket/multipart-abort.txtIDHOST' `; -snapshot[`Proxy/objects/get/non-existent body 1`] = `'NoSuchKeyThe specified key does not exist.'`; - -snapshot[`Swift/objects/get/non-existent metadata 1`] = ` +snapshot[`Proxy/objects/multipart/abort metadata 1`] = ` { headers: { "content-type": "application/xml", @@ -96,66 +50,9 @@ snapshot[`Swift/objects/get/non-existent metadata 1`] = ` } `; -snapshot[`Swift/objects/get/non-existent body 1`] = `'NoSuchKeyNot Found'`; - -snapshot[`Baseline/objects/head/existing metadata 1`] = ` -{ - headers: { - "accept-ranges": "bytes", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 204, -} -`; - -snapshot[`Proxy/objects/head/existing metadata 1`] = ` -{ - headers: {}, - status: 204, -} -`; - -snapshot[`Swift/objects/head/existing metadata 1`] = ` -{ - headers: {}, - status: 204, -} -`; - -snapshot[`Baseline/objects/head/non-existent metadata 1`] = ` -{ - headers: { - "accept-ranges": "bytes", - "content-length": "0", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "x-content-type-options": "nosniff", - "x-minio-error-code": "NoSuchKey", - "x-minio-error-desc": '"The specified key does not exist."', - "x-xss-protection": "1; mode=block", - vary: "Origin, Accept-Encoding", - }, - status: 404, -} -`; - -snapshot[`Proxy/objects/head/non-existent metadata 1`] = ` -{ - headers: {}, - status: 404, -} -`; - -snapshot[`Swift/objects/head/non-existent metadata 1`] = ` -{ - headers: {}, - status: 404, -} -`; +snapshot[`Proxy/objects/multipart/abort body 1`] = `'NoSuchUploadThe specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.'`; -snapshot[`Baseline/objects/delete/existing metadata 1`] = ` +snapshot[`Baseline/objects/multipart/list-parts metadata 1`] = ` { headers: { "accept-ranges": "bytes", @@ -168,14 +65,14 @@ snapshot[`Baseline/objects/delete/existing metadata 1`] = ` } `; -snapshot[`Proxy/objects/delete/existing metadata 1`] = ` +snapshot[`Proxy/objects/multipart/list-parts metadata 1`] = ` { headers: {}, status: 204, } `; -snapshot[`Swift/objects/delete/existing metadata 1`] = ` +snapshot[`Proxy/objects/multipart/empty metadata 1`] = ` { headers: {}, status: 204, diff --git a/tests/integration/objects.test.ts b/tests/integration/objects.test.ts index e2799f1..d419bfa 100644 --- a/tests/integration/objects.test.ts +++ b/tests/integration/objects.test.ts @@ -1,12 +1,17 @@ import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, CreateBucketCommand, + CreateMultipartUploadCommand, DeleteBucketCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, + ListPartsCommand, PutObjectCommand, type S3Client, S3ServiceException, + UploadPartCommand, } from "@aws-sdk/client-s3"; import { harness, type ProxyTestCase } from "../utils.ts"; import type { GlobalConfig } from "../../src/Domain/Config.ts"; @@ -121,6 +126,180 @@ const specs: ObjectTestSpec[] = [ ); }, }, + { + name: "objects/multipart/basic", + fn: async (c) => { + const key = "multipart-basic.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + const partSize = 5 * 1024 * 1024 + 1; + const body1 = new Uint8Array(partSize).fill(97); // 'a' + const body2 = new Uint8Array(10).fill(98); // 'b' + + const { ETag: etag1 } = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: body1, + }), + ); + const { ETag: etag2 } = await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 2, + Body: body2, + }), + ); + + await c.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { + Parts: [ + { ETag: etag1, PartNumber: 1 }, + { ETag: etag2, PartNumber: 2 }, + ], + }, + }), + ); + + const { ContentLength } = await c.send( + new HeadObjectCommand({ Bucket: BUCKET, Key: key }), + ); + if (ContentLength !== partSize + 10) { + throw new Error( + `Size mismatch: expected ${partSize + 10}, got ${ContentLength}`, + ); + } + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ + Bucket: BUCKET, + Key: "multipart-basic.txt", + }), + ); + } catch { /* ignore */ } + }, + }, + { + name: "objects/multipart/abort", + fn: async (c) => { + const key = "multipart-abort.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: "part 1", + }), + ); + + await c.send( + new AbortMultipartUploadCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + + try { + await c.send( + new ListPartsCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + throw new Error("ListParts should have failed after Abort"); + } catch (e) { + if (!(e instanceof S3ServiceException && e.name === "NoSuchUpload")) { + throw e; + } + } + }, + }, + { + name: "objects/multipart/list-parts", + fn: async (c) => { + const key = "multipart-list.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + await c.send( + new UploadPartCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + PartNumber: 1, + Body: "part 1", + }), + ); + + const { Parts } = await c.send( + new ListPartsCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + + if (!Parts || Parts.length !== 1 || Parts[0].PartNumber !== 1) { + throw new Error(`Unexpected parts list: ${JSON.stringify(Parts)}`); + } + + await c.send( + new AbortMultipartUploadCommand({ Bucket: BUCKET, Key: key, UploadId }), + ); + }, + }, + { + name: "objects/multipart/empty", + fn: async (c) => { + const key = "multipart-empty.txt"; + const { UploadId } = await c.send( + new CreateMultipartUploadCommand({ Bucket: BUCKET, Key: key }), + ); + if (!UploadId) throw new Error("No UploadId"); + + try { + await c.send( + new CompleteMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + MultipartUpload: { Parts: [] }, + }), + ); + throw new Error("Complete should have failed for empty parts"); + } catch (e) { + if (!(e instanceof S3ServiceException && e.name === "MalformedXML")) { + // AWS S3 returns MalformedXML for empty parts list + // Some other implementations might return InvalidPart + if (e instanceof S3ServiceException && e.name === "InvalidPart") { + return; + } + throw e; + } + } finally { + try { + await c.send( + new AbortMultipartUploadCommand({ + Bucket: BUCKET, + Key: key, + UploadId, + }), + ); + } catch { /* ignore */ } + } + }, + }, ]; async function runObjectTest(tc: ObjectTestSpec, client: S3Client) { diff --git a/x/s3-tests.ts b/x/s3-tests.ts index 9c9bc41..d4e958e 100755 --- a/x/s3-tests.ts +++ b/x/s3-tests.ts @@ -179,9 +179,6 @@ const program = Effect.gen(function* () { const FileLoggingLive = Logger.replace( Logger.defaultLogger, Logger.make(({ message, logLevel: currentLogLevel }) => { - if (currentLogLevel.syslog > minLogLevel.syslog) { - return; - } const timestamp = new Date().toISOString(); const level = currentLogLevel.label; const msg = typeof message === "string" ? message : String(message); From f3f9a4d94a131d17c289254d5e18efae39b506c9 Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Tue, 20 Jan 2026 09:02:20 +0300 Subject: [PATCH 2/2] refactor: code quality cleanup (#81) --- .github/workflows/checks.yml | 94 ++ .github/workflows/pre-commit.yml | 24 - .github/workflows/release-request.yml | 159 --- .github/workflows/tests.yml | 119 --- .pre-commit-config.yaml | 9 +- AGENTS.md | 2 + CONTRIBUTING.md | 4 +- chart/values.yaml | 7 +- deno.jsonc | 5 + deno.lock | 2 + flake.nix | 4 +- src/Api.ts | 13 +- src/Backends/S3/Backend.ts | 998 +----------------- src/Backends/S3/Buckets.ts | 74 ++ src/Backends/S3/Client.ts | 174 ++- src/Backends/S3/Objects.ts | 757 +++++++++++++ src/Backends/S3/Utils.ts | 147 +++ src/Backends/Swift/Backend.ts | 667 +----------- src/Backends/Swift/Buckets.ts | 143 +++ src/Backends/Swift/Client.ts | 289 +++-- src/Backends/Swift/Objects.ts | 529 ++++++++++ src/Backends/Swift/Utils.ts | 74 ++ src/Config/Layer.ts | 8 +- src/Domain/Config.ts | 2 +- src/Frontend/Api.ts | 2 +- src/Frontend/Buckets/Create.ts | 63 +- src/Frontend/Buckets/Delete.ts | 16 +- src/Frontend/Buckets/Head.ts | 16 +- src/Frontend/Buckets/List.ts | 4 +- src/Frontend/Health/Api.ts | 2 +- src/Frontend/Health/Http.ts | 4 +- src/Frontend/Http.ts | 30 +- src/Frontend/Objects/Delete.ts | 33 +- src/Frontend/Objects/Get.ts | 55 +- src/Frontend/Objects/Head.ts | 38 +- src/Frontend/Objects/List.ts | 90 +- src/Frontend/Objects/Post.ts | 258 +++-- src/Frontend/Objects/Put.ts | 60 +- src/Frontend/Utils.ts | 127 ++- src/Http.ts | 19 +- src/Logging/Layer.ts | 4 +- src/Services/Backend.ts | 4 + src/Services/BackendResolver.ts | 125 ++- src/Services/S3Xml.ts | 3 + src/main.ts | 10 +- tests/config.test.ts | 8 +- tests/health.test.ts | 16 +- .../__snapshots__/buckets.test.ts.snap | 4 +- .../__snapshots__/objects.test.ts.snap | 195 ++++ tests/integration/objects.test.ts | 14 +- tests/utils.ts | 10 +- tools/compose.yml | 2 + x/compose-down.ts | 4 +- x/compose-up.ts | 4 +- x/purge-minio.ts | 1 - x/swift-debug.ts | 7 +- 56 files changed, 2876 insertions(+), 2656 deletions(-) create mode 100644 .github/workflows/checks.yml delete mode 100644 .github/workflows/pre-commit.yml delete mode 100644 .github/workflows/release-request.yml delete mode 100644 .github/workflows/tests.yml create mode 100644 src/Backends/S3/Buckets.ts create mode 100644 src/Backends/S3/Objects.ts create mode 100644 src/Backends/S3/Utils.ts create mode 100644 src/Backends/Swift/Buckets.ts create mode 100644 src/Backends/Swift/Objects.ts create mode 100644 src/Backends/Swift/Utils.ts diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..13c559e --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,94 @@ +name: checks + +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + +env: + DOCKER_CMD: docker + UV_CACHE_DIR: /tmp/.uv-cache + +jobs: + checks: + runs-on: ubuntu-latest + env: + HERALD_SWIFTTEST_OS_USERNAME: ${{ secrets.OPENSTACK_USERNAME }} + HERALD_SWIFTTEST_OS_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD }} + HERALD_SWIFTTEST_OS_PROJECT_NAME: ${{ secrets.OPENSTACK_PROJECT }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v16 + + - name: Set up Nix cache + uses: DeterminateSystems/magic-nix-cache-action@v9 + + - name: Run pre-commit hooks via prek + run: nix develop --command prek run --all-files + + - name: Cache Deno + uses: actions/cache@v4 + with: + path: ~/.cache/deno + key: ${{ runner.os }}-deno-${{ hashFiles('deno.lock') }} + restore-keys: | + ${{ runner.os }}-deno- + + - name: Restore uv cache + uses: actions/cache@v5 + with: + path: /tmp/.uv-cache + key: uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }} + restore-keys: | + uv-${{ runner.os }}-${{ hashFiles('s3-tests/requirements.txt') }} + uv-${{ runner.os }} + + - name: Start services + run: nix develop --command deno run --allow-all x/compose-up.ts s3 db + + - name: Wait for MinIO + run: | + for i in {1..30}; do + if curl -f http://localhost:9000/minio/health/live; then + echo "MinIO is ready" + exit 0 + fi + echo "Waiting for MinIO..." + sleep 2 + done + echo "MinIO failed to start" + exit 1 + + - name: Integration tests + run: nix develop --command deno task test + + - name: S3 Compatibility (MinIO) + run: nix develop --command deno run --allow-all x/s3-tests.ts --backend minio + + - name: S3 Compatibility (Swift) + if: env.HERALD_SWIFTTEST_OS_USERNAME != '' + env: + HERALD_SWIFTTEST_OS_REGION_NAME: dc3-a + HERALD_SWIFTTEST_AUTH_URL: https://api.pub1.infomaniak.cloud/identity/v3 + run: nix develop --command deno run --allow-all x/s3-tests.ts --backend swift + + - name: Minimize uv cache + run: nix develop --command uv cache prune --ci + + - name: Dump logs on failure + if: failure() + run: | + echo "--- s3-tests/s3-tests.log ---" + cat s3-tests/s3-tests.log || true + echo "--- s3-tests/herald-proxy.log ---" + cat s3-tests/herald-proxy.log || true + echo "--- s3-tests/herald-proxy-swift.log ---" + cat s3-tests/herald-proxy-swift.log || true diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index 630ff40..0000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: pre-commit - -on: - push: - branches: [main] - pull_request: - branches: [main] - workflow_dispatch: - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install Nix - uses: DeterminateSystems/nix-installer-action@main - - - name: Set up Nix cache - uses: DeterminateSystems/magic-nix-cache-action@main - - - name: Run pre-commit hooks via prek - run: nix develop --command prek run --all-files diff --git a/.github/workflows/release-request.yml b/.github/workflows/release-request.yml deleted file mode 100644 index e11d30b..0000000 --- a/.github/workflows/release-request.yml +++ /dev/null @@ -1,159 +0,0 @@ -name: Prepare Release - -on: - workflow_dispatch: - push: - branches: - - main - -jobs: - check-version: - name: Check Commitizen Version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.version.outputs.version }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Configure Git - run: | - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - - - name: Get current version (without bumping or pushing) - id: version - uses: commitizen-tools/commitizen-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - push: false - dry_run: true - changelog: false - - prepare-release-pr: - name: Create Release Branch and PR - needs: check-version - if: ${{ needs.check-version.outputs.version != '' && github.ref == 'refs/heads/main' }} - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: main - - - name: Bump version using Commitizen - id: cz - uses: commitizen-tools/commitizen-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - git_name: ${{ github.actor }} - git_email: ${{ github.actor }}@users.noreply.github.com - push: false - changelog: true - dry_run: false - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v8 - with: - title: "Release ${{ steps.cz.outputs.version }}" - body: "Automated PR for version bump to ${{ steps.cz.outputs.version }}" - branch: "release-v${{ steps.cz.outputs.version }}" - delete-branch: true - - check-release: - runs-on: ubuntu-latest - # if: github.ref == 'refs/heads/main' && github.event_name == 'push' - outputs: - release: ${{ steps.check.outputs.release }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: main - - - name: Configure Git - run: | - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - - - name: Get current version - id: version - run: | - VERSION=$(yq '.commitizen.version' .cz.yaml) - echo "version=$VERSION" >> $GITHUB_OUTPUT - - - name: Check if GitHub release already exists - id: check - run: | - VERSION=${{ steps.version.outputs.version }} - echo "Detected version: $VERSION" - - RELEASE_EXISTS=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - https://api.github.com/repos/${{ github.repository }}/releases/tags/v$VERSION \ - | jq -r '.tag_name // empty') - - if [[ "$RELEASE_EXISTS" == "v$VERSION" ]]; then - echo "Release v$VERSION already exists." - echo "release=" >> $GITHUB_OUTPUT - else - echo "Release v$VERSION does not exist yet." - echo "release=$VERSION" >> $GITHUB_OUTPUT - fi - finalize-release: - name: Finalize Release - needs: check-release - if: ${{ needs.check-release.outputs.release != '' }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Tag and Push - run: | - git config user.name "${{ github.actor }}" - git config user.email "${{ github.actor }}@users.noreply.github.com" - git tag -a "v${{ needs.check-release.outputs.release }}" -m "Release v${{ needs.check-release.outputs.release }}" - git push origin "v${{ needs.check-release.outputs.release }}" - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - tag_name: "v${{ needs.check-release.outputs.release }}" - name: "Release v${{ needs.check-release.outputs.release }}" - body_path: "CHANGELOG.md" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - build-docker: - name: Build and Push Docker - needs: check-release - if: ${{ needs.check-release.outputs.release != '' }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and Push Docker - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ghcr.io/${{ github.repository_owner }}/herald:v${{ needs.check-release.outputs.release }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index f09fed5..0000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: test suite -run-name: test suite for ${{ github.event.pull_request.title || github.ref }} -on: - workflow_dispatch: - push: - branches: - - main - pull_request: - types: - - opened - - reopened - - synchronize - - ready_for_review - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - DENO_V: 2.3.5 - GHJK_VERSION: "v0.3.2" - GHJK_ENV: "ci" - -jobs: - changes: - runs-on: ubuntu-latest - permissions: - pull-requests: read - steps: - - uses: actions/checkout@v6 - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - full: - - '.github/workflows/tests.yml' - - 'src/**' - - 'tests/**' - - 'examples/**' - outputs: - full: ${{ steps.filter.outputs.full }} - - pre-commit: - needs: changes - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - uses: denoland/setup-deno@v2 - with: - deno-version: ${{ env.DENO_V }} - - name: Install tofu - run: | - curl --proto '=https' --tlsv1.2 -fsSL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh - chmod +x install-opentofu.sh - ./install-opentofu.sh --install-method deb - rm -f install-opentofu.sh - - - shell: bash - run: | - python -m pip install --upgrade pip - pip install pre-commit - pre-commit install - deno --version - pre-commit run --all-files - - test-full: - needs: [changes] - if: ${{ needs.changes.outputs.full == 'true' && github.event.pull_request.draft == false }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: denoland/setup-deno@v2 - with: - deno-version: ${{ env.DENO_V }} - - name: Download Install Script - run: curl -fsSL "https://raw.github.com/metatypedev/ghjk/$GHJK_VERSION/install.sh" -o install.sh - - name: Execute Install Script - run: yes | bash install.sh - - run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - run: echo "BASH_ENV=$HOME/.local/share/ghjk/env.sh" >> "$GITHUB_ENV" - - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - name: Install tofu - run: | - curl --proto '=https' --tlsv1.2 -fsSL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh - chmod +x install-opentofu.sh - ./install-opentofu.sh --install-method deb - rm -f install-opentofu.sh - - uses: actions/setup-node@v6 - with: - node-version: 18 - - name: setup start-server-and-test - run: npm install -g start-server-and-test - - shell: bash - env: - AUTH_TYPE: "default" - LOG_LEVEL: "DEBUG" - ENV: "DEV" - S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} - S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} - OPENSTACK_USERNAME: ${{ secrets.OPENSTACK_USERNAME }} - OPENSTACK_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD }} - OPENSTACK_PROJECT: ${{ secrets.OPENSTACK_PROJECT }} - AWS_ACCESS_KEY_ID: ${{ secrets.OPENSTACK_USERNAME }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.OPENSTACK_PASSWORD }} - run: | - # run all tests - deno --version - ghjk x dev-compose s3 - sleep 20 - - deno install - - # ghjk x setup-auth - npx start-server-and-test 'deno serve -A --unstable-kv src/main.ts' http://0.0.0.0:8000/ 'deno test -A' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b600640..96b71d2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v6.0.0 hooks: - id: check-added-large-files exclude: tests/res @@ -39,9 +39,14 @@ repos: types: - ts - repo: https://github.com/tofuutils/pre-commit-opentofu - rev: v1.0.3 + rev: v2.2.2 hooks: - id: tofu_fmt + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.36.0 + hooks: + - id: check-dependabot + - id: check-github-workflows # - repo: https://github.com/shellcheck-py/shellcheck-py # rev: v0.10.0.1 # hooks: diff --git a/AGENTS.md b/AGENTS.md index fbc11d0..4410833 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,7 @@ - We're using the effects library https://effect.website/llms.txt - Their HTTP implementation is described in ./HTTP_PLATFORM.md + - **ALWAYS** use `@effect/platform/HttpClient` instead of native `fetch` for + all HTTP requests. - Prefer generators over effect piping. - Use methods on `Effect.Option` like `Option.isNone` instead of looking at _tag. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96c5f88..219781a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,8 +5,8 @@ - `src/Domain`: Core logic and data models. Contains Effect Schemas for global configuration and logic for bucket matching. -- `src/Config`: Application configuration loading. Defines the AppConfig service - layer. +- `src/Config`: Application configuration loading. Defines the HeraldConfig + service layer. - `src/Services`: Shared service abstractions and implementations. diff --git a/chart/values.yaml b/chart/values.yaml index e55ba44..1691974 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1,4 +1,3 @@ - name: herald namespace: herald @@ -87,9 +86,9 @@ ingress: - path: / pathType: ImplementationSpecific tls: [] - # - secretName: web-tls - # hosts: - # - chart-example.local +# - secretName: web-tls +# hosts: +# - chart-example.local volumeMounts: - name: herald diff --git a/deno.jsonc b/deno.jsonc index c600c50..646ee69 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -30,6 +30,11 @@ "cliffy/ansi/": "https://deno.land/x/cliffy@v1.0.0-rc.3/ansi/" }, "compilerOptions": {}, + "fmt": { + "exclude": [ + "./chart/" + ] + }, "lint": { "exclude": [ "x", diff --git a/deno.lock b/deno.lock index da816a9..766c865 100644 --- a/deno.lock +++ b/deno.lock @@ -26,12 +26,14 @@ "npm:@aws-sdk/client-s3@3": "3.937.0", "npm:@effect/opentelemetry@~0.56.2": "0.56.6_@effect+platform@0.90.10__effect@3.19.14_@opentelemetry+sdk-trace-base@2.3.0__@opentelemetry+api@1.9.0_@opentelemetry+sdk-trace-node@2.3.0__@opentelemetry+api@1.9.0_@opentelemetry+semantic-conventions@1.38.0_effect@3.19.14", "npm:@effect/platform-node@0.96": "0.96.1_@effect+cluster@0.48.16__@effect+platform@0.90.10___effect@3.19.14__@effect+rpc@0.69.5___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+sql@0.44.2___@effect+experimental@0.54.6____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+workflow@0.9.6___@effect+platform@0.90.10____effect@3.19.14___@effect+rpc@0.69.5____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___effect@3.19.14__effect@3.19.14_@effect+platform@0.90.10__effect@3.19.14_@effect+rpc@0.69.5__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_@effect+sql@0.44.2__@effect+experimental@0.54.6___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_effect@3.19.14", + "npm:@effect/platform@*": "0.90.10_effect@3.19.14", "npm:@effect/platform@~0.90.3": "0.90.10_effect@3.19.14", "npm:@opentelemetry/exporter-trace-otlp-http@0.203": "0.203.0_@opentelemetry+api@1.9.0", "npm:@opentelemetry/sdk-trace-base@^2.0.1": "2.3.0_@opentelemetry+api@1.9.0", "npm:@opentelemetry/sdk-trace-node@^2.0.1": "2.3.0_@opentelemetry+api@1.9.0", "npm:@smithy/signature-v4@^4.2.0": "4.2.4", "npm:@smithy/types@^3.7.0": "3.7.2", + "npm:effect@*": "3.19.14", "npm:effect@^3.17.7": "3.19.14", "npm:jest-diff@*": "29.7.0", "npm:jest-diff@^29.7.0": "29.7.0", diff --git a/flake.nix b/flake.nix index 1e0d33f..a1e07ab 100644 --- a/flake.nix +++ b/flake.nix @@ -41,7 +41,9 @@ ]; shellHook = '' export PATH=$PATH:$PWD/x/ - exec $(getent passwd $USER | cut -d: -f7) + if [[ -t 0 ]]; then + exec $(getent passwd $USER | cut -d: -f7) + fi ''; }; diff --git a/src/Api.ts b/src/Api.ts index 2b3afe2..b7a62ec 100644 --- a/src/Api.ts +++ b/src/Api.ts @@ -1,8 +1,11 @@ import { HttpApi, OpenApi } from "@effect/platform"; -import { HealthApi } from "./Frontend/Health/Api.ts"; -import { S3Api } from "./Frontend/Api.ts"; +import { HealthHttpApi } from "./Frontend/Health/Api.ts"; +import { HttpS3Api } from "./Frontend/Api.ts"; -export class Api extends HttpApi.make("api") - .add(HealthApi) - .add(S3Api) +// the http interface is declared first and separately +// and the impl is to adhere to it +// used for openAPI +export class HttpHeraldApi extends HttpApi.make("HeraldHttpApi") + .add(HealthHttpApi) + .add(HttpS3Api) .annotate(OpenApi.Title, "Herald API") {} diff --git a/src/Backends/S3/Backend.ts b/src/Backends/S3/Backend.ts index 3aa7309..fc930c7 100644 --- a/src/Backends/S3/Backend.ts +++ b/src/Backends/S3/Backend.ts @@ -1,988 +1,24 @@ -import { Chunk, Effect, Option, Stream } from "effect"; -import { - AbortMultipartUploadCommand, - CompleteMultipartUploadCommand, - CreateBucketCommand, - CreateMultipartUploadCommand, - DeleteBucketCommand, - DeleteObjectCommand, - DeleteObjectsCommand, - GetObjectCommand, - HeadBucketCommand, - HeadObjectCommand, - ListBucketsCommand, - type ListBucketsCommandOutput, - ListMultipartUploadsCommand, - ListObjectsCommand, - type ListObjectsCommandOutput, - ListObjectsV2Command, - type ListObjectsV2CommandOutput, - ListObjectVersionsCommand, - ListPartsCommand, - PutObjectCommand, - UploadPartCommand, -} from "@aws-sdk/client-s3"; +import { Effect } from "effect"; import type { MaterializedBucket } from "../../Domain/Config.ts"; -import { AppConfig } from "../../Config/Layer.ts"; -import { - AccessDenied, - type BackendError, - type BackendService, - BucketAlreadyExists, - BucketAlreadyOwnedByYou, - type BucketInfo, - BucketNotEmpty, - type CommonPrefix, - type DeleteObjectsResult, - EntityTooSmall, - InternalError, - InvalidPart, - InvalidPartOrder, - InvalidRequest, - type ListObjectsResult, - MalformedXML, - NoSuchBucket, - NoSuchKey, - NoSuchUpload, - type ObjectInfo, -} from "../../Services/Backend.ts"; -import { S3Client } from "./Client.ts"; - -/** - * Strips MinIO metadata suffixes like [minio_cache:v2,return:] from strings. - */ -function stripMinioMetadata(s: string): string { - return s.replace(/\[minio_cache:[^\]]+\]/g, ""); -} - -/** - * Maps S3 SDK exceptions to internal BackendError types. - */ -function mapS3Error(e: unknown, bucketName?: string): BackendError { - const err = e as { - name?: string; - Code?: string; - Message?: string; - message?: string; - $metadata?: { httpStatusCode?: number }; - }; - const name = err?.name || err?.Code || - (e instanceof Error ? e.name : "UnknownError"); - const message = err?.message || err?.Message || - "An unknown S3 error occurred"; - const bucket = bucketName ?? "unknown-bucket"; - - switch (name) { - case "NoSuchBucket": - case "NotFound": - return new NoSuchBucket({ bucketName: bucket, message }); - case "NoSuchKey": - return new NoSuchKey({ - bucketName: bucket, - key: "unknown", - message: message, - }); - case "NoSuchUpload": - return new NoSuchUpload({ - uploadId: "unknown", - message: message, - }); - case "InvalidPart": - case "InvalidPartNumber": - return new InvalidPart({ message }); - case "InvalidPartOrder": - return new InvalidPartOrder({ message }); - case "EntityTooSmall": - return new EntityTooSmall({ message }); - case "InvalidRequest": - if (message.includes("at least one part")) { - return new MalformedXML({ message }); - } - return new InvalidRequest({ message }); - case "MalformedXML": - return new MalformedXML({ message }); - case "BucketAlreadyExists": - return new BucketAlreadyExists({ bucketName: bucket, message }); - case "BucketAlreadyOwnedByYou": - return new BucketAlreadyOwnedByYou({ bucketName: bucket, message }); - case "AccessDenied": - case "Forbidden": - return new AccessDenied({ message }); - case "BucketNotEmpty": - case "Conflict": - return new BucketNotEmpty({ bucketName: bucket, message }); - } - - // Handle case where it might be a raw 404 from HEAD request - if (err?.$metadata?.httpStatusCode === 404) { - return new NoSuchKey({ - bucketName: bucket, - key: "unknown", - message: "Not Found", - }); - } - - return new InternalError({ - message: e instanceof Error ? `${e.name}: ${e.message}` : String(e), - }); -} +import type { BackendError, BackendService } from "../../Services/Backend.ts"; +import { makeBucketOps } from "./Buckets.ts"; +import { makeObjectOps } from "./Objects.ts"; +import { getTarget } from "./Utils.ts"; +import type { S3Client } from "./Client.ts"; +import type { HeraldConfig } from "../../Config/Layer.ts"; /** * Creates an S3-specific Backend implementation for a given configuration context. + * Composes bucket and object operations modularly. + * Resolves the target once per backend creation (request-scoped). */ export const makeS3Backend = ( bucket: MaterializedBucket | { backend_id: string }, -): Effect.Effect => - Effect.all({ - s3Service: S3Client, - config: AppConfig, - }).pipe( - Effect.map(({ s3Service, config }) => { - const getTargetBucket = (): MaterializedBucket => { - if ("bucket_name" in bucket) return bucket as MaterializedBucket; - - const backendConfig = config.raw.backends[bucket.backend_id]; - if (backendConfig && backendConfig.protocol === "s3") { - return { - name: "", - backend_id: bucket.backend_id, - protocol: "s3" as const, - endpoint: backendConfig.endpoint, - region: backendConfig.region, - bucket_name: "", - credentials: backendConfig.credentials, - }; - } - throw new Error(`Backend ${bucket.backend_id} is not an S3 backend`); - }; - - const targetBucket = getTargetBucket(); - - const service: BackendService = { - listBuckets: () => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send(new ListBucketsCommand({})) as Promise< - ListBucketsCommandOutput - >, - catch: (e) => mapS3Error(e, targetBucket.name), - }) - ), - Effect.flatMap((result) => { - const buckets: BucketInfo[] = []; - for (const b of (result.Buckets ?? [])) { - if (b.Name === undefined) { - return Effect.fail( - new InternalError({ - message: "S3 returned bucket without Name", - }), - ); - } - buckets.push({ - name: b.Name, - creationDate: b.CreationDate, - }); - } - - return Effect.succeed({ - buckets, - owner: { - id: result.Owner?.ID ?? "unknown-owner-id", - displayName: result.Owner?.DisplayName ?? - "unknown-owner-name", - }, - }); - }), - ), - - createBucket: () => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new CreateBucketCommand({ - Bucket: targetBucket.bucket_name, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map(() => undefined), - ), - - deleteBucket: () => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new DeleteBucketCommand({ - Bucket: targetBucket.bucket_name, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map(() => undefined), - ), - - headBucket: () => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new HeadBucketCommand({ Bucket: targetBucket.bucket_name }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map(() => undefined), - ), - - listObjects: (args) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => { - if (args.listType === 2) { - return Effect.tryPromise({ - try: () => - client.send( - new ListObjectsV2Command({ - Bucket: targetBucket.bucket_name, - Prefix: args.prefix, - Delimiter: args.delimiter, - MaxKeys: args.maxKeys, - ContinuationToken: args.continuationToken, - StartAfter: args.startAfter, - }), - ) as Promise, - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }).pipe( - Effect.map((result): ListObjectsResult => ({ - name: result.Name ?? targetBucket.bucket_name, - prefix: result.Prefix, - maxKeys: result.MaxKeys ?? 1000, - delimiter: result.Delimiter, - isTruncated: result.IsTruncated ?? false, - encodingType: args.encodingType, - continuationToken: result.ContinuationToken, - nextContinuationToken: result.NextContinuationToken, - keyCount: result.KeyCount, - listType: 2, - contents: (result.Contents ?? []).map((c): ObjectInfo => ({ - key: stripMinioMetadata(c.Key ?? ""), - lastModified: c.LastModified ?? new Date(), - etag: c.ETag ?? "", - size: c.Size ?? 0, - storageClass: c.StorageClass, - owner: c.Owner - ? { - id: c.Owner.ID ?? "unknown", - displayName: c.Owner.DisplayName ?? "unknown", - } - : undefined, - })), - commonPrefixes: (result.CommonPrefixes ?? []).map(( - cp, - ): CommonPrefix => ({ - prefix: stripMinioMetadata(cp.Prefix ?? ""), - })), - })), - ); - } else { - return Effect.tryPromise({ - try: () => - client.send( - new ListObjectsCommand({ - Bucket: targetBucket.bucket_name, - Prefix: args.prefix, - Delimiter: args.delimiter, - Marker: args.marker, - MaxKeys: args.maxKeys, - }), - ) as Promise, - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }).pipe( - Effect.map((result): ListObjectsResult => ({ - name: result.Name ?? targetBucket.bucket_name, - prefix: result.Prefix, - marker: result.Marker, - nextMarker: result.NextMarker, - maxKeys: result.MaxKeys ?? 1000, - delimiter: result.Delimiter, - isTruncated: result.IsTruncated ?? false, - encodingType: args.encodingType, - listType: 1, - contents: (result.Contents ?? []).map((c): ObjectInfo => ({ - key: stripMinioMetadata(c.Key ?? ""), - lastModified: c.LastModified ?? new Date(), - etag: c.ETag ?? "", - size: c.Size ?? 0, - storageClass: c.StorageClass, - owner: c.Owner - ? { - id: c.Owner.ID ?? "unknown", - displayName: c.Owner.DisplayName ?? "unknown", - } - : undefined, - })), - commonPrefixes: (result.CommonPrefixes ?? []).map(( - cp, - ): CommonPrefix => ({ - prefix: stripMinioMetadata(cp.Prefix ?? ""), - })), - })), - ); - } - }), - ), - - listVersions: (args) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new ListObjectVersionsCommand({ - Bucket: targetBucket.bucket_name, - Prefix: args.prefix, - Delimiter: args.delimiter, - KeyMarker: args.keyMarker, - VersionIdMarker: args.versionIdMarker, - MaxKeys: args.maxKeys, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map((result): ListObjectsResult => ({ - name: result.Name ?? targetBucket.bucket_name, - prefix: result.Prefix, - marker: result.KeyMarker, - nextMarker: result.NextKeyMarker, - maxKeys: result.MaxKeys ?? 1000, - delimiter: result.Delimiter, - isTruncated: result.IsTruncated ?? false, - encodingType: args.encodingType, - listType: 1, // listVersions is similar to V1 - contents: [ - ...(result.Versions ?? []).map((v): ObjectInfo => ({ - key: stripMinioMetadata(v.Key ?? ""), - lastModified: v.LastModified ?? new Date(), - etag: v.ETag ?? "", - size: v.Size ?? 0, - storageClass: v.StorageClass, - versionId: v.VersionId, - isDeleteMarker: false, - isLatest: v.IsLatest, - owner: v.Owner - ? { - id: v.Owner.ID ?? "unknown", - displayName: v.Owner.DisplayName ?? "unknown", - } - : undefined, - })), - ...(result.DeleteMarkers ?? []).map((dm): ObjectInfo => ({ - key: stripMinioMetadata(dm.Key ?? ""), - lastModified: dm.LastModified ?? new Date(), - etag: "", - size: 0, - versionId: dm.VersionId, - isDeleteMarker: true, - isLatest: dm.IsLatest, - owner: dm.Owner - ? { - id: dm.Owner.ID ?? "unknown", - displayName: dm.Owner.DisplayName ?? "unknown", - } - : undefined, - })), - ], - commonPrefixes: (result.CommonPrefixes ?? []).map(( - cp, - ): CommonPrefix => ({ - prefix: stripMinioMetadata(cp.Prefix ?? ""), - })), - })), - ), - - getObject: (key, headers) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new GetObjectCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - Range: (headers["range"] || headers["Range"]) as string, - PartNumber: (headers["part-number"] || - headers["Part-Number"] || - headers["x-amz-part-number"]) - ? parseInt( - (headers["part-number"] || - headers["Part-Number"] || - headers["x-amz-part-number"]) as string, - ) - : undefined, - IfMatch: - (headers["if-match"] || headers["If-Match"]) as string, - IfNoneMatch: (headers["if-none-match"] || - headers["If-None-Match"]) as string, - IfModifiedSince: (headers["if-modified-since"] || - headers["If-Modified-Since"]) - ? new Date( - (headers["if-modified-since"] || - headers["If-Modified-Since"]) as string, - ) - : undefined, - IfUnmodifiedSince: (headers["if-unmodified-since"] || - headers["If-Unmodified-Since"]) - ? new Date( - (headers["if-unmodified-since"] || - headers["If-Unmodified-Since"]) as string, - ) - : undefined, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.flatMap((result) => { - const body = result.Body; - if (!body) { - return Effect.fail( - new InternalError({ - message: "S3 returned empty body for GetObject", - }), - ); - } - - // AWS SDK Body can be many things. In Deno/Browser it has transformToWebStream() - // Use a type-safe check to avoid 'any' - const getWebStream = (): ReadableStream => { - if ( - body && typeof body === "object" && - "transformToWebStream" in body - ) { - const b = body as { transformToWebStream: unknown }; - if (typeof b.transformToWebStream === "function") { - return b.transformToWebStream() as ReadableStream< - Uint8Array - >; - } - } - return body as ReadableStream; - }; - - const stream = Stream.fromReadableStream( - getWebStream, - (e) => new Error(String(e)), - ); - - const metadata: Record = {}; - if (result.Metadata) { - for (const [k, v] of Object.entries(result.Metadata)) { - metadata[k] = Option.liftThrowable(decodeURIComponent)( - v ?? "", - ).pipe( - Option.getOrElse(() => v ?? ""), - ); - } - } - - const headers: Record = {}; - if (result.ContentType) { - headers["content-type"] = result.ContentType; - } - if (result.ContentLength !== undefined) { - headers["content-length"] = String(result.ContentLength); - } - if (result.ETag) headers["etag"] = result.ETag; - if (result.PartsCount !== undefined) { - headers["x-amz-mp-parts-count"] = String(result.PartsCount); - } - if (result.VersionId) { - headers["x-amz-version-id"] = result.VersionId; - } - if (result.LastModified) { - headers["last-modified"] = result.LastModified.toUTCString(); - } - - for (const [k, v] of Object.entries(metadata)) { - headers[`x-amz-meta-${k}`] = v; - } - - // Buffer the entire stream to ensure it's fully read and connection is closed - // This also addresses issues where the SDK's Body might not be a standard ReadableStream - return Stream.runCollect(stream).pipe( - Effect.mapError((e) => - new InternalError({ message: String(e) }) - ), - Effect.map((chunks) => { - const totalLength = Chunk.reduce( - chunks, - 0, - (acc, chunk) => acc + chunk.length, - ); - const all = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of chunks) { - all.set(chunk, offset); - offset += chunk.length; - } - return { - stream: Stream.succeed(all), - contentType: result.ContentType, - contentLength: all.length, - etag: result.ETag, - lastModified: result.LastModified, - metadata, - headers, - }; - }), - ); - }), - ), - - headObject: (key, headers) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => { - const commandInput = { - Bucket: targetBucket.bucket_name, - Key: key, - PartNumber: (headers["part-number"] || - headers["Part-Number"] || - headers["x-amz-part-number"]) - ? parseInt( - (headers["part-number"] || - headers["Part-Number"] || - headers["x-amz-part-number"]) as string, - ) - : undefined, - }; - return Effect.tryPromise({ - try: () => client.send(new HeadObjectCommand(commandInput)), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }); - }), - Effect.map((result) => { - const metadata: Record = {}; - if (result.Metadata) { - for (const [k, v] of Object.entries(result.Metadata)) { - metadata[k] = Option.liftThrowable(decodeURIComponent)( - v ?? "", - ).pipe( - Option.getOrElse(() => v ?? ""), - ); - } - } - - const headers: Record = {}; - if (result.ContentType) { - headers["content-type"] = result.ContentType; - } - if (result.ContentLength !== undefined) { - headers["content-length"] = String(result.ContentLength); - } - if (result.ETag) headers["etag"] = result.ETag; - if (result.PartsCount !== undefined) { - headers["x-amz-mp-parts-count"] = String(result.PartsCount); - } - if (result.VersionId) { - headers["x-amz-version-id"] = result.VersionId; - } - if (result.LastModified) { - headers["last-modified"] = result - .LastModified.toUTCString(); - } - - for (const [k, v] of Object.entries(metadata)) { - headers[`x-amz-meta-${k}`] = v; - } - - return { - contentType: result.ContentType, - contentLength: result.ContentLength, - etag: result.ETag, - lastModified: result.LastModified, - metadata, - headers, - }; - }), - ), - - putObject: (key, bodyStream, headers) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Stream.runCollect(bodyStream).pipe( - Effect.mapError((e) => - new InternalError({ message: String(e) }) - ), - Effect.flatMap((chunks) => { - const totalLength = Chunk.reduce( - chunks, - 0, - (acc, chunk) => acc + chunk.length, - ); - const body = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of chunks) { - body.set(chunk, offset); - offset += chunk.length; - } - - const metadata: Record = {}; - for (const [k, v] of Object.entries(headers)) { - if (k.toLowerCase().startsWith("x-amz-meta-")) { - const metaKey = k.substring("x-amz-meta-".length); - const value = String(v); - metadata[metaKey] = /[^\x20-\x7E]/.test(value) - ? encodeURIComponent(value) - : value; - } - } - - const contentType = headers["content-type"]; - - return Effect.tryPromise({ - try: () => - client.send( - new PutObjectCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - Body: body, - ContentType: contentType - ? String(contentType) - : undefined, - Metadata: metadata, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }); - }), - ) - ), - Effect.map((result) => ({ - etag: result.ETag, - versionId: result.VersionId, - })), - ), - - deleteObject: (key) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new DeleteObjectCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map(() => undefined), - ), - - deleteObjects: ( - objects, - ): Effect.Effect => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new DeleteObjectsCommand({ - Bucket: targetBucket.bucket_name, - Delete: { - Objects: objects.map((o) => ({ - Key: o.key, - VersionId: o.versionId === "null" - ? undefined - : o.versionId, - })), - }, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map((result) => ({ - deleted: (result.Deleted ?? []).map((d) => d.Key ?? ""), - errors: (result.Errors ?? []).map((e) => ({ - key: e.Key ?? "unknown", - code: e.Code ?? "InternalError", - message: e.Message ?? "Unknown error", - })), - })), - ), - - createMultipartUpload: (key, headers) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => { - const metadata: Record = {}; - for (const [k, v] of Object.entries(headers)) { - if (k.toLowerCase().startsWith("x-amz-meta-")) { - const metaKey = k.substring("x-amz-meta-".length); - metadata[metaKey] = String(v); - } - } - const contentType = headers["content-type"]; - - return Effect.tryPromise({ - try: () => - client.send( - new CreateMultipartUploadCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - Metadata: metadata, - ContentType: contentType - ? String(contentType) - : undefined, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }); - }), - Effect.flatMap((result) => { - if (!result.UploadId) { - return Effect.fail( - new InternalError({ - message: "S3 returned empty UploadId", - }), - ); - } - return Effect.succeed({ uploadId: result.UploadId }); - }), - ), - - uploadPart: (key, uploadId, partNumber, bodyStream) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Stream.runCollect(bodyStream).pipe( - Effect.mapError((e) => - new InternalError({ message: String(e) }) - ), - Effect.flatMap((chunks) => { - const totalLength = Chunk.reduce( - chunks, - 0, - (acc, chunk) => acc + chunk.length, - ); - const body = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of chunks) { - body.set(chunk, offset); - offset += chunk.length; - } - - return Effect.tryPromise({ - try: () => - client.send( - new UploadPartCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - UploadId: uploadId, - PartNumber: partNumber, - Body: body, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }); - }), - ) - ), - Effect.flatMap((result) => { - if (!result.ETag) { - return Effect.fail( - new InternalError({ - message: "S3 returned empty ETag for UploadPart", - }), - ); - } - return Effect.succeed({ etag: result.ETag }); - }), - ), - - completeMultipartUpload: (key, uploadId, parts) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new CompleteMultipartUploadCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - UploadId: uploadId, - MultipartUpload: { - Parts: parts.map((p) => ({ - ETag: p.etag, - PartNumber: p.partNumber, - })), - }, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.flatMap((result) => { - if ( - !result.Location || !result.Bucket || !result.Key || - !result.ETag - ) { - return Effect.fail( - new InternalError({ - message: - "S3 returned incomplete CompleteMultipartUploadResult", - }), - ); - } - return Effect.succeed({ - location: result.Location, - bucket: result.Bucket, - key: result.Key, - etag: result.ETag, - versionId: result.VersionId, - }); - }), - ), - - abortMultipartUpload: (key, uploadId) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new AbortMultipartUploadCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - UploadId: uploadId, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map(() => undefined), - ), - - listMultipartUploads: (args) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new ListMultipartUploadsCommand({ - Bucket: targetBucket.bucket_name, - Prefix: args.prefix, - Delimiter: args.delimiter, - KeyMarker: args.keyMarker, - UploadIdMarker: args.uploadIdMarker, - MaxUploads: args.maxUploads, - EncodingType: args.encodingType as "url" | undefined, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map((result) => ({ - bucket: result.Bucket ?? targetBucket.bucket_name, - prefix: result.Prefix, - keyMarker: result.KeyMarker, - uploadIdMarker: result.UploadIdMarker, - nextKeyMarker: result.NextKeyMarker, - nextUploadIdMarker: result.NextUploadIdMarker, - maxUploads: result.MaxUploads ?? 1000, - delimiter: result.Delimiter, - isTruncated: result.IsTruncated ?? false, - encodingType: result.EncodingType as string, - uploads: (result.Uploads ?? []).map((u) => ({ - key: u.Key ?? "", - uploadId: u.UploadId ?? "", - owner: { - id: u.Owner?.ID ?? "", - displayName: u.Owner?.DisplayName ?? "", - }, - initiator: { - id: u.Initiator?.ID ?? "", - displayName: u.Initiator?.DisplayName ?? "", - }, - storageClass: u.StorageClass ?? "STANDARD", - initiated: u.Initiated ?? new Date(), - })), - commonPrefixes: (result.CommonPrefixes ?? []).map((cp) => ({ - prefix: cp.Prefix ?? "", - })), - })), - ), - - listParts: (key, uploadId) => - s3Service.getClient(targetBucket).pipe( - Effect.mapError((e) => mapS3Error(e, targetBucket.name)), - Effect.flatMap((client) => - Effect.tryPromise({ - try: () => - client.send( - new ListPartsCommand({ - Bucket: targetBucket.bucket_name, - Key: key, - UploadId: uploadId, - }), - ), - catch: (e) => mapS3Error(e, targetBucket.bucket_name), - }) - ), - Effect.map((result) => ({ - bucket: result.Bucket ?? targetBucket.bucket_name, - key: result.Key ?? key, - uploadId: result.UploadId ?? uploadId, - owner: { - id: result.Owner?.ID ?? "", - displayName: result.Owner?.DisplayName ?? "", - }, - initiator: { - id: result.Initiator?.ID ?? "", - displayName: result.Initiator?.DisplayName ?? "", - }, - storageClass: result.StorageClass ?? "STANDARD", - partNumberMarker: result.PartNumberMarker - ? parseInt(String(result.PartNumberMarker)) - : 0, - nextPartNumberMarker: result.NextPartNumberMarker - ? parseInt(String(result.NextPartNumberMarker)) - : 0, - maxParts: result.MaxParts ?? 1000, - isTruncated: result.IsTruncated ?? false, - parts: (result.Parts ?? []).map((p) => ({ - partNumber: p.PartNumber ?? 0, - lastModified: p.LastModified ?? new Date(), - etag: p.ETag ?? "", - size: p.Size ?? 0, - })), - })), - ), - }; - - return service; - }), - ); +): Effect.Effect => + Effect.gen(function* () { + const target = yield* getTarget(bucket); + return { + ...makeBucketOps(target), + ...makeObjectOps(target), + } satisfies BackendService; + }); diff --git a/src/Backends/S3/Buckets.ts b/src/Backends/S3/Buckets.ts new file mode 100644 index 0000000..b0ff891 --- /dev/null +++ b/src/Backends/S3/Buckets.ts @@ -0,0 +1,74 @@ +import { Effect } from "effect"; +import { + CreateBucketCommand, + DeleteBucketCommand, + HeadBucketCommand, + ListBucketsCommand, + type ListBucketsCommandOutput, +} from "@aws-sdk/client-s3"; +import { type BucketInfo, InternalError } from "../../Services/Backend.ts"; +import { mapS3Error, type S3Target } from "./Utils.ts"; + +export const makeBucketOps = (target: S3Target) => ({ + listBuckets: () => + Effect.gen(function* () { + const { client, name } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send(new ListBucketsCommand({})) as Promise< + ListBucketsCommandOutput + >, + catch: (e) => mapS3Error(e, name), + }); + + const buckets: BucketInfo[] = []; + for (const b of (result.Buckets ?? [])) { + if (b.Name === undefined) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned bucket without Name", + }), + ); + } + buckets.push({ + name: b.Name, + creationDate: b.CreationDate, + }); + } + + return { + buckets, + owner: { + id: result.Owner?.ID ?? "unknown-owner-id", + displayName: result.Owner?.DisplayName ?? "unknown-owner-name", + }, + }; + }), + + createBucket: () => + Effect.gen(function* () { + const { client, bucketName, name } = target; + yield* Effect.tryPromise({ + try: () => client.send(new CreateBucketCommand({ Bucket: bucketName })), + catch: (e) => mapS3Error(e, bucketName || name), + }); + }), + + deleteBucket: () => + Effect.gen(function* () { + const { client, bucketName, name } = target; + yield* Effect.tryPromise({ + try: () => client.send(new DeleteBucketCommand({ Bucket: bucketName })), + catch: (e) => mapS3Error(e, bucketName || name), + }); + }), + + headBucket: () => + Effect.gen(function* () { + const { client, bucketName, name } = target; + yield* Effect.tryPromise({ + try: () => client.send(new HeadBucketCommand({ Bucket: bucketName })), + catch: (e) => mapS3Error(e, bucketName || name), + }); + }), +}); diff --git a/src/Backends/S3/Client.ts b/src/Backends/S3/Client.ts index bf79c56..376de09 100644 --- a/src/Backends/S3/Client.ts +++ b/src/Backends/S3/Client.ts @@ -1,7 +1,7 @@ -import { Context, Effect, Layer } from "effect"; +import { Cache, Context, Effect, Layer } from "effect"; import { S3Client as S3ClientSDK } from "@aws-sdk/client-s3"; import type { MaterializedBucket } from "../../Domain/Config.ts"; -import { AppConfig } from "../../Config/Layer.ts"; +import { HeraldConfig } from "../../Config/Layer.ts"; export class S3Client extends Context.Tag("S3Client")< S3Client, @@ -14,106 +14,100 @@ export class S3Client extends Context.Tag("S3Client")< export const S3ClientLive = Layer.effect( S3Client, - AppConfig.pipe( - Effect.flatMap((appConfig) => { - // A simple cache for SDK clients - const clients = new Map(); + Effect.gen(function* () { + const appConfig = yield* HeraldConfig; - return Effect.succeed( - S3Client.of({ - getClient: (bucket: MaterializedBucket | { backend_id: string }) => { - // Resolve full bucket if only backend_id provided - let resolved: MaterializedBucket; - if ("bucket_name" in bucket) { - resolved = bucket; - } else { - const backendConfig = appConfig.raw.backends[bucket.backend_id]; - if (backendConfig && backendConfig.protocol === "s3") { - resolved = { - name: "", - backend_id: bucket.backend_id, - protocol: "s3" as const, - endpoint: backendConfig.endpoint, - region: backendConfig.region, - bucket_name: "", - credentials: backendConfig.credentials, - }; - } else { - return Effect.fail( - new Error( - `Backend ${bucket.backend_id} is not an S3 backend or not found`, - ), - ); - } - } + const cache = yield* Cache.make({ + capacity: 100, + timeToLive: "24 hours", // S3 clients can live a long time + lookup: (resolved: MaterializedBucket) => + Effect.gen(function* () { + if (resolved.endpoint === undefined) { + return yield* Effect.fail( + new Error( + `Missing endpoint for backend ${resolved.backend_id}`, + ), + ); + } + + if (resolved.region === undefined) { + return yield* Effect.fail( + new Error(`Missing region for backend ${resolved.backend_id}`), + ); + } + + let accessKeyId: string | undefined; + let secretAccessKey: string | undefined; - const key = - `${resolved.backend_id}:${resolved.endpoint}:${resolved.region}`; - const existing = clients.get(key); - if (existing) { - return Effect.succeed(existing); + if (resolved.credentials) { + const creds = resolved.credentials; + if ("accessKeyId" in creds) { + accessKeyId = creds.accessKeyId; + secretAccessKey = creds.secretAccessKey; + } else if ("username" in creds) { + accessKeyId = creds.username; + secretAccessKey = creds.password; } - if (resolved.endpoint === undefined) { - return Effect.fail( + if (accessKeyId === undefined) { + return yield* Effect.fail( new Error( - `Missing endpoint for backend ${resolved.backend_id}`, + `Missing accessKeyId/username for backend ${resolved.backend_id}`, ), ); } - - if (resolved.region === undefined) { - return Effect.fail( - new Error(`Missing region for backend ${resolved.backend_id}`), + if (secretAccessKey === undefined) { + return yield* Effect.fail( + new Error( + `Missing secretAccessKey/password for backend ${resolved.backend_id}`, + ), ); } + } - let accessKeyId: string | undefined; - let secretAccessKey: string | undefined; - - if (resolved.credentials) { - const creds = resolved.credentials; - if ("accessKeyId" in creds) { - accessKeyId = creds.accessKeyId; - secretAccessKey = creds.secretAccessKey; - } else if ("username" in creds) { - accessKeyId = creds.username; - secretAccessKey = creds.password; - } - - if (accessKeyId === undefined) { - return Effect.fail( - new Error( - `Missing accessKeyId/username for backend ${resolved.backend_id}`, - ), - ); - } - if (secretAccessKey === undefined) { - return Effect.fail( - new Error( - `Missing secretAccessKey/password for backend ${resolved.backend_id}`, - ), - ); + return new S3ClientSDK({ + endpoint: resolved.endpoint, + region: resolved.region, + credentials: accessKeyId && secretAccessKey + ? { + accessKeyId, + secretAccessKey, } - } + : undefined, + forcePathStyle: true, + }); + }), + }); - const sdkClient = new S3ClientSDK({ - endpoint: resolved.endpoint, - region: resolved.region, - credentials: accessKeyId && secretAccessKey - ? { - accessKeyId, - secretAccessKey, - } - : undefined, - forcePathStyle: true, - }); + return S3Client.of({ + getClient: (bucket: MaterializedBucket | { backend_id: string }) => { + // Resolve full bucket if only backend_id provided + let resolved: MaterializedBucket; + if ("bucket_name" in bucket) { + resolved = bucket; + } else { + const backendConfig = appConfig.raw.backends[bucket.backend_id]; + if (backendConfig && backendConfig.protocol === "s3") { + resolved = { + name: "", + backend_id: bucket.backend_id, + protocol: "s3" as const, + endpoint: backendConfig.endpoint, + region: backendConfig.region, + bucket_name: "", + credentials: backendConfig.credentials, + }; + } else { + return Effect.fail( + new Error( + `Backend ${bucket.backend_id} is not an S3 backend or not found`, + ), + ); + } + } - clients.set(key, sdkClient); - return Effect.succeed(sdkClient); - }, - }), - ); - }), - ), + return cache.get(resolved); + }, + }); + }), ); diff --git a/src/Backends/S3/Objects.ts b/src/Backends/S3/Objects.ts new file mode 100644 index 0000000..58a1f06 --- /dev/null +++ b/src/Backends/S3/Objects.ts @@ -0,0 +1,757 @@ +import { Chunk, Effect, Option, Stream } from "effect"; +import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CreateMultipartUploadCommand, + DeleteObjectCommand, + DeleteObjectsCommand, + GetObjectCommand, + HeadObjectCommand, + ListMultipartUploadsCommand, + ListObjectsCommand, + type ListObjectsCommandOutput, + ListObjectsV2Command, + type ListObjectsV2CommandOutput, + ListObjectVersionsCommand, + ListPartsCommand, + PutObjectCommand, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import { + type CommonPrefix, + InternalError, + type ListObjectsResult, + type ObjectInfo, + type ObjectResponse, +} from "../../Services/Backend.ts"; +import { mapS3Error, type S3Target, stripMinioMetadata } from "./Utils.ts"; + +export const makeObjectOps = (target: S3Target) => ({ + listObjects: (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + encodingType?: string; + continuationToken?: string; + startAfter?: string; + listType?: 1 | 2; + }) => + Effect.gen(function* () { + const { client, bucketName } = target; + if (args.listType === 2) { + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + MaxKeys: args.maxKeys, + ContinuationToken: args.continuationToken, + StartAfter: args.startAfter, + }), + ) as Promise, + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + name: result.Name ?? bucketName, + prefix: result.Prefix, + maxKeys: result.MaxKeys ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: args.encodingType, + continuationToken: result.ContinuationToken, + nextContinuationToken: result.NextContinuationToken, + keyCount: result.KeyCount, + listType: 2, + contents: (result.Contents ?? []).map((c): ObjectInfo => ({ + key: stripMinioMetadata(c.Key ?? ""), + lastModified: c.LastModified ?? new Date(), + etag: c.ETag ?? "", + size: c.Size ?? 0, + storageClass: c.StorageClass, + owner: c.Owner + ? { + id: c.Owner.ID ?? "unknown", + displayName: c.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + commonPrefixes: (result.CommonPrefixes ?? []).map(( + cp, + ): CommonPrefix => ({ + prefix: stripMinioMetadata(cp.Prefix ?? ""), + })), + } satisfies ListObjectsResult; + } else { + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListObjectsCommand({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + Marker: args.marker, + MaxKeys: args.maxKeys, + }), + ) as Promise, + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + name: result.Name ?? bucketName, + prefix: result.Prefix, + marker: result.Marker, + nextMarker: result.NextMarker, + maxKeys: result.MaxKeys ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: args.encodingType, + listType: 1, + contents: (result.Contents ?? []).map((c): ObjectInfo => ({ + key: stripMinioMetadata(c.Key ?? ""), + lastModified: c.LastModified ?? new Date(), + etag: c.ETag ?? "", + size: c.Size ?? 0, + storageClass: c.StorageClass, + owner: c.Owner + ? { + id: c.Owner.ID ?? "unknown", + displayName: c.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + commonPrefixes: (result.CommonPrefixes ?? []).map(( + cp, + ): CommonPrefix => ({ + prefix: stripMinioMetadata(cp.Prefix ?? ""), + })), + } satisfies ListObjectsResult; + } + }), + + listVersions: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + versionIdMarker?: string; + maxKeys?: number; + encodingType?: string; + }) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListObjectVersionsCommand({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + KeyMarker: args.keyMarker, + VersionIdMarker: args.versionIdMarker, + MaxKeys: args.maxKeys, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + name: result.Name ?? bucketName, + prefix: result.Prefix, + marker: result.KeyMarker, + nextMarker: result.NextKeyMarker, + maxKeys: result.MaxKeys ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: args.encodingType, + listType: 1, + contents: [ + ...(result.Versions ?? []).map((v): ObjectInfo => ({ + key: stripMinioMetadata(v.Key ?? ""), + lastModified: v.LastModified ?? new Date(), + etag: v.ETag ?? "", + size: v.Size ?? 0, + storageClass: v.StorageClass, + versionId: v.VersionId, + isDeleteMarker: false, + isLatest: v.IsLatest, + owner: v.Owner + ? { + id: v.Owner.ID ?? "unknown", + displayName: v.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + ...(result.DeleteMarkers ?? []).map((dm): ObjectInfo => ({ + key: stripMinioMetadata(dm.Key ?? ""), + lastModified: dm.LastModified ?? new Date(), + etag: "", + size: 0, + versionId: dm.VersionId, + isDeleteMarker: true, + isLatest: dm.IsLatest, + owner: dm.Owner + ? { + id: dm.Owner.ID ?? "unknown", + displayName: dm.Owner.DisplayName ?? "unknown", + } + : undefined, + })), + ], + commonPrefixes: (result.CommonPrefixes ?? []).map(( + cp, + ): CommonPrefix => ({ + prefix: stripMinioMetadata(cp.Prefix ?? ""), + })), + } satisfies ListObjectsResult; + }), + + getObject: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: key, + Range: (headers["range"] || headers["Range"]) as string, + PartNumber: (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) + ? parseInt( + (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) as string, + ) + : undefined, + IfMatch: (headers["if-match"] || headers["If-Match"]) as string, + IfNoneMatch: (headers["if-none-match"] || + headers["If-None-Match"]) as string, + IfModifiedSince: (headers["if-modified-since"] || + headers["If-Modified-Since"]) + ? new Date( + (headers["if-modified-since"] || + headers["If-Modified-Since"]) as string, + ) + : undefined, + IfUnmodifiedSince: (headers["if-unmodified-since"] || + headers["If-Unmodified-Since"]) + ? new Date( + (headers["if-unmodified-since"] || + headers["If-Unmodified-Since"]) as string, + ) + : undefined, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + const body = result.Body; + if (!body) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned empty body for GetObject", + }), + ); + } + + const getWebStream = (): ReadableStream => { + if ( + body && typeof body === "object" && + "transformToWebStream" in body + ) { + const b = body as { transformToWebStream: unknown }; + if (typeof b.transformToWebStream === "function") { + return b.transformToWebStream() as ReadableStream< + Uint8Array + >; + } + } + return body as ReadableStream; + }; + + const stream = Stream.fromReadableStream( + getWebStream, + (e) => new Error(String(e)), + ); + + const metadata: Record = {}; + if (result.Metadata) { + for (const [k, v] of Object.entries(result.Metadata)) { + metadata[k] = Option.liftThrowable(decodeURIComponent)( + v ?? "", + ).pipe( + Option.getOrElse(() => v ?? ""), + ); + } + } + + const s3Headers: Record = {}; + if (result.ContentType) { + s3Headers["content-type"] = result.ContentType; + } + if (result.ContentLength !== undefined) { + s3Headers["content-length"] = String(result.ContentLength); + } + if (result.ETag) s3Headers["etag"] = result.ETag; + if (result.PartsCount !== undefined) { + s3Headers["x-amz-mp-parts-count"] = String(result.PartsCount); + } + if (result.VersionId) { + s3Headers["x-amz-version-id"] = result.VersionId; + } + if (result.LastModified) { + s3Headers["last-modified"] = result.LastModified.toUTCString(); + } + + for (const [k, v] of Object.entries(metadata)) { + s3Headers[`x-amz-meta-${k}`] = v; + } + + return yield* Stream.runCollect(stream).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + Effect.map((chunks) => { + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const all = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + all.set(chunk, offset); + offset += chunk.length; + } + return { + stream: Stream.succeed(all), + contentType: result.ContentType, + contentLength: all.length, + etag: result.ETag, + lastModified: result.LastModified, + metadata, + headers: s3Headers, + } satisfies ObjectResponse; + }), + ); + }), + + headObject: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const commandInput = { + Bucket: bucketName, + Key: key, + PartNumber: (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) + ? parseInt( + (headers["part-number"] || + headers["Part-Number"] || + headers["x-amz-part-number"]) as string, + ) + : undefined, + }; + const result = yield* Effect.tryPromise({ + try: () => client.send(new HeadObjectCommand(commandInput)), + catch: (e) => mapS3Error(e, bucketName), + }); + + const metadata: Record = {}; + if (result.Metadata) { + for (const [k, v] of Object.entries(result.Metadata)) { + metadata[k] = Option.liftThrowable(decodeURIComponent)( + v ?? "", + ).pipe( + Option.getOrElse(() => v ?? ""), + ); + } + } + + const s3Headers: Record = {}; + if (result.ContentType) { + s3Headers["content-type"] = result.ContentType; + } + if (result.ContentLength !== undefined) { + s3Headers["content-length"] = String(result.ContentLength); + } + if (result.ETag) s3Headers["etag"] = result.ETag; + if (result.PartsCount !== undefined) { + s3Headers["x-amz-mp-parts-count"] = String(result.PartsCount); + } + if (result.VersionId) { + s3Headers["x-amz-version-id"] = result.VersionId; + } + if (result.LastModified) { + s3Headers["last-modified"] = result + .LastModified.toUTCString(); + } + + for (const [k, v] of Object.entries(metadata)) { + s3Headers[`x-amz-meta-${k}`] = v; + } + + return { + contentType: result.ContentType, + contentLength: result.ContentLength, + etag: result.ETag, + lastModified: result.LastModified, + metadata, + headers: s3Headers, + }; + }), + + putObject: ( + key: string, + bodyStream: Stream.Stream, + headers: Record, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const chunks = yield* Stream.runCollect(bodyStream).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + ); + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const body = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.length; + } + + const metadata: Record = {}; + for (const [k, v] of Object.entries(headers)) { + if (k.toLowerCase().startsWith("x-amz-meta-")) { + const metaKey = k.substring("x-amz-meta-".length); + const value = String(v); + metadata[metaKey] = /[^\x20-\x7E]/.test(value) + ? encodeURIComponent(value) + : value; + } + } + + const contentType = headers["content-type"]; + + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: key, + Body: body, + ContentType: contentType ? String(contentType) : undefined, + Metadata: metadata, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + etag: result.ETag, + versionId: result.VersionId, + }; + }), + + deleteObject: (key: string) => + Effect.gen(function* () { + const { client, bucketName } = target; + yield* Effect.tryPromise({ + try: () => + client.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: key, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + }), + + deleteObjects: (objects: readonly { key: string; versionId?: string }[]) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects.map((o) => ({ + Key: o.key, + VersionId: o.versionId === "null" ? undefined : o.versionId, + })), + }, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + deleted: (result.Deleted ?? []).map((d) => d.Key ?? ""), + errors: (result.Errors ?? []).map((e) => ({ + key: e.Key ?? "unknown", + code: e.Code ?? "InternalError", + message: e.Message ?? "Unknown error", + })), + }; + }), + + createMultipartUpload: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const metadata: Record = {}; + for (const [k, v] of Object.entries(headers)) { + if (k.toLowerCase().startsWith("x-amz-meta-")) { + const metaKey = k.substring("x-amz-meta-".length); + metadata[metaKey] = String(v); + } + } + const contentType = headers["content-type"]; + + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + Metadata: metadata, + ContentType: contentType ? String(contentType) : undefined, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + if (!result.UploadId) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned empty UploadId", + }), + ); + } + return { uploadId: result.UploadId }; + }), + + uploadPart: ( + key: string, + uploadId: string, + partNumber: number, + bodyStream: Stream.Stream, + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const chunks = yield* Stream.runCollect(bodyStream).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + ); + const totalLength = Chunk.reduce( + chunks, + 0, + (acc, chunk) => acc + chunk.length, + ); + const body = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.length; + } + + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + PartNumber: partNumber, + Body: body, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + if (!result.ETag) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned empty ETag for UploadPart", + }), + ); + } + return { etag: result.ETag }; + }), + + completeMultipartUpload: ( + key: string, + uploadId: string, + parts: readonly { etag: string; partNumber: number }[], + ) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: parts.map((p) => ({ + ETag: p.etag, + PartNumber: p.partNumber, + })), + }, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + if ( + !result.Location || !result.Bucket || !result.Key || + !result.ETag + ) { + return yield* Effect.fail( + new InternalError({ + message: "S3 returned incomplete CompleteMultipartUploadResult", + }), + ); + } + return { + location: result.Location, + bucket: result.Bucket, + key: result.Key, + etag: result.ETag, + versionId: result.VersionId, + }; + }), + + abortMultipartUpload: (key: string, uploadId: string) => + Effect.gen(function* () { + const { client, bucketName } = target; + yield* Effect.tryPromise({ + try: () => + client.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + }), + + listMultipartUploads: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + uploadIdMarker?: string; + maxUploads?: number; + encodingType?: string; + }) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListMultipartUploadsCommand({ + Bucket: bucketName, + Prefix: args.prefix, + Delimiter: args.delimiter, + KeyMarker: args.keyMarker, + UploadIdMarker: args.uploadIdMarker, + MaxUploads: args.maxUploads, + EncodingType: args.encodingType as "url" | undefined, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + bucket: result.Bucket ?? bucketName, + prefix: result.Prefix, + keyMarker: result.KeyMarker, + uploadIdMarker: result.UploadIdMarker, + nextKeyMarker: result.NextKeyMarker, + nextUploadIdMarker: result.NextUploadIdMarker, + maxUploads: result.MaxUploads ?? 1000, + delimiter: result.Delimiter, + isTruncated: result.IsTruncated ?? false, + encodingType: result.EncodingType as string, + uploads: (result.Uploads ?? []).map((u) => ({ + key: u.Key ?? "", + uploadId: u.UploadId ?? "", + owner: { + id: u.Owner?.ID ?? "", + displayName: u.Owner?.DisplayName ?? "", + }, + initiator: { + id: u.Initiator?.ID ?? "", + displayName: u.Initiator?.DisplayName ?? "", + }, + storageClass: u.StorageClass ?? "STANDARD", + initiated: u.Initiated ?? new Date(), + })), + commonPrefixes: (result.CommonPrefixes ?? []).map((cp) => ({ + prefix: cp.Prefix ?? "", + })), + }; + }), + + listParts: (key: string, uploadId: string) => + Effect.gen(function* () { + const { client, bucketName } = target; + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new ListPartsCommand({ + Bucket: bucketName, + Key: key, + UploadId: uploadId, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + bucket: result.Bucket ?? bucketName, + key: result.Key ?? key, + uploadId: result.UploadId ?? uploadId, + owner: { + id: result.Owner?.ID ?? "", + displayName: result.Owner?.DisplayName ?? "", + }, + initiator: { + id: result.Initiator?.ID ?? "", + displayName: result.Initiator?.DisplayName ?? "", + }, + storageClass: result.StorageClass ?? "STANDARD", + partNumberMarker: result.PartNumberMarker + ? parseInt(String(result.PartNumberMarker)) + : 0, + nextPartNumberMarker: result.NextPartNumberMarker + ? parseInt(String(result.NextPartNumberMarker)) + : 0, + maxParts: result.MaxParts ?? 1000, + isTruncated: result.IsTruncated ?? false, + parts: (result.Parts ?? []).map((p) => ({ + partNumber: p.PartNumber ?? 0, + lastModified: p.LastModified ?? new Date(), + etag: p.ETag ?? "", + size: p.Size ?? 0, + })), + }; + }), +}); diff --git a/src/Backends/S3/Utils.ts b/src/Backends/S3/Utils.ts new file mode 100644 index 0000000..f11d486 --- /dev/null +++ b/src/Backends/S3/Utils.ts @@ -0,0 +1,147 @@ +import { Effect } from "effect"; +import type { S3Client as S3ClientSDK } from "@aws-sdk/client-s3"; +import type { MaterializedBucket } from "../../Domain/Config.ts"; +import { HeraldConfig } from "../../Config/Layer.ts"; +import { + AccessDenied, + type BackendError, + BucketAlreadyExists, + BucketAlreadyOwnedByYou, + BucketNotEmpty, + EntityTooSmall, + InternalError, + InvalidPart, + InvalidPartOrder, + InvalidRequest, + MalformedXML, + NoSuchBucket, + NoSuchKey, + NoSuchUpload, +} from "../../Services/Backend.ts"; +import { S3Client } from "./Client.ts"; + +export interface S3Target { + readonly client: S3ClientSDK; + readonly bucketName: string; + readonly name: string; +} + +/** + * Strips MinIO metadata suffixes like [minio_cache:v2,return:] from strings. + */ +export function stripMinioMetadata(s: string): string { + return s.replace(/\[minio_cache:[^\]]+\]/g, ""); +} + +/** + * Maps S3 SDK exceptions to internal BackendError types. + */ +export function mapS3Error(e: unknown, bucketName?: string): BackendError { + const err = e as { + name?: string; + Code?: string; + Message?: string; + message?: string; + $metadata?: { httpStatusCode?: number }; + }; + const name = err?.name || err?.Code || + (e instanceof Error ? e.name : "UnknownError"); + const message = err?.message || err?.Message || + "An unknown S3 error occurred"; + const bucket = bucketName ?? "unknown-bucket"; + + switch (name) { + case "NoSuchBucket": + case "NotFound": + return new NoSuchBucket({ bucketName: bucket, message }); + case "NoSuchKey": + return new NoSuchKey({ + bucketName: bucket, + key: "unknown", + message: message, + }); + case "NoSuchUpload": + return new NoSuchUpload({ + uploadId: "unknown", + message: message, + }); + case "InvalidPart": + case "InvalidPartNumber": + return new InvalidPart({ message }); + case "InvalidPartOrder": + return new InvalidPartOrder({ message }); + case "EntityTooSmall": + return new EntityTooSmall({ message }); + case "InvalidRequest": + if (message.includes("at least one part")) { + return new MalformedXML({ message }); + } + return new InvalidRequest({ message }); + case "MalformedXML": + return new MalformedXML({ message }); + case "BucketAlreadyExists": + return new BucketAlreadyExists({ bucketName: bucket, message }); + case "BucketAlreadyOwnedByYou": + return new BucketAlreadyOwnedByYou({ bucketName: bucket, message }); + case "AccessDenied": + case "Forbidden": + return new AccessDenied({ message }); + case "BucketNotEmpty": + case "Conflict": + return new BucketNotEmpty({ bucketName: bucket, message }); + } + + // Handle case where it might be a raw 404 from HEAD request + if (err?.$metadata?.httpStatusCode === 404) { + return new NoSuchKey({ + bucketName: bucket, + key: "unknown", + message: "Not Found", + }); + } + + return new InternalError({ + message: e instanceof Error ? `${e.name}: ${e.message}` : String(e), + }); +} + +/** + * Resolves the target bucket configuration and acquires the S3 client. + * This ensures the backend remains a stateless proxy that picks up request-local configuration and clients. + */ +export const getTarget = ( + bucket: MaterializedBucket | { backend_id: string }, +): Effect.Effect => + Effect.gen(function* () { + const s3Service = yield* S3Client; + const config = yield* HeraldConfig; + + const resolveTargetBucket = (): MaterializedBucket => { + if ("bucket_name" in bucket) return bucket as MaterializedBucket; + + const backendConfig = config.raw.backends[bucket.backend_id]; + if (backendConfig && backendConfig.protocol === "s3") { + return { + name: "", + backend_id: bucket.backend_id, + protocol: "s3" as const, + endpoint: backendConfig.endpoint, + region: backendConfig.region, + bucket_name: "", + credentials: backendConfig.credentials, + }; + } + throw new Error(`Backend ${bucket.backend_id} is not an S3 backend`); + }; + + const targetBucket = resolveTargetBucket(); + const client = yield* s3Service.getClient(targetBucket).pipe( + Effect.mapError((e) => mapS3Error(e, targetBucket.name)), + ); + + return { + client, + bucketName: targetBucket.bucket_name, + name: targetBucket.name, + }; + }); diff --git a/src/Backends/Swift/Backend.ts b/src/Backends/Swift/Backend.ts index 8689888..1c50070 100644 --- a/src/Backends/Swift/Backend.ts +++ b/src/Backends/Swift/Backend.ts @@ -1,650 +1,29 @@ -import { Effect, Option, Stream } from "effect"; -import { - type BackendError, - type BackendService, - BucketAlreadyExists, - BucketAlreadyOwnedByYou, - type BucketInfo, - BucketNotEmpty, - type CommonPrefix, - type DeleteObjectsResult, - type HeadObjectResult, - InternalError, - type ListObjectsResult, - NoSuchBucket, - NoSuchKey, - type ObjectInfo, - type ObjectResponse, - type OwnerInfo, - type PutObjectResult, -} from "../../Services/Backend.ts"; +import { Effect } from "effect"; +import { HttpClient } from "@effect/platform"; +import type { BackendError, BackendService } from "../../Services/Backend.ts"; import type { MaterializedBucket } from "../../Domain/Config.ts"; -import { SwiftClient } from "./Client.ts"; -import { fixHeaderEncoding } from "../../Frontend/Utils.ts"; - -interface SwiftContainer { - readonly name: string; - readonly last_modified?: string; -} - -interface SwiftObject { - readonly name?: string; - readonly hash?: string; - readonly bytes?: number; - readonly content_type?: string; - readonly last_modified?: string; - readonly subdir?: string; -} - +import { makeBucketOps } from "./Buckets.ts"; +import { makeObjectOps } from "./Objects.ts"; +import { getTarget } from "./Utils.ts"; +import type { SwiftClient } from "./Client.ts"; + +/** + * Creates a Swift-specific Backend implementation for a given configuration context. + * Composes bucket and object operations modularly. + * Resolves the target and client once per backend creation (request-scoped). + */ export const makeSwiftBackend = ( bucket: MaterializedBucket | { backend_id: string }, -): Effect.Effect => +): Effect.Effect< + BackendService, + BackendError, + SwiftClient | HttpClient.HttpClient +> => Effect.gen(function* () { - const swiftClient = yield* SwiftClient; - - const getTarget = () => - Effect.gen(function* () { - const auth = yield* swiftClient.getAuthMeta(bucket).pipe( - Effect.mapError((e) => new InternalError({ message: e.message })), - ); - const container = "bucket_name" in bucket ? bucket.bucket_name : ""; - const encodedContainer = container ? encodeURIComponent(container) : ""; - return { - storageUrl: auth.storageUrl, - token: auth.token, - container, - url: encodedContainer - ? `${auth.storageUrl}/${encodedContainer}` - : auth.storageUrl, - }; - }); - - const mapError = ( - status: number, - message: string, - bucketName: string, - method?: string, - key?: string, - ): BackendError => { - switch (status) { - case 404: - if (key) { - return new NoSuchKey({ bucketName, key, message }); - } - return new NoSuchBucket({ bucketName, message }); - case 409: - if (method === "DELETE") { - return new BucketNotEmpty({ bucketName, message }); - } - return new BucketAlreadyExists({ bucketName, message }); - case 202: - if (method === "PUT") { - return new BucketAlreadyOwnedByYou({ bucketName, message }); - } - return new InternalError({ - message: `Swift error (${status}): ${message}`, - }); - default: - return new InternalError({ - message: `Swift error (${status}): ${message}`, - }); - } - }; - - const listObjects = (args: { - prefix?: string; - delimiter?: string; - marker?: string; - maxKeys?: number; - encodingType?: string; - continuationToken?: string; - startAfter?: string; - listType?: 1 | 2; - }) => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const limit = args.maxKeys ?? 1000; - const query = new URLSearchParams({ format: "json" }); - if (args.prefix) query.set("prefix", args.prefix); - if (args.delimiter) query.set("delimiter", args.delimiter); - if (args.marker) query.set("marker", args.marker); - query.set("limit", String(limit + 1)); - if (args.continuationToken) query.set("marker", args.continuationToken); - if (args.startAfter) query.set("marker", args.startAfter); - - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${url}?${query.toString()}`, { - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - yield* Effect.logDebug( - `Swift listObjects query=[${query.toString()}] status=${response.status}`, - ); - - if (!response.ok) { - return yield* Effect.fail( - mapError(response.status, response.statusText, container, "GET"), - ); - } - - const rawObjects = (yield* Effect.tryPromise({ - try: () => response.json(), - catch: (e) => - new InternalError({ - message: `Failed to parse Swift response: ${e}`, - }), - })) as readonly SwiftObject[]; - - const isTruncated = rawObjects.length > limit; - const objects = isTruncated ? rawObjects.slice(0, limit) : rawObjects; - - const contents: ObjectInfo[] = []; - const commonPrefixes: CommonPrefix[] = []; - - for (const obj of objects) { - if (obj.subdir) { - commonPrefixes.push({ prefix: obj.subdir }); - } else if (obj.name) { - contents.push({ - key: obj.name, - lastModified: obj.last_modified - ? new Date(obj.last_modified) - : new Date(), - etag: obj.hash ? `"${obj.hash}"` : "", - size: obj.bytes ?? 0, - storageClass: "STANDARD", - owner: { id: "swift", displayName: "Swift User" }, - }); - } - } - - const nextMarker = isTruncated && objects.length > 0 - ? objects[objects.length - 1].name || - objects[objects.length - 1].subdir - : undefined; - - return { - name: container, - prefix: args.prefix, - maxKeys: limit, - delimiter: args.delimiter, - isTruncated, - marker: args.marker, - nextMarker, - contents, - commonPrefixes, - encodingType: args.encodingType, - listType: args.listType ?? 1, - nextContinuationToken: args.listType === 2 ? nextMarker : undefined, - keyCount: contents.length + commonPrefixes.length, - } satisfies ListObjectsResult; - }); - + const target = yield* getTarget(bucket); + const client = yield* HttpClient.HttpClient; return { - listBuckets: () => - Effect.gen(function* () { - const { storageUrl, token } = yield* getTarget(); - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${storageUrl}?format=json`, { - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - if (!response.ok) { - return yield* Effect.fail( - mapError(response.status, response.statusText, "", "GET"), - ); - } - - const buckets = (yield* Effect.tryPromise({ - try: () => response.json(), - catch: (e) => - new InternalError({ - message: `Failed to parse Swift response: ${e}`, - }), - })) as readonly SwiftContainer[]; - - const bucketInfos: BucketInfo[] = buckets.map((b) => ({ - name: b.name, - creationDate: b.last_modified - ? new Date(b.last_modified) - : undefined, - })); - - const owner: OwnerInfo = { id: "swift", displayName: "Swift User" }; - - return { buckets: bucketInfos, owner }; - }), - - createBucket: () => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const response = yield* Effect.tryPromise({ - try: () => - fetch(url, { - method: "PUT", - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - if (response.status === 201) { - return yield* Effect.void; - } - - if (response.status === 202) { - return yield* Effect.fail( - new BucketAlreadyOwnedByYou({ - bucketName: container, - message: "Bucket already exists", - }), - ); - } - - if (!response.ok) { - return yield* Effect.fail( - mapError(response.status, response.statusText, container, "PUT"), - ); - } - - return yield* Effect.void; - }), - - deleteBucket: () => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const response = yield* Effect.tryPromise({ - try: () => - fetch(url, { - method: "DELETE", - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - yield* Effect.logDebug( - `Swift deleteBucket container=[${container}] status=${response.status}`, - ); - - if (response.status === 204) { - return yield* Effect.void; - } - - if (!response.ok) { - return yield* Effect.fail( - mapError( - response.status, - response.statusText, - container, - "DELETE", - ), - ); - } - - return yield* Effect.void; - }), - - headBucket: () => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const response = yield* Effect.tryPromise({ - try: () => - fetch(url, { - method: "HEAD", - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - if (!response.ok) { - return yield* Effect.fail( - mapError(response.status, response.statusText, container, "HEAD"), - ); - } - - return yield* Effect.void; - }), - - listObjects, - - listVersions: (args) => - Effect.gen(function* () { - const result = yield* listObjects({ - prefix: args.prefix, - delimiter: args.delimiter, - marker: args.keyMarker, - maxKeys: args.maxKeys, - }); - return { - ...result, - contents: result.contents.map((c) => ({ - ...c, - versionId: "null", - isLatest: true, - })), - }; - }), - - getObject: ( - key: string, - headers: Record, - ) => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const encodedKey = key.split("/").map(encodeURIComponent).join("/"); - const swiftHeaders: Record = { - "X-Auth-Token": token, - }; - if (headers["range"] || headers["Range"]) { - swiftHeaders["Range"] = String( - headers["range"] || headers["Range"], - ); - } - if (headers["if-match"] || headers["If-Match"]) { - swiftHeaders["If-Match"] = String( - headers["if-match"] || - headers["If-Match"], - ); - } - if (headers["if-none-match"] || headers["If-None-Match"]) { - swiftHeaders["If-None-Match"] = String( - headers["if-none-match"] || - headers["If-None-Match"], - ); - } - if (headers["if-modified-since"] || headers["If-Modified-Since"]) { - swiftHeaders["If-Modified-Since"] = String( - headers["if-modified-since"] || - headers["If-Modified-Since"], - ); - } - if ( - headers["if-unmodified-since"] || headers["If-Unmodified-Since"] - ) { - swiftHeaders["If-Unmodified-Since"] = String( - headers["if-unmodified-since"] || - headers["If-Unmodified-Since"], - ); - } - - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${url}/${encodedKey}`, { - headers: swiftHeaders, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - if (!response.ok) { - return yield* Effect.fail( - mapError( - response.status, - response.statusText, - container, - "GET", - key, - ), - ); - } - - const metadata: Record = {}; - const s3Headers: Record = {}; - response.headers.forEach((v, k) => { - const lowK = k.toLowerCase(); - if (lowK.startsWith("x-object-meta-")) { - const metaKey = lowK.substring("x-object-meta-".length); - const value = (v.includes("%")) - ? Option.liftThrowable(decodeURIComponent)(v).pipe( - Option.getOrElse(() => v), - ) - : v; - metadata[metaKey] = value; - s3Headers[`x-amz-meta-${metaKey}`] = value; - } else if (lowK === "content-type") { - s3Headers["Content-Type"] = v; - } else if (lowK === "content-length") { - s3Headers["Content-Length"] = v; - } else if (lowK === "etag") { - s3Headers["ETag"] = v; - } else if (lowK === "last-modified") { - s3Headers["Last-Modified"] = v; - } - }); - - return { - stream: Stream.fromReadableStream( - () => response.body!, - (e) => new InternalError({ message: String(e) }), - ), - contentType: response.headers.get("Content-Type") || undefined, - contentLength: parseInt( - response.headers.get("Content-Length") || "0", - ), - etag: response.headers.get("ETag") || undefined, - lastModified: response.headers.get("Last-Modified") - ? new Date(response.headers.get("Last-Modified")!) - : undefined, - metadata, - headers: s3Headers, - } satisfies ObjectResponse; - }), - - headObject: ( - key: string, - _headers: Record, - ) => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const encodedKey = key.split("/").map(encodeURIComponent).join("/"); - const swiftHeaders: Record = { - "X-Auth-Token": token, - }; - // ... handle headers if needed - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${url}/${encodedKey}`, { - method: "HEAD", - headers: swiftHeaders, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - if (!response.ok) { - return yield* Effect.fail( - mapError( - response.status, - response.statusText, - container, - "HEAD", - key, - ), - ); - } - - const metadata: Record = {}; - const s3Headers: Record = {}; - response.headers.forEach((v, k) => { - const lowK = k.toLowerCase(); - if (lowK.startsWith("x-object-meta-")) { - const metaKey = lowK.substring("x-object-meta-".length); - const value = (v.includes("%")) - ? Option.liftThrowable(decodeURIComponent)(v).pipe( - Option.getOrElse(() => v), - ) - : v; - metadata[metaKey] = value; - s3Headers[`x-amz-meta-${metaKey}`] = value; - } else if (lowK === "content-type") { - s3Headers["Content-Type"] = v; - } else if (lowK === "content-length") { - s3Headers["Content-Length"] = v; - } else if (lowK === "etag") { - s3Headers["ETag"] = v; - } else if (lowK === "last-modified") { - s3Headers["Last-Modified"] = v; - } - }); - - return { - contentType: response.headers.get("Content-Type") || undefined, - contentLength: parseInt( - response.headers.get("Content-Length") || "0", - ), - etag: response.headers.get("ETag") || undefined, - lastModified: response.headers.get("Last-Modified") - ? new Date(response.headers.get("Last-Modified")!) - : undefined, - metadata, - headers: s3Headers, - } satisfies HeadObjectResult; - }), - - putObject: (key, stream, headers) => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const encodedKey = key.split("/").map(encodeURIComponent).join("/"); - const contentLength = headers["content-length"] || - headers["Content-Length"]; - - const swiftHeaders: Record = { - "X-Auth-Token": token, - "Content-Type": - (headers["content-type"] || headers["Content-Type"] || - "application/octet-stream") as string, - ...(contentLength - ? { "Content-Length": String(contentLength) } - : {}), - }; - - for (const [k, v] of Object.entries(headers)) { - const lowK = k.toLowerCase(); - if (lowK.startsWith("x-amz-meta-")) { - const metaKey = lowK.substring("x-amz-meta-".length); - const value = fixHeaderEncoding(String(v)); - swiftHeaders[`X-Object-Meta-${metaKey}`] = - /[^\x20-\x7E]/.test(value) ? encodeURIComponent(value) : value; - } - } - - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${url}/${encodedKey}`, { - method: "PUT", - headers: swiftHeaders, - body: Stream.toReadableStream(stream), - // @ts-ignore: duplex is required for streaming body in fetch - duplex: "half", - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - yield* Effect.logDebug( - `Swift putObject key=[${key}] status=${response.status}`, - ); - - if (!response.ok) { - return yield* Effect.fail( - mapError( - response.status, - response.statusText, - container, - "PUT", - key, - ), - ); - } - - return { - etag: response.headers.get("ETag") || undefined, - } satisfies PutObjectResult; - }), - - deleteObject: (key: string) => - Effect.gen(function* () { - const { url, token, container } = yield* getTarget(); - const encodedKey = key.split("/").map(encodeURIComponent).join("/"); - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${url}/${encodedKey}`, { - method: "DELETE", - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - if (!response.ok && response.status !== 204) { - return yield* Effect.fail( - mapError( - response.status, - response.statusText, - container, - "DELETE", - key, - ), - ); - } - - return yield* Effect.void; - }), - - deleteObjects: (objects) => - Effect.gen(function* () { - const { url, token, container: _container } = yield* getTarget(); - const deleted: string[] = []; - const errors: { key: string; code: string; message: string }[] = []; - - for (const obj of objects) { - const encodedKey = obj.key.split("/").map(encodeURIComponent).join( - "/", - ); - const response = yield* Effect.tryPromise({ - try: () => - fetch(`${url}/${encodedKey}`, { - method: "DELETE", - headers: { "X-Auth-Token": token }, - }), - catch: (e) => new InternalError({ message: String(e) }), - }); - - yield* Effect.logDebug( - `Swift deleteObject key=[${obj.key}] status=${response.status}`, - ); - - if ( - response.ok || response.status === 204 || response.status === 404 - ) { - deleted.push(obj.key); - } else { - const errorBody = yield* Effect.tryPromise(() => response.text()) - .pipe( - Effect.orElseSucceed(() => "Unknown error"), - ); - errors.push({ - key: obj.key, - code: String(response.status), - message: errorBody, - }); - } - } - - return { deleted, errors } satisfies DeleteObjectsResult; - }), - - createMultipartUpload: (_key, _headers) => - Effect.fail(new InternalError({ message: "Not implemented" })), - uploadPart: (_key, _uploadId, _partNumber, _body) => - Effect.fail(new InternalError({ message: "Not implemented" })), - completeMultipartUpload: (_key, _uploadId, _parts) => - Effect.fail(new InternalError({ message: "Not implemented" })), - abortMultipartUpload: (_key, _uploadId) => - Effect.fail(new InternalError({ message: "Not implemented" })), - listMultipartUploads: (_args) => - Effect.fail(new InternalError({ message: "Not implemented" })), - listParts: (_key, _uploadId) => - Effect.fail(new InternalError({ message: "Not implemented" })), - }; + ...makeBucketOps(target, client), + ...makeObjectOps(target, client), + } satisfies BackendService; }); diff --git a/src/Backends/Swift/Buckets.ts b/src/Backends/Swift/Buckets.ts new file mode 100644 index 0000000..c6371fd --- /dev/null +++ b/src/Backends/Swift/Buckets.ts @@ -0,0 +1,143 @@ +import { Effect } from "effect"; +import { type HttpClient, HttpClientRequest } from "@effect/platform"; +import { + BucketAlreadyOwnedByYou, + type BucketInfo, + type OwnerInfo, +} from "../../Services/Backend.ts"; +import { mapError, type SwiftTarget } from "./Utils.ts"; + +export interface SwiftContainer { + readonly name: string; + readonly last_modified?: string; +} + +export const makeBucketOps = ( + target: SwiftTarget, + client: HttpClient.HttpClient, +) => ({ + listBuckets: () => + Effect.gen(function* () { + const { storageUrl, token } = target; + const response = yield* client.execute( + HttpClientRequest.get(`${storageUrl}?format=json`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), "")), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", "", "GET"), + ); + } + + const containers = (yield* response.json.pipe( + Effect.mapError((e) => + mapError(500, `Failed to parse Swift response: ${e}`, "") + ), + )) as readonly SwiftContainer[]; + + const bucketInfos: BucketInfo[] = containers.map((b) => ({ + name: b.name, + creationDate: b.last_modified ? new Date(b.last_modified) : undefined, + })); + + const owner: OwnerInfo = { id: "swift", displayName: "Swift User" }; + + return { buckets: bucketInfos, owner }; + }), + + createBucket: () => + Effect.gen(function* () { + const { url, token, container } = target; + const response = yield* client.execute( + HttpClientRequest.put(url).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status === 201) { + return; + } + + if (response.status === 202) { + return yield* Effect.fail( + new BucketAlreadyOwnedByYou({ + bucketName: container, + message: "Bucket already exists", + }), + ); + } + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", container, "PUT"), + ); + } + }), + + deleteBucket: () => + Effect.gen(function* () { + const { url, token, container } = target; + const response = yield* client.execute( + HttpClientRequest.del(url).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + yield* Effect.logDebug( + `Swift deleteBucket container=[${container}] status=${response.status}`, + ); + + if (response.status === 204) { + return; + } + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "DELETE", + ), + ); + } + }), + + headBucket: () => + Effect.gen(function* () { + const { url, token, container } = target; + const response = yield* client.execute( + HttpClientRequest.head(url).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", container, "HEAD"), + ); + } + }), +}); diff --git a/src/Backends/Swift/Client.ts b/src/Backends/Swift/Client.ts index 7be8123..0feda08 100644 --- a/src/Backends/Swift/Client.ts +++ b/src/Backends/Swift/Client.ts @@ -1,6 +1,7 @@ -import { Context, Effect, Layer, type Schema } from "effect"; +import { Cache, Context, Effect, Layer, type Schema } from "effect"; +import { HttpClient, HttpClientRequest } from "@effect/platform"; import type { MaterializedBucket, SwiftConfig } from "../../Domain/Config.ts"; -import { AppConfig } from "../../Config/Layer.ts"; +import { HeraldConfig } from "../../Config/Layer.ts"; export interface SwiftAuthMeta { readonly token: string; @@ -35,160 +36,152 @@ interface SwiftTokenResponse { export const SwiftClientLive = Layer.effect( SwiftClient, - AppConfig.pipe( - Effect.flatMap((appConfig) => { - const cache = new Map(); + Effect.gen(function* () { + const appConfig = yield* HeraldConfig; + const client = yield* HttpClient.HttpClient; + + const fetchAuthMeta = ( + config: Schema.Schema.Type, + ): Effect.Effect => { + const { auth_url, credentials, region } = config; + + if (!credentials || !("username" in credentials)) { + return Effect.fail( + new Error( + "Swift credentials (username, password, etc.) are required", + ), + ); + } + + const { + username, + password, + project_name, + user_domain_name = "Default", + project_domain_name = "Default", + } = credentials; + + const requestBody = { + auth: { + identity: { + methods: ["password"], + password: { + user: { + name: username, + domain: { name: user_domain_name }, + password: password, + }, + }, + }, + scope: { + project: { + domain: { name: project_domain_name }, + name: project_name, + }, + }, + }, + }; - const fetchAuthMeta = ( - config: Schema.Schema.Type, - ): Effect.Effect => { - const { auth_url, credentials, region } = config; + return Effect.gen(function* () { + const request = yield* HttpClientRequest.post(`${auth_url}/auth/tokens`) + .pipe( + HttpClientRequest.bodyJson(requestBody), + Effect.mapError((e) => new Error(String(e))), + ); + const response = yield* client.execute(request).pipe( + Effect.mapError((e) => new Error(String(e))), + ); - if (!auth_url) { - return Effect.fail( - new Error("auth_url is required for Swift backend"), + if (response.status < 200 || response.status >= 300) { + const msg = yield* response.text.pipe( + Effect.orElseSucceed(() => "Unknown error"), + ); + return yield* Effect.fail( + new Error(`Failed to authenticate with Swift: ${msg}`), ); } - if (!credentials || !("username" in credentials)) { - return Effect.fail( + + const token = response.headers["x-subject-token"]; + const tokenStr = Array.isArray(token) ? token[0] : token; + + if (!tokenStr) { + return yield* Effect.fail( new Error( - "Swift credentials (username, password, etc.) are required", + "X-Subject-Token header missing from Swift response", ), ); } - const { - username, - password, - project_name, - user_domain_name = "Default", - project_domain_name = "Default", - } = credentials; - - const requestBody = JSON.stringify({ - auth: { - identity: { - methods: ["password"], - password: { - user: { - name: username, - domain: { name: user_domain_name }, - password: password, - }, - }, - }, - scope: { - project: { - domain: { name: project_domain_name }, - name: project_name, - }, - }, - }, - }); - - return Effect.tryPromise({ - try: async () => { - const response = await fetch(`${auth_url}/auth/tokens`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: requestBody, - }); - - if (!response.ok) { - const msg = await response.text(); - throw new Error(`Failed to authenticate with Swift: ${msg}`); - } - - const token = response.headers.get("X-Subject-Token"); - if (!token) { - throw new Error( - "X-Subject-Token header missing from Swift response", - ); - } - - const body = (await response.json()) as SwiftTokenResponse; - const catalog = body.token.catalog; - const storageService = catalog.find((s) => - s.type === "object-store" - ); - - if (!storageService) { - throw new Error( - "Object Store service not found in Swift catalog", - ); - } - - const endpoint = storageService.endpoints.find( - (e) => - (region ? e.region === region : true) && - e.interface === "public", - ); - - if (!endpoint) { - throw new Error( - `Public Swift endpoint not found (region: ${region ?? "any"})`, - ); - } - - return { - token, - storageUrl: endpoint.url, - }; - }, - catch: (e) => e as Error, - }); - }; + const body = (yield* response.json.pipe( + Effect.mapError((e) => new Error(String(e))), + )) as SwiftTokenResponse; - return Effect.succeed( - SwiftClient.of({ - getAuthMeta: ( - bucket: MaterializedBucket | { backend_id: string }, - ) => { - let backend_id: string; - let config: Schema.Schema.Type; - - if ("protocol" in bucket) { - backend_id = bucket.backend_id; - config = appConfig.raw.backends[backend_id] as Schema.Schema.Type< - typeof SwiftConfig - >; - } else { - backend_id = bucket.backend_id; - config = appConfig.raw.backends[backend_id] as Schema.Schema.Type< - typeof SwiftConfig - >; - } - - if (!config || config.protocol !== "swift") { - return Effect.fail( - new Error(`Backend ${backend_id} is not a Swift backend`), - ); - } - - const cacheKey = - `${backend_id}:${config.auth_url}:${config.region}`; - const cached = cache.get(cacheKey); - const now = Date.now(); - - if (cached && cached.expires > now) { - return Effect.succeed({ - token: cached.token, - storageUrl: cached.storageUrl, - }); - } - - return fetchAuthMeta(config).pipe( - Effect.tap((meta) => { - // Cache for 50 minutes (Swift tokens usually last 1h) - cache.set(cacheKey, { - ...meta, - expires: now + 50 * 60 * 1000, - }); - }), - ); - }, - }), - ); - }), - ), + const catalog = body.token.catalog; + const storageService = catalog.find((s) => s.type === "object-store"); + + if (!storageService) { + return yield* Effect.fail( + new Error( + "Object Store service not found in Swift catalog", + ), + ); + } + + const endpoint = storageService.endpoints.find( + (e) => + (region ? e.region === region : true) && + e.interface === "public", + ); + + if (!endpoint) { + return yield* Effect.fail( + new Error( + `Public Swift endpoint not found (region: ${region ?? "any"})`, + ), + ); + } + + return { + token: tokenStr, + storageUrl: endpoint.url, + }; + }); + }; + + const cache = yield* Cache.make({ + capacity: 100, + timeToLive: "50 minutes", // Swift tokens usually last 1h + lookup: (config: Schema.Schema.Type) => + fetchAuthMeta(config), + }); + + return SwiftClient.of({ + getAuthMeta: ( + bucket: MaterializedBucket | { backend_id: string }, + ) => { + let backend_id: string; + let config: Schema.Schema.Type; + + if ("protocol" in bucket) { + backend_id = bucket.backend_id; + config = appConfig.raw.backends[backend_id] as Schema.Schema.Type< + typeof SwiftConfig + >; + } else { + backend_id = bucket.backend_id; + config = appConfig.raw.backends[backend_id] as Schema.Schema.Type< + typeof SwiftConfig + >; + } + + if (!config || config.protocol !== "swift") { + return Effect.fail( + new Error(`Backend ${backend_id} is not a Swift backend`), + ); + } + + return cache.get(config); + }, + }); + }), ); diff --git a/src/Backends/Swift/Objects.ts b/src/Backends/Swift/Objects.ts new file mode 100644 index 0000000..431394b --- /dev/null +++ b/src/Backends/Swift/Objects.ts @@ -0,0 +1,529 @@ +import { Effect, Option, type Stream } from "effect"; +import { type HttpClient, HttpClientRequest } from "@effect/platform"; +import { + type CommonPrefix, + type DeleteObjectsResult, + InternalError, + type ListObjectsResult, + type ObjectInfo, + type ObjectResponse, + type PutObjectResult, +} from "../../Services/Backend.ts"; +import { mapError, type SwiftTarget } from "./Utils.ts"; +import { fixHeaderEncoding } from "../../Frontend/Utils.ts"; + +export interface SwiftObject { + readonly name?: string; + readonly hash?: string; + readonly bytes?: number; + readonly content_type?: string; + readonly last_modified?: string; + readonly subdir?: string; +} + +export const makeObjectOps = ( + target: SwiftTarget, + client: HttpClient.HttpClient, +) => { + const listObjects = (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + encodingType?: string; + continuationToken?: string; + startAfter?: string; + listType?: 1 | 2; + }) => + Effect.gen(function* () { + const { url, token, container } = target; + const limit = args.maxKeys ?? 1000; + const query = new URLSearchParams({ format: "json" }); + if (args.prefix) query.set("prefix", args.prefix); + if (args.delimiter) query.set("delimiter", args.delimiter); + if (args.marker) query.set("marker", args.marker); + query.set("limit", String(limit + 1)); + if (args.continuationToken) query.set("marker", args.continuationToken); + if (args.startAfter) query.set("marker", args.startAfter); + + const response = yield* client.execute( + HttpClientRequest.get(`${url}?${query.toString()}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + yield* Effect.logDebug( + `Swift listObjects query=[${query.toString()}] status=${response.status}`, + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError(response.status, message || "Error", container, "GET"), + ); + } + + const rawObjects = (yield* response.json.pipe( + Effect.mapError((e) => + mapError(500, `Failed to parse Swift response: ${e}`, container) + ), + )) as readonly SwiftObject[]; + + const isTruncated = rawObjects.length > limit; + const objects = isTruncated ? rawObjects.slice(0, limit) : rawObjects; + + const contents: ObjectInfo[] = []; + const commonPrefixes: CommonPrefix[] = []; + + for (const obj of objects) { + if (obj.subdir) { + commonPrefixes.push({ prefix: obj.subdir }); + } else if (obj.name) { + contents.push({ + key: obj.name, + lastModified: obj.last_modified + ? new Date(obj.last_modified) + : new Date(), + etag: obj.hash ? `"${obj.hash}"` : "", + size: obj.bytes ?? 0, + storageClass: "STANDARD", + owner: { id: "swift", displayName: "Swift User" }, + }); + } + } + + const nextMarker = isTruncated && objects.length > 0 + ? objects[objects.length - 1].name || + objects[objects.length - 1].subdir + : undefined; + + return { + name: container, + prefix: args.prefix, + maxKeys: limit, + delimiter: args.delimiter, + isTruncated, + marker: args.marker, + nextMarker, + contents, + commonPrefixes, + encodingType: args.encodingType, + listType: args.listType ?? 1, + nextContinuationToken: args.listType === 2 ? nextMarker : undefined, + keyCount: contents.length + commonPrefixes.length, + } satisfies ListObjectsResult; + }); + + return { + listObjects: (args: { + prefix?: string; + delimiter?: string; + marker?: string; + maxKeys?: number; + encodingType?: string; + continuationToken?: string; + startAfter?: string; + listType?: 1 | 2; + }) => listObjects(args), + + listVersions: (args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + versionIdMarker?: string; + maxKeys?: number; + encodingType?: string; + }) => + Effect.gen(function* () { + const result = yield* listObjects({ + prefix: args.prefix, + delimiter: args.delimiter, + marker: args.keyMarker, + maxKeys: args.maxKeys, + }); + return { + ...result, + contents: result.contents.map((c) => ({ + ...c, + versionId: "null", + isLatest: true, + })), + }; + }), + + getObject: ( + key: string, + headers: Record, + ) => + Effect.gen(function* () { + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const swiftHeaders: Record = { + "X-Auth-Token": token, + }; + if (headers["range"] || headers["Range"]) { + swiftHeaders["Range"] = String( + headers["range"] || headers["Range"], + ); + } + if (headers["if-match"] || headers["If-Match"]) { + swiftHeaders["If-Match"] = String( + headers["if-match"] || + headers["If-Match"], + ); + } + if (headers["if-none-match"] || headers["If-None-Match"]) { + swiftHeaders["If-None-Match"] = String( + headers["if-none-match"] || + headers["If-None-Match"], + ); + } + if (headers["if-modified-since"] || headers["If-Modified-Since"]) { + swiftHeaders["If-Modified-Since"] = String( + headers["if-modified-since"] || + headers["If-Modified-Since"], + ); + } + if ( + headers["if-unmodified-since"] || headers["If-Unmodified-Since"] + ) { + swiftHeaders["If-Unmodified-Since"] = String( + headers["if-unmodified-since"] || + headers["If-Unmodified-Since"], + ); + } + + const response = yield* client.execute( + HttpClientRequest.get(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders(swiftHeaders), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "GET", + key, + ), + ); + } + + const metadata: Record = {}; + const s3Headers: Record = {}; + + for (const [k, v] of Object.entries(response.headers)) { + const lowK = k.toLowerCase(); + const value = Array.isArray(v) ? v.join(", ") : v; + if (lowK.startsWith("x-object-meta-")) { + const metaKey = lowK.substring("x-object-meta-".length); + const decodedValue = (value.includes("%")) + ? Option.liftThrowable(decodeURIComponent)(value).pipe( + Option.getOrElse(() => value), + ) + : value; + metadata[metaKey] = decodedValue; + s3Headers[`x-amz-meta-${metaKey}`] = decodedValue; + } else if (lowK === "content-type") { + s3Headers["Content-Type"] = value; + } else if (lowK === "content-length") { + s3Headers["Content-Length"] = value; + } else if (lowK === "etag") { + s3Headers["ETag"] = value; + } else if (lowK === "last-modified") { + s3Headers["Last-Modified"] = value; + } + } + + const contentLengthHeader = response.headers["content-length"]; + const contentLength = Array.isArray(contentLengthHeader) + ? parseInt(contentLengthHeader[0] || "0") + : parseInt(contentLengthHeader || "0"); + + const etagHeader = response.headers["etag"]; + const etag = Array.isArray(etagHeader) ? etagHeader[0] : etagHeader; + + const lastModifiedHeader = response.headers["last-modified"]; + const lastModified = Array.isArray(lastModifiedHeader) + ? lastModifiedHeader[0] + : lastModifiedHeader; + + return { + stream: response.stream, + contentType: (Array.isArray(response.headers["content-type"]) + ? response.headers["content-type"][0] + : response.headers["content-type"]) || undefined, + contentLength, + etag: etag || undefined, + lastModified: lastModified ? new Date(lastModified) : undefined, + metadata, + headers: s3Headers, + } satisfies ObjectResponse; + }), + + headObject: ( + key: string, + _headers: Record, + ) => + Effect.gen(function* () { + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const swiftHeaders: Record = { + "X-Auth-Token": token, + }; + // ... handle headers if needed + const response = yield* client.execute( + HttpClientRequest.head(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders(swiftHeaders), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "HEAD", + key, + ), + ); + } + + const metadata: Record = {}; + const s3Headers: Record = {}; + + for (const [k, v] of Object.entries(response.headers)) { + const lowK = k.toLowerCase(); + const value = Array.isArray(v) ? v.join(", ") : v; + if (lowK.startsWith("x-object-meta-")) { + const metaKey = lowK.substring("x-object-meta-".length); + const decodedValue = (value.includes("%")) + ? Option.liftThrowable(decodeURIComponent)(value).pipe( + Option.getOrElse(() => value), + ) + : value; + metadata[metaKey] = decodedValue; + s3Headers[`x-amz-meta-${metaKey}`] = decodedValue; + } else if (lowK === "content-type") { + s3Headers["Content-Type"] = value; + } else if (lowK === "content-length") { + s3Headers["Content-Length"] = value; + } else if (lowK === "etag") { + s3Headers["ETag"] = value; + } else if (lowK === "last-modified") { + s3Headers["Last-Modified"] = value; + } + } + + const contentLengthHeader = response.headers["content-length"]; + const contentLength = Array.isArray(contentLengthHeader) + ? parseInt(contentLengthHeader[0] || "0") + : parseInt(contentLengthHeader || "0"); + + const etagHeader = response.headers["etag"]; + const etag = Array.isArray(etagHeader) ? etagHeader[0] : etagHeader; + + const lastModifiedHeader = response.headers["last-modified"]; + const lastModified = Array.isArray(lastModifiedHeader) + ? lastModifiedHeader[0] + : lastModifiedHeader; + + return { + contentType: (Array.isArray(response.headers["content-type"]) + ? response.headers["content-type"][0] + : response.headers["content-type"]) || undefined, + contentLength, + etag: etag || undefined, + lastModified: lastModified ? new Date(lastModified) : undefined, + metadata, + headers: s3Headers, + }; + }), + + putObject: ( + key: string, + stream: Stream.Stream, + headers: Record, + ) => + Effect.gen(function* () { + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const contentLength = headers["content-length"] || + headers["Content-Length"]; + + const swiftHeaders: Record = { + "X-Auth-Token": token, + "Content-Type": (headers["content-type"] || headers["Content-Type"] || + "application/octet-stream") as string, + ...(contentLength ? { "Content-Length": String(contentLength) } : {}), + }; + + for (const [k, v] of Object.entries(headers)) { + const lowK = k.toLowerCase(); + if (lowK.startsWith("x-amz-meta-")) { + const metaKey = lowK.substring("x-amz-meta-".length); + const value = fixHeaderEncoding(String(v)); + swiftHeaders[`X-Object-Meta-${metaKey}`] = + /[^\x20-\x7E]/.test(value) + ? encodeURIComponent(value) + : value; + } + } + + const request = HttpClientRequest.put(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders(swiftHeaders), + HttpClientRequest.bodyStream(stream), + ); + + const response = yield* client.execute(request).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + yield* Effect.logDebug( + `Swift putObject key=[${key}] status=${response.status}`, + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "PUT", + key, + ), + ); + } + + const etagHeader = response.headers["etag"]; + const etagValue = Array.isArray(etagHeader) + ? etagHeader[0] + : etagHeader; + + return { + etag: etagValue || undefined, + } satisfies PutObjectResult; + }), + + deleteObject: (key: string) => + Effect.gen(function* () { + const { url, token, container } = target; + const encodedKey = key.split("/").map(encodeURIComponent).join("/"); + const response = yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + if (response.status < 200 || response.status >= 300) { + if (response.status === 404) { + return; + } + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "DELETE", + key, + ), + ); + } + }), + + deleteObjects: (objects: readonly { key: string; versionId?: string }[]) => + Effect.gen(function* () { + const { url, token, container } = target; + const deleted: string[] = []; + const errors: { key: string; code: string; message: string }[] = []; + + for (const obj of objects) { + const encodedKey = obj.key.split("/").map(encodeURIComponent).join( + "/", + ); + const response = yield* client.execute( + HttpClientRequest.del(`${url}/${encodedKey}`).pipe( + HttpClientRequest.setHeaders({ "X-Auth-Token": token }), + ), + ).pipe( + Effect.mapError((e) => mapError(500, String(e), container)), + ); + + yield* Effect.logDebug( + `Swift deleteObject key=[${obj.key}] status=${response.status}`, + ); + + if ( + (response.status >= 200 && response.status < 300) || + response.status === 204 || response.status === 404 + ) { + deleted.push(obj.key); + } else { + const errorBody = yield* response.text.pipe( + Effect.orElseSucceed(() => "Unknown error"), + ); + errors.push({ + key: obj.key, + code: String(response.status), + message: errorBody, + }); + } + } + + return { deleted, errors } satisfies DeleteObjectsResult; + }), + + createMultipartUpload: ( + _key: string, + _headers: Record, + ) => Effect.fail(new InternalError({ message: "Not implemented" })), + uploadPart: ( + _key: string, + _uploadId: string, + _partNumber: number, + _body: Stream.Stream, + ) => Effect.fail(new InternalError({ message: "Not implemented" })), + completeMultipartUpload: ( + _key: string, + _uploadId: string, + _parts: readonly { etag: string; partNumber: number }[], + ) => Effect.fail(new InternalError({ message: "Not implemented" })), + abortMultipartUpload: (_key: string, _uploadId: string) => + Effect.fail(new InternalError({ message: "Not implemented" })), + listMultipartUploads: (_args: { + prefix?: string; + delimiter?: string; + keyMarker?: string; + uploadIdMarker?: string; + maxUploads?: number; + encodingType?: string; + }) => Effect.fail(new InternalError({ message: "Not implemented" })), + listParts: (_key: string, _uploadId: string) => + Effect.fail(new InternalError({ message: "Not implemented" })), + }; +}; diff --git a/src/Backends/Swift/Utils.ts b/src/Backends/Swift/Utils.ts new file mode 100644 index 0000000..62295b0 --- /dev/null +++ b/src/Backends/Swift/Utils.ts @@ -0,0 +1,74 @@ +import { Effect } from "effect"; +import { + type BackendError, + BucketAlreadyExists, + BucketAlreadyOwnedByYou, + BucketNotEmpty, + InternalError, + NoSuchBucket, + NoSuchKey, +} from "../../Services/Backend.ts"; +import type { MaterializedBucket } from "../../Domain/Config.ts"; +import { SwiftClient } from "./Client.ts"; + +export interface SwiftTarget { + readonly storageUrl: string; + readonly token: string; + readonly container: string; + readonly url: string; +} + +export const mapError = ( + status: number, + message: string, + bucketName: string, + method?: string, + key?: string, +): BackendError => { + switch (status) { + case 404: + if (key) { + return new NoSuchKey({ bucketName, key, message }); + } + return new NoSuchBucket({ bucketName, message }); + case 409: + if (method === "DELETE") { + return new BucketNotEmpty({ bucketName, message }); + } + return new BucketAlreadyExists({ bucketName, message }); + case 202: + if (method === "PUT") { + return new BucketAlreadyOwnedByYou({ bucketName, message }); + } + return new InternalError({ + message: `Swift error (${status}): ${message}`, + }); + default: + return new InternalError({ + message: `Swift error (${status}): ${message}`, + }); + } +}; + +/** + * Resolves the target container and acquires the Swift token dynamically. + */ +export const getTarget = ( + bucket: MaterializedBucket | { backend_id: string }, +): Effect.Effect => + Effect.gen(function* () { + const swiftClient = yield* SwiftClient; + const auth = yield* swiftClient.getAuthMeta(bucket).pipe( + Effect.mapError((e) => new InternalError({ message: e.message })), + ); + const container = "bucket_name" in bucket ? bucket.bucket_name : ""; + const encodedContainer = container ? encodeURIComponent(container) : ""; + return { + storageUrl: auth.storageUrl, + token: auth.token, + container, + url: encodedContainer + ? `${auth.storageUrl}/${encodedContainer}` + : auth.storageUrl, + }; + }); diff --git a/src/Config/Layer.ts b/src/Config/Layer.ts index 4acdfeb..8efd2d0 100644 --- a/src/Config/Layer.ts +++ b/src/Config/Layer.ts @@ -6,8 +6,8 @@ import { type MaterializedBucket, } from "../Domain/Config.ts"; -export class AppConfig extends Context.Tag("AppConfig")< - AppConfig, +export class HeraldConfig extends Context.Tag("HeraldConfig")< + HeraldConfig, { readonly raw: GlobalConfig; readonly lookupBucket: (name: string) => Option.Option; @@ -109,8 +109,8 @@ export function parseConfig( return Schema.decodeUnknownSync(GlobalConfig)({ backends }); } -export const AppConfigLive = Layer.effect( - AppConfig, +export const HeraldConfigLive = Layer.effect( + HeraldConfig, Effect.gen(function* () { const configPath = yield* Config.string("HERALD_CONFIG_PATH").pipe( Config.orElse(() => Config.string("CONFIG_PATH")), diff --git a/src/Domain/Config.ts b/src/Domain/Config.ts index 337f560..77f32a0 100644 --- a/src/Domain/Config.ts +++ b/src/Domain/Config.ts @@ -41,7 +41,7 @@ export const S3Config = Schema.Struct({ export const SwiftConfig = Schema.Struct({ protocol: Schema.Literal("swift"), - auth_url: Schema.optional(Schema.String), + auth_url: Schema.String, region: Schema.optional(Schema.String), container: Schema.optional(Schema.String), credentials: Schema.optional(SwiftCredentials), diff --git a/src/Frontend/Api.ts b/src/Frontend/Api.ts index 91a78de..f4a7fdc 100644 --- a/src/Frontend/Api.ts +++ b/src/Frontend/Api.ts @@ -5,7 +5,7 @@ export class BadGateway extends Schema.TaggedError()("BadGateway", { message: Schema.String, }) {} -export const S3Api = HttpApiGroup.make("s3") +export const HttpS3Api = HttpApiGroup.make("s3") .add( HttpApiEndpoint.post("postRoot", "/") .addError(BadGateway, { status: 502 }), diff --git a/src/Frontend/Buckets/Create.ts b/src/Frontend/Buckets/Create.ts index 7d42506..68c32a9 100644 --- a/src/Frontend/Buckets/Create.ts +++ b/src/Frontend/Buckets/Create.ts @@ -1,40 +1,37 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; -export const createBucket = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const url = new URL(request.url, "http://localhost"); - yield* Effect.logDebug( - `createBucket bucket=[${bucket}] url=[${request.url}]`, - ); +export const createBucket = () => + Effect.gen(function* () { + const { backend, bucket, params, request } = yield* RequestContext; - if (url.searchParams.has("acl")) { - // PutBucketAcl - // Check for canned ACL validity if present - const cannedAcl = request.headers["x-amz-acl"]; - const validCannedAcls = [ - "private", - "public-read", - "public-read-write", - "authenticated-read", - ]; - if (cannedAcl && !validCannedAcls.includes(cannedAcl)) { - return HttpServerResponse.text( - `InvalidArgumentArgument x-amz-acl is invalid.`, - { status: 400, headers: { "Content-Type": "application/xml" } }, - ); - } + yield* Effect.logDebug( + `createBucket bucket=[${bucket}] url=[${request.url}]`, + ); - // For now, we just return 200 OK if the bucket exists - yield* backend.headBucket(); - return HttpServerResponse.text("", { status: 200 }); + if (params.acl !== undefined) { + // PutBucketAcl + // Check for canned ACL validity if present + const cannedAcl = request.headers["x-amz-acl"]; + const validCannedAcls = [ + "private", + "public-read", + "public-read-write", + "authenticated-read", + ]; + if (cannedAcl && !validCannedAcls.includes(cannedAcl)) { + return HttpServerResponse.text( + `InvalidArgumentArgument x-amz-acl is invalid.`, + { status: 400, headers: { "Content-Type": "application/xml" } }, + ); } - yield* backend.createBucket(); + // For now, we just return 200 OK if the bucket exists + yield* backend.headBucket(); return HttpServerResponse.text("", { status: 200 }); - })); + } + + yield* backend.createBucket(); + return HttpServerResponse.text("", { status: 200 }); + }); diff --git a/src/Frontend/Buckets/Delete.ts b/src/Frontend/Buckets/Delete.ts index de3a301..6c7fbe1 100644 --- a/src/Frontend/Buckets/Delete.ts +++ b/src/Frontend/Buckets/Delete.ts @@ -1,12 +1,10 @@ import { Effect } from "effect"; import { HttpServerResponse } from "@effect/platform"; -import { resolveBucket } from "../Utils.ts"; +import { RequestContext } from "../Utils.ts"; -export const deleteBucket = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - yield* backend.deleteBucket(); - return HttpServerResponse.empty({ status: 204 }); - })); +export const deleteBucket = () => + Effect.gen(function* () { + const { backend } = yield* RequestContext; + yield* backend.deleteBucket(); + return HttpServerResponse.empty({ status: 204 }); + }); diff --git a/src/Frontend/Buckets/Head.ts b/src/Frontend/Buckets/Head.ts index fb371d8..a076d71 100644 --- a/src/Frontend/Buckets/Head.ts +++ b/src/Frontend/Buckets/Head.ts @@ -1,12 +1,10 @@ import { Effect } from "effect"; import { HttpServerResponse } from "@effect/platform"; -import { resolveBucket } from "../Utils.ts"; +import { RequestContext } from "../Utils.ts"; -export const headBucket = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - yield* backend.headBucket(); - return HttpServerResponse.empty({ status: 200 }); - })); +export const headBucket = () => + Effect.gen(function* () { + const { backend } = yield* RequestContext; + yield* backend.headBucket(); + return HttpServerResponse.empty({ status: 200 }); + }); diff --git a/src/Frontend/Buckets/List.ts b/src/Frontend/Buckets/List.ts index a6759fc..4bb13f5 100644 --- a/src/Frontend/Buckets/List.ts +++ b/src/Frontend/Buckets/List.ts @@ -1,11 +1,11 @@ import { Effect } from "effect"; -import { AppConfig } from "../../Config/Layer.ts"; +import { HeraldConfig } from "../../Config/Layer.ts"; import { S3Xml } from "../../Services/S3Xml.ts"; import { resolveBackend } from "../Utils.ts"; export const listBuckets = () => Effect.gen(function* () { - const config = yield* AppConfig; + const config = yield* HeraldConfig; // For ListBuckets, we need to decide which backend to proxy to. // We prefer an S3 backend if available, otherwise we take the first one. diff --git a/src/Frontend/Health/Api.ts b/src/Frontend/Health/Api.ts index 70032e5..829ae5f 100644 --- a/src/Frontend/Health/Api.ts +++ b/src/Frontend/Health/Api.ts @@ -1,7 +1,7 @@ import { HttpApiEndpoint, HttpApiGroup, OpenApi } from "@effect/platform"; import { Schema } from "effect"; -export class HealthApi extends HttpApiGroup.make("health") +export class HealthHttpApi extends HttpApiGroup.make("health") .add( HttpApiEndpoint.get("getStatus", "/health") .addSuccess(Schema.Struct({ status: Schema.Literal("ok") })), diff --git a/src/Frontend/Health/Http.ts b/src/Frontend/Health/Http.ts index 52e64f2..0f186df 100644 --- a/src/Frontend/Health/Http.ts +++ b/src/Frontend/Health/Http.ts @@ -1,9 +1,9 @@ import { HttpApiBuilder } from "@effect/platform"; import { Effect } from "effect"; -import { Api } from "../../Api.ts"; +import { HttpHeraldApi } from "../../Api.ts"; export const HttpHealthLive = HttpApiBuilder.group( - Api, + HttpHeraldApi, "health", (handlers) => handlers.handle( diff --git a/src/Frontend/Http.ts b/src/Frontend/Http.ts index 28825cc..90b4541 100644 --- a/src/Frontend/Http.ts +++ b/src/Frontend/Http.ts @@ -1,6 +1,6 @@ import { HttpApiBuilder, HttpServerResponse } from "@effect/platform"; import { Effect, Layer } from "effect"; -import { Api } from "../Api.ts"; +import { HttpHeraldApi } from "../Api.ts"; import { listBuckets } from "./Buckets/List.ts"; import { createBucket } from "./Buckets/Create.ts"; import { deleteBucket } from "./Buckets/Delete.ts"; @@ -15,28 +15,34 @@ import { S3ClientLive } from "../Backends/S3/Client.ts"; import { SwiftClientLive } from "../Backends/Swift/Client.ts"; import { S3XmlLive } from "../Services/S3Xml.ts"; import { BackendResolverLive } from "../Services/BackendResolver.ts"; +import { provideRequestContext } from "./Utils.ts"; export const HttpS3Live = HttpApiBuilder.group( - Api, + HttpHeraldApi, "s3", (handlers) => handlers + // handleRaw is preferred througout since + // we want to return XML directly + // after setting our own .handleRaw("postRoot", (_handlers) => Effect.gen(function* () { yield* Effect.logDebug("POST / received"); + // FIXME: what's the purose of this handler? + // 200 diverges from 502 as defiend in the openapi return HttpServerResponse.text("", { status: 200 }); })) .handleRaw("listBuckets", listBuckets) - .handleRaw("createBucket", createBucket) - .handleRaw("deleteBucket", deleteBucket) - .handleRaw("headBucket", headBucket) - .handleRaw("listObjects", listObjects) - .handleRaw("postBucket", postObject) - .handleRaw("getObject", getObject) - .handleRaw("putObject", putObject) - .handleRaw("postObject", postObject) - .handleRaw("deleteObject", deleteObject) - .handleRaw("headObject", headObject), + .handleRaw("createBucket", provideRequestContext(createBucket)) + .handleRaw("deleteBucket", provideRequestContext(deleteBucket)) + .handleRaw("headBucket", provideRequestContext(headBucket)) + .handleRaw("listObjects", provideRequestContext(listObjects)) + .handleRaw("postBucket", provideRequestContext(postObject)) + .handleRaw("getObject", provideRequestContext(getObject)) + .handleRaw("putObject", provideRequestContext(putObject)) + .handleRaw("postObject", provideRequestContext(postObject)) + .handleRaw("deleteObject", provideRequestContext(deleteObject)) + .handleRaw("headObject", provideRequestContext(headObject)), ).pipe( Layer.provide(BackendResolverLive), Layer.provide(S3ClientLive), diff --git a/src/Frontend/Objects/Delete.ts b/src/Frontend/Objects/Delete.ts index daa07ac..3e1856b 100644 --- a/src/Frontend/Objects/Delete.ts +++ b/src/Frontend/Objects/Delete.ts @@ -1,27 +1,20 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; /** * Handler for DeleteObject (DELETE /:bucket/*) */ -export const deleteObject = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const key = extractKey(request.url, bucket); - const url = new URL(request.url, "http://localhost"); - const searchParams = url.searchParams; +export const deleteObject = () => + Effect.gen(function* () { + const { backend, key, params } = yield* RequestContext; - if (searchParams.has("uploadId")) { - // Abort Multipart Upload - const uploadId = searchParams.get("uploadId")!; - yield* backend.abortMultipartUpload(key, uploadId); - return HttpServerResponse.empty({ status: 204 }); - } - - yield* backend.deleteObject(key); + if (params.uploadId) { + // Abort Multipart Upload + yield* backend.abortMultipartUpload(key, params.uploadId); return HttpServerResponse.empty({ status: 204 }); - })); + } + + yield* backend.deleteObject(key); + return HttpServerResponse.empty({ status: 204 }); + }); diff --git a/src/Frontend/Objects/Get.ts b/src/Frontend/Objects/Get.ts index 65e0a28..c223fec 100644 --- a/src/Frontend/Objects/Get.ts +++ b/src/Frontend/Objects/Get.ts @@ -1,40 +1,35 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; import { S3Xml } from "../../Services/S3Xml.ts"; /** * Handler for GetObject (GET /:bucket/*) * Also handles ListParts (?uploadId=...). */ -export const getObject = ({ path: { bucket } }: { path: { bucket: string } }) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const s3Xml = yield* S3Xml; - const key = extractKey(request.url, bucket); - const url = new URL(request.url, "http://localhost"); - const searchParams = url.searchParams; +export const getObject = () => + Effect.gen(function* () { + const { backend, key, params, request } = yield* RequestContext; + const s3Xml = yield* S3Xml; - if (searchParams.has("uploadId")) { - // List Parts - const uploadId = searchParams.get("uploadId")!; - const result = yield* backend.listParts(key, uploadId); - return s3Xml.formatListParts(result); - } + if (params.uploadId) { + // List Parts + const result = yield* backend.listParts(key, params.uploadId); + return s3Xml.formatListParts(result); + } - const combinedHeaders = { ...request.headers }; - if (searchParams.has("partNumber")) { - combinedHeaders["x-amz-part-number"] = searchParams.get("partNumber")!; - } + const combinedHeaders = { ...request.headers }; + if (params.partNumber) { + combinedHeaders["x-amz-part-number"] = String(params.partNumber); + } - const result = yield* backend.getObject(key, combinedHeaders); - const status = (request.headers["range"] || request.headers["Range"]) - ? 206 - : 200; - return HttpServerResponse.stream(result.stream, { - status, - headers: result.headers, - contentType: result.contentType, - }); - })); + const result = yield* backend.getObject(key, combinedHeaders); + const status = (request.headers["range"] || request.headers["Range"]) + ? 206 + : 200; + return HttpServerResponse.stream(result.stream, { + status, + headers: result.headers, + contentType: result.contentType, + }); + }); diff --git a/src/Frontend/Objects/Head.ts b/src/Frontend/Objects/Head.ts index 483d854..b3daa57 100644 --- a/src/Frontend/Objects/Head.ts +++ b/src/Frontend/Objects/Head.ts @@ -1,28 +1,22 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; /** * Handler for HeadObject (HEAD /:bucket/*) */ -export const headObject = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const key = extractKey(request.url, bucket); - const url = new URL(request.url, "http://localhost"); - const combinedHeaders = { ...request.headers }; - if (url.searchParams.has("partNumber")) { - combinedHeaders["x-amz-part-number"] = url.searchParams.get( - "partNumber", - )!; - } +export const headObject = () => + Effect.gen(function* () { + const { backend, key, params, request } = yield* RequestContext; - const result = yield* backend.headObject(key, combinedHeaders); - return HttpServerResponse.empty({ - status: 200, - headers: result.headers, - }); - })); + const combinedHeaders = { ...request.headers }; + if (params.partNumber) { + combinedHeaders["x-amz-part-number"] = String(params.partNumber); + } + + const result = yield* backend.headObject(key, combinedHeaders); + return HttpServerResponse.empty({ + status: 200, + headers: result.headers, + }); + }); diff --git a/src/Frontend/Objects/List.ts b/src/Frontend/Objects/List.ts index f92cdc1..883f247 100644 --- a/src/Frontend/Objects/List.ts +++ b/src/Frontend/Objects/List.ts @@ -1,61 +1,49 @@ import { Effect } from "effect"; -import { HttpServerRequest } from "@effect/platform"; -import { resolveBucket } from "../Utils.ts"; +import { RequestContext } from "../Utils.ts"; import { S3Xml } from "../../Services/S3Xml.ts"; /** * Handler for ListObjects (GET /:bucket) */ -export const listObjects = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const s3Xml = yield* S3Xml; - const url = new URL(request.url, "http://localhost"); - const searchParams = url.searchParams; +export const listObjects = () => + Effect.gen(function* () { + const { backend, params } = yield* RequestContext; + const s3Xml = yield* S3Xml; - if (searchParams.has("versions")) { - const result = yield* backend.listVersions({ - prefix: searchParams.get("prefix") ?? undefined, - delimiter: searchParams.get("delimiter") ?? undefined, - keyMarker: searchParams.get("key-marker") ?? undefined, - versionIdMarker: searchParams.get("version-id-marker") ?? undefined, - maxKeys: searchParams.has("max-keys") - ? parseInt(searchParams.get("max-keys")!) - : undefined, - encodingType: searchParams.get("encoding-type") ?? undefined, - }); - return s3Xml.formatListVersions(result); - } - - if (searchParams.has("uploads")) { - const result = yield* backend.listMultipartUploads({ - prefix: searchParams.get("prefix") ?? undefined, - delimiter: searchParams.get("delimiter") ?? undefined, - keyMarker: searchParams.get("key-marker") ?? undefined, - uploadIdMarker: searchParams.get("upload-id-marker") ?? undefined, - maxUploads: searchParams.has("max-uploads") - ? parseInt(searchParams.get("max-uploads")!) - : undefined, - encodingType: searchParams.get("encoding-type") ?? undefined, - }); - return s3Xml.formatListMultipartUploads(result); - } + if (params.versions !== undefined) { + const result = yield* backend.listVersions({ + prefix: params.prefix, + delimiter: params.delimiter, + keyMarker: params["key-marker"], + versionIdMarker: params["version-id-marker"], + maxKeys: params["max-keys"], + encodingType: params["encoding-type"], + }); + return s3Xml.formatListVersions(result); + } - const result = yield* backend.listObjects({ - prefix: searchParams.get("prefix") ?? undefined, - delimiter: searchParams.get("delimiter") ?? undefined, - marker: searchParams.get("marker") ?? undefined, - maxKeys: searchParams.has("max-keys") - ? parseInt(searchParams.get("max-keys")!) - : undefined, - encodingType: searchParams.get("encoding-type") ?? undefined, - continuationToken: searchParams.get("continuation-token") ?? undefined, - startAfter: searchParams.get("start-after") ?? undefined, - listType: searchParams.get("list-type") === "2" ? 2 : 1, + if (params.uploads !== undefined) { + const result = yield* backend.listMultipartUploads({ + prefix: params.prefix, + delimiter: params.delimiter, + keyMarker: params["key-marker"], + uploadIdMarker: params["upload-id-marker"], + maxUploads: params["max-uploads"], + encodingType: params["encoding-type"], }); + return s3Xml.formatListMultipartUploads(result); + } + + const result = yield* backend.listObjects({ + prefix: params.prefix, + delimiter: params.delimiter, + marker: params.marker, + maxKeys: params["max-keys"], + encodingType: params["encoding-type"], + continuationToken: params["continuation-token"], + startAfter: params["start-after"], + listType: params["list-type"] === "2" ? 2 : 1, + }); - return s3Xml.formatListObjects(result); - })); + return s3Xml.formatListObjects(result); + }); diff --git a/src/Frontend/Objects/Post.ts b/src/Frontend/Objects/Post.ts index 7287c3a..dffade7 100644 --- a/src/Frontend/Objects/Post.ts +++ b/src/Frontend/Objects/Post.ts @@ -1,6 +1,6 @@ import { Effect, Option, Stream } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; import { S3Xml } from "../../Services/S3Xml.ts"; /** @@ -8,152 +8,144 @@ import { S3Xml } from "../../Services/S3Xml.ts"; * Primarily used for Multi-Object Delete (POST /:bucket?delete). * Also handles InitiateMultipartUpload (?uploads) and CompleteMultipartUpload (?uploadId=...). */ -export const postObject = ( - { path: { bucket } }: { path: { bucket: string } }, -) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const s3Xml = yield* S3Xml; - const url = new URL(request.url, "http://localhost"); - const searchParams = url.searchParams; - const key = extractKey(request.url, bucket); +export const postObject = () => + Effect.gen(function* () { + const { backend, bucket, key, params, request } = yield* RequestContext; + const s3Xml = yield* S3Xml; - if (searchParams.has("delete")) { - // ... (Multi-Object Delete logic) - // Multi-Object Delete - const bodyChunks = yield* Stream.runCollect(request.stream); - let totalLength = 0; - for (const chunk of Array.from(bodyChunks)) { - totalLength += chunk.length; - } - const bodyBytes = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of Array.from(bodyChunks)) { - bodyBytes.set(chunk, offset); - offset += chunk.length; - } - const bodyText = new TextDecoder().decode(bodyBytes); + if (params.delete !== undefined) { + // Multi-Object Delete + const bodyChunks = yield* Stream.runCollect(request.stream); + let totalLength = 0; + for (const chunk of Array.from(bodyChunks)) { + totalLength += chunk.length; + } + const bodyBytes = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of Array.from(bodyChunks)) { + bodyBytes.set(chunk, offset); + offset += chunk.length; + } + const bodyText = new TextDecoder().decode(bodyBytes); - const objects: { key: string; versionId?: string }[] = []; - // Simple XML parsing for Multi-Object Delete - const objectMatches = Array.from( - bodyText.matchAll(/(.*?)<\/Object>/gs), - ); - for (const match of objectMatches) { - const content = match[1]; - const keyMatch = content.match(/(.*?)<\/Key>/); - const versionIdMatch = content.match(/(.*?)<\/VersionId>/); - if (keyMatch) { - const rawKey = keyMatch[1]; - const key = Option.liftThrowable(decodeURIComponent)(rawKey).pipe( - Option.getOrElse(() => rawKey), - ); - yield* Effect.logDebug(`DeleteObjects extracted key=[${key}]`); - objects.push({ - key, - versionId: versionIdMatch ? versionIdMatch[1] : undefined, - }); - } + const objects: { key: string; versionId?: string }[] = []; + // Simple XML parsing for Multi-Object Delete + const objectMatches = Array.from( + bodyText.matchAll(/(.*?)<\/Object>/gs), + ); + for (const match of objectMatches) { + const content = match[1]; + const keyMatch = content.match(/(.*?)<\/Key>/); + const versionIdMatch = content.match(/(.*?)<\/VersionId>/); + if (keyMatch) { + const rawKey = keyMatch[1]; + const key = Option.liftThrowable(decodeURIComponent)(rawKey).pipe( + Option.getOrElse(() => rawKey), + ); + yield* Effect.logDebug(`DeleteObjects extracted key=[${key}]`); + objects.push({ + key, + versionId: versionIdMatch ? versionIdMatch[1] : undefined, + }); } + } - if (objects.length > 0) { - const deleteResult = yield* backend.deleteObjects(objects); - const deletedXml = deleteResult.deleted.map((k) => - `${k}` - ).join(""); - const errorsXml = deleteResult.errors.map((e) => - `${e.key}${e.code}${e.message}` - ).join(""); + if (objects.length > 0) { + const deleteResult = yield* backend.deleteObjects(objects); + const deletedXml = deleteResult.deleted.map((k) => + `${k}` + ).join(""); + const errorsXml = deleteResult.errors.map((e) => + `${e.key}${e.code}${e.message}` + ).join(""); - const xml = - `${deletedXml}${errorsXml}`; - return HttpServerResponse.text(xml, { - headers: { "Content-Type": "application/xml" }, - }); - } - // If no keys, still return empty result const xml = - ``; + `${deletedXml}${errorsXml}`; return HttpServerResponse.text(xml, { headers: { "Content-Type": "application/xml" }, }); } + // If no keys, still return empty result + const xml = + ``; + return HttpServerResponse.text(xml, { + headers: { "Content-Type": "application/xml" }, + }); + } - if (searchParams.has("uploads")) { - // Initiate Multipart Upload - const result = yield* backend.createMultipartUpload( - key, - request.headers, - ); - return s3Xml.formatInitiateMultipartUpload( - bucket, - key, - result.uploadId, - ); - } + if (params.uploads !== undefined) { + // Initiate Multipart Upload + const result = yield* backend.createMultipartUpload( + key, + request.headers, + ); + return s3Xml.formatInitiateMultipartUpload( + bucket, + key, + result.uploadId, + ); + } - if (searchParams.has("uploadId")) { - // Complete Multipart Upload - const uploadId = searchParams.get("uploadId")!; - const bodyChunks = yield* Stream.runCollect(request.stream); - let totalLength = 0; - for (const chunk of Array.from(bodyChunks)) { - totalLength += chunk.length; - } - const bodyBytes = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of Array.from(bodyChunks)) { - bodyBytes.set(chunk, offset); - offset += chunk.length; - } - const bodyText = new TextDecoder().decode(bodyBytes); + if (params.uploadId) { + // Complete Multipart Upload + const bodyChunks = yield* Stream.runCollect(request.stream); + let totalLength = 0; + for (const chunk of Array.from(bodyChunks)) { + totalLength += chunk.length; + } + const bodyBytes = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of Array.from(bodyChunks)) { + bodyBytes.set(chunk, offset); + offset += chunk.length; + } + const bodyText = new TextDecoder().decode(bodyBytes); - const parts: { etag: string; partNumber: number }[] = []; - const partMatches = Array.from( - bodyText.matchAll(/(.*?)<\/Part>/gs), + const parts: { etag: string; partNumber: number }[] = []; + const partMatches = Array.from( + bodyText.matchAll(/(.*?)<\/Part>/gs), + ); + for (const match of partMatches) { + const content = match[1]; + const partNumberMatch = content.match( + /(.*?)<\/PartNumber>/, ); - for (const match of partMatches) { - const content = match[1]; - const partNumberMatch = content.match( - /(.*?)<\/PartNumber>/, - ); - const etagMatch = content.match(/(.*?)<\/ETag>/); - if (partNumberMatch && etagMatch) { - parts.push({ - partNumber: parseInt(partNumberMatch[1]), - etag: etagMatch[1].replace(/"/g, '"'), - }); - } + const etagMatch = content.match(/(.*?)<\/ETag>/); + if (partNumberMatch && etagMatch) { + parts.push({ + partNumber: parseInt(partNumberMatch[1]), + etag: etagMatch[1].replace(/"/g, '"'), + }); } - - const result = yield* backend.completeMultipartUpload( - key, - uploadId, - parts, - ).pipe( - Effect.catchTag("NoSuchUpload", (e) => - Effect.gen(function* () { - // Idempotency: check if object already exists - const head = yield* backend.headObject(key, {}).pipe( - Effect.orElseFail(() => e), - ); - if (head.etag) { - return { - location: `http://localhost/${bucket}/${key}`, // Approximate - bucket, - key, - etag: head.etag, - versionId: head.headers["x-amz-version-id"], - }; - } - return yield* Effect.fail(e); - })), - ); - return s3Xml.formatCompleteMultipartUpload(result); } - return yield* Effect.fail( - new Error(`Method POST for key [${key}] not implemented`), + const result = yield* backend.completeMultipartUpload( + key, + params.uploadId, + parts, + ).pipe( + Effect.catchTag("NoSuchUpload", (e) => + Effect.gen(function* () { + // Idempotency: check if object already exists + const head = yield* backend.headObject(key, {}).pipe( + Effect.orElseFail(() => e), + ); + if (head.etag) { + return { + location: `http://localhost/${bucket}/${key}`, // Approximate + bucket, + key, + etag: head.etag, + versionId: head.headers["x-amz-version-id"], + }; + } + return yield* Effect.fail(e); + })), ); - })); + return s3Xml.formatCompleteMultipartUpload(result); + } + + return yield* Effect.fail( + new Error(`Method POST for key [${key}] not implemented`), + ); + }); diff --git a/src/Frontend/Objects/Put.ts b/src/Frontend/Objects/Put.ts index 4b9638b..08421cd 100644 --- a/src/Frontend/Objects/Put.ts +++ b/src/Frontend/Objects/Put.ts @@ -1,45 +1,39 @@ import { Effect } from "effect"; -import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; -import { extractKey, resolveBucket } from "../Utils.ts"; +import { HttpServerResponse } from "@effect/platform"; +import { RequestContext } from "../Utils.ts"; /** * Handler for PutObject (PUT /:bucket/*) */ -export const putObject = ({ path: { bucket } }: { path: { bucket: string } }) => - resolveBucket(bucket, (backend) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const key = extractKey(request.url, bucket); - const url = new URL(request.url, "http://localhost"); - const searchParams = url.searchParams; +export const putObject = () => + Effect.gen(function* () { + const { backend, key, params, request } = yield* RequestContext; - if (searchParams.has("partNumber") && searchParams.has("uploadId")) { - // Upload Part - const partNumber = parseInt(searchParams.get("partNumber")!); - const uploadId = searchParams.get("uploadId")!; - const result = yield* backend.uploadPart( - key, - uploadId, - partNumber, - request.stream, - ); - return HttpServerResponse.empty({ - status: 200, - headers: { ETag: result.etag }, - }); - } - - const result = yield* backend.putObject( + if (params.partNumber && params.uploadId) { + // Upload Part + const result = yield* backend.uploadPart( key, + params.uploadId, + params.partNumber, request.stream, - request.headers, ); - const headers: Record = {}; - if (result.etag) headers["ETag"] = result.etag; - if (result.versionId) headers["x-amz-version-id"] = result.versionId; - return HttpServerResponse.empty({ status: 200, - headers, + headers: { ETag: result.etag }, }); - })); + } + + const result = yield* backend.putObject( + key, + request.stream, + request.headers, + ); + const headers: Record = {}; + if (result.etag) headers["ETag"] = result.etag; + if (result.versionId) headers["x-amz-version-id"] = result.versionId; + + return HttpServerResponse.empty({ + status: 200, + headers, + }); + }); diff --git a/src/Frontend/Utils.ts b/src/Frontend/Utils.ts index 58733d4..972743c 100644 --- a/src/Frontend/Utils.ts +++ b/src/Frontend/Utils.ts @@ -1,4 +1,4 @@ -import { Effect, Option } from "effect"; +import { Context, Effect, Either, Option, Schema } from "effect"; import { BackendResolver } from "../Services/BackendResolver.ts"; import { S3Xml } from "../Services/S3Xml.ts"; import { @@ -18,8 +18,12 @@ import { NoSuchKey, NoSuchUpload, } from "../Services/Backend.ts"; -import { HttpServerRequest, type HttpServerResponse } from "@effect/platform"; -import type { AppConfig } from "../Config/Layer.ts"; +import { + HttpServerRequest, + type HttpServerResponse, + Url, +} from "@effect/platform"; +import type { HeraldConfig } from "../Config/Layer.ts"; import type { S3Client } from "../Backends/S3/Client.ts"; import type { SwiftClient } from "../Backends/Swift/Client.ts"; import { BadGateway } from "./Api.ts"; @@ -45,9 +49,10 @@ export function fixHeaderEncoding(value: string): string { * Extracts the object key from the request URL, given the bucket name. */ export function extractKey(requestUrl: string, bucket: string): string { - const pathname = requestUrl.startsWith("/") - ? requestUrl - : new URL(requestUrl).pathname; + const urlResult = Url.fromString(requestUrl, "http://localhost"); + const pathname = Either.isRight(urlResult) + ? urlResult.right.pathname + : requestUrl; const [pathOnly] = pathname.split("?"); const bucketPrefixWithSlash = `/${bucket}/`; @@ -61,6 +66,112 @@ export function extractKey(requestUrl: string, bucket: string): string { return ""; } +/** + * Context for S3 operations (bucket or object). + */ +export class RequestContext extends Context.Tag("RequestContext")< + RequestContext, + { + readonly backend: typeof Backend.Service; + readonly bucket: string; + readonly key: string; + readonly params: S3QueryParams; + readonly request: HttpServerRequest.HttpServerRequest; + } +>() {} + +/** + * Higher-order function to handle S3 context. + */ +export function provideRequestContext< + A extends HttpServerResponse.HttpServerResponse, + E, + R, +>( + fn: () => Effect.Effect, +): ( + args: { path: { bucket: string } }, +) => Effect.Effect< + HttpServerResponse.HttpServerResponse, + BadGateway, + | Exclude + | BackendResolver + | S3Xml + | HeraldConfig + | S3Client + | SwiftClient + | HttpServerRequest.HttpServerRequest +> { + return ({ path: { bucket } }) => + resolveBucket(bucket, (backend) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const urlResult = Url.fromString(request.url, "http://localhost"); + if (Either.isLeft(urlResult)) { + return yield* Effect.fail( + new InternalError({ message: String(urlResult.left) }), + ); + } + const url = urlResult.right; + const key = extractKey(request.url, bucket); + const params = yield* parseQueryParams(url.searchParams, S3QueryParams); + const ctx = { + backend, + bucket, + key, + params, + request, + }; + return yield* fn().pipe(Effect.provideService(RequestContext, ctx)); + }) as unknown as Effect.Effect< + HttpServerResponse.HttpServerResponse, + BadGateway, + Exclude + >); +} + +/** + * Common S3 Query Parameters Schema + */ +export const S3QueryParams = Schema.Struct({ + uploadId: Schema.optional(Schema.String), + partNumber: Schema.optional(Schema.NumberFromString), + prefix: Schema.optional(Schema.String), + delimiter: Schema.optional(Schema.String), + marker: Schema.optional(Schema.String), + "max-keys": Schema.optional(Schema.NumberFromString), + "max-uploads": Schema.optional(Schema.NumberFromString), + "encoding-type": Schema.optional(Schema.String), + "continuation-token": Schema.optional(Schema.String), + "start-after": Schema.optional(Schema.String), + "list-type": Schema.optional(Schema.String), + "version-id-marker": Schema.optional(Schema.String), + "key-marker": Schema.optional(Schema.String), + "upload-id-marker": Schema.optional(Schema.String), + versions: Schema.optional(Schema.String), + uploads: Schema.optional(Schema.String), + delete: Schema.optional(Schema.String), + acl: Schema.optional(Schema.String), +}); + +export type S3QueryParams = Schema.Schema.Type; + +/** + * Utility to parse search params using a Schema. + */ +export function parseQueryParams( + searchParams: URLSearchParams, + schema: Schema.Schema, +): Effect.Effect { + const paramsRecord: Record = {}; + searchParams.forEach((value, key) => { + paramsRecord[key] = value; + }); + return Schema.decodeUnknown(schema)(paramsRecord).pipe( + Effect.mapError((e) => new InternalError({ message: String(e) })), + ); +} + /** * Resolves a bucket by name and runs the provided effect with the resolved backend. * Centralizes error handling via S3Xml.formatError. @@ -78,7 +189,7 @@ export function resolveBucket< | R | BackendResolver | S3Xml - | AppConfig + | HeraldConfig | S3Client | SwiftClient | HttpServerRequest.HttpServerRequest @@ -171,7 +282,7 @@ export function resolveBackend< | R | BackendResolver | S3Xml - | AppConfig + | HeraldConfig | S3Client | SwiftClient | HttpServerRequest.HttpServerRequest diff --git a/src/Http.ts b/src/Http.ts index e77671f..3c3c693 100644 --- a/src/Http.ts +++ b/src/Http.ts @@ -9,33 +9,38 @@ import { Config, Effect, Layer } from "effect"; // deno-lint-ignore no-external-import import { createServer } from "node:http"; -export { Api } from "./Api.ts"; +export { HttpHeraldApi as HeraldHttpApi } from "./Api.ts"; export { HttpHealthLive } from "./Frontend/Health/Http.ts"; export { HttpS3Live } from "./Frontend/Http.ts"; -import { AppConfigLive } from "./Config/Layer.ts"; +import { HeraldConfigLive } from "./Config/Layer.ts"; import { HttpHealthLive } from "./Frontend/Health/Http.ts"; import { HttpS3Live } from "./Frontend/Http.ts"; -import { Api } from "./Api.ts"; +import { HttpHeraldApi } from "./Api.ts"; -export const ApiLive = HttpApiBuilder.api(Api).pipe( +export const HttpHeraldLive = HttpApiBuilder.api(HttpHeraldApi).pipe( Layer.provide(HttpHealthLive), Layer.provide(HttpS3Live), ); -export const HttpLive = Layer.unwrapEffect( +export const HttpServerHeraldLive = Layer.unwrapEffect( Effect.gen(function* () { const port = yield* Config.withDefault( Config.integer("PORT"), 3000, ); return HttpApiBuilder.serve(HttpMiddleware.logger).pipe( + // provides swagger ui for http api Layer.provide(HttpApiSwagger.layer()), + // provides openapi.json endpoint Layer.provide(HttpApiBuilder.middlewareOpenApi()), + // adds cors support + // FIXME: config support Layer.provide(HttpApiBuilder.middlewareCors()), - Layer.provide(ApiLive), + Layer.provide(HttpHeraldLive), + // log address at startup HttpServer.withLogAddress, Layer.provide(NodeHttpServer.layer(createServer, { port })), - Layer.provide(AppConfigLive), + Layer.provide(HeraldConfigLive), ); }), ); diff --git a/src/Logging/Layer.ts b/src/Logging/Layer.ts index ab0c5ed..1c2c233 100644 --- a/src/Logging/Layer.ts +++ b/src/Logging/Layer.ts @@ -1,4 +1,4 @@ -import { Config, Effect, Layer, Logger, LogLevel } from "effect"; +import { Config, Effect, Layer, Logger, LogLevel, Option } from "effect"; export const LoggingLive = Layer.mergeAll( Layer.unwrapEffect( @@ -7,7 +7,7 @@ export const LoggingLive = Layer.mergeAll( Config.string("HERALD_LOG_LEVEL"), ); - if (logLevelStr._tag === "None") { + if (Option.isNone(logLevelStr)) { return Logger.minimumLogLevel(LogLevel.Info); } diff --git a/src/Services/Backend.ts b/src/Services/Backend.ts index 9a6f5a5..9f3f1f4 100644 --- a/src/Services/Backend.ts +++ b/src/Services/Backend.ts @@ -1,3 +1,7 @@ +/** + * The `Backend` service represents a single impl that herald can proxy to. + */ + import { Context, type Effect, Schema, type Stream } from "effect"; export interface BucketInfo { diff --git a/src/Services/BackendResolver.ts b/src/Services/BackendResolver.ts index 979866d..4ef9801 100644 --- a/src/Services/BackendResolver.ts +++ b/src/Services/BackendResolver.ts @@ -1,10 +1,11 @@ -import { Context, Effect, Layer, Option } from "effect"; -import { AppConfig } from "../Config/Layer.ts"; -import { Backend, type BackendService } from "./Backend.ts"; +import { Cache, Context, Effect, Layer, Option } from "effect"; +import { HeraldConfig } from "../Config/Layer.ts"; +import { Backend } from "./Backend.ts"; import type { S3Client } from "../Backends/S3/Client.ts"; import { makeS3Backend } from "../Backends/S3/Backend.ts"; import { makeSwiftBackend } from "../Backends/Swift/Backend.ts"; import type { SwiftClient } from "../Backends/Swift/Client.ts"; +import type { MaterializedBucket } from "../Domain/Config.ts"; /** * BackendResolver handles dynamic resolution and provisioning of Backend implementations @@ -19,7 +20,7 @@ export class BackendResolver extends Context.Tag("BackendResolver")< ) => Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client | SwiftClient + Exclude | HeraldConfig | S3Client | SwiftClient >; readonly provideForBackendId: ( @@ -28,7 +29,7 @@ export class BackendResolver extends Context.Tag("BackendResolver")< ) => Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client | SwiftClient + Exclude | HeraldConfig | S3Client | SwiftClient >; } >() {} @@ -36,52 +37,78 @@ export class BackendResolver extends Context.Tag("BackendResolver")< export const BackendResolverLive = Layer.effect( BackendResolver, Effect.gen(function* () { - const config = yield* AppConfig; + const config = yield* HeraldConfig; - // Dynamic provision logic with memoization. - const bucketCache = new Map(); - const backendCache = new Map(); + const makeBackend = ( + bucketConfig: MaterializedBucket | { backend_id: string }, + ) => + Effect.gen(function* () { + const protocol = "protocol" in bucketConfig + ? bucketConfig.protocol + : config.raw.backends[bucketConfig.backend_id]?.protocol; - return { - provideForBucket: ( - bucketName: string, - effect: Effect.Effect, - ) => - Effect.gen(function* () { - if (bucketCache.has(bucketName)) { - return yield* Effect.provideService( - effect, - Backend, - bucketCache.get(bucketName)!, - ); - } + if (protocol === "s3") { + return yield* makeS3Backend(bucketConfig); + } else if (protocol === "swift") { + return yield* makeSwiftBackend(bucketConfig); + } else { + return yield* Effect.fail( + new Error(`Unsupported protocol: ${protocol}`), + ); + } + }); + + // We cache by the string identifier (bucket name or backend ID). + // The BackendService itself is request-scoped because makeBackend yields requirements + // that are resolved from the current context when the cache is lookep up. + // Wait, Cache.get(key) will execute the lookup if not present. + // If we want the BackendService to be truly request-scoped but cached, + // we have a conflict if the requirements (like HeraldConfig) change per request. + // However, in Herald, HeraldConfig is usually a singleton for the app. + // If it's a singleton, then caching the BackendService is fine. + const bucketCache = yield* Cache.make({ + capacity: 100, + timeToLive: "24 hours", + lookup: (bucketName: string) => + Effect.gen(function* () { const matched = config.lookupBucket(bucketName); if (Option.isNone(matched)) { return yield* Effect.fail( new Error(`No configuration found for bucket: ${bucketName}`), ); } + return yield* makeBackend(matched.value); + }), + }); - const bucketConfig = matched.value; - let backendImpl: BackendService; - - if (bucketConfig.protocol === "s3") { - backendImpl = yield* makeS3Backend(bucketConfig); - } else if (bucketConfig.protocol === "swift") { - backendImpl = yield* makeSwiftBackend(bucketConfig); - } else { + const backendCache = yield* Cache.make({ + capacity: 100, + timeToLive: "24 hours", + lookup: (backendId: string) => + Effect.gen(function* () { + const backendConfig = config.raw.backends[backendId]; + if (!backendConfig) { return yield* Effect.fail( - new Error(`Unsupported protocol: ${bucketConfig.protocol}`), + new Error(`No configuration found for backend: ${backendId}`), ); } + return yield* makeBackend({ backend_id: backendId }); + }), + }); - bucketCache.set(bucketName, backendImpl); + return { + provideForBucket: ( + bucketName: string, + effect: Effect.Effect, + ) => + Effect.gen(function* () { + const backendImpl = yield* bucketCache.get(bucketName); return yield* Effect.provideService(effect, Backend, backendImpl); }) as Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client | SwiftClient + Exclude | HeraldConfig | S3Client | SwiftClient >, provideForBackendId: ( @@ -89,40 +116,12 @@ export const BackendResolverLive = Layer.effect( effect: Effect.Effect, ) => Effect.gen(function* () { - if (backendCache.has(backendId)) { - return yield* Effect.provideService( - effect, - Backend, - backendCache.get(backendId)!, - ); - } - - const backendConfig = config.raw.backends[backendId]; - if (!backendConfig) { - return yield* Effect.fail( - new Error(`No configuration found for backend: ${backendId}`), - ); - } - - let backendImpl: BackendService; - - if (backendConfig.protocol === "s3") { - backendImpl = yield* makeS3Backend({ backend_id: backendId }); - } else if (backendConfig.protocol === "swift") { - backendImpl = yield* makeSwiftBackend({ backend_id: backendId }); - } else { - const protocol = (backendConfig as { protocol: string }).protocol; - return yield* Effect.fail( - new Error(`Unsupported protocol: ${protocol}`), - ); - } - - backendCache.set(backendId, backendImpl); + const backendImpl = yield* backendCache.get(backendId); return yield* Effect.provideService(effect, Backend, backendImpl); }) as Effect.Effect< A, E | Error, - Exclude | AppConfig | S3Client | SwiftClient + Exclude | HeraldConfig | S3Client | SwiftClient >, }; }), diff --git a/src/Services/S3Xml.ts b/src/Services/S3Xml.ts index 991c657..06815fb 100644 --- a/src/Services/S3Xml.ts +++ b/src/Services/S3Xml.ts @@ -21,6 +21,9 @@ import { type OwnerInfo, } from "./Backend.ts"; +/** + * This service centeralizes XML authoring logic. + */ export class S3Xml extends Context.Tag("S3Xml")< S3Xml, { diff --git a/src/main.ts b/src/main.ts index cff9dea..56ccc3b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,13 +2,19 @@ import { FetchHttpClient } from "@effect/platform"; import { NodeRuntime } from "@effect/platform-node"; import { Layer } from "effect"; // our http server impl layer -import { HttpLive } from "./Http.ts"; +import { HttpServerHeraldLive } from "./Http.ts"; // otel tracing layer import { TracingLive } from "./Tracing.ts"; -HttpLive.pipe( +HttpServerHeraldLive.pipe( Layer.provide(TracingLive), + // provider an HttpClient impl based on `fetch` + // used to talk the the swift impl Layer.provide(FetchHttpClient.layer), + // run layer until interrupted Layer.launch, + // add support for Cli goodies like + // signal mgmt, teardown, exit codes and stdio impl + // for Logger NodeRuntime.runMain, ); diff --git a/tests/config.test.ts b/tests/config.test.ts index 2e1f485..eb5981a 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -6,7 +6,7 @@ import { BackendResolver, BackendResolverLive, } from "../src/Services/BackendResolver.ts"; -import { AppConfig, parseConfig } from "../src/Config/Layer.ts"; +import { HeraldConfig, parseConfig } from "../src/Config/Layer.ts"; import { S3Client } from "../src/Backends/S3/Client.ts"; import { SwiftClient } from "../src/Backends/Swift/Client.ts"; import type { S3Client as S3ClientSDK } from "@aws-sdk/client-s3"; @@ -333,7 +333,7 @@ interface ResolverTestCase { config: GlobalConfig; op: ( resolver: Context.Tag.Service, - ) => Effect.Effect; + ) => Effect.Effect; expectedError?: string; } @@ -404,7 +404,7 @@ const resolverCases: ResolverTestCase[] = [ for (const tc of resolverCases) { testEffect(`resolver/${tc.id}`, () => Effect.gen(function* () { - const AppConfigLive = Layer.succeed(AppConfig, { + const HeraldConfigLive = Layer.succeed(HeraldConfig, { raw: tc.config, lookupBucket: (name: string) => lookupBucket(tc.config, name), }); @@ -425,7 +425,7 @@ for (const tc of resolverCases) { return yield* tc.op(resolver); }).pipe( Effect.provide(BackendResolverLive), - Effect.provide(AppConfigLive), + Effect.provide(HeraldConfigLive), Effect.provide(S3ClientLive), Effect.provide(SwiftClientLive), Effect.either, diff --git a/tests/health.test.ts b/tests/health.test.ts index 8ce8e58..296d6c5 100644 --- a/tests/health.test.ts +++ b/tests/health.test.ts @@ -5,27 +5,29 @@ import { HttpApiClient, HttpServer, } from "@effect/platform"; -import { Api, HttpHealthLive, HttpS3Live } from "../src/Http.ts"; -import { AppConfig } from "../src/Config/Layer.ts"; +import { HeraldHttpApi, HttpHealthLive, HttpS3Live } from "../src/Http.ts"; +import { HeraldConfig } from "../src/Config/Layer.ts"; import { S3ClientLive } from "../src/Backends/S3/Client.ts"; +import { SwiftClientLive } from "../src/Backends/Swift/Client.ts"; import { S3XmlLive } from "../src/Services/S3Xml.ts"; import { BackendResolverLive } from "../src/Services/BackendResolver.ts"; import { EffectAssert, testEffect } from "./utils.ts"; testEffect("health/getStatus", () => Effect.gen(function* () { - const AppConfigLive = Layer.succeed(AppConfig, { + const HeraldConfigLive = Layer.succeed(HeraldConfig, { raw: { backends: {} }, lookupBucket: () => Option.none(), }); - const ApiWithRequirements = HttpApiBuilder.api(Api).pipe( + const ApiWithRequirements = HttpApiBuilder.api(HeraldHttpApi).pipe( Layer.provide(HttpHealthLive), Layer.provide(HttpS3Live), - Layer.provide(S3ClientLive), Layer.provide(BackendResolverLive), + Layer.provide(S3ClientLive), + Layer.provide(SwiftClientLive), Layer.provide(S3XmlLive), - Layer.provide(AppConfigLive), + Layer.provide(HeraldConfigLive), Layer.provide(FetchHttpClient.layer), Layer.provideMerge(HttpServer.layerContext), ); @@ -34,7 +36,7 @@ testEffect("health/getStatus", () => const webHandler = HttpApiBuilder.toWebHandler(ApiWithRequirements); const clientProgram = Effect.gen(function* () { - const client = yield* HttpApiClient.make(Api, { + const client = yield* HttpApiClient.make(HeraldHttpApi, { baseUrl: "http://localhost", }); return yield* client.health.getStatus(); diff --git a/tests/integration/__snapshots__/buckets.test.ts.snap b/tests/integration/__snapshots__/buckets.test.ts.snap index c982eb5..76add56 100644 --- a/tests/integration/__snapshots__/buckets.test.ts.snap +++ b/tests/integration/__snapshots__/buckets.test.ts.snap @@ -123,7 +123,7 @@ snapshot[`Swift/buckets/delete/non-existent metadata 1`] = ` } `; -snapshot[`Swift/buckets/delete/non-existent body 1`] = `'NoSuchBucketNot Found'`; +snapshot[`Swift/buckets/delete/non-existent body 1`] = `'NoSuchBucket

Not Found

The resource could not be found.

'`; snapshot[`Baseline/buckets/head/existing metadata 1`] = ` { @@ -222,4 +222,4 @@ snapshot[`Swift/buckets/list metadata 1`] = ` } `; -snapshot[`Swift/buckets/list body 1`] = `'swiftSwift User192.168.5.1232026-01-15T00:00:00.000Za2026-01-15T00:00:00.000Zaa2026-01-15T00:00:00.000Zbuilds2026-01-15T00:00:00.000Zfoo-2026-01-15T00:00:00.000Zfoo-.bar2026-01-15T00:00:00.000Zfoo.-bar2026-01-15T00:00:00.000Zfoo..bar2026-01-15T00:00:00.000Zfoo_bar2026-01-15T00:00:00.000Zherald-swift-2w97l75ompcxiypo-12026-01-15T00:00:00.000Zherald-swift-5dfyoor543wddfpb-12026-01-15T00:00:00.000Zherald-swift-5m8pru9nzpno98zp-1572026-01-15T00:00:00.000Zherald-swift-5txup4vs19i8tr6s-292026-01-15T00:00:00.000Zherald-swift-cze1vw7y05q33782-1462026-01-15T00:00:00.000Zherald-swift-fd0oi5radob46p39-12026-01-15T00:00:00.000Zherald-swift-oda2k1hu2ds0wir6-22026-01-15T00:00:00.000Zherald-swift-sy6d1ftl2i7g78jj-12026-01-15T00:00:00.000Zherald-swift-yx0xlhaeebv1g9c7-12026-01-15T00:00:00.000Zherald-task-store-mr-120-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-127-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-130-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-131-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-132-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-137-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-139-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-143-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-144-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-145-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-146-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-147-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-149-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-150-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-151-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-154-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-155-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-157-vivavox2026-01-15T00:00:00.000Zherald-task-store-prd-vivavox2026-01-15T00:00:00.000Zherald-task-store-stg-vivavox2026-01-15T00:00:00.000Ziac-swift2026-01-15T00:00:00.000Zmr-101-vivavox2026-01-15T00:00:00.000Zmr-109-vivavox2026-01-15T00:00:00.000Zmr-111-vivavox2026-01-15T00:00:00.000Zmr-115-vivavox2026-01-15T00:00:00.000Zmr-116-vivavox2026-01-15T00:00:00.000Zmr-120-vivavox2026-01-15T00:00:00.000Zmr-121-vivavox2026-01-15T00:00:00.000Zmr-122-vivavox2026-01-15T00:00:00.000Zmr-124-vivavox2026-01-15T00:00:00.000Zmr-126-vivavox2026-01-15T00:00:00.000Zmr-127-vivavox2026-01-15T00:00:00.000Zmr-130-vivavox2026-01-15T00:00:00.000Zmr-131-vivavox2026-01-15T00:00:00.000Zmr-132-vivavox2026-01-15T00:00:00.000Zmr-137-vivavox2026-01-15T00:00:00.000Zmr-139-vivavox2026-01-15T00:00:00.000Zmr-143-vivavox2026-01-15T00:00:00.000Zmr-144-vivavox2026-01-15T00:00:00.000Zmr-145-vivavox2026-01-15T00:00:00.000Zmr-146-vivavox2026-01-15T00:00:00.000Zmr-147-vivavox2026-01-15T00:00:00.000Zmr-149-vivavox2026-01-15T00:00:00.000Zmr-150-vivavox2026-01-15T00:00:00.000Zmr-151-vivavox2026-01-15T00:00:00.000Zmr-154-vivavox2026-01-15T00:00:00.000Zmr-155-vivavox2026-01-15T00:00:00.000Zmr-157-vivavox2026-01-15T00:00:00.000Zprd-vivavox2026-01-15T00:00:00.000Zstg-vivavox2026-01-15T00:00:00.000Zstg-vivavox+segments2026-01-15T00:00:00.000Z'`; +snapshot[`Swift/buckets/list body 1`] = `'swiftSwift User192.168.5.1232026-01-15T00:00:00.000Za2026-01-15T00:00:00.000Zaa2026-01-15T00:00:00.000Zbuilds2026-01-15T00:00:00.000Zfoo-2026-01-15T00:00:00.000Zfoo-.bar2026-01-15T00:00:00.000Zfoo.-bar2026-01-15T00:00:00.000Zfoo..bar2026-01-15T00:00:00.000Zfoo_bar2026-01-15T00:00:00.000Zherald-swift-2w97l75ompcxiypo-12026-01-15T00:00:00.000Zherald-swift-5dfyoor543wddfpb-12026-01-15T00:00:00.000Zherald-swift-5m8pru9nzpno98zp-1572026-01-15T00:00:00.000Zherald-swift-5txup4vs19i8tr6s-292026-01-15T00:00:00.000Zherald-swift-cze1vw7y05q33782-1462026-01-15T00:00:00.000Zherald-swift-fd0oi5radob46p39-12026-01-15T00:00:00.000Zherald-swift-m55m3lqytoaxuxro-132026-01-15T00:00:00.000Zherald-swift-oda2k1hu2ds0wir6-22026-01-15T00:00:00.000Zherald-swift-sy6d1ftl2i7g78jj-12026-01-15T00:00:00.000Zherald-swift-yx0xlhaeebv1g9c7-12026-01-15T00:00:00.000Zherald-task-store-mr-120-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-127-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-130-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-131-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-132-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-137-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-139-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-143-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-144-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-145-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-146-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-147-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-149-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-150-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-151-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-154-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-155-vivavox2026-01-15T00:00:00.000Zherald-task-store-mr-157-vivavox2026-01-15T00:00:00.000Zherald-task-store-prd-vivavox2026-01-15T00:00:00.000Zherald-task-store-stg-vivavox2026-01-15T00:00:00.000Ziac-swift2026-01-15T00:00:00.000Zmr-101-vivavox2026-01-15T00:00:00.000Zmr-109-vivavox2026-01-15T00:00:00.000Zmr-111-vivavox2026-01-15T00:00:00.000Zmr-115-vivavox2026-01-15T00:00:00.000Zmr-116-vivavox2026-01-15T00:00:00.000Zmr-120-vivavox2026-01-15T00:00:00.000Zmr-121-vivavox2026-01-15T00:00:00.000Zmr-122-vivavox2026-01-15T00:00:00.000Zmr-124-vivavox2026-01-15T00:00:00.000Zmr-126-vivavox2026-01-15T00:00:00.000Zmr-127-vivavox2026-01-15T00:00:00.000Zmr-130-vivavox2026-01-15T00:00:00.000Zmr-131-vivavox2026-01-15T00:00:00.000Zmr-132-vivavox2026-01-15T00:00:00.000Zmr-137-vivavox2026-01-15T00:00:00.000Zmr-139-vivavox2026-01-15T00:00:00.000Zmr-143-vivavox2026-01-15T00:00:00.000Zmr-144-vivavox2026-01-15T00:00:00.000Zmr-145-vivavox2026-01-15T00:00:00.000Zmr-146-vivavox2026-01-15T00:00:00.000Zmr-147-vivavox2026-01-15T00:00:00.000Zmr-149-vivavox2026-01-15T00:00:00.000Zmr-150-vivavox2026-01-15T00:00:00.000Zmr-151-vivavox2026-01-15T00:00:00.000Zmr-154-vivavox2026-01-15T00:00:00.000Zmr-155-vivavox2026-01-15T00:00:00.000Zmr-157-vivavox2026-01-15T00:00:00.000Zprd-vivavox2026-01-15T00:00:00.000Zstg-vivavox2026-01-15T00:00:00.000Zstg-vivavox+segments2026-01-15T00:00:00.000Ztest-objects-bucket2026-01-15T00:00:00.000Z'`; diff --git a/tests/integration/__snapshots__/objects.test.ts.snap b/tests/integration/__snapshots__/objects.test.ts.snap index 798fa82..63b70f8 100644 --- a/tests/integration/__snapshots__/objects.test.ts.snap +++ b/tests/integration/__snapshots__/objects.test.ts.snap @@ -1,5 +1,187 @@ export const snapshot = {}; +snapshot[`Baseline/objects/put metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/put metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/put metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/get/existing metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/get/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/get/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/get/non-existent metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "359", + "content-type": "application/xml", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Baseline/objects/get/non-existent body 1`] = ` +' +NoSuchKeyThe specified key does not exist.no-suchtest-objects-bucket/test-objects-bucket/no-suchIDHOST' +`; + +snapshot[`Proxy/objects/get/non-existent metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/objects/get/non-existent body 1`] = `'NoSuchKeyThe specified key does not exist.'`; + +snapshot[`Swift/objects/get/non-existent metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Swift/objects/get/non-existent body 1`] = `'NoSuchKey

Not Found

The resource could not be found.

'`; + +snapshot[`Baseline/objects/head/existing metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/head/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/head/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/head/non-existent metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "0", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-minio-error-code": "NoSuchKey", + "x-minio-error-desc": '"The specified key does not exist."', + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/objects/head/non-existent metadata 1`] = ` +{ + headers: {}, + status: 404, +} +`; + +snapshot[`Swift/objects/head/non-existent metadata 1`] = ` +{ + headers: {}, + status: 404, +} +`; + +snapshot[`Baseline/objects/delete/existing metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/delete/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/delete/existing metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + snapshot[`Baseline/objects/multipart/basic metadata 1`] = ` { headers: { @@ -72,6 +254,19 @@ snapshot[`Proxy/objects/multipart/list-parts metadata 1`] = ` } `; +snapshot[`Baseline/objects/multipart/empty metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + snapshot[`Proxy/objects/multipart/empty metadata 1`] = ` { headers: {}, diff --git a/tests/integration/objects.test.ts b/tests/integration/objects.test.ts index d419bfa..a4f2de9 100644 --- a/tests/integration/objects.test.ts +++ b/tests/integration/objects.test.ts @@ -279,14 +279,14 @@ const specs: ObjectTestSpec[] = [ ); throw new Error("Complete should have failed for empty parts"); } catch (e) { - if (!(e instanceof S3ServiceException && e.name === "MalformedXML")) { - // AWS S3 returns MalformedXML for empty parts list - // Some other implementations might return InvalidPart - if (e instanceof S3ServiceException && e.name === "InvalidPart") { - return; - } - throw e; + if ( + e instanceof S3ServiceException && + (e.name === "MalformedXML" || e.name === "InvalidPart" || + e.name === "InvalidRequest") + ) { + return; } + throw e; } finally { try { await c.send( diff --git a/tests/utils.ts b/tests/utils.ts index 6748cd1..e5446b5 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -1,7 +1,7 @@ import { S3Client } from "@aws-sdk/client-s3"; import { Config, Effect, Layer, Logger, LogLevel, Option } from "effect"; -import { ApiLive } from "../src/Http.ts"; -import { AppConfig } from "../src/Config/Layer.ts"; +import { HttpHeraldLive } from "../src/Http.ts"; +import { HeraldConfig } from "../src/Config/Layer.ts"; import { lookupBucket } from "../src/Domain/Config.ts"; import { BackendResolverLive } from "../src/Services/BackendResolver.ts"; import { S3ClientLive } from "../src/Backends/S3/Client.ts"; @@ -39,17 +39,17 @@ export const makeTestHarness = ( ), ) => Effect.gen(function* () { - const AppConfigLive = Layer.succeed(AppConfig, { + const HeraldConfigLive = Layer.succeed(HeraldConfig, { raw: config, lookupBucket: (name: string) => lookupBucket(config, name), }); - const ApiWithRequirements = ApiLive.pipe( + const ApiWithRequirements = HttpHeraldLive.pipe( Layer.provide(BackendResolverLive), Layer.provide(S3ClientLive), Layer.provide(SwiftClientLive), Layer.provide(S3XmlLive), - Layer.provide(AppConfigLive), + Layer.provide(HeraldConfigLive), Layer.provide(FetchHttpClient.layer), Layer.provideMerge(HttpServer.layerContext), Layer.provideMerge(loggingLayer), diff --git a/tools/compose.yml b/tools/compose.yml index 0b5de2d..0067818 100644 --- a/tools/compose.yml +++ b/tools/compose.yml @@ -1,6 +1,7 @@ name: herald services: redis: + profiles: ["db"] image: docker.io/library/redis:alpine command: --save 60 1 --loglevel warning healthcheck: @@ -15,6 +16,7 @@ services: - redisdata:/data minio: + profiles: ["s3"] image: docker.io/minio/minio:latest command: server /data --console-address ":9001" ports: diff --git a/x/compose-down.ts b/x/compose-down.ts index 4094166..04f52d5 100755 --- a/x/compose-down.ts +++ b/x/compose-down.ts @@ -2,4 +2,6 @@ import { $, DOCKER_CMD } from "./utils.ts"; -await $.raw`${DOCKER_CMD} compose down`.cwd($.relativeDir("../tools/")); +await $.raw`${DOCKER_CMD} compose -f compose.yml down`.cwd( + $.path(import.meta.resolve("../tools/")), +); diff --git a/x/compose-up.ts b/x/compose-up.ts index 367e9a6..e6cfe3d 100755 --- a/x/compose-up.ts +++ b/x/compose-up.ts @@ -6,6 +6,6 @@ const profiles = $.argv .map((prof) => `--profile ${prof}`) .join(" "); -await $.raw`${DOCKER_CMD} compose ${profiles} up -d`.cwd( - $.relativeDir("../tools/"), +await $.raw`${DOCKER_CMD} compose -f compose.yml ${profiles} up -d`.cwd( + $.path(import.meta.resolve("../tools/")), ); diff --git a/x/purge-minio.ts b/x/purge-minio.ts index 204714a..3bf8ccd 100644 --- a/x/purge-minio.ts +++ b/x/purge-minio.ts @@ -2,7 +2,6 @@ import { DeleteBucketCommand, DeleteObjectsCommand, ListBucketsCommand, - ListObjectsV2Command, ListObjectVersionsCommand, S3Client, } from "npm:@aws-sdk/client-s3"; diff --git a/x/swift-debug.ts b/x/swift-debug.ts index 986d71b..76e888c 100644 --- a/x/swift-debug.ts +++ b/x/swift-debug.ts @@ -1,9 +1,9 @@ #!/usr/bin/env -S deno run --allow-all import { Effect, Logger, LogLevel } from "effect"; import { SwiftClient, SwiftClientLive } from "../src/Backends/Swift/Client.ts"; -import { AppConfigLive } from "../src/Config/Layer.ts"; +import { HeraldConfigLive } from "../src/Config/Layer.ts"; import { makeSwiftBackend } from "../src/Backends/Swift/Backend.ts"; -import { Backend } from "../src/Services/Backend.ts"; +import { FetchHttpClient } from "@effect/platform"; const program = Effect.gen(function* () { console.log("Checking Swift connection..."); @@ -27,7 +27,8 @@ const program = Effect.gen(function* () { } }).pipe( Effect.provide(SwiftClientLive), - Effect.provide(AppConfigLive), + Effect.provide(HeraldConfigLive), + Effect.provide(FetchHttpClient.layer), Effect.provide(Logger.minimumLogLevel(LogLevel.Debug)), );