diff --git a/README.md b/README.md index b26a2fc..7bdf8dc 100644 --- a/README.md +++ b/README.md @@ -264,8 +264,8 @@ operations. The following are **not** currently supported (or are partial): (`?acl`), website (`?website`), public access block (`?publicAccessBlock`), replication, logging, inventory, metrics, ownership controls. - **Object subresources:** Object ACLs, tagging, legal hold, retention (Object - Lock), S3 Select. Copy Object (`x-amz-copy-source`) and Multi-Object Delete - (`POST ?delete`) are not implemented. + Lock), S3 Select. Multi-Object Delete (`POST ?delete`) is not implemented. + Copy Object (`PUT` with `x-amz-copy-source`) is supported. - **Object operations:** GetObjectAttributes (`?attributes`) is not implemented. Checksum headers (`x-amz-checksum-*`) and conditional requests (`If-Match`, etc.) are not fully supported. diff --git a/TODO.md b/TODO.md index 5be06f6..0356ad2 100644 --- a/TODO.md +++ b/TODO.md @@ -72,7 +72,7 @@ implementation. - [ ] **Unicode Metadata**: Fix support for non-ASCII characters in object metadata. Currently failing across all backends. _(Focus tests: `test_object_set_get_unicode_metadata`)_ -- [ ] **Copy Object**: Support for `PUT` with `x-amz-copy-source` header. +- [x] **Copy Object**: Support for `PUT` with `x-amz-copy-source` header. _(Focus tests: `test_object_copy`)_ - [ ] **Tagging**: Implementation of `GET/PUT/DELETE /?tagging` for objects. _(Focus tests: `test_object_tagging`)_ @@ -88,7 +88,7 @@ implementation. - [ ] **Checksums**: Support for `x-amz-checksum-sha1`, `x-amz-checksum-sha256`, `x-amz-checksum-crc32`, and `x-amz-checksum-crc32c`. Currently failing validation tests. _(Focus tests: `test_object_checksum_sha256`)_ - - [ ] **Fix S3 Buffering**: Refactor S3 `putObject` and `uploadPart` to stream + - [x] **Fix S3 Buffering**: Refactor S3 `putObject` and `uploadPart` to stream directly to the AWS SDK instead of collecting chunks into a `Uint8Array`. - [ ] **Fix Swift Validation Timing**: Move Swift checksum validation before diff --git a/deno.jsonc b/deno.jsonc index c18560e..cd65225 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -23,6 +23,7 @@ "@smithy/types": "npm:@smithy/types@^3.7.0", "@aws-crypto/sha256": "npm:@aws-crypto/sha256-js@^5.2.0", "@aws-sdk/client-s3": "npm:@aws-sdk/client-s3@^3.x", + "@aws-sdk/s3-request-presigner": "npm:@aws-sdk/s3-request-presigner@^3.x", "effect": "npm:effect@^3.17.7", "xml2js": "npm:xml2js@0.6.2", "node-http": "node:http", diff --git a/deno.lock b/deno.lock index fcc418a..8d6aabb 100644 --- a/deno.lock +++ b/deno.lock @@ -26,6 +26,7 @@ "npm:@aws-crypto/sha256-js@^5.2.0": "5.2.0", "npm:@aws-sdk/client-s3@*": "3.937.0", "npm:@aws-sdk/client-s3@3": "3.937.0", + "npm:@aws-sdk/s3-request-presigner@3": "3.937.0", "npm:@effect/opentelemetry@~0.56.2": "0.56.6_@effect+platform@0.90.10__effect@3.19.14_@opentelemetry+sdk-trace-base@2.3.0__@opentelemetry+api@1.9.0_@opentelemetry+sdk-trace-node@2.3.0__@opentelemetry+api@1.9.0_@opentelemetry+semantic-conventions@1.38.0_effect@3.19.14", "npm:@effect/platform-node@0.96": "0.96.1_@effect+cluster@0.48.16__@effect+platform@0.90.10___effect@3.19.14__@effect+rpc@0.69.5___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+sql@0.44.2___@effect+experimental@0.54.6____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+workflow@0.9.6___@effect+platform@0.90.10____effect@3.19.14___@effect+rpc@0.69.5____@effect+platform@0.90.10_____effect@3.19.14____effect@3.19.14___effect@3.19.14__effect@3.19.14_@effect+platform@0.90.10__effect@3.19.14_@effect+rpc@0.69.5__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_@effect+sql@0.44.2__@effect+experimental@0.54.6___@effect+platform@0.90.10____effect@3.19.14___effect@3.19.14__@effect+platform@0.90.10___effect@3.19.14__effect@3.19.14_effect@3.19.14", "npm:@effect/platform@*": "0.90.10_effect@3.19.14", @@ -583,6 +584,19 @@ "tslib" ] }, + "@aws-sdk/s3-request-presigner@3.937.0": { + "integrity": "sha512-AvsCt6FnnKTpkmzDA1pFzmXPyxbGBdtllOIY0mL1iNSVZ3d7SoJKZH4NaqlcgUtbYG9zVh6QfLWememj1yEAmw==", + "dependencies": [ + "@aws-sdk/signature-v4-multi-region", + "@aws-sdk/types", + "@aws-sdk/util-format-url", + "@smithy/middleware-endpoint", + "@smithy/protocol-http@5.3.8", + "@smithy/smithy-client", + "@smithy/types@4.12.0", + "tslib" + ] + }, "@aws-sdk/signature-v4-multi-region@3.936.0": { "integrity": "sha512-8qS0GFUqkmwO7JZ0P8tdluBmt1UTfYUah8qJXGzNh9n1Pcb0AIeT117cCSiCUtwk+gDbJvd4hhRIhJCNr5wgjg==", "dependencies": [ @@ -629,6 +643,15 @@ "tslib" ] }, + "@aws-sdk/util-format-url@3.936.0": { + "integrity": "sha512-MS5eSEtDUFIAMHrJaMERiHAvDPdfxc/T869ZjDNFAIiZhyc037REw0aoTNeimNXDNy2txRNZJaAUn/kE4RwN+g==", + "dependencies": [ + "@aws-sdk/types", + "@smithy/querystring-builder", + "@smithy/types@4.12.0", + "tslib" + ] + }, "@aws-sdk/util-locate-window@3.893.0": { "integrity": "sha512-T89pFfgat6c8nMmpI8eKjBcDcgJq36+m9oiXbcUzeU55MP9ZuGgBomGjGnHaEyF36jenW9gmg3NfZDm0AO2XPg==", "dependencies": [ @@ -1811,6 +1834,7 @@ "jsr:@std/yaml@^1.0.5", "npm:@aws-crypto/sha256-js@^5.2.0", "npm:@aws-sdk/client-s3@3", + "npm:@aws-sdk/s3-request-presigner@3", "npm:@effect/opentelemetry@~0.56.2", "npm:@effect/platform-node@0.96", "npm:@effect/platform@~0.90.3", diff --git a/src/Backends/S3/Objects.ts b/src/Backends/S3/Objects.ts index 879f561..b45c4a0 100644 --- a/src/Backends/S3/Objects.ts +++ b/src/Backends/S3/Objects.ts @@ -1,4 +1,5 @@ import { + CopyObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, GetObjectAttributesCommand, @@ -251,6 +252,7 @@ export const makeObjectOps = ( Key: key, Range: normalized["range"], PartNumber: s3Params.partNumber, + VersionId: s3Params.versionId, ChecksumMode: s3Params.checksumMode as "ENABLED", IfMatch: normalized["if-match"], IfNoneMatch: normalized["if-none-match"], @@ -649,4 +651,40 @@ export const makeObjectOps = ( storageClass: result.StorageClass, }; }), + + copyObject: ( + sourceKey: string, + destKey: string, + metadataDirective: "COPY" | "REPLACE", + headers: Record, + sourceBucket?: string, + ) => + Effect.gen(function* () { + const srcBucket = sourceBucket || bucketName; + const { s3Params, metadata } = headerService.fromRequestHeaders(headers); + + const result = yield* Effect.tryPromise({ + try: () => + client.send( + new CopyObjectCommand({ + Bucket: bucketName, + Key: destKey, + CopySource: `${encodeURIComponent(srcBucket)}/${ + encodeURIComponent( + sourceKey, + ) + }${s3Params.versionId ? `?versionId=${s3Params.versionId}` : ""}`, + MetadataDirective: metadataDirective, + Metadata: metadataDirective === "REPLACE" ? metadata : undefined, + }), + ), + catch: (e) => mapS3Error(e, bucketName), + }); + + return { + etag: result.CopyObjectResult?.ETag, + versionId: result.VersionId, + lastModified: result.CopyObjectResult?.LastModified, + }; + }), }); diff --git a/src/Backends/Swift/Objects.ts b/src/Backends/Swift/Objects.ts index ca16e93..dd71284 100644 --- a/src/Backends/Swift/Objects.ts +++ b/src/Backends/Swift/Objects.ts @@ -24,6 +24,41 @@ import { type SwiftTarget, } from "./Utils.ts"; +/** + * Resolves Content-Type from Swift response headers with multiple fallbacks. + */ +function resolveContentType( + response: HttpClientResponse.HttpClientResponse, + normalizedResp: Record, + s3Headers: Record, +): string | undefined { + let contentType = normalizedResp["content-type"]; + + // Platform may wrap Fetch Response; try native Response.headers first (case-insensitive get). + if ( + contentType === undefined && + (response as unknown as { source?: unknown }).source instanceof Response + ) { + const src = (response as unknown as { source: Response }).source; + contentType = src.headers.get("content-type") ?? undefined; + } + + if (contentType === undefined) { + const h = response.headers as unknown as { + get?: (n: string) => string | null; + }; + if (typeof h.get === "function") { + contentType = h.get("content-type") ?? h.get("Content-Type") ?? undefined; + } + } + + if (contentType === undefined) { + contentType = s3Headers["Content-Type"] ?? s3Headers["content-type"]; + } + + return contentType; +} + export interface SwiftObject { readonly name?: string; readonly hash?: string; @@ -315,24 +350,17 @@ export const makeObjectOps = ( ); } + const normalizedResp = normalizeHeaders(response.headers); const { metadata, s3Headers, checksums, partsCount } = headerService - .fromSwiftHeaders(response.headers); + .fromSwiftHeaders(normalizedResp); - const contentLengthHeader = response.headers["content-length"]; - const contentLengthRaw = Array.isArray(contentLengthHeader) - ? contentLengthHeader[0] - : contentLengthHeader; + const contentLengthRaw = normalizedResp["content-length"]; const contentLength = contentLengthRaw ? parseInt(contentLengthRaw, 10) : NaN; - const etagHeader = response.headers["etag"]; - const etag = Array.isArray(etagHeader) ? etagHeader[0] : etagHeader; - - const lastModifiedHeader = response.headers["last-modified"]; - const lastModified = Array.isArray(lastModifiedHeader) - ? lastModifiedHeader[0] - : lastModifiedHeader; + const etag = normalizedResp["etag"]; + const lastModified = normalizedResp["last-modified"]; // S3 clients (e.g. Restate) require Content-Length on GetObject; match old impl and fail if Swift omits it if ( @@ -379,12 +407,16 @@ export const makeObjectOps = ( ? (response as unknown as { source: Response }).source.body : undefined; + const contentType = resolveContentType( + response, + normalizedResp, + s3Headers, + ); + return { stream: response.stream, nativeStream: nativeStream || undefined, - contentType: (Array.isArray(response.headers["content-type"]) - ? response.headers["content-type"][0] - : response.headers["content-type"]) || undefined, + contentType, contentLength, etag: etag || undefined, lastModified: lastModified ? new Date(lastModified) : undefined, @@ -418,8 +450,6 @@ export const makeObjectOps = ( const swiftHeaders: Record = { "X-Auth-Token": token, - "Content-Type": (normalized["content-type"] || - "application/octet-stream") as string, ...headerService.toSwiftHeaders(metadata, checksums), }; @@ -469,8 +499,13 @@ export const makeObjectOps = ( : validatedStream; const request = HttpClientRequest.put(`${url}/${encodedKey}`).pipe( - HttpClientRequest.setHeaders(swiftHeaders), HttpClientRequest.bodyStream(bodyStream), + HttpClientRequest.setHeaders(swiftHeaders), + HttpClientRequest.setHeader( + "Content-Type", + (normalized["content-type"] || + "application/octet-stream") as string, + ), ); const response: HttpClientResponse.HttpClientResponse = yield* client @@ -761,5 +796,73 @@ export const makeObjectOps = ( return result; }), + + copyObject: ( + sourceKey: string, + destKey: string, + metadataDirective: "COPY" | "REPLACE", + headers: Record, + sourceBucket?: string, + ) => { + const encodedDestKey = encodeObjectKeyForSwift(destKey); + const srcBucket = sourceBucket || container; + const srcPath = `/${srcBucket}/${encodeObjectKeyForSwift(sourceKey)}`; + + return Effect.gen(function* () { + const { checksums, metadata } = headerService.fromRequestHeaders( + headers, + ); + const normalized = normalizeHeaders(headers); + + const swiftHeaders: Record = { + "X-Auth-Token": token, + "X-Copy-From": srcPath, + "Content-Length": "0", // Swift COPY/X-Copy-From requires 0 length or no body + }; + + if (metadataDirective === "REPLACE") { + swiftHeaders["X-Fresh-Metadata"] = "True"; + swiftHeaders["content-type"] = (normalized["content-type"] || + "application/octet-stream") as string; + Object.assign( + swiftHeaders, + headerService.toSwiftHeaders(metadata, checksums), + ); + } + + const request = HttpClientRequest.put(`${url}/${encodedDestKey}`).pipe( + HttpClientRequest.setHeaders(swiftHeaders), + ); + + const response: HttpClientResponse.HttpClientResponse = yield* client + .execute(request).pipe( + Effect.mapError((e) => + mapError(500, formatSwiftTransportError(e), container) + ), + ); + + if (response.status < 200 || response.status >= 300) { + const message = yield* response.text.pipe( + Effect.orElseSucceed(() => "Error"), + ); + return yield* Effect.fail( + mapError( + response.status, + message || "Error", + container, + "PUT", + destKey, + ), + ); + } + + const etagHeader = response.headers["etag"]; + const etag = Array.isArray(etagHeader) ? etagHeader[0] : etagHeader; + + return { + etag: etag || undefined, + }; + }); + }, }; }; diff --git a/src/Frontend/Multipart/Delete.ts b/src/Frontend/Multipart/Delete.ts index 8a0c44c..4eac1de 100644 --- a/src/Frontend/Multipart/Delete.ts +++ b/src/Frontend/Multipart/Delete.ts @@ -21,6 +21,6 @@ export const abortMultipartUpload = Effect.gen(function* () { yield* backend.abortMultipartUpload(key, s3Params.uploadId); return HttpServerResponse.empty({ status: 204, - headers: { "Content-Length": "0" }, + headers: {}, }); }); diff --git a/src/Frontend/Multipart/Put.ts b/src/Frontend/Multipart/Put.ts index dcebb0e..b1aa625 100644 --- a/src/Frontend/Multipart/Put.ts +++ b/src/Frontend/Multipart/Put.ts @@ -51,9 +51,6 @@ export const uploadPart = Effect.gen(function* () { ); const headers = headerService.toResponseHeaders(result); - if (headers["Content-Length"] === undefined) { - headers["Content-Length"] = "0"; - } return HttpServerResponse.empty({ status: 200, headers, diff --git a/src/Frontend/Objects/Delete.ts b/src/Frontend/Objects/Delete.ts index 762d8d6..abb0162 100644 --- a/src/Frontend/Objects/Delete.ts +++ b/src/Frontend/Objects/Delete.ts @@ -18,6 +18,6 @@ export const deleteObject = Effect.gen(function* () { yield* backend.deleteObject(key); return HttpServerResponse.empty({ status: 204, - headers: { "Content-Length": "0" }, + headers: {}, }); }); diff --git a/src/Frontend/Objects/Put.ts b/src/Frontend/Objects/Put.ts index cd33e1e..f32ef76 100644 --- a/src/Frontend/Objects/Put.ts +++ b/src/Frontend/Objects/Put.ts @@ -1,12 +1,223 @@ import { HttpServerRequest, HttpServerResponse } from "@effect/platform"; import { Effect } from "effect"; -import { Backend } from "../../Services/Backend.ts"; +import { + Backend, + InternalError, + InvalidRequest, +} from "../../Services/Backend.ts"; +import { BackendResolver } from "../../Services/BackendResolver.ts"; +import { S3Xml } from "../../Services/S3Xml.ts"; import { S3RequestParser } from "../Utils.ts"; import { S3HeaderService } from "../../Services/S3HeaderService.ts"; +import { RequestContext } from "../Utils.ts"; import { uploadPart } from "../Multipart/Put.ts"; +function getHeader( + headers: Record, + name: string, +): string | undefined { + const lower = name.toLowerCase(); + const entry = Object.entries(headers).find( + ([k]) => k.toLowerCase() === lower, + ); + if (!entry) return undefined; + const v = entry[1]; + return Array.isArray(v) ? v[0] : v; +} + +/** + * Parse x-amz-copy-source header. Format: /bucket/key or bucket/key, optional ?versionId=xxx. + * Returns sourceBucket and sourceKey; fails with InvalidRequest if missing or malformed. + */ +function parseCopySource( + value: string, +): Effect.Effect< + { sourceBucket: string; sourceKey: string; versionId?: string }, + InvalidRequest +> { + return Effect.gen(function* () { + const trimmed = value.trim(); + if (!trimmed) { + return yield* Effect.fail( + new InvalidRequest({ + message: "x-amz-copy-source must be non-empty", + }), + ); + } + let decoded: string; + try { + decoded = decodeURIComponent(trimmed); + } catch { + return yield* Effect.fail( + new InvalidRequest({ + message: "x-amz-copy-source is not valid URL-encoded", + }), + ); + } + const withoutQuery = decoded.includes("?") + ? decoded.split("?")[0] + : decoded; + const path = withoutQuery.startsWith("/") + ? withoutQuery.slice(1) + : withoutQuery; + const firstSlash = path.indexOf("/"); + if (firstSlash === -1) { + return yield* Effect.fail( + new InvalidRequest({ + message: "x-amz-copy-source must be /bucket/key or bucket/key", + }), + ); + } + const sourceBucket = path.slice(0, firstSlash); + const sourceKey = path.slice(firstSlash + 1); + if (!sourceBucket) { + return yield* Effect.fail( + new InvalidRequest({ + message: "x-amz-copy-source source bucket is empty", + }), + ); + } + if (!sourceKey) { + return yield* Effect.fail( + new InvalidRequest({ + message: "x-amz-copy-source source key is empty", + }), + ); + } + let versionId: string | undefined; + if (decoded.includes("?versionId=")) { + const versionPart = decoded.split("?versionId=")[1]; + versionId = versionPart?.split("&")[0]; + } + return { sourceBucket, sourceKey, versionId }; + }); +} + +/** + * CopyObject: GET source object from source backend, PUT stream to destination. + * Uses x-amz-metadata-directive: COPY (default) = use source metadata; REPLACE = use request headers. + */ +const copyObject = Effect.gen(function* () { + const backend = yield* Backend; + const request = yield* HttpServerRequest.HttpServerRequest; + const { bucket } = yield* RequestContext; + const { key } = yield* S3RequestParser; + const headerService = yield* S3HeaderService; + const resolver = yield* BackendResolver; + const s3Xml = yield* S3Xml; + + const copySourceRaw = getHeader(request.headers, "x-amz-copy-source"); + if (!copySourceRaw) { + return yield* Effect.fail( + new InvalidRequest({ + message: "CopyObject requires x-amz-copy-source header", + }), + ); + } + + const { sourceBucket, sourceKey, versionId } = yield* parseCopySource( + copySourceRaw, + ); + + if (sourceBucket === bucket && sourceKey === key) { + return yield* Effect.fail( + new InvalidRequest({ + message: "CopyObject to the same key is not allowed", + }), + ); + } + + const metadataDirective = (getHeader( + request.headers, + "x-amz-metadata-directive", + )?.toUpperCase() || "COPY") as "COPY" | "REPLACE"; + + const sourceBackend = yield* resolver.getLayerForBucket(sourceBucket); + + // If source and dest backends are the same instance, use native copy. + if (sourceBackend === backend) { + const result = yield* backend.copyObject( + sourceKey, + key, + metadataDirective, + request.headers, + sourceBucket, + ); + const lastModified = result.lastModified !== undefined + ? result.lastModified + : new Date(); + return s3Xml.formatCopyObjectResult({ + etag: result.etag || "", + lastModified, + }); + } + + // Cross-backend copy: GET then PUT + const getHeaders: Record = { + ...request.headers, + }; + if (versionId) { + getHeaders["x-amz-version-id"] = versionId; + } + + const sourceResponse = yield* sourceBackend.getObject( + sourceKey, + getHeaders, + ); + + const isReplace = metadataDirective === "REPLACE"; + + const putHeaders: Record = {}; + if (sourceResponse.contentLength !== undefined) { + putHeaders["content-length"] = String(sourceResponse.contentLength); + } + if (isReplace) { + const ct = getHeader(request.headers, "content-type"); + if (ct) putHeaders["content-type"] = ct; + const parsed = headerService.fromRequestHeaders(request.headers); + for (const [k, v] of Object.entries(parsed.metadata)) { + putHeaders[`x-amz-meta-${k}`] = v; + } + } else { + const sourceContentType = sourceResponse.contentType ?? + (() => { + const lower = "content-type"; + const entry = Object.entries(sourceResponse.headers).find( + ([k]) => k.toLowerCase() === lower, + ); + return entry ? entry[1] : undefined; + })(); + if (sourceContentType) { + putHeaders["content-type"] = sourceContentType; + } + for (const [k, v] of Object.entries(sourceResponse.metadata)) { + if (!k.toLowerCase().startsWith("s3-checksum-")) { + putHeaders[`x-amz-meta-${k}`] = v; + } + } + } + + const putResult = yield* backend.putObject( + key, + sourceResponse.stream, + putHeaders, + ); + + const etag = putResult.etag; + if (!etag) { + return yield* Effect.fail( + new InternalError({ message: "CopyObject: no ETag from put" }), + ); + } + return s3Xml.formatCopyObjectResult({ + etag, + lastModified: new Date(), + }); +}); + /** * Handler for PutObject (PUT /:bucket/*) + * If x-amz-copy-source is present, performs CopyObject (server-side copy) instead. */ export const putObject = Effect.gen(function* () { const backend = yield* Backend; @@ -18,6 +229,11 @@ export const putObject = Effect.gen(function* () { return yield* uploadPart; } + const copySource = getHeader(request.headers, "x-amz-copy-source"); + if (copySource) { + return yield* copyObject; + } + const result = yield* backend.putObject( key, request.stream, diff --git a/src/Services/Backend.ts b/src/Services/Backend.ts index 00214c6..a999ad4 100644 --- a/src/Services/Backend.ts +++ b/src/Services/Backend.ts @@ -200,6 +200,10 @@ export interface PutObjectResult extends ChecksumInfo { readonly versionId?: string; } +export interface CopyObjectResult extends PutObjectResult { + readonly lastModified?: Date; +} + export interface MultipartUploadResult extends ChecksumInfo { readonly uploadId: string; } @@ -344,6 +348,14 @@ export class Backend extends Context.Tag("Backend")< headers: Record, ) => Effect.Effect; + copyObject: ( + sourceKey: string, + destKey: string, + metadataDirective: "COPY" | "REPLACE", + headers: Record, + sourceBucket?: string, + ) => Effect.Effect; + // Multipart Upload createMultipartUpload: ( key: string, diff --git a/src/Services/BackendResolver.ts b/src/Services/BackendResolver.ts index 6825564..6c6bb7a 100644 --- a/src/Services/BackendResolver.ts +++ b/src/Services/BackendResolver.ts @@ -3,6 +3,7 @@ import { makeS3Backend } from "../Backends/S3/Backend.ts"; import { makeSwiftBackend } from "../Backends/Swift/Backend.ts"; import { HeraldConfig } from "../Config/Layer.ts"; import type { MaterializedBucket } from "../Domain/Config.ts"; +import { NoSuchBucket } from "./Backend.ts"; export class BackendResolver extends Effect.Service()("BackendResolver", { @@ -36,7 +37,10 @@ export class BackendResolver const matched = config.lookupBucket(bucketName); if (Option.isNone(matched)) { return yield* Effect.fail( - new Error(`No configuration found for bucket: ${bucketName}`), + new NoSuchBucket({ + bucket: bucketName, + message: `No configuration found for bucket: ${bucketName}`, + }), ); } return yield* makeBackend(matched.value); diff --git a/src/Services/S3HeaderService.ts b/src/Services/S3HeaderService.ts index c8d0bcd..1b75610 100644 --- a/src/Services/S3HeaderService.ts +++ b/src/Services/S3HeaderService.ts @@ -9,11 +9,39 @@ import type { import { ChecksumHeaders } from "./S3Schema.ts"; export const normalizeHeaders = ( - raw: Record, + raw: Record | Headers | unknown, ): Record => { const normalized: Record = {}; - for (const [key, value] of Object.entries(raw)) { - normalized[key.toLowerCase()] = Array.isArray(value) ? value[0] : value; + if ( + raw != null && + typeof raw === "object" && + "entries" in raw && + typeof (raw as Headers).entries === "function" + ) { + for (const [key, value] of (raw as Headers).entries()) { + normalized[key.toLowerCase()] = value; + } + return normalized; + } + + if (raw == null || typeof raw !== "object") { + return normalized; + } + + for (const [key, value] of Object.entries(raw as Record)) { + const lowerKey = key.toLowerCase(); + if (value === undefined) { + normalized[lowerKey] = undefined; + continue; + } + if (Array.isArray(value)) { + const first = value[0]; + normalized[lowerKey] = first === undefined || typeof first === "string" + ? first + : String(first); + continue; + } + normalized[lowerKey] = typeof value === "string" ? value : String(value); } return normalized; }; diff --git a/src/Services/S3Xml.ts b/src/Services/S3Xml.ts index 655badd..f6e5878 100644 --- a/src/Services/S3Xml.ts +++ b/src/Services/S3Xml.ts @@ -74,6 +74,10 @@ export class S3Xml extends Context.Tag("S3Xml")< key: string; etag: string; }) => HttpServerResponse.HttpServerResponse; + formatCopyObjectResult: (args: { + etag: string; + lastModified: Date; + }) => HttpServerResponse.HttpServerResponse; } >() {} @@ -467,6 +471,20 @@ export const makeS3Xml = Effect.sync(() => { }); }, + formatCopyObjectResult: (args) => { + const xml = + `${ + encode(args.etag) + }${args.lastModified.toISOString()}`; + return HttpServerResponse.text(xml, { + status: 200, + headers: { + "Content-Type": "application/xml", + "Content-Length": String(new TextEncoder().encode(xml).length), + }, + }); + }, + formatObjectAttributes: (result: ObjectAttributes) => { const checksumXml = result.checksum ? `${ diff --git a/tests/integration/__snapshots__/objects.test.ts.snap b/tests/integration/__snapshots__/objects.test.ts.snap index e3d5f90..18e8524 100644 --- a/tests/integration/__snapshots__/objects.test.ts.snap +++ b/tests/integration/__snapshots__/objects.test.ts.snap @@ -306,3 +306,253 @@ snapshot[`Swift/objects/multipart/empty metadata 1`] = ` status: 204, } `; + +snapshot[`Baseline/objects/copy/same_bucket metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/copy/same_bucket metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/copy/same_bucket metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/copy/zero_size metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/copy/zero_size metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/copy/zero_size metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/copy/copy_to_itself metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/copy/copy_to_itself metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/copy/copy_to_itself metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/copy/source_key_not_found metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "376", + "content-type": "application/xml", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Baseline/objects/copy/source_key_not_found body 1`] = ` +' +NoSuchKeyThe specified key does not exist.no-such-source-keytest-objects-bucket/test-objects-bucket/copy-dest-anyIDHOST' +`; + +snapshot[`Proxy/objects/copy/source_key_not_found metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/objects/copy/source_key_not_found body 1`] = `'NoSuchKeyThe specified key does not exist.'`; + +snapshot[`Swift/objects/copy/source_key_not_found metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Swift/objects/copy/source_key_not_found body 1`] = `'NoSuchKey

Not Found

The resource could not be found.

'`; + +snapshot[`Baseline/objects/copy/source_bucket_not_found metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "content-length": "377", + "content-type": "application/xml", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Baseline/objects/copy/source_bucket_not_found body 1`] = ` +' +NoSuchBucketThe specified bucket does not existany-keynonexistent-bucket-xyz-123/test-objects-bucket/copy-dest-anyIDHOST' +`; + +snapshot[`Proxy/objects/copy/source_bucket_not_found metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Proxy/objects/copy/source_bucket_not_found body 1`] = `'NoSuchBucketThe specified bucket does not exist'`; + +snapshot[`Swift/objects/copy/source_bucket_not_found metadata 1`] = ` +{ + headers: { + "content-type": "application/xml", + vary: "Accept-Encoding", + }, + status: 404, +} +`; + +snapshot[`Swift/objects/copy/source_bucket_not_found body 1`] = `'NoSuchKey

Not Found

The resource could not be found.

'`; + +snapshot[`Baseline/objects/copy/diff_bucket metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/copy/diff_bucket metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/copy/diff_bucket metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/copy/verify_content_type metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/copy/verify_content_type metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/copy/verify_content_type metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Baseline/objects/copy/replace_metadata metadata 1`] = ` +{ + headers: { + "accept-ranges": "bytes", + "strict-transport-security": "max-age=31536000; includeSubDomains", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block", + vary: "Origin, Accept-Encoding", + }, + status: 204, +} +`; + +snapshot[`Proxy/objects/copy/replace_metadata metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; + +snapshot[`Swift/objects/copy/replace_metadata metadata 1`] = ` +{ + headers: {}, + status: 204, +} +`; diff --git a/tests/integration/objects.test.ts b/tests/integration/objects.test.ts index 1d115ec..9e254ad 100644 --- a/tests/integration/objects.test.ts +++ b/tests/integration/objects.test.ts @@ -1,6 +1,7 @@ import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, + CopyObjectCommand, CreateBucketCommand, CreateMultipartUploadCommand, DeleteBucketCommand, @@ -44,6 +45,7 @@ interface ObjectTestSpec { } const BUCKET = "test-objects-bucket"; +const BUCKET_COPY_DIFF = "test-objects-bucket-copy-dest"; const specs: ObjectTestSpec[] = [ { @@ -412,6 +414,308 @@ const specs: ObjectTestSpec[] = [ skipSnapshot: true, ignoreBaseline: true, }, + // CopyObject: same bucket + { + name: "objects/copy/same_bucket", + fn: async (c) => { + const srcKey = "copy-src-key"; + const destKey = "copy-dest-key"; + await c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: srcKey, + Body: "content to copy", + }), + ); + await c.send( + new CopyObjectCommand({ + Bucket: BUCKET, + Key: destKey, + CopySource: `${BUCKET}/${srcKey}`, + }), + ); + const out = await c.send( + new GetObjectCommand({ Bucket: BUCKET, Key: destKey }), + ); + const body = await out.Body?.transformToByteArray(); + if (!body || new TextDecoder().decode(body) !== "content to copy") { + throw new Error( + `Copy body mismatch: expected "content to copy", got ${ + body ? new TextDecoder().decode(body) : "null" + }`, + ); + } + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "copy-src-key" }), + ); + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "copy-dest-key" }), + ); + } catch { /* ignore */ } + }, + }, + // CopyObject: zero size + { + name: "objects/copy/zero_size", + fn: async (c) => { + const srcKey = "copy-zero-src"; + const destKey = "copy-zero-dest"; + await c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: srcKey, + Body: new Uint8Array(0), + }), + ); + await c.send( + new CopyObjectCommand({ + Bucket: BUCKET, + Key: destKey, + CopySource: `${BUCKET}/${srcKey}`, + }), + ); + const head = await c.send( + new HeadObjectCommand({ Bucket: BUCKET, Key: destKey }), + ); + if (head.ContentLength !== 0) { + throw new Error( + `Expected ContentLength 0, got ${head.ContentLength}`, + ); + } + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "copy-zero-src" }), + ); + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "copy-zero-dest" }), + ); + } catch { /* ignore */ } + }, + }, + // CopyObject: copy to self -> 400 InvalidRequest + { + name: "objects/copy/copy_to_itself", + fn: async (c) => { + const key = "copy-self-key"; + await c.send( + new PutObjectCommand({ Bucket: BUCKET, Key: key, Body: "x" }), + ); + try { + await c.send( + new CopyObjectCommand({ + Bucket: BUCKET, + Key: key, + CopySource: `${BUCKET}/${key}`, + }), + ); + throw new Error("Expected CopyObject to self to fail with 400"); + } catch (e) { + if ( + e instanceof S3ServiceException && + e.name === "InvalidRequest" && + e.$metadata?.httpStatusCode === 400 + ) { + return; + } + throw e; + } finally { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: key }), + ); + } catch { /* ignore */ } + } + }, + }, + // CopyObject: source key not found -> 404 NoSuchKey + { + name: "objects/copy/source_key_not_found", + fn: (c) => + c.send( + new CopyObjectCommand({ + Bucket: BUCKET, + Key: "copy-dest-any", + CopySource: `${BUCKET}/no-such-source-key`, + }), + ), + expectedErrorCode: "NoSuchKey", + }, + // CopyObject: source bucket not found -> 404 (NoSuchBucket or backend 404) + { + name: "objects/copy/source_bucket_not_found", + fn: async (c) => { + try { + await c.send( + new CopyObjectCommand({ + Bucket: BUCKET, + Key: "copy-dest-any", + CopySource: "nonexistent-bucket-xyz-123/any-key", + }), + ); + throw new Error("Expected CopyObject to fail with 404"); + } catch (e) { + if (e instanceof S3ServiceException) { + if (e.$metadata?.httpStatusCode !== 404) { + throw new Error( + `Expected 404, got ${e.$metadata?.httpStatusCode} (${e.name})`, + ); + } + return; + } + throw e; + } + }, + }, + // CopyObject: different bucket (same backend) + { + name: "objects/copy/diff_bucket", + fn: async (c) => { + const srcKey = "copy-diff-src"; + const destKey = "copy-diff-dest"; + await c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: srcKey, + Body: "foo from A", + }), + ); + await c.send( + new CopyObjectCommand({ + Bucket: BUCKET_COPY_DIFF, + Key: destKey, + CopySource: `${BUCKET}/${srcKey}`, + }), + ); + const out = await c.send( + new GetObjectCommand({ + Bucket: BUCKET_COPY_DIFF, + Key: destKey, + }), + ); + const body = await out.Body?.transformToByteArray(); + if (!body || new TextDecoder().decode(body) !== "foo from A") { + throw new Error( + `Copy diff_bucket body mismatch: got ${ + body ? new TextDecoder().decode(body) : "null" + }`, + ); + } + }, + setup: async (c) => { + try { + await c.send(new CreateBucketCommand({ Bucket: BUCKET_COPY_DIFF })); + } catch { /* ignore if exists */ } + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "copy-diff-src" }), + ); + await c.send( + new DeleteObjectCommand({ + Bucket: BUCKET_COPY_DIFF, + Key: "copy-diff-dest", + }), + ); + await c.send( + new DeleteBucketCommand({ Bucket: BUCKET_COPY_DIFF }), + ); + } catch { /* ignore */ } + }, + }, + // CopyObject: verify Content-Type preserved (metadata COPY). + { + name: "objects/copy/verify_content_type", + fn: async (c) => { + const srcKey = "copy-ct-src"; + const destKey = "copy-ct-dest"; + const contentType = "application/x-custom-test"; + await c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: srcKey, + Body: "data", + ContentType: contentType, + }), + ); + await c.send( + new CopyObjectCommand({ + Bucket: BUCKET, + Key: destKey, + CopySource: `${BUCKET}/${srcKey}`, + }), + ); + const head = await c.send( + new HeadObjectCommand({ Bucket: BUCKET, Key: destKey }), + ); + if (head.ContentType !== contentType) { + throw new Error( + `Expected Content-Type ${contentType}, got ${head.ContentType}`, + ); + } + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "copy-ct-src" }), + ); + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "copy-ct-dest" }), + ); + } catch { /* ignore */ } + }, + }, + // CopyObject: replace metadata + { + name: "objects/copy/replace_metadata", + fn: async (c) => { + const srcKey = "copy-replace-src"; + const destKey = "copy-replace-dest"; + await c.send( + new PutObjectCommand({ + Bucket: BUCKET, + Key: srcKey, + Body: "data", + Metadata: { "old-meta": "old-value" }, + }), + ); + await c.send( + new CopyObjectCommand({ + Bucket: BUCKET, + Key: destKey, + CopySource: `${BUCKET}/${srcKey}`, + MetadataDirective: "REPLACE", + Metadata: { "new-meta": "new-value" }, + }), + ); + const head = await c.send( + new HeadObjectCommand({ Bucket: BUCKET, Key: destKey }), + ); + if (head.Metadata?.["new-meta"] !== "new-value") { + throw new Error( + `Expected new-meta: new-value, got ${head.Metadata?.["new-meta"]}`, + ); + } + if (head.Metadata?.["old-meta"]) { + throw new Error("old-meta should have been replaced"); + } + }, + teardown: async (c) => { + try { + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "copy-replace-src" }), + ); + await c.send( + new DeleteObjectCommand({ Bucket: BUCKET, Key: "copy-replace-dest" }), + ); + } catch { /* ignore */ } + }, + }, ]; async function runObjectTest(tc: ObjectTestSpec, client: S3Client) { diff --git a/x/s3-tests.ts b/x/s3-tests.ts index 7f4cc05..43625d4 100755 --- a/x/s3-tests.ts +++ b/x/s3-tests.ts @@ -35,6 +35,8 @@ import { GlobalConfig } from "../src/Domain/Config.ts"; const DEFAULT_TAGS = "not appendobject and not bucket_policy and not copy and not cors and not encryption and not fails_strict_rfc2616 and not iam_tenant and not iam_user and not iam_account and not lifecycle and not object_lock and not policy and not policy_status and not s3select and not s3website and not sse_s3 and not tagging and not test_of_sts and not user_policy and not versioning and not webidentity_test"; +// To run only copy tests: S3TEST_TAGS=copy ./x/s3-tests.ts or ./x/s3-tests.ts -- -m copy + function getMinioConfig(): GlobalConfig { return { backends: {