diff --git a/packages/host/tests/helpers/adapter.ts b/packages/host/tests/helpers/adapter.ts index fb2c71aff2..8d3fa14b57 100644 --- a/packages/host/tests/helpers/adapter.ts +++ b/packages/host/tests/helpers/adapter.ts @@ -48,7 +48,7 @@ interface Dir { interface File { kind: 'file'; - content: string | object; + content: string | object | Uint8Array; } type CardAPI = typeof import('https://cardstack.com/base/card-api'); @@ -253,7 +253,7 @@ export class TestRealmAdapter implements RealmAdapter { let value = content.content; - let fileRefContent = ''; + let fileRefContent: string | Uint8Array = ''; if (path.endsWith('.json')) { let cardApi = await this.#loader.import( @@ -272,6 +272,8 @@ export class TestRealmAdapter implements RealmAdapter { } else { fileRefContent = shimmedModuleIndicator; } + } else if (value instanceof Uint8Array) { + fileRefContent = value; } else { fileRefContent = value as string; } @@ -291,7 +293,7 @@ export class TestRealmAdapter implements RealmAdapter { async write( path: LocalPath, - contents: string | object, + contents: string | object | Uint8Array, ): Promise { let segments = path.split('/'); let name = segments.pop()!; @@ -326,9 +328,11 @@ export class TestRealmAdapter implements RealmAdapter { dir.contents[name] = { kind: 'file', content: - typeof contents === 'string' + contents instanceof Uint8Array ? contents - : JSON.stringify(contents, null, 2), + : typeof contents === 'string' + ? contents + : JSON.stringify(contents, null, 2), }; this.postUpdateEvent(updateEvent); diff --git a/packages/realm-server/node-realm.ts b/packages/realm-server/node-realm.ts index a83d100aa5..825f69da4c 100644 --- a/packages/realm-server/node-realm.ts +++ b/packages/realm-server/node-realm.ts @@ -179,7 +179,10 @@ export class NodeAdapter implements RealmAdapter { }; } - async write(path: string, contents: string): Promise { + async write( + path: string, + contents: string | Uint8Array, + ): Promise { let absolutePath = join(this.realmDir, path); ensureFileSync(absolutePath); writeFileSync(absolutePath, contents); diff --git a/packages/realm-server/tests/card-source-endpoints-test.ts b/packages/realm-server/tests/card-source-endpoints-test.ts index 676863b4eb..5450943b72 100644 --- a/packages/realm-server/tests/card-source-endpoints-test.ts +++ b/packages/realm-server/tests/card-source-endpoints-test.ts @@ -1063,6 +1063,188 @@ module(basename(__filename), function () { }); }); }); + + module('binary file POST request', function (_hooks) { + module('public writable realm', function (hooks) { + setupPermissionedRealmAtURL(hooks, realmURL, { + permissions: { + '*': ['read', 'write'], + }, + onRealmSetup, + }); + + let { getMessagesSince } = setupMatrixRoom(hooks, getRealmSetup); + + test('serves a binary file POST request', async function (assert) { + let bytes = new Uint8Array([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0xff, 0xfe, + ]); + let response = await request + .post('/test-image.png') + .set('Content-Type', 'application/octet-stream') + .send(Buffer.from(bytes)); + + assert.strictEqual(response.status, 204, 'HTTP 204 status'); + assert.ok( + response.headers['x-created'], + 'created date should be set for new binary file', + ); + + let filePath = join( + dir.name, + 'realm_server_1', + 'test', + 'test-image.png', + ); + assert.ok(existsSync(filePath), 'binary file exists on disk'); + let fileBytes = readFileSync(filePath); + assert.deepEqual( + new Uint8Array(fileBytes), + bytes, + 'file bytes match uploaded bytes', + ); + }); + + test('creates file metadata for binary upload', async function (assert) { + let bytes = new Uint8Array([0x00, 0x01, 0x02, 0x03]); + await request + .post('/meta-test.bin') + .set('Content-Type', 'application/octet-stream') + .send(Buffer.from(bytes)); + + let rows = await query(dbAdapter, [ + 'SELECT content_hash FROM realm_file_meta WHERE realm_url =', + param(testRealmHref), + 'AND file_path =', + param('meta-test.bin'), + ]); + assert.strictEqual(rows.length, 1, 'file meta row exists'); + assert.ok(rows[0].content_hash, 'content hash is set'); + }); + + test('overwrites existing binary file', async function (assert) { + let bytes1 = new Uint8Array([0x01, 0x02, 0x03]); + let bytes2 = new Uint8Array([0x04, 0x05, 0x06]); + + let response1 = await request + .post('/overwrite-test.bin') + .set('Content-Type', 'application/octet-stream') + .send(Buffer.from(bytes1)); + assert.strictEqual(response1.status, 204, 'first upload returns 204'); + + let response2 = await request + .post('/overwrite-test.bin') + .set('Content-Type', 'application/octet-stream') + .send(Buffer.from(bytes2)); + assert.strictEqual( + response2.status, + 204, + 'second upload returns 204', + ); + + let filePath = join( + dir.name, + 'realm_server_1', + 'test', + 'overwrite-test.bin', + ); + let fileBytes = readFileSync(filePath); + assert.deepEqual( + new Uint8Array(fileBytes), + bytes2, + 'file contains second upload bytes', + ); + }); + + test('broadcasts realm events for binary upload', async function (assert) { + let realmEventTimestampStart = Date.now(); + + await request + .post('/event-test.bin') + .set('Content-Type', 'application/octet-stream') + .send(Buffer.from(new Uint8Array([0xca, 0xfe]))); + + await expectIncrementalIndexEvent( + `${testRealmURL}event-test.bin`, + realmEventTimestampStart, + { + assert, + getMessagesSince, + realm: testRealmHref, + }, + ); + }); + }); + + module( + 'public writable realm with size limit for binary', + function (hooks) { + setupPermissionedRealmAtURL(hooks, realmURL, { + permissions: { + '*': ['read', 'write'], + }, + cardSizeLimitBytes: 512, + onRealmSetup, + }); + + test('returns 413 when binary payload exceeds size limit', async function (assert) { + let oversized = new Uint8Array(2048).fill(0xff); + let response = await request + .post('/too-large.bin') + .set('Content-Type', 'application/octet-stream') + .send(Buffer.from(oversized)); + + assert.strictEqual(response.status, 413, 'HTTP 413 status'); + assert.strictEqual( + response.body.errors[0].title, + 'Payload Too Large', + 'error title is correct', + ); + }); + }, + ); + + module('permissioned realm for binary', function (hooks) { + setupPermissionedRealmAtURL(hooks, realmURL, { + permissions: { + john: ['read', 'write'], + }, + onRealmSetup, + }); + + test('401 without a JWT for binary upload', async function (assert) { + let response = await request + .post('/secret.bin') + .set('Content-Type', 'application/octet-stream') + .send(Buffer.from(new Uint8Array([0x01]))); + + assert.strictEqual(response.status, 401, 'HTTP 401 status'); + }); + + test('403 without permission for binary upload', async function (assert) { + let response = await request + .post('/secret.bin') + .set('Content-Type', 'application/octet-stream') + .send(Buffer.from(new Uint8Array([0x01]))) + .set('Authorization', `Bearer ${createJWT(testRealm, 'not-john')}`); + + assert.strictEqual(response.status, 403, 'HTTP 403 status'); + }); + + test('204 with permission for binary upload', async function (assert) { + let response = await request + .post('/secret.bin') + .set('Content-Type', 'application/octet-stream') + .send(Buffer.from(new Uint8Array([0x01]))) + .set( + 'Authorization', + `Bearer ${createJWT(testRealm, 'john', ['read', 'write'])}`, + ); + + assert.strictEqual(response.status, 204, 'HTTP 204 status'); + }); + }); + }); }); }); diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 2ffe301e64..0d94323b5c 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -325,7 +325,10 @@ export interface RealmAdapter { exists(path: LocalPath): Promise; - write(path: LocalPath, contents: string): Promise; + write( + path: LocalPath, + contents: string | Uint8Array, + ): Promise; remove(path: LocalPath): Promise; @@ -644,6 +647,11 @@ export class Realm { SupportedMimeType.CardSource, this.upsertCardSource.bind(this), ) + .post( + '/.*', + SupportedMimeType.OctetStream, + this.upsertBinaryFile.bind(this), + ) .get('/.*', SupportedMimeType.FileMeta, this.getFileMeta.bind(this)) .head( '/.*', @@ -750,7 +758,7 @@ export class Realm { async write( path: LocalPath, - contents: string, + contents: string | Uint8Array, options?: WriteOptions, ): Promise { let results = await this._batchWrite(new Map([[path, contents]]), options); @@ -758,14 +766,14 @@ export class Realm { } async writeMany( - files: Map, + files: Map, options?: WriteOptions, ): Promise { return this._batchWrite(files, options); } private async _batchWrite( - files: Map, + files: Map, options?: WriteOptions, ): Promise { await this.indexing(); @@ -794,38 +802,46 @@ export class Realm { let currentWriteType: 'module' | 'instance' | undefined = hasExecutableExtension(path) ? 'module' - : path.endsWith('.json') && isCardDocumentString(content) + : typeof content === 'string' && + path.endsWith('.json') && + isCardDocumentString(content) ? 'instance' : undefined; - try { - let doc = JSON.parse(content); - if (isCardResource(doc.data) && options?.serializeFile) { - let serialized = await this.fileSerialization( - { data: merge(doc.data, { meta: { realmURL: this.url } }) }, - url, - ); - content = JSON.stringify(serialized, null, 2); - } - } catch (e: any) { - if ( - e.message?.includes?.('not found') || - isFilterRefersToNonexistentTypeError(e) - ) { - throw e; + if (typeof content === 'string') { + try { + let doc = JSON.parse(content); + if (isCardResource(doc.data) && options?.serializeFile) { + let serialized = await this.fileSerialization( + { data: merge(doc.data, { meta: { realmURL: this.url } }) }, + url, + ); + content = JSON.stringify(serialized, null, 2); + } + } catch (e: any) { + if ( + e.message?.includes?.('not found') || + isFilterRefersToNonexistentTypeError(e) + ) { + throw e; + } } } let sizeType: 'card' | 'file' = - path.endsWith('.json') && isCardDocumentString(content) + typeof content === 'string' && + path.endsWith('.json') && + isCardDocumentString(content) ? 'card' : 'file'; this.assertWriteSize(content, sizeType); - let existingFile = await readFileAsText(path, (p) => - this.#adapter.openFile(p), - ); - if (existingFile?.content === content) { - results.push({ path, lastModified: existingFile.lastModified }); - fileMetaRows.push({ path }); - continue; + if (typeof content === 'string') { + let existingFile = await readFileAsText(path, (p) => + this.#adapter.openFile(p), + ); + if (existingFile?.content === content) { + results.push({ path, lastModified: existingFile.lastModified }); + fileMetaRows.push({ path }); + continue; + } } let contentHash = computeContentHash(content); if (lastWriteType === 'module' && currentWriteType === 'instance') { @@ -1933,7 +1949,33 @@ export class Realm { }); } - private assertWriteSize(content: string, type: 'card' | 'file') { + private async upsertBinaryFile( + request: Request, + requestContext: RequestContext, + ): Promise { + let bytes = new Uint8Array(await request.arrayBuffer()); + let { lastModified, created } = await this.write( + this.paths.local(new URL(request.url)), + bytes, + { + clientRequestId: request.headers.get('X-Boxel-Client-Request-Id'), + serializeFile: false, + }, + ); + return createResponse({ + body: null, + init: { + status: 204, + headers: { + 'last-modified': formatRFC7231(lastModified * 1000), + ...(created ? { 'x-created': formatRFC7231(created * 1000) } : {}), + }, + }, + requestContext, + }); + } + + private assertWriteSize(content: string | Uint8Array, type: 'card' | 'file') { try { validateWriteSize(content, this.#cardSizeLimitBytes, type); } catch (error: any) { diff --git a/packages/runtime-common/router.ts b/packages/runtime-common/router.ts index 60805e6ca4..8f4112a0ad 100644 --- a/packages/runtime-common/router.ts +++ b/packages/runtime-common/router.ts @@ -34,6 +34,7 @@ export enum SupportedMimeType { JSON = 'application/json', CardDependencies = 'application/json', CardTypeSummary = 'application/json', + OctetStream = 'application/octet-stream', All = '*/*', } /* eslint-enable @typescript-eslint/no-duplicate-enum-values */ @@ -73,15 +74,30 @@ export function lookupRouteTable( let acceptMimeType = extractSupportedMimeType( request.headers.get('Accept') as unknown as null | string | [string], ); - if (!acceptMimeType) { - return; - } if (!isHTTPMethod(request.method)) { return; } - let routes = routeTable.get(acceptMimeType)?.get(request.method); + let routes = acceptMimeType + ? routeTable.get(acceptMimeType)?.get(request.method) + : undefined; + // Fall back to Content-Type when Accept doesn't match a route. This + // supports POST/PATCH routes where the request body type (e.g. + // application/octet-stream) is the meaningful discriminator rather than the + // desired response type. if (!routes) { - return; + let contentType = extractSupportedMimeType( + request.headers.get('Content-Type') as unknown as + | null + | string + | [string], + ); + if (!contentType) { + return; + } + routes = routeTable.get(contentType)?.get(request.method); + if (!routes) { + return; + } } // we construct a new URL within RealmPath.local() param that strips off the query string diff --git a/packages/runtime-common/write-size-validation.ts b/packages/runtime-common/write-size-validation.ts index f9f7e87314..04b5f8ddb1 100644 --- a/packages/runtime-common/write-size-validation.ts +++ b/packages/runtime-common/write-size-validation.ts @@ -1,11 +1,14 @@ const textEncoder = new TextEncoder(); export function validateWriteSize( - content: string, + content: string | Uint8Array, maxSizeBytes: number, type: 'card' | 'file', ): void { - const actualSize = textEncoder.encode(content).length; + const actualSize = + content instanceof Uint8Array + ? content.length + : textEncoder.encode(content).length; if (actualSize > maxSizeBytes) { throw new Error( `${type === 'card' ? 'Card' : 'File'} size (${actualSize} bytes) exceeds maximum allowed size (${maxSizeBytes} bytes)`,