From 2fdf5c853d7bf4fa56f6b13d396747290e2ec8f6 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Tue, 28 Oct 2025 15:18:26 -0400 Subject: [PATCH 01/19] ts-rewrite, toolchain update --- .eslintrc | 67 ----------------- .gitignore | 8 +- .npmignore | 9 --- .travis.yml | 6 -- benchmarks/{array.js => array.ts} | 0 benchmarks/{boolean.js => boolean.ts} | 0 benchmarks/{double.js => double.ts} | 0 benchmarks/index.ts | 27 +++++++ benchmarks/{integer.js => integer.ts} | 0 benchmarks/{object.js => object.ts} | 0 benchmarks/{string.js => string.ts} | 0 eslint.config.mjs | 23 ++++++ package.json | 73 ++++++++++++++----- rollup.config.mjs | 22 ++++++ src/{converter.js => converter.ts} | 4 +- src/{decoder.js => decoder.ts} | 4 +- src/{encoder.js => encoder.ts} | 2 +- index.js => src/index.ts | 8 +- src/{reader.js => reader.ts} | 8 +- src/{schema.js => schema.ts} | 16 ++-- src/{writer.js => writer.ts} | 6 +- tests/{index.js => integration/index.ts} | 71 +++++++++--------- tsconfig.json | 23 ++++++ types.d.ts | 93 ++++++++++++++++++++++++ 24 files changed, 304 insertions(+), 166 deletions(-) delete mode 100644 .eslintrc delete mode 100644 .npmignore delete mode 100644 .travis.yml rename benchmarks/{array.js => array.ts} (100%) rename benchmarks/{boolean.js => boolean.ts} (100%) rename benchmarks/{double.js => double.ts} (100%) create mode 100644 benchmarks/index.ts rename benchmarks/{integer.js => integer.ts} (100%) rename benchmarks/{object.js => object.ts} (100%) rename benchmarks/{string.js => string.ts} (100%) create mode 100644 eslint.config.mjs create mode 100644 rollup.config.mjs rename src/{converter.js => converter.ts} (97%) rename src/{decoder.js => decoder.ts} (99%) rename src/{encoder.js => encoder.ts} (99%) rename index.js => src/index.ts (59%) rename src/{reader.js => reader.ts} (92%) rename src/{schema.js => schema.ts} (90%) rename src/{writer.js => writer.ts} (96%) rename tests/{index.js => integration/index.ts} (79%) create mode 100644 tsconfig.json create mode 100644 types.d.ts diff --git a/.eslintrc b/.eslintrc deleted file mode 100644 index 8be354d..0000000 --- a/.eslintrc +++ /dev/null @@ -1,67 +0,0 @@ -{ - "parserOptions": { - "ecmaVersion": 9 - }, - "rules": { - "arrow-body-style": "off", - "camelcase": "off", - "class-methods-use-this": "off", - "comma-dangle": [ - "error", - { - "arrays": "always-multiline", - "objects": "always-multiline", - "imports": "always-multiline", - "exports": "always-multiline", - "functions": "never" - } - ], - "default-case": "off", - "consistent-return": "off", - "func-names": "off", - "import/extensions": "off", - "import/first": "off", - "import/newline-after-import": "off", - "import/no-named-as-default": "off", - "import/no-named-as-default-member": "off", - "import/prefer-default-export": "off", - "indent": ["error", 2], - "max-len": "off", - "no-mixed-operators": "off", - "no-param-reassign": "off", - "no-plusplus": "off", - "no-prototype-builtins": "off", - "no-restricted-syntax": "off", - "no-underscore-dangle": "off", - "no-useless-escape": "off", - "no-use-before-define": [ - "error", - { - "functions": false, - "classes": true, - "variables": true - } - ], - "no-var": "off", - "object-property-newline": "off", - "operator-assignment": "off", - "prefer-arrow-callback": "off", - "prefer-rest-params": "off", - "prefer-spread": "off", - "prefer-template": "off", - "object-curly-newline": "off", - "prefer-destructuring": "off", - "no-restricted-globals": "off", - "radix": "off", - "space-before-function-paren": [ - "error", - { - "anonymous": "never", - "named": "never" - } - ], - "linebreak-style": "off", - "no-lonely-if": "off" - } -} - \ No newline at end of file diff --git a/.gitignore b/.gitignore index b6cf311..2500e3a 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,10 @@ api_docs /benchmarks/compare.js yarn.lock package-lock.json -.clinic \ No newline at end of file +.clinic + +# Build output +dist/ + +# TypeScript +*.tsbuildinfo \ No newline at end of file diff --git a/.npmignore b/.npmignore deleted file mode 100644 index 9747b89..0000000 --- a/.npmignore +++ /dev/null @@ -1,9 +0,0 @@ -tests -README.md -.travis.yml -benchmarks -yarn.lock -package-lock.json -docs -.eslintrc -.clinic \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 91edb73..0000000 --- a/.travis.yml +++ /dev/null @@ -1,6 +0,0 @@ -language: node_js -node_js: - - "10" - - "12" -script: "npm run test && npm run bench" -cache: yarn diff --git a/benchmarks/array.js b/benchmarks/array.ts similarity index 100% rename from benchmarks/array.js rename to benchmarks/array.ts diff --git a/benchmarks/boolean.js b/benchmarks/boolean.ts similarity index 100% rename from benchmarks/boolean.js rename to benchmarks/boolean.ts diff --git a/benchmarks/double.js b/benchmarks/double.ts similarity index 100% rename from benchmarks/double.js rename to benchmarks/double.ts diff --git a/benchmarks/index.ts b/benchmarks/index.ts new file mode 100644 index 0000000..840a258 --- /dev/null +++ b/benchmarks/index.ts @@ -0,0 +1,27 @@ +#!/usr/bin/env node +/** Benchmark runner script */ + +import { execSync } from 'child_process'; + +const benchmarks = [ + 'array', + 'boolean', + 'double', + 'integer', + 'object', + 'string', +]; + +console.log('Running Compactr benchmarks...\n'); + +for (const benchmark of benchmarks) { + console.log(`\n=== ${benchmark.toUpperCase()} BENCHMARK ===\n`); + try { + execSync(`node ./${benchmark}.ts`, { stdio: 'inherit' }); + } + catch (error) { + console.error(`Failed to run ${benchmark} benchmark:`, error); + } +} + +console.log('\nAll benchmarks completed!'); diff --git a/benchmarks/integer.js b/benchmarks/integer.ts similarity index 100% rename from benchmarks/integer.js rename to benchmarks/integer.ts diff --git a/benchmarks/object.js b/benchmarks/object.ts similarity index 100% rename from benchmarks/object.js rename to benchmarks/object.ts diff --git a/benchmarks/string.js b/benchmarks/string.ts similarity index 100% rename from benchmarks/string.js rename to benchmarks/string.ts diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 0000000..7ed04c6 --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,23 @@ +import eslint from '@eslint/js'; +import tseslint from 'typescript-eslint'; +import jestConfig from 'eslint-plugin-jest'; +import spacing from '@stylistic/eslint-plugin'; + +export default tseslint.config( + eslint.configs.recommended, + tseslint.configs.recommended, + jestConfig.configs['flat/recommended'], + spacing.configs.recommended, + { + rules: { + '@stylistic/semi': [2, 'always'], + '@typescript-eslint/no-explicit-any': 0, + '@typescript-eslint/no-require-imports': 1, + 'jest/no-done-callback': 0, + 'jest/no-conditional-expect': 0, + }, + }, + { + ignores: ['**/dist', '**/benchmarks', '**/*.js'], + }, +); diff --git a/package.json b/package.json index a539dcd..8034cca 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,19 @@ { "name": "compactr", - "version": "2.4.2", + "version": "3.0.0", "description": "Schema based serialization made easy", - "main": "index.js", + "main": "dist/compactr.js", "scripts": { "lint": "eslint .", "lint:fix": "eslint . --fix", - "test": "mocha tests/index.js --exit", - "bench": "node benchmarks/array.js && node benchmarks/boolean.js && node benchmarks/double.js && node benchmarks/integer.js && node benchmarks/object.js && node benchmarks/string.js" + "test": "jest ./tests", + "build": "npm run clean && rollup -c rollup.config.mjs", + "clean": "rm -rf ./dist/*", + "bench": "node ./benchmarks/index.ts", + "prepublishOnly": "npm run build" + }, + "engines": { + "node": ">=20" }, "funding": { "type": "Github", @@ -18,21 +24,23 @@ "url": "git+https://github.com/compactr/compactr-js.git" }, "keywords": [ - "compactr", - "compact", - "encode", - "encoding", + "serialize", "serializing", "buffer", "byte", - "decode", - "decoding", "compress", - "serialize" + "protobuff", + "snappy", + "msgpack" + ], + "files": [ + "dist", + "types.d.ts" ], + "typings": "./types.d.ts", "husky": { "hooks": { - "pre-commit": "yarn lint" + "pre-commit": "npm run lint" } }, "author": "frederic charette ", @@ -41,12 +49,43 @@ "url": "https://github.com/compactr/compactr.js/issues" }, "homepage": "https://github.com/compactr/compactr.js#readme", + "jest": { + "preset": "ts-jest", + "testEnvironment": "node", + "testMatch": [ + "**/tests/**/*.ts", + "**/tests/**/*.js" + ], + "transform": { + "^.+\\.tsx?$": [ + "ts-jest", + { + "diagnostics": false + } + ], + "^.+\\.jsx?$": [ + "ts-jest", + { + "diagnostics": false + } + ] + } + }, "devDependencies": { + "@rollup/plugin-node-resolve": "^16.0.0", + "@rollup/plugin-sucrase": "^5.0.0", + "@stylistic/eslint-plugin": "^5.5.0", + "@types/jest": "^30.0.0", + "@types/node": "^24.9.0", "benchmark": "^2.1.4", - "chai": "^4.2.0", - "eslint": "^6.7.1", - "husky": "^3.1.0", - "mocha": "^6.2.0", - "protobufjs": "^6.8.8" + "eslint": "^9.38.0", + "eslint-plugin-jest": "^29.0.0", + "husky": "^9.1.0", + "jest": "^30.2.0", + "protobufjs": "^7.5.0", + "rollup": "^4.52.0", + "ts-jest": "^29.4.0", + "typescript": "^5.9.0", + "typescript-eslint": "^8.46.0" } } diff --git a/rollup.config.mjs b/rollup.config.mjs new file mode 100644 index 0000000..69cce84 --- /dev/null +++ b/rollup.config.mjs @@ -0,0 +1,22 @@ +import resolve from '@rollup/plugin-node-resolve'; +import sucrase from '@rollup/plugin-sucrase'; + +export default { + input: 'src/index.ts', + plugins: [ + resolve({ + extensions: ['.ts'], + preferBuiltins: true, + browser: false, + }), + sucrase({ + include: ['src/**'], + transforms: ['typescript'], + }), + ], + output: { + file: 'dist/compactr.js', + name: 'compactr', + format: 'umd', + }, +}; diff --git a/src/converter.js b/src/converter.ts similarity index 97% rename from src/converter.js rename to src/converter.ts index 36859e3..a24bbfe 100644 --- a/src/converter.js +++ b/src/converter.ts @@ -45,7 +45,7 @@ function array(value) { /* Exports -------------------------------------------------------------------*/ -module.exports = { +export default { int8, int16, int32, @@ -58,4 +58,4 @@ module.exports = { boolean, array, object, -}; +} diff --git a/src/decoder.js b/src/decoder.ts similarity index 99% rename from src/decoder.js rename to src/decoder.ts index 8be8685..ab4173e 100644 --- a/src/decoder.js +++ b/src/decoder.ts @@ -108,7 +108,7 @@ function double(bytes) { /* Exports -------------------------------------------------------------------*/ -module.exports = { +export default { boolean, number: double, int8, @@ -125,4 +125,4 @@ module.exports = { unsigned8: uint8, unsigned16: uint16, unsigned32: int32, -}; \ No newline at end of file +} diff --git a/src/encoder.js b/src/encoder.ts similarity index 99% rename from src/encoder.js rename to src/encoder.ts index 91013b6..f88bd69 100644 --- a/src/encoder.js +++ b/src/encoder.ts @@ -151,7 +151,7 @@ function getSize(count, byteLength) { /* Exports -------------------------------------------------------------------*/ -module.exports = { +export default { boolean, number: double, int8, diff --git a/index.js b/src/index.ts similarity index 59% rename from index.js rename to src/index.ts index 2c75d03..4ed873d 100644 --- a/index.js +++ b/src/index.ts @@ -1,9 +1,9 @@ /** Entry point */ -/* Requires ------------------------------------------------------------------*/ +/* Requires ------------------------------------------------------------------ */ -const schema = require('./src/schema'); +import schema from './schema'; -/* Exports -------------------------------------------------------------------*/ +/* Exports ------------------------------------------------------------------- */ -module.exports = { schema }; +export { schema }; diff --git a/src/reader.js b/src/reader.ts similarity index 92% rename from src/reader.js rename to src/reader.ts index be52414..7e543ae 100644 --- a/src/reader.js +++ b/src/reader.ts @@ -2,11 +2,11 @@ /* Requires ------------------------------------------------------------------*/ -const Decoder = require('./decoder'); +import Decoder from './decoder'; /* Methods -------------------------------------------------------------------*/ -function Reader(scope) { +export default function Reader(scope) { /** * Decodes an encoded buffer. Requires header bytes. @@ -75,7 +75,3 @@ function Reader(scope) { return { read, readHeader, readContent }; } - -/* Exports -------------------------------------------------------------------*/ - -module.exports = Reader; diff --git a/src/schema.js b/src/schema.ts similarity index 90% rename from src/schema.js rename to src/schema.ts index 490958f..966547d 100644 --- a/src/schema.js +++ b/src/schema.ts @@ -2,11 +2,11 @@ /* Requires ------------------------------------------------------------------*/ -const Encoder = require('./encoder'); -const Decoder = require('./decoder'); -const Reader = require('./reader'); -const Writer = require('./writer'); -const Converter = require('./converter'); +import Encoder from './encoder'; +import Decoder from './decoder'; +import Reader from './reader'; +import Writer from './writer'; +import Converter from './converter'; /* Methods -------------------------------------------------------------------*/ @@ -15,7 +15,7 @@ const Converter = require('./converter'); * @param {*} schema The schema to use * @param {Object (keyOrder: {boolean})} options The options for the schema */ -function Schema(schema, options = { keyOrder: false }) { +export default function Schema(schema, options = { keyOrder: false }) { const sizeRef = { boolean: 1, number: 8, @@ -128,7 +128,3 @@ function Schema(schema, options = { keyOrder: false }) { return Object.assign({}, writer, reader); } - -/* Exports -------------------------------------------------------------------*/ - -module.exports = Schema; \ No newline at end of file diff --git a/src/writer.js b/src/writer.ts similarity index 96% rename from src/writer.js rename to src/writer.ts index 80c429c..9a7149a 100644 --- a/src/writer.js +++ b/src/writer.ts @@ -2,7 +2,7 @@ /* Methods -------------------------------------------------------------------*/ -function Writer(scope) { +export default function Writer(scope) { /** * Start writing some data against a schema @@ -104,7 +104,3 @@ function Writer(scope) { return { write, headerBuffer, contentBuffer, buffer, typedArray, sizes }; } - -/* Exports -------------------------------------------------------------------*/ - -module.exports = Writer; \ No newline at end of file diff --git a/tests/index.js b/tests/integration/index.ts similarity index 79% rename from tests/index.js rename to tests/integration/index.ts index 354b3a0..dd6aeec 100644 --- a/tests/index.js +++ b/tests/integration/index.ts @@ -2,27 +2,26 @@ * Unit test suite */ -/* Requires ------------------------------------------------------------------*/ +/* Requires ------------------------------------------------------------------ */ -const expect = require('chai').expect; -const Compactr = require('../'); +import Compactr from '../../src'; -/* Tests ---------------------------------------------------------------------*/ +/* Tests --------------------------------------------------------------------- */ describe('Data integrity - simple', () => { describe('Boolean', () => { const Schema = Compactr.schema({ test: { type: 'boolean' } }); it('should preserve boolean value and type - true', () => { - expect(Schema.read(Schema.write({ test: true }).buffer())).to.deep.equal({ test: true }); + expect(Schema.read(Schema.write({ test: true }).buffer())).toEqual({ test: true }); }); it('should preserve boolean value and type - false', () => { - expect(Schema.read(Schema.write({ test: false }).buffer())).to.deep.equal({ test: false }); + expect(Schema.read(Schema.write({ test: false }).buffer())).toEqual({ test: false }); }); it('should skip null or undefined values', () => { - expect(Schema.read(Schema.write({ test: null }).buffer())).to.deep.equal({}); + expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({}); }); }); @@ -30,11 +29,11 @@ describe('Data integrity - simple', () => { const Schema = Compactr.schema({ test: { type: 'number' } }); it('should preserve number value and type', () => { - expect(Schema.read(Schema.write({ test: 23.23 }).buffer())).to.deep.equal({ test: 23.23 }); + expect(Schema.read(Schema.write({ test: 23.23 }).buffer())).toEqual({ test: 23.23 }); }); it('should preserve number value and type for negative values', () => { - expect(Schema.read(Schema.write({ test: -23.23 }).buffer())).to.deep.equal({ test: -23.23 }); + expect(Schema.read(Schema.write({ test: -23.23 }).buffer())).toEqual({ test: -23.23 }); }); }); @@ -42,7 +41,7 @@ describe('Data integrity - simple', () => { const Schema = Compactr.schema({ test: { type: 'string' } }); it('should preserve string value and type', () => { - expect(Schema.read(Schema.write({ test: 'hello world' }).buffer())).to.deep.equal({ test: 'hello world' }); + expect(Schema.read(Schema.write({ test: 'hello world' }).buffer())).toEqual({ test: 'hello world' }); }); }); @@ -50,7 +49,7 @@ describe('Data integrity - simple', () => { const Schema = Compactr.schema({ test: { type: 'array', items: { type: 'string' } } }); it('should preserve array values and types', () => { - expect(Schema.read(Schema.write({ test: ['a', 'b', 'c'] }).buffer())).to.deep.equal({ test: ['a', 'b', 'c'] }); + expect(Schema.read(Schema.write({ test: ['a', 'b', 'c'] }).buffer())).toEqual({ test: ['a', 'b', 'c'] }); }); }); @@ -58,7 +57,7 @@ describe('Data integrity - simple', () => { const Schema = Compactr.schema({ test: { type: 'object', schema: { test: { type: 'number' } } } }); it('should preserve object values and types', () => { - expect(Schema.read(Schema.write({ test: { test: 23.23 } }).buffer())).to.deep.equal({ test: { test: 23.23 } }); + expect(Schema.read(Schema.write({ test: { test: 23.23 } }).buffer())).toEqual({ test: { test: 23.23 } }); }); }); }); @@ -68,11 +67,11 @@ describe('Data integrity - multi simple', () => { const Schema = Compactr.schema({ test: { type: 'boolean' }, test2: { type: 'boolean' } }); it('should preserve boolean value and type - false', () => { - expect(Schema.read(Schema.write({ test: false, test2: true }).buffer())).to.deep.equal({ test: false, test2: true }); + expect(Schema.read(Schema.write({ test: false, test2: true }).buffer())).toEqual({ test: false, test2: true }); }); it('should skip null or undefined values', () => { - expect(Schema.read(Schema.write({ test: null, test2: false }).buffer())).to.deep.equal({ test2: false }); + expect(Schema.read(Schema.write({ test: null, test2: false }).buffer())).toEqual({ test2: false }); }); }); @@ -80,7 +79,7 @@ describe('Data integrity - multi simple', () => { const Schema = Compactr.schema({ test: { type: 'number' }, test2: { type: 'number' } }); it('should preserve number value and type', () => { - expect(Schema.read(Schema.write({ test: 23.23, test2: -97.7 }).buffer())).to.deep.equal({ test: 23.23, test2: -97.7 }); + expect(Schema.read(Schema.write({ test: 23.23, test2: -97.7 }).buffer())).toEqual({ test: 23.23, test2: -97.7 }); }); }); @@ -88,7 +87,7 @@ describe('Data integrity - multi simple', () => { const Schema = Compactr.schema({ test: { type: 'string' }, test2: { type: 'string' } }); it('should preserve string value and type', () => { - expect(Schema.read(Schema.write({ test: 'hello world', test2: 'écho' }).buffer())).to.deep.equal({ test: 'hello world', test2: 'écho' }); + expect(Schema.read(Schema.write({ test: 'hello world', test2: 'écho' }).buffer())).toEqual({ test: 'hello world', test2: 'écho' }); }); }); @@ -96,7 +95,7 @@ describe('Data integrity - multi simple', () => { const Schema = Compactr.schema({ test: { type: 'array', items: { type: 'string' } }, test2: { type: 'array', items: { type: 'string' } } }); it('should preserve array values and types', () => { - expect(Schema.read(Schema.write({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }).buffer())).to.deep.equal({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }); + expect(Schema.read(Schema.write({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }).buffer())).toEqual({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }); }); }); @@ -104,7 +103,7 @@ describe('Data integrity - multi simple', () => { const Schema = Compactr.schema({ test: { type: 'object', schema: { test: { type: 'number' } } }, test2: { type: 'object', schema: { test: { type: 'number' } } } }); it('should preserve object values and types', () => { - expect(Schema.read(Schema.write({ test: { test: 23.23 }, test2: { test: -97.7 } }).buffer())).to.deep.equal({ test: { test: 23.23 }, test2: { test: -97.7 } }); + expect(Schema.read(Schema.write({ test: { test: 23.23 }, test2: { test: -97.7 } }).buffer())).toEqual({ test: { test: 23.23 }, test2: { test: -97.7 } }); }); }); }); @@ -120,27 +119,27 @@ describe('Data integrity - multi mixed', () => { }); it('should preserve values and types', () => { - expect(Schema.read(Schema.write({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }).buffer())).to.deep.equal({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }); + expect(Schema.read(Schema.write({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }).buffer())).toEqual({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }); }); }); }); -/* Partial -------------------------------------------------------------------*/ +/* Partial ------------------------------------------------------------------- */ describe('Data integrity - partial - simple', () => { describe('Boolean', () => { const Schema = Compactr.schema({ test: { type: 'boolean' } }); it('should preserve boolean value and type - true', () => { - expect(Schema.readContent(Schema.write({ test: true }).contentBuffer())).to.deep.equal({ test: true }); + expect(Schema.readContent(Schema.write({ test: true }).contentBuffer())).toEqual({ test: true }); }); it('should preserve boolean value and type - false', () => { - expect(Schema.readContent(Schema.write({ test: false }).contentBuffer())).to.deep.equal({ test: false }); + expect(Schema.readContent(Schema.write({ test: false }).contentBuffer())).toEqual({ test: false }); }); it('should still send one 0 byte in case of null (coersed)', () => { - expect(Schema.readContent(Schema.write({ test: null }).contentBuffer())).to.deep.equal({ test: false }); + expect(Schema.readContent(Schema.write({ test: null }).contentBuffer())).toEqual({ test: false }); }); }); @@ -148,11 +147,11 @@ describe('Data integrity - partial - simple', () => { const Schema = Compactr.schema({ test: { type: 'number' } }); it('should preserve number value and type', () => { - expect(Schema.readContent(Schema.write({ test: 23.23 }).contentBuffer())).to.deep.equal({ test: 23.23 }); + expect(Schema.readContent(Schema.write({ test: 23.23 }).contentBuffer())).toEqual({ test: 23.23 }); }); it('should preserve number value and type for negative values', () => { - expect(Schema.readContent(Schema.write({ test: -23.23 }).contentBuffer())).to.deep.equal({ test: -23.23 }); + expect(Schema.readContent(Schema.write({ test: -23.23 }).contentBuffer())).toEqual({ test: -23.23 }); }); }); @@ -160,7 +159,7 @@ describe('Data integrity - partial - simple', () => { const Schema = Compactr.schema({ test: { type: 'string', size: 22 } }); it('should preserve string value and type', () => { - expect(Schema.readContent(Schema.write({ test: 'hello world' }).contentBuffer())).to.deep.equal({ test: 'hello world' }); + expect(Schema.readContent(Schema.write({ test: 'hello world' }).contentBuffer())).toEqual({ test: 'hello world' }); }); }); @@ -168,7 +167,7 @@ describe('Data integrity - partial - simple', () => { const Schema = Compactr.schema({ test: { type: 'array', size: 12, items: { type: 'string' } } }); it('should preserve array values and types', () => { - expect(Schema.readContent(Schema.write({ test: ['a', 'b', 'c'] }).contentBuffer())).to.deep.equal({ test: ['a', 'b', 'c', '', '', ''] }); + expect(Schema.readContent(Schema.write({ test: ['a', 'b', 'c'] }).contentBuffer())).toEqual({ test: ['a', 'b', 'c', '', '', ''] }); }); }); @@ -176,7 +175,7 @@ describe('Data integrity - partial - simple', () => { const Schema = Compactr.schema({ test: { type: 'object', size: 20, schema: { test: { type: 'number' } } } }); it('should preserve object values and types', () => { - expect(Schema.readContent(Schema.write({ test: { test: 23.23 } }).contentBuffer())).to.deep.equal({ test: { test: 23.23 } }); + expect(Schema.readContent(Schema.write({ test: { test: 23.23 } }).contentBuffer())).toEqual({ test: { test: 23.23 } }); }); }); }); @@ -186,11 +185,11 @@ describe('Data integrity - partial - multi simple', () => { const Schema = Compactr.schema({ test: { type: 'boolean' }, test2: { type: 'boolean' } }); it('should preserve boolean value and type - false', () => { - expect(Schema.readContent(Schema.write({ test: false, test2: true }).contentBuffer())).to.deep.equal({ test: false, test2: true }); + expect(Schema.readContent(Schema.write({ test: false, test2: true }).contentBuffer())).toEqual({ test: false, test2: true }); }); it('should skip null or undefined values', () => { - expect(Schema.readContent(Schema.write({ test: null, test2: false }).contentBuffer())).to.deep.equal({ test: false, test2: false }); + expect(Schema.readContent(Schema.write({ test: null, test2: false }).contentBuffer())).toEqual({ test: false, test2: false }); }); }); @@ -198,7 +197,7 @@ describe('Data integrity - partial - multi simple', () => { const Schema = Compactr.schema({ test: { type: 'number' }, test2: { type: 'number' } }); it('should preserve number value and type', () => { - expect(Schema.readContent(Schema.write({ test: 23.23, test2: -97.7 }).contentBuffer())).to.deep.equal({ test: 23.23, test2: -97.7 }); + expect(Schema.readContent(Schema.write({ test: 23.23, test2: -97.7 }).contentBuffer())).toEqual({ test: 23.23, test2: -97.7 }); }); }); @@ -206,7 +205,7 @@ describe('Data integrity - partial - multi simple', () => { const Schema = Compactr.schema({ test: { type: 'string', size: 22 }, test2: { type: 'string', size: 8 } }); it('should preserve string value and type', () => { - expect(Schema.readContent(Schema.write({ test: 'hello world', test2: 'écho' }).contentBuffer())).to.deep.equal({ test: 'hello world', test2: 'écho' }); + expect(Schema.readContent(Schema.write({ test: 'hello world', test2: 'écho' }).contentBuffer())).toEqual({ test: 'hello world', test2: 'écho' }); }); }); @@ -214,7 +213,7 @@ describe('Data integrity - partial - multi simple', () => { const Schema = Compactr.schema({ test: { type: 'array', size: 9, items: { type: 'string' } }, test2: { type: 'array', size: 9, items: { type: 'string' } } }); it('should preserve array values and types', () => { - expect(Schema.readContent(Schema.write({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }).contentBuffer())).to.deep.equal({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }); + expect(Schema.readContent(Schema.write({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }).contentBuffer())).toEqual({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }); }); }); @@ -222,7 +221,7 @@ describe('Data integrity - partial - multi simple', () => { const Schema = Compactr.schema({ test: { type: 'object', size: 11, schema: { test: { type: 'number' } } }, test2: { type: 'object', size: 11, schema: { test: { type: 'number' } } } }); it('should preserve object values and types', () => { - expect(Schema.readContent(Schema.write({ test: { test: 23.23 }, test2: { test: -97.7 } }).contentBuffer())).to.deep.equal({ test: { test: 23.23 }, test2: { test: -97.7 } }); + expect(Schema.readContent(Schema.write({ test: { test: 23.23 }, test2: { test: -97.7 } }).contentBuffer())).toEqual({ test: { test: 23.23 }, test2: { test: -97.7 } }); }); }); }); @@ -238,7 +237,7 @@ describe('Data integrity - partial - multi mixed', () => { }); it('should preserve values and types', () => { - expect(Schema.readContent(Schema.write({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }).contentBuffer())).to.deep.equal({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }); + expect(Schema.readContent(Schema.write({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }).contentBuffer())).toEqual({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }); }); }); -}); \ No newline at end of file +}); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..0306e9a --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "module": "ESNext", + "moduleResolution": "node", + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "allowUmdGlobalAccess": true, + "noImplicitAny": false, + "removeComments": true, + "preserveConstEnums": true, + "sourceMap": false, + "allowJs": true, + "target": "esnext", + "outDir": "dist" + }, + "include": [ + "./src", + "./types.d.ts" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/types.d.ts b/types.d.ts new file mode 100644 index 0000000..92bfaa3 --- /dev/null +++ b/types.d.ts @@ -0,0 +1,93 @@ +/** + * Type definitions for Compactr + * Schema based serialization made easy + */ + +export interface SchemaFieldDefinition { + type: 'boolean' | 'number' | 'int8' | 'int16' | 'int32' | 'double' | 'string' | 'char8' | 'char16' | 'char32' | 'array' | 'object' | 'unsigned' | 'unsigned8' | 'unsigned16' | 'unsigned32' + count?: number + size?: number + schema?: SchemaDefinition + items?: { + type: string + count?: number + schema?: SchemaDefinition + } +} + +export interface SchemaDefinition { + [key: string]: SchemaFieldDefinition +} + +export interface SchemaOptions { + keyOrder?: boolean +} + +export interface WriteOptions { + coerse?: boolean + validate?: boolean +} + +export interface SchemaInstance { + /** + * Start writing some data against a schema + * @param data The data to be encoded + * @param options The options for the encoding + * @returns Self reference + */ + write(data: any, options?: WriteOptions): this + + /** + * Returns the bytes from the header of the encoded data buffer. + * A fresh schema with no written data will return a blank, usable for partial encodings. + * @returns The header buffer + */ + headerBuffer(): Buffer + + /** + * Returns the bytes from the content of the encoded data buffer. + * @returns The content buffer + */ + contentBuffer(): Buffer + + /** + * Returns the bytes from the header AND content of the encoded data buffer. + * @returns The data buffer + */ + buffer(): Buffer + + /** + * Returns the typedArray from the header AND content of the encoded data buffer. + * @returns The typed array + */ + typedArray(): number[] + + /** + * Returns the byte sizes of a data object, for insight or troubleshooting + * @param data The data to extract size information of + * @returns The detailed sizes information + */ + sizes(data: any): Record + + /** + * Reads data from a buffer and decodes it according to the schema + * @param buffer The buffer to decode + * @param options The options for the decoding + * @returns The decoded data + */ + read(buffer: Buffer | number[]): any +} + +/** + * Creates a new schema definition, with a reader and writer attached + * @param schema The schema to use + * @param options The options for the schema + * @returns A schema instance + */ +export function schema(schema: SchemaDefinition, options?: SchemaOptions): SchemaInstance; + +declare const compactr: { + schema: typeof schema +}; + +export default compactr; From 596abcb4aaef630fc7474db28e8abcc197566885 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Tue, 28 Oct 2025 15:35:23 -0400 Subject: [PATCH 02/19] cleaned types, old jsdocs, module exports --- src/converter.ts | 6 +- src/decoder.ts | 22 +++--- src/encoder.ts | 38 +++++----- src/reader.ts | 22 +----- src/schema.ts | 17 ++--- src/writer.ts | 41 +++-------- tests/integration/index.ts | 46 ++++++------ types.d.ts | 143 +++++++++++++++++-------------------- 8 files changed, 140 insertions(+), 195 deletions(-) diff --git a/src/converter.ts b/src/converter.ts index a24bbfe..91de159 100644 --- a/src/converter.ts +++ b/src/converter.ts @@ -1,6 +1,6 @@ /** Type Coersion utilities */ -/* Methods -------------------------------------------------------------------*/ +/* Methods ------------------------------------------------------------------- */ /** @private */ function int8(value) { @@ -43,7 +43,7 @@ function array(value) { return (value.concat !== undefined) ? value : [value]; } -/* Exports -------------------------------------------------------------------*/ +/* Exports ------------------------------------------------------------------- */ export default { int8, @@ -58,4 +58,4 @@ export default { boolean, array, object, -} +}; diff --git a/src/decoder.ts b/src/decoder.ts index ab4173e..84ba438 100644 --- a/src/decoder.ts +++ b/src/decoder.ts @@ -1,10 +1,10 @@ /** Decoding utilities */ -/* Local variables -----------------------------------------------------------*/ +/* Local variables ----------------------------------------------------------- */ const fromChar = String.fromCharCode; -/* Methods -------------------------------------------------------------------*/ +/* Methods ------------------------------------------------------------------- */ /** @private */ function boolean(bytes) { @@ -13,7 +13,7 @@ function boolean(bytes) { /** @private */ function int8(bytes) { - return (!(bytes[0] & 0x80))?bytes[0]:((0xff - bytes[0] + 1) * -1); + return (!(bytes[0] & 0x80)) ? bytes[0] : ((0xff - bytes[0] + 1) * -1); } /** @private */ @@ -24,7 +24,7 @@ function int16(bytes) { /** @private */ function int32(bytes) { - return (bytes[0] << 24) | (bytes[1] << 16) | (bytes[2] << 8) | (bytes[3]) + return (bytes[0] << 24) | (bytes[1] << 16) | (bytes[2] << 8) | (bytes[3]); } function uint8(bytes) { @@ -44,7 +44,7 @@ function unsigned(bytes) { /** @private */ function string(bytes) { - let res = []; + const res = []; for (let i = 0; i < bytes.length; i += 2) { res.push(unsigned([bytes[i], bytes[i + 1]])); } @@ -58,7 +58,7 @@ function char8(bytes) { /** @private */ function char32(bytes) { - let res = []; + const res = []; for (let i = 0; i < bytes.length; i += 4) { res.push(int32([bytes[i], bytes[i + 1], bytes[i + 2], bytes[i + 3]])); } @@ -92,7 +92,7 @@ function double(bytes) { let e = (s & 127); e = e * 256 + bytes[1]; let m = e & 15; - s >>= 7; + s >>= 7; e >>= 4; for (let im = 2; im <= 7; im++) { m = m * 256 + bytes[im]; @@ -106,9 +106,9 @@ function double(bytes) { return (s ? -1 : 1) * m * Math.pow(2, e - 52); } -/* Exports -------------------------------------------------------------------*/ +/* Exports ------------------------------------------------------------------- */ -export default { +export default { boolean, number: double, int8, @@ -119,10 +119,10 @@ export default { char8, char16: string, char32, - array, + array, object, unsigned, unsigned8: uint8, unsigned16: uint16, unsigned32: int32, -} +}; diff --git a/src/encoder.ts b/src/encoder.ts index f88bd69..17cc673 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -1,6 +1,6 @@ /** Encoding utilities */ -/* Local variables -----------------------------------------------------------*/ +/* Local variables ----------------------------------------------------------- */ const intMap = [null, unsigned8, unsigned16, null, unsigned32]; const abs = Math.abs; @@ -12,7 +12,7 @@ const bias = pow(2, 52); const eIn = pow(2, -1022); const eOut = pow(2, 1022) * bias; -/* Methods -------------------------------------------------------------------*/ +/* Methods ------------------------------------------------------------------- */ /** @private */ function boolean(val) { @@ -75,7 +75,7 @@ function string(encoding, val) { function array(schema, val) { const ret = []; for (let i = 0; i < val.length; i++) { - let encoded = schema.transformIn(val[i]); + const encoded = schema.transformIn(val[i]); ret.push(...schema.getSize(encoded.length), ...encoded); } return ret; @@ -86,19 +86,19 @@ function object(schema, val) { return schema.write(val).typedArray(); } -/** - * Credit to @feross' ieee754 module +/** + * Credit to @feross' ieee754 module * @private */ function double(val) { - let buffer = []; + const buffer = []; let e, m, c; - let eMax = 2047; - let eBias = 1023; - let rt = 0; + const eMax = 2047; + const eBias = 1023; + const rt = 0; let i = 7; - let d = -1; - let s = val <= 0 ? 1 : 0; + const d = -1; + const s = val <= 0 ? 1 : 0; val = abs(val); e = floor(log(val) / ln2); c = pow(2, -e); @@ -109,7 +109,7 @@ function double(val) { if (e + eBias >= 1) val += rt / c; else val += rt * eIn; - + if (val * c >= 2) { e++; c /= 2; @@ -118,14 +118,16 @@ function double(val) { if (e + eBias >= eMax) { m = 0; e = eMax; - } else if (e + eBias >= 1) { - m = (val * c - 1) * bias + } + else if (e + eBias >= 1) { + m = (val * c - 1) * bias; e = e + eBias; - } else { + } + else { m = val * eOut; e = 0; } - + for (let a = 0; a < 6; a++) { buffer[i] = m & 0xff; i += d; @@ -149,7 +151,7 @@ function getSize(count, byteLength) { return intMap[count](byteLength); } -/* Exports -------------------------------------------------------------------*/ +/* Exports ------------------------------------------------------------------- */ export default { boolean, @@ -168,4 +170,4 @@ export default { unsigned8, unsigned16, unsigned32, -}; \ No newline at end of file +}; diff --git a/src/reader.ts b/src/reader.ts index 7e543ae..063f336 100644 --- a/src/reader.ts +++ b/src/reader.ts @@ -1,27 +1,17 @@ /** Data reader component */ -/* Requires ------------------------------------------------------------------*/ +/* Requires ------------------------------------------------------------------ */ import Decoder from './decoder'; -/* Methods -------------------------------------------------------------------*/ +/* Methods ------------------------------------------------------------------- */ export default function Reader(scope) { - - /** - * Decodes an encoded buffer. Requires header bytes. - * @param {Buffer} bytes - * @returns {Object} The decoded buffer - */ function read(bytes) { readHeader(bytes); return readContent(bytes, scope.contentBegins); } - /** - * Reads only the header of an encoded buffer - * @param {*} bytes - */ function readHeader(bytes) { scope.header = []; let caret = 1; @@ -52,13 +42,7 @@ export default function Reader(scope) { } } - /** - * Reads only a content buffer and returns an object with the decoded values - * @param {Buffer} bytes The content buffer - * @param {Integer} caret The content bytes offset, if the bytes also include an header - * @returns {Object} An object with the decoded values - */ - function readContent(bytes, caret) { + function readContent(bytes, caret?) { caret = caret || 0; const ret = {}; if (scope.options.keyOrder === true) { diff --git a/src/schema.ts b/src/schema.ts index 966547d..2057f3e 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -1,6 +1,6 @@ /** Schema parsing component */ -/* Requires ------------------------------------------------------------------*/ +/* Requires ------------------------------------------------------------------ */ import Encoder from './encoder'; import Decoder from './decoder'; @@ -8,13 +8,8 @@ import Reader from './reader'; import Writer from './writer'; import Converter from './converter'; -/* Methods -------------------------------------------------------------------*/ +/* Methods ------------------------------------------------------------------- */ -/** - * Creates a new schema definition, with a reader and writer attached - * @param {*} schema The schema to use - * @param {Object (keyOrder: {boolean})} options The options for the schema - */ export default function Schema(schema, options = { keyOrder: false }) { const sizeRef = { boolean: 1, @@ -72,7 +67,7 @@ export default function Schema(schema, options = { keyOrder: false }) { .forEach((key, index) => { const keyType = schema[key].type; const count = schema[key].count || 1; - const childSchema = computeNested(schema, key, keyType); + const childSchema = computeNested(schema, key); ret[key] = { name: key, @@ -82,7 +77,7 @@ export default function Schema(schema, options = { keyOrder: false }) { transformOut: (childSchema !== undefined) ? Decoder[keyType].bind(null, childSchema) : Decoder[keyType], coerse: Converter[keyType], getSize: Encoder.getSize.bind(null, count), - fixedSize: defaultSizes[keyType] && Encoder.getSize(count, defaultSizes[keyType]) || null, + fixedSize: (defaultSizes[keyType] && Encoder.getSize(count, defaultSizes[keyType])) || null, size: schema[key].size || defaultSizes[keyType] || null, count, nested: childSchema, @@ -94,7 +89,7 @@ export default function Schema(schema, options = { keyOrder: false }) { /** @private */ function applyBlank() { - for (let key in scope.schema) { + for (const key in scope.schema) { scope.header.push({ key: scope.indices[key], size: scope.indices[key].size || sizeRef[scope.indices[key].type], @@ -122,7 +117,7 @@ export default function Schema(schema, options = { keyOrder: false }) { }; } } - + return childSchema; } diff --git a/src/writer.ts b/src/writer.ts index 9a7149a..e1bcce5 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -1,16 +1,9 @@ /** Data writer component */ -/* Methods -------------------------------------------------------------------*/ +/* Methods ------------------------------------------------------------------- */ export default function Writer(scope) { - - /** - * Start writing some data against a schema - * @param {*} data The data to be encoded - * @param {Object (coerce: {boolean}, validate: {boolean})} options The options for the encoding - * @returns {Writer} Self reference - */ - function write(data, options) { + function write(data, options?) { scope.headerBytes = [0]; scope.contentBytes = []; @@ -35,9 +28,9 @@ export default function Writer(scope) { } else { scope.headerBytes.push(scope.indices[key].index, ...scope.indices[key].getSize(encoded.length)); - if(scope.indices[key].size !== encoded.length && scope.indices[key].size !== null) { - let fixedSize = new Array(scope.indices[key].size).fill(0); - let smallestSize = Math.min(encoded.length, fixedSize.length); + if (scope.indices[key].size !== encoded.length && scope.indices[key].size !== null) { + const fixedSize = new Array(scope.indices[key].size).fill(0); + const smallestSize = Math.min(encoded.length, fixedSize.length); fixedSize.splice(0, smallestSize, ...encoded.slice(0, smallestSize)); return scope.contentBytes.push(...fixedSize); } @@ -45,14 +38,9 @@ export default function Writer(scope) { scope.contentBytes.push(...encoded); } - /** - * Returns the byte sizes of a data object, for insight or troubleshooting - * @param {*} data The data to extract size information of - * @returns {Object} The detailed sizes information - */ function sizes(data) { - const s = {}; - for (let key in data) { + const s: any = {}; + for (const key in data) { if (data[key] instanceof Object) { s[key] = scope.indices[key].nested.sizes(data[key]); s.size = scope.indices[key].transformIn(data[key]).length; @@ -66,7 +54,7 @@ export default function Writer(scope) { /** @private */ function filterKeys(data) { const res = []; - for (let key in data) { + for (const key in data) { if (scope.items.indexOf(key) !== -1 && data[key] !== null && data[key] !== undefined) res.push(key); } return res; @@ -77,27 +65,14 @@ export default function Writer(scope) { return [...scope.headerBytes, ...scope.contentBytes]; } - /** - * Returns the bytes from the header of the encoded data buffer. - * A fresh schema with no written data will return a blank, usable for partial encodings. - * @returns {Buffer} The header buffer - */ function headerBuffer() { return Buffer.from(scope.headerBytes); } - /** - * Returns the bytes from the content of the encoded data buffer. - * @returns {Buffer} The content buffer - */ function contentBuffer() { return Buffer.from(scope.contentBytes); } - /** - * Returns the bytes from the header AND content of the encoded data buffer. - * @returns {Buffer} The data buffer - */ function buffer() { return Buffer.from(typedArray()); } diff --git a/tests/integration/index.ts b/tests/integration/index.ts index dd6aeec..25fa1dd 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -4,13 +4,13 @@ /* Requires ------------------------------------------------------------------ */ -import Compactr from '../../src'; +import { schema } from '../../src'; /* Tests --------------------------------------------------------------------- */ describe('Data integrity - simple', () => { describe('Boolean', () => { - const Schema = Compactr.schema({ test: { type: 'boolean' } }); + const Schema = schema({ test: { type: 'boolean' } }); it('should preserve boolean value and type - true', () => { expect(Schema.read(Schema.write({ test: true }).buffer())).toEqual({ test: true }); @@ -26,7 +26,7 @@ describe('Data integrity - simple', () => { }); describe('Number', () => { - const Schema = Compactr.schema({ test: { type: 'number' } }); + const Schema = schema({ test: { type: 'number' } }); it('should preserve number value and type', () => { expect(Schema.read(Schema.write({ test: 23.23 }).buffer())).toEqual({ test: 23.23 }); @@ -38,7 +38,7 @@ describe('Data integrity - simple', () => { }); describe('String', () => { - const Schema = Compactr.schema({ test: { type: 'string' } }); + const Schema = schema({ test: { type: 'string' } }); it('should preserve string value and type', () => { expect(Schema.read(Schema.write({ test: 'hello world' }).buffer())).toEqual({ test: 'hello world' }); @@ -46,7 +46,7 @@ describe('Data integrity - simple', () => { }); describe('Array', () => { - const Schema = Compactr.schema({ test: { type: 'array', items: { type: 'string' } } }); + const Schema = schema({ test: { type: 'array', items: { type: 'string' } } }); it('should preserve array values and types', () => { expect(Schema.read(Schema.write({ test: ['a', 'b', 'c'] }).buffer())).toEqual({ test: ['a', 'b', 'c'] }); @@ -54,7 +54,7 @@ describe('Data integrity - simple', () => { }); describe('Schema', () => { - const Schema = Compactr.schema({ test: { type: 'object', schema: { test: { type: 'number' } } } }); + const Schema = schema({ test: { type: 'object', schema: { test: { type: 'number' } } } }); it('should preserve object values and types', () => { expect(Schema.read(Schema.write({ test: { test: 23.23 } }).buffer())).toEqual({ test: { test: 23.23 } }); @@ -64,7 +64,7 @@ describe('Data integrity - simple', () => { describe('Data integrity - multi simple', () => { describe('Booleans', () => { - const Schema = Compactr.schema({ test: { type: 'boolean' }, test2: { type: 'boolean' } }); + const Schema = schema({ test: { type: 'boolean' }, test2: { type: 'boolean' } }); it('should preserve boolean value and type - false', () => { expect(Schema.read(Schema.write({ test: false, test2: true }).buffer())).toEqual({ test: false, test2: true }); @@ -76,7 +76,7 @@ describe('Data integrity - multi simple', () => { }); describe('Numbers', () => { - const Schema = Compactr.schema({ test: { type: 'number' }, test2: { type: 'number' } }); + const Schema = schema({ test: { type: 'number' }, test2: { type: 'number' } }); it('should preserve number value and type', () => { expect(Schema.read(Schema.write({ test: 23.23, test2: -97.7 }).buffer())).toEqual({ test: 23.23, test2: -97.7 }); @@ -84,7 +84,7 @@ describe('Data integrity - multi simple', () => { }); describe('Strings', () => { - const Schema = Compactr.schema({ test: { type: 'string' }, test2: { type: 'string' } }); + const Schema = schema({ test: { type: 'string' }, test2: { type: 'string' } }); it('should preserve string value and type', () => { expect(Schema.read(Schema.write({ test: 'hello world', test2: 'écho' }).buffer())).toEqual({ test: 'hello world', test2: 'écho' }); @@ -92,7 +92,7 @@ describe('Data integrity - multi simple', () => { }); describe('Arrays', () => { - const Schema = Compactr.schema({ test: { type: 'array', items: { type: 'string' } }, test2: { type: 'array', items: { type: 'string' } } }); + const Schema = schema({ test: { type: 'array', items: { type: 'string' } }, test2: { type: 'array', items: { type: 'string' } } }); it('should preserve array values and types', () => { expect(Schema.read(Schema.write({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }).buffer())).toEqual({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }); @@ -100,7 +100,7 @@ describe('Data integrity - multi simple', () => { }); describe('Schemas', () => { - const Schema = Compactr.schema({ test: { type: 'object', schema: { test: { type: 'number' } } }, test2: { type: 'object', schema: { test: { type: 'number' } } } }); + const Schema = schema({ test: { type: 'object', schema: { test: { type: 'number' } } }, test2: { type: 'object', schema: { test: { type: 'number' } } } }); it('should preserve object values and types', () => { expect(Schema.read(Schema.write({ test: { test: 23.23 }, test2: { test: -97.7 } }).buffer())).toEqual({ test: { test: 23.23 }, test2: { test: -97.7 } }); @@ -110,7 +110,7 @@ describe('Data integrity - multi simple', () => { describe('Data integrity - multi mixed', () => { describe('Boolean + number + string + array + object', () => { - const Schema = Compactr.schema({ + const Schema = schema({ bool: { type: 'boolean' }, num: { type: 'number' }, str: { type: 'string' }, @@ -128,7 +128,7 @@ describe('Data integrity - multi mixed', () => { describe('Data integrity - partial - simple', () => { describe('Boolean', () => { - const Schema = Compactr.schema({ test: { type: 'boolean' } }); + const Schema = schema({ test: { type: 'boolean' } }); it('should preserve boolean value and type - true', () => { expect(Schema.readContent(Schema.write({ test: true }).contentBuffer())).toEqual({ test: true }); @@ -144,7 +144,7 @@ describe('Data integrity - partial - simple', () => { }); describe('Number', () => { - const Schema = Compactr.schema({ test: { type: 'number' } }); + const Schema = schema({ test: { type: 'number' } }); it('should preserve number value and type', () => { expect(Schema.readContent(Schema.write({ test: 23.23 }).contentBuffer())).toEqual({ test: 23.23 }); @@ -156,7 +156,7 @@ describe('Data integrity - partial - simple', () => { }); describe('String', () => { - const Schema = Compactr.schema({ test: { type: 'string', size: 22 } }); + const Schema = schema({ test: { type: 'string', size: 22 } }); it('should preserve string value and type', () => { expect(Schema.readContent(Schema.write({ test: 'hello world' }).contentBuffer())).toEqual({ test: 'hello world' }); @@ -164,7 +164,7 @@ describe('Data integrity - partial - simple', () => { }); describe('Array', () => { - const Schema = Compactr.schema({ test: { type: 'array', size: 12, items: { type: 'string' } } }); + const Schema = schema({ test: { type: 'array', size: 12, items: { type: 'string' } } }); it('should preserve array values and types', () => { expect(Schema.readContent(Schema.write({ test: ['a', 'b', 'c'] }).contentBuffer())).toEqual({ test: ['a', 'b', 'c', '', '', ''] }); @@ -172,7 +172,7 @@ describe('Data integrity - partial - simple', () => { }); describe('Schema', () => { - const Schema = Compactr.schema({ test: { type: 'object', size: 20, schema: { test: { type: 'number' } } } }); + const Schema = schema({ test: { type: 'object', size: 20, schema: { test: { type: 'number' } } } }); it('should preserve object values and types', () => { expect(Schema.readContent(Schema.write({ test: { test: 23.23 } }).contentBuffer())).toEqual({ test: { test: 23.23 } }); @@ -182,7 +182,7 @@ describe('Data integrity - partial - simple', () => { describe('Data integrity - partial - multi simple', () => { describe('Booleans', () => { - const Schema = Compactr.schema({ test: { type: 'boolean' }, test2: { type: 'boolean' } }); + const Schema = schema({ test: { type: 'boolean' }, test2: { type: 'boolean' } }); it('should preserve boolean value and type - false', () => { expect(Schema.readContent(Schema.write({ test: false, test2: true }).contentBuffer())).toEqual({ test: false, test2: true }); @@ -194,7 +194,7 @@ describe('Data integrity - partial - multi simple', () => { }); describe('Numbers', () => { - const Schema = Compactr.schema({ test: { type: 'number' }, test2: { type: 'number' } }); + const Schema = schema({ test: { type: 'number' }, test2: { type: 'number' } }); it('should preserve number value and type', () => { expect(Schema.readContent(Schema.write({ test: 23.23, test2: -97.7 }).contentBuffer())).toEqual({ test: 23.23, test2: -97.7 }); @@ -202,7 +202,7 @@ describe('Data integrity - partial - multi simple', () => { }); describe('Strings', () => { - const Schema = Compactr.schema({ test: { type: 'string', size: 22 }, test2: { type: 'string', size: 8 } }); + const Schema = schema({ test: { type: 'string', size: 22 }, test2: { type: 'string', size: 8 } }); it('should preserve string value and type', () => { expect(Schema.readContent(Schema.write({ test: 'hello world', test2: 'écho' }).contentBuffer())).toEqual({ test: 'hello world', test2: 'écho' }); @@ -210,7 +210,7 @@ describe('Data integrity - partial - multi simple', () => { }); describe('Arrays', () => { - const Schema = Compactr.schema({ test: { type: 'array', size: 9, items: { type: 'string' } }, test2: { type: 'array', size: 9, items: { type: 'string' } } }); + const Schema = schema({ test: { type: 'array', size: 9, items: { type: 'string' } }, test2: { type: 'array', size: 9, items: { type: 'string' } } }); it('should preserve array values and types', () => { expect(Schema.readContent(Schema.write({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }).contentBuffer())).toEqual({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }); @@ -218,7 +218,7 @@ describe('Data integrity - partial - multi simple', () => { }); describe('Schemas', () => { - const Schema = Compactr.schema({ test: { type: 'object', size: 11, schema: { test: { type: 'number' } } }, test2: { type: 'object', size: 11, schema: { test: { type: 'number' } } } }); + const Schema = schema({ test: { type: 'object', size: 11, schema: { test: { type: 'number' } } }, test2: { type: 'object', size: 11, schema: { test: { type: 'number' } } } }); it('should preserve object values and types', () => { expect(Schema.readContent(Schema.write({ test: { test: 23.23 }, test2: { test: -97.7 } }).contentBuffer())).toEqual({ test: { test: 23.23 }, test2: { test: -97.7 } }); @@ -228,7 +228,7 @@ describe('Data integrity - partial - multi simple', () => { describe('Data integrity - partial - multi mixed', () => { describe('Boolean + number + string + array + object', () => { - const Schema = Compactr.schema({ + const Schema = schema({ bool: { type: 'boolean' }, num: { type: 'number' }, str: { type: 'string', size: 22 }, diff --git a/types.d.ts b/types.d.ts index 92bfaa3..e6abb2c 100644 --- a/types.d.ts +++ b/types.d.ts @@ -2,92 +2,81 @@ * Type definitions for Compactr * Schema based serialization made easy */ +declare module 'compactr' { + export const schema: (schema: SchemaDefinition, options?: SchemaOptions) => SchemaInstance; -export interface SchemaFieldDefinition { - type: 'boolean' | 'number' | 'int8' | 'int16' | 'int32' | 'double' | 'string' | 'char8' | 'char16' | 'char32' | 'array' | 'object' | 'unsigned' | 'unsigned8' | 'unsigned16' | 'unsigned32' - count?: number - size?: number - schema?: SchemaDefinition - items?: { - type: string - count?: number - schema?: SchemaDefinition - } -} + export interface SchemaInstance { + /** + * Start writing some data against a schema + * @param data The data to be encoded + * @param options The options for the encoding + * @returns Self reference + */ + write(data: any, options?: WriteOptions): this -export interface SchemaDefinition { - [key: string]: SchemaFieldDefinition -} + /** + * Returns the bytes from the header of the encoded data buffer. + * A fresh schema with no written data will return a blank, usable for partial encodings. + * @returns The header buffer + */ + headerBuffer(): Buffer -export interface SchemaOptions { - keyOrder?: boolean -} + /** + * Returns the bytes from the content of the encoded data buffer. + * @returns The content buffer + */ + contentBuffer(): Buffer -export interface WriteOptions { - coerse?: boolean - validate?: boolean -} + /** + * Returns the bytes from the header AND content of the encoded data buffer. + * @returns The data buffer + */ + buffer(): Buffer -export interface SchemaInstance { - /** - * Start writing some data against a schema - * @param data The data to be encoded - * @param options The options for the encoding - * @returns Self reference - */ - write(data: any, options?: WriteOptions): this + /** + * Returns the typedArray from the header AND content of the encoded data buffer. + * @returns The typed array + */ + typedArray(): number[] - /** - * Returns the bytes from the header of the encoded data buffer. - * A fresh schema with no written data will return a blank, usable for partial encodings. - * @returns The header buffer - */ - headerBuffer(): Buffer + /** + * Returns the byte sizes of a data object, for insight or troubleshooting + * @param data The data to extract size information of + * @returns The detailed sizes information + */ + sizes(data: any): Record - /** - * Returns the bytes from the content of the encoded data buffer. - * @returns The content buffer - */ - contentBuffer(): Buffer + /** + * Reads data from a buffer and decodes it according to the schema + * @param buffer The buffer to decode + * @param options The options for the decoding + * @returns The decoded data + */ + read(buffer: Buffer | number[]): any + } - /** - * Returns the bytes from the header AND content of the encoded data buffer. - * @returns The data buffer - */ - buffer(): Buffer + export interface SchemaFieldDefinition { + type: 'boolean' | 'number' | 'int8' | 'int16' | 'int32' | 'double' | 'string' | 'char8' | 'char16' | 'char32' | 'array' | 'object' | 'unsigned' | 'unsigned8' | 'unsigned16' | 'unsigned32' + count?: number + size?: number + schema?: SchemaDefinition + items?: { + type: string + count?: number + schema?: SchemaDefinition + } + } - /** - * Returns the typedArray from the header AND content of the encoded data buffer. - * @returns The typed array - */ - typedArray(): number[] + export interface SchemaDefinition { + [key: string]: SchemaFieldDefinition + } - /** - * Returns the byte sizes of a data object, for insight or troubleshooting - * @param data The data to extract size information of - * @returns The detailed sizes information - */ - sizes(data: any): Record + export interface SchemaOptions { + keyOrder?: boolean + } - /** - * Reads data from a buffer and decodes it according to the schema - * @param buffer The buffer to decode - * @param options The options for the decoding - * @returns The decoded data - */ - read(buffer: Buffer | number[]): any + export interface WriteOptions { + coerse?: boolean + validate?: boolean + } } - -/** - * Creates a new schema definition, with a reader and writer attached - * @param schema The schema to use - * @param options The options for the schema - * @returns A schema instance - */ -export function schema(schema: SchemaDefinition, options?: SchemaOptions): SchemaInstance; - -declare const compactr: { - schema: typeof schema -}; - -export default compactr; From a7cad0e0914fb3681cf1c4bd46e4bde46249c647 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Tue, 28 Oct 2025 16:10:37 -0400 Subject: [PATCH 03/19] Modified numbers to match openapi spec --- README.md | 10 +++---- src/converter.ts | 11 -------- src/decoder.ts | 17 ------------ src/encoder.ts | 17 ------------ src/schema.ts | 54 ++++++++++++++++++++++---------------- tests/integration/index.ts | 32 +++++++++++++++------- types.d.ts | 4 ++- 7 files changed, 60 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 8f5a257..0901cf4 100644 --- a/README.md +++ b/README.md @@ -84,20 +84,18 @@ Type | Count bytes | Byte size --- | --- | --- boolean | 0 | 1 number | 0 | 8 -int8 | 0 | 1 -int16 | 0 | 2 +integer | 0 | 8 int32 | 0 | 4 +int64 | 0 | 8 double | 0 | 8 +float | 0 | 8 string | 1 | 2/char char8 | 1 | 1/char char16 | 1 | 2/char char32 | 1 | 4/char array | 1 | (x)/entry object | 1 | (x) -unsigned | 0 | 8 -unsigned8 | 0 | 1 -unsigned16 | 0 | 2 -unsigned32 | 0 | 4 + * Count bytes range can be specified per-item in the schema* diff --git a/src/converter.ts b/src/converter.ts index 91de159..7293101 100644 --- a/src/converter.ts +++ b/src/converter.ts @@ -3,14 +3,6 @@ /* Methods ------------------------------------------------------------------- */ /** @private */ -function int8(value) { - return Number(value) & 0xff; -} - -/** @private */ -function int16(value) { - return Number(value) & 0xffff; -} /** @private */ function int32(value) { @@ -46,10 +38,7 @@ function array(value) { /* Exports ------------------------------------------------------------------- */ export default { - int8, - int16, int32, - number: double, double, string, char8: string, diff --git a/src/decoder.ts b/src/decoder.ts index 84ba438..4c4a50d 100644 --- a/src/decoder.ts +++ b/src/decoder.ts @@ -11,17 +11,6 @@ function boolean(bytes) { return !!bytes[0]; } -/** @private */ -function int8(bytes) { - return (!(bytes[0] & 0x80)) ? bytes[0] : ((0xff - bytes[0] + 1) * -1); -} - -/** @private */ -function int16(bytes) { - const val = (bytes[0] << 8) | bytes[1]; - return (val & 0x8000) ? val | 0xFFFF0000 : val; -} - /** @private */ function int32(bytes) { return (bytes[0] << 24) | (bytes[1] << 16) | (bytes[2] << 8) | (bytes[3]); @@ -110,9 +99,6 @@ function double(bytes) { export default { boolean, - number: double, - int8, - int16, int32, double, string, @@ -122,7 +108,4 @@ export default { array, object, unsigned, - unsigned8: uint8, - unsigned16: uint16, - unsigned32: int32, }; diff --git a/src/encoder.ts b/src/encoder.ts index 17cc673..afdc0e4 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -19,17 +19,6 @@ function boolean(val) { return [val ? 1 : 0]; } -/** @private */ -function int8(val) { - return [(val < 0) ? 256 + val : val]; -} - -/** @private */ -function int16(val) { - if (val < 0) val = 0xffff + val + 1; - return [val >> 8, val & 0xff]; -} - /** @private */ function int32(val) { if (val < 0) val = 0xffffffff + val + 1; @@ -155,9 +144,6 @@ function getSize(count, byteLength) { export default { boolean, - number: double, - int8, - int16, int32, double, string: string.bind(null, unsigned16), @@ -167,7 +153,4 @@ export default { array, object, getSize, - unsigned8, - unsigned16, - unsigned32, }; diff --git a/src/schema.ts b/src/schema.ts index 2057f3e..15168da 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -10,12 +10,26 @@ import Converter from './converter'; /* Methods ------------------------------------------------------------------- */ +/** + * Resolves the internal type based on OpenAPI type and format + * @private + */ +function resolveType(type, format) { + if (type === 'integer') { + const fmt = format || 'int32'; + return fmt === 'int64' ? 'double' : 'int32'; + } + + if (type === 'number') { + return 'double'; // Both float and double use double (8 bytes) + } + + return type; +} + export default function Schema(schema, options = { keyOrder: false }) { const sizeRef = { boolean: 1, - number: 8, - int8: 1, - int16: 2, int32: 4, double: 8, string: 2, @@ -24,23 +38,12 @@ export default function Schema(schema, options = { keyOrder: false }) { char32: 4, array: 2, object: 1, - unsigned: 8, - unsigned8: 1, - unsigned16: 2, - unsigned32: 4, }; const defaultSizes = { boolean: 1, - number: 8, - int8: 1, - int16: 2, int32: 4, double: 8, - unsigned: 8, - unsigned8: 1, - unsigned16: 2, - unsigned32: 4, }; const scope = { @@ -65,20 +68,22 @@ export default function Schema(schema, options = { keyOrder: false }) { Object.keys(schema) .sort() .forEach((key, index) => { - const keyType = schema[key].type; + const fieldType = schema[key].type; + const fieldFormat = schema[key].format; + const internalType = resolveType(fieldType, fieldFormat); const count = schema[key].count || 1; const childSchema = computeNested(schema, key); ret[key] = { name: key, index, - type: keyType, - transformIn: (childSchema !== undefined) ? Encoder[keyType].bind(null, childSchema) : Encoder[keyType], - transformOut: (childSchema !== undefined) ? Decoder[keyType].bind(null, childSchema) : Decoder[keyType], - coerse: Converter[keyType], + type: internalType, + transformIn: (childSchema !== undefined) ? Encoder[internalType].bind(null, childSchema) : Encoder[internalType], + transformOut: (childSchema !== undefined) ? Decoder[internalType].bind(null, childSchema) : Decoder[internalType], + coerse: Converter[internalType], getSize: Encoder.getSize.bind(null, count), - fixedSize: (defaultSizes[keyType] && Encoder.getSize(count, defaultSizes[keyType])) || null, - size: schema[key].size || defaultSizes[keyType] || null, + fixedSize: (defaultSizes[internalType] && Encoder.getSize(count, defaultSizes[internalType])) || null, + size: schema[key].size || defaultSizes[internalType] || null, count, nested: childSchema, }; @@ -108,12 +113,15 @@ export default function Schema(schema, options = { keyOrder: false }) { if (isObject === true) childSchema = Schema(schema[key].schema, options); if (isArray === true) { const itemChildSchema = computeNested(schema[key], 'items'); + const itemType = schema[key].items.type; + const itemFormat = schema[key].items.format; + const internalItemType = resolveType(itemType, itemFormat); childSchema = { count: schema[key].items.count || 1, getSize: Encoder.getSize.bind(null, schema[key].items.count || 1), - transformIn: (itemChildSchema !== undefined) ? Encoder[schema[key].items.type].bind(null, itemChildSchema) : Encoder[schema[key].items.type], - transformOut: (itemChildSchema !== undefined) ? Decoder[schema[key].items.type].bind(null, itemChildSchema) : Decoder[schema[key].items.type], + transformIn: (itemChildSchema !== undefined) ? Encoder[internalItemType].bind(null, itemChildSchema) : Encoder[internalItemType], + transformOut: (itemChildSchema !== undefined) ? Decoder[internalItemType].bind(null, itemChildSchema) : Decoder[internalItemType], }; } } diff --git a/tests/integration/index.ts b/tests/integration/index.ts index 25fa1dd..3692916 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -26,7 +26,7 @@ describe('Data integrity - simple', () => { }); describe('Number', () => { - const Schema = schema({ test: { type: 'number' } }); + const Schema = schema({ test: { type: 'number', format: 'double' } }); it('should preserve number value and type', () => { expect(Schema.read(Schema.write({ test: 23.23 }).buffer())).toEqual({ test: 23.23 }); @@ -37,6 +37,18 @@ describe('Data integrity - simple', () => { }); }); + describe('Integer', () => { + const Schema = schema({ test: { type: 'integer', format: 'int32' } }); + + it('should preserve integer value and type', () => { + expect(Schema.read(Schema.write({ test: 123 }).buffer())).toEqual({ test: 123 }); + }); + + it('should preserve integer value and type for negative values', () => { + expect(Schema.read(Schema.write({ test: -456 }).buffer())).toEqual({ test: -456 }); + }); + }); + describe('String', () => { const Schema = schema({ test: { type: 'string' } }); @@ -54,7 +66,7 @@ describe('Data integrity - simple', () => { }); describe('Schema', () => { - const Schema = schema({ test: { type: 'object', schema: { test: { type: 'number' } } } }); + const Schema = schema({ test: { type: 'object', schema: { test: { type: 'number', format: 'double' } } } }); it('should preserve object values and types', () => { expect(Schema.read(Schema.write({ test: { test: 23.23 } }).buffer())).toEqual({ test: { test: 23.23 } }); @@ -76,7 +88,7 @@ describe('Data integrity - multi simple', () => { }); describe('Numbers', () => { - const Schema = schema({ test: { type: 'number' }, test2: { type: 'number' } }); + const Schema = schema({ test: { type: 'number', format: 'double' }, test2: { type: 'number', format: 'double' } }); it('should preserve number value and type', () => { expect(Schema.read(Schema.write({ test: 23.23, test2: -97.7 }).buffer())).toEqual({ test: 23.23, test2: -97.7 }); @@ -100,7 +112,7 @@ describe('Data integrity - multi simple', () => { }); describe('Schemas', () => { - const Schema = schema({ test: { type: 'object', schema: { test: { type: 'number' } } }, test2: { type: 'object', schema: { test: { type: 'number' } } } }); + const Schema = schema({ test: { type: 'object', schema: { test: { type: 'number', format: 'double' } } }, test2: { type: 'object', schema: { test: { type: 'number', format: 'double' } } } }); it('should preserve object values and types', () => { expect(Schema.read(Schema.write({ test: { test: 23.23 }, test2: { test: -97.7 } }).buffer())).toEqual({ test: { test: 23.23 }, test2: { test: -97.7 } }); @@ -112,7 +124,7 @@ describe('Data integrity - multi mixed', () => { describe('Boolean + number + string + array + object', () => { const Schema = schema({ bool: { type: 'boolean' }, - num: { type: 'number' }, + num: { type: 'number', format: 'double' }, str: { type: 'string' }, arr: { type: 'array', items: { type: 'string' } }, obj: { type: 'object', schema: { sub: { type: 'string' } } }, @@ -144,7 +156,7 @@ describe('Data integrity - partial - simple', () => { }); describe('Number', () => { - const Schema = schema({ test: { type: 'number' } }); + const Schema = schema({ test: { type: 'number', format: 'double' } }); it('should preserve number value and type', () => { expect(Schema.readContent(Schema.write({ test: 23.23 }).contentBuffer())).toEqual({ test: 23.23 }); @@ -172,7 +184,7 @@ describe('Data integrity - partial - simple', () => { }); describe('Schema', () => { - const Schema = schema({ test: { type: 'object', size: 20, schema: { test: { type: 'number' } } } }); + const Schema = schema({ test: { type: 'object', size: 20, schema: { test: { type: 'number', format: 'double' } } } }); it('should preserve object values and types', () => { expect(Schema.readContent(Schema.write({ test: { test: 23.23 } }).contentBuffer())).toEqual({ test: { test: 23.23 } }); @@ -194,7 +206,7 @@ describe('Data integrity - partial - multi simple', () => { }); describe('Numbers', () => { - const Schema = schema({ test: { type: 'number' }, test2: { type: 'number' } }); + const Schema = schema({ test: { type: 'number', format: 'double' }, test2: { type: 'number', format: 'double' } }); it('should preserve number value and type', () => { expect(Schema.readContent(Schema.write({ test: 23.23, test2: -97.7 }).contentBuffer())).toEqual({ test: 23.23, test2: -97.7 }); @@ -218,7 +230,7 @@ describe('Data integrity - partial - multi simple', () => { }); describe('Schemas', () => { - const Schema = schema({ test: { type: 'object', size: 11, schema: { test: { type: 'number' } } }, test2: { type: 'object', size: 11, schema: { test: { type: 'number' } } } }); + const Schema = schema({ test: { type: 'object', size: 11, schema: { test: { type: 'number', format: 'double' } } }, test2: { type: 'object', size: 11, schema: { test: { type: 'number', format: 'double' } } } }); it('should preserve object values and types', () => { expect(Schema.readContent(Schema.write({ test: { test: 23.23 }, test2: { test: -97.7 } }).contentBuffer())).toEqual({ test: { test: 23.23 }, test2: { test: -97.7 } }); @@ -230,7 +242,7 @@ describe('Data integrity - partial - multi mixed', () => { describe('Boolean + number + string + array + object', () => { const Schema = schema({ bool: { type: 'boolean' }, - num: { type: 'number' }, + num: { type: 'number', format: 'double' }, str: { type: 'string', size: 22 }, arr: { type: 'array', items: { type: 'string' }, size: 9 }, obj: { type: 'object', size: 9, schema: { sub: { type: 'string' } } }, diff --git a/types.d.ts b/types.d.ts index e6abb2c..705f914 100644 --- a/types.d.ts +++ b/types.d.ts @@ -56,12 +56,14 @@ declare module 'compactr' { } export interface SchemaFieldDefinition { - type: 'boolean' | 'number' | 'int8' | 'int16' | 'int32' | 'double' | 'string' | 'char8' | 'char16' | 'char32' | 'array' | 'object' | 'unsigned' | 'unsigned8' | 'unsigned16' | 'unsigned32' + type: 'boolean' | 'integer' | 'number' | 'string' | 'char8' | 'char16' | 'char32' | 'array' | 'object' + format?: 'int32' | 'int64' | 'float' | 'double' count?: number size?: number schema?: SchemaDefinition items?: { type: string + format?: string count?: number schema?: SchemaDefinition } From 81ffa8bf3ddc1dec954e44cca9080720c2633f3c Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Tue, 28 Oct 2025 16:28:01 -0400 Subject: [PATCH 04/19] finished migrating numbers, all formats and modernized double/float implementations --- src/converter.ts | 14 ++++++ src/decoder.ts | 50 ++++++++++++++------- src/encoder.ts | 92 +++++++++++++++----------------------- src/schema.ts | 9 +++- tests/integration/index.ts | 84 ++++++++++++++++++++++++++++++++++ 5 files changed, 173 insertions(+), 76 deletions(-) diff --git a/src/converter.ts b/src/converter.ts index 7293101..e48f097 100644 --- a/src/converter.ts +++ b/src/converter.ts @@ -9,12 +9,24 @@ function int32(value) { return Number(value) & 0xffffffff; } +/** @private */ +function float(value) { + const ret = Number(value); + return (Number.isFinite(ret)) ? ret : 0; +} + /** @private */ function double(value) { const ret = Number(value); return (Number.isFinite(ret)) ? ret : 0; } +/** @private */ +function int64(value) { + const ret = Number(value); + return (Number.isFinite(ret)) ? Math.trunc(ret) : 0; +} + /** @private */ function string(value) { return '' + value; @@ -39,6 +51,8 @@ function array(value) { export default { int32, + int64, + float, double, string, char8: string, diff --git a/src/decoder.ts b/src/decoder.ts index 4c4a50d..0aa4f08 100644 --- a/src/decoder.ts +++ b/src/decoder.ts @@ -73,26 +73,40 @@ function object(schema, bytes) { } /** - * Credit to @feross' ieee754 module + * IEEE 754 single precision (32-bit float) decoder + * Simplified implementation using JavaScript's Float32Array + * @private + */ +function float(bytes) { + // Bytes come in big-endian order, convert to little-endian for typed array + const byteArray = new Uint8Array([bytes[3], bytes[2], bytes[1], bytes[0]]); + const floatArray = new Float32Array(byteArray.buffer); + + return floatArray[0]; +} + +/** + * IEEE 754 double precision (64-bit float) decoder + * Simplified implementation using JavaScript's Float64Array * @private */ function double(bytes) { - let s = bytes[0]; - let e = (s & 127); - e = e * 256 + bytes[1]; - let m = e & 15; - s >>= 7; - e >>= 4; - for (let im = 2; im <= 7; im++) { - m = m * 256 + bytes[im]; - } - if (e === 0) e = -1022; - else if (e === 2047) return NaN; - else { - m += 4503599627370496; - e -= 1023; - } - return (s ? -1 : 1) * m * Math.pow(2, e - 52); + // Bytes come in big-endian order, convert to little-endian for typed array + const byteArray = new Uint8Array([ + bytes[7], bytes[6], bytes[5], bytes[4], + bytes[3], bytes[2], bytes[1], bytes[0] + ]); + const doubleArray = new Float64Array(byteArray.buffer); + + return doubleArray[0]; +} + +/** + * 64-bit integer decoder (uses double for JavaScript compatibility) + * @private + */ +function int64(bytes) { + return double(bytes); } /* Exports ------------------------------------------------------------------- */ @@ -100,6 +114,8 @@ function double(bytes) { export default { boolean, int32, + int64, + float, double, string, char8, diff --git a/src/encoder.ts b/src/encoder.ts index afdc0e4..c3f77de 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -3,14 +3,6 @@ /* Local variables ----------------------------------------------------------- */ const intMap = [null, unsigned8, unsigned16, null, unsigned32]; -const abs = Math.abs; -const pow = Math.pow; -const ln2 = Math.LN2; -const log = Math.log; -const floor = Math.floor; -const bias = pow(2, 52); -const eIn = pow(2, -1022); -const eOut = pow(2, 1022) * bias; /* Methods ------------------------------------------------------------------- */ @@ -76,63 +68,47 @@ function object(schema, val) { } /** - * Credit to @feross' ieee754 module + * IEEE 754 single precision (32-bit float) + * Simplified implementation using JavaScript's Float32Array * @private */ -function double(val) { - const buffer = []; - let e, m, c; - const eMax = 2047; - const eBias = 1023; - const rt = 0; - let i = 7; - const d = -1; - const s = val <= 0 ? 1 : 0; - val = abs(val); - e = floor(log(val) / ln2); - c = pow(2, -e); - if (val * c < 1) { - e--; - c *= 2; - } +function float(val) { + // Use Float32Array to get proper IEEE 754 single precision encoding + const floatArray = new Float32Array(1); + const byteArray = new Uint8Array(floatArray.buffer); - if (e + eBias >= 1) val += rt / c; - else val += rt * eIn; + floatArray[0] = val; - if (val * c >= 2) { - e++; - c /= 2; - } - - if (e + eBias >= eMax) { - m = 0; - e = eMax; - } - else if (e + eBias >= 1) { - m = (val * c - 1) * bias; - e = e + eBias; - } - else { - m = val * eOut; - e = 0; - } + // Return bytes in big-endian order to match double implementation + return [byteArray[3], byteArray[2], byteArray[1], byteArray[0]]; +} - for (let a = 0; a < 6; a++) { - buffer[i] = m & 0xff; - i += d; - m /= 256; - } +/** + * IEEE 754 double precision (64-bit float) + * Simplified implementation using JavaScript's Float64Array + * @private + */ +function double(val) { + // Use Float64Array to get proper IEEE 754 double precision encoding + const doubleArray = new Float64Array(1); + const byteArray = new Uint8Array(doubleArray.buffer); - e = (e << 4) | m; - for (let b = 0; b < 2; b++) { - buffer[i] = e & 0xff; - i += d; - e /= 256; - } + doubleArray[0] = val; - buffer[i - d] |= s * 128; + // Return bytes in big-endian order + return [ + byteArray[7], byteArray[6], byteArray[5], byteArray[4], + byteArray[3], byteArray[2], byteArray[1], byteArray[0] + ]; +} - return buffer; +/** + * 64-bit integer encoding (uses double for JavaScript compatibility) + * JavaScript's Number type can safely represent integers up to 2^53-1 + * @private + */ +function int64(val) { + return double(val); } /** @private */ @@ -145,6 +121,8 @@ function getSize(count, byteLength) { export default { boolean, int32, + int64, + float, double, string: string.bind(null, unsigned16), char8, diff --git a/src/schema.ts b/src/schema.ts index 15168da..c70bb5e 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -17,11 +17,12 @@ import Converter from './converter'; function resolveType(type, format) { if (type === 'integer') { const fmt = format || 'int32'; - return fmt === 'int64' ? 'double' : 'int32'; + return fmt === 'int64' ? 'int64' : 'int32'; } if (type === 'number') { - return 'double'; // Both float and double use double (8 bytes) + const fmt = format || 'double'; + return fmt === 'float' ? 'float' : 'double'; } return type; @@ -31,6 +32,8 @@ export default function Schema(schema, options = { keyOrder: false }) { const sizeRef = { boolean: 1, int32: 4, + int64: 8, + float: 4, double: 8, string: 2, char8: 1, @@ -43,6 +46,8 @@ export default function Schema(schema, options = { keyOrder: false }) { const defaultSizes = { boolean: 1, int32: 4, + int64: 8, + float: 4, double: 8, }; diff --git a/tests/integration/index.ts b/tests/integration/index.ts index 3692916..dcf1ab6 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -49,6 +49,56 @@ describe('Data integrity - simple', () => { }); }); + describe('Integer (int64)', () => { + const Schema = schema({ test: { type: 'integer', format: 'int64' } }); + + it('should preserve int64 value and type', () => { + expect(Schema.read(Schema.write({ test: 9007199254740991 }).buffer())).toEqual({ test: 9007199254740991 }); + }); + + it('should preserve int64 value and type for negative values', () => { + expect(Schema.read(Schema.write({ test: -9007199254740991 }).buffer())).toEqual({ test: -9007199254740991 }); + }); + }); + + describe('Number (float)', () => { + const Schema = schema({ test: { type: 'number', format: 'float' } }); + + it('should preserve float value (with precision loss)', () => { + const result = Schema.read(Schema.write({ test: 3.14159 }).buffer()); + expect(result.test).toBeCloseTo(3.14159, 5); + }); + + it('should preserve float value for negative values', () => { + const result = Schema.read(Schema.write({ test: -2.71828 }).buffer()); + expect(result.test).toBeCloseTo(-2.71828, 5); + }); + }); + + describe('Plain Integer (no format)', () => { + const Schema = schema({ test: { type: 'integer' } }); + + it('should default to int32 format', () => { + expect(Schema.read(Schema.write({ test: 42 }).buffer())).toEqual({ test: 42 }); + }); + + it('should handle negative values', () => { + expect(Schema.read(Schema.write({ test: -42 }).buffer())).toEqual({ test: -42 }); + }); + }); + + describe('Plain Number (no format)', () => { + const Schema = schema({ test: { type: 'number' } }); + + it('should default to double format', () => { + expect(Schema.read(Schema.write({ test: 3.141592653589793 }).buffer())).toEqual({ test: 3.141592653589793 }); + }); + + it('should handle negative values', () => { + expect(Schema.read(Schema.write({ test: -2.718281828459045 }).buffer())).toEqual({ test: -2.718281828459045 }); + }); + }); + describe('String', () => { const Schema = schema({ test: { type: 'string' } }); @@ -253,3 +303,37 @@ describe('Data integrity - partial - multi mixed', () => { }); }); }); + +/* Size comparison tests ----------------------------------------------------- */ + +describe('Format size differences', () => { + describe('Float vs Double', () => { + const FloatSchema = schema({ value: { type: 'number', format: 'float' } }); + const DoubleSchema = schema({ value: { type: 'number', format: 'double' } }); + + it('float should use 4 bytes for content', () => { + const buffer = FloatSchema.write({ value: 3.14 }).contentBuffer(); + expect(buffer.length).toBe(4); + }); + + it('double should use 8 bytes for content', () => { + const buffer = DoubleSchema.write({ value: 3.14 }).contentBuffer(); + expect(buffer.length).toBe(8); + }); + }); + + describe('Int32 vs Int64', () => { + const Int32Schema = schema({ value: { type: 'integer', format: 'int32' } }); + const Int64Schema = schema({ value: { type: 'integer', format: 'int64' } }); + + it('int32 should use 4 bytes for content', () => { + const buffer = Int32Schema.write({ value: 12345 }).contentBuffer(); + expect(buffer.length).toBe(4); + }); + + it('int64 should use 8 bytes for content', () => { + const buffer = Int64Schema.write({ value: 12345 }).contentBuffer(); + expect(buffer.length).toBe(8); + }); + }); +}); From 34354c7f2d06034ef5dbb98fac77db3c3d622b8e Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Tue, 28 Oct 2025 16:37:57 -0400 Subject: [PATCH 05/19] migrated strings --- benchmarks/array.ts | 4 ++-- benchmarks/string.ts | 8 ++++---- src/converter.ts | 3 --- src/decoder.ts | 22 +++------------------- src/encoder.ts | 22 +++++----------------- src/schema.ts | 3 --- tests/integration/index.ts | 8 ++++++++ types.d.ts | 2 +- 8 files changed, 23 insertions(+), 49 deletions(-) diff --git a/benchmarks/array.ts b/benchmarks/array.ts index c1e15c7..96b5112 100644 --- a/benchmarks/array.ts +++ b/benchmarks/array.ts @@ -9,8 +9,8 @@ const Compactr = require('../'); let User = Compactr.schema({ - id: { type: 'int32', size: 4 }, - arr: { type: 'array', size: 6, items: { type: 'char8', size: 1 }}, + id: { type: 'integer', format: 'int32', size: 4 }, + arr: { type: 'array', size: 6, items: { type: 'string', size: 1 }}, }); const mult = 32; diff --git a/benchmarks/string.ts b/benchmarks/string.ts index 8650e66..f638fe9 100644 --- a/benchmarks/string.ts +++ b/benchmarks/string.ts @@ -9,10 +9,10 @@ const protobuf = require('protobufjs'); /* Local variables -----------------------------------------------------------*/ -let User = Compactr.schema({ - id: { type: 'int32', size: 4 }, - str: { type: 'char8', size: 6 }, - special: { type: 'char32', size: 4 }, +let User = Compactr.schema({ + id: { type: 'integer', format: 'int32', size: 4 }, + str: { type: 'string', size: 6 }, + special: { type: 'string', size: 4 }, }); let root = protobuf.Root.fromJSON({ diff --git a/src/converter.ts b/src/converter.ts index e48f097..13b90af 100644 --- a/src/converter.ts +++ b/src/converter.ts @@ -55,9 +55,6 @@ export default { float, double, string, - char8: string, - char16: string, - char32: string, boolean, array, object, diff --git a/src/decoder.ts b/src/decoder.ts index 0aa4f08..3860663 100644 --- a/src/decoder.ts +++ b/src/decoder.ts @@ -35,21 +35,8 @@ function unsigned(bytes) { function string(bytes) { const res = []; for (let i = 0; i < bytes.length; i += 2) { - res.push(unsigned([bytes[i], bytes[i + 1]])); - } - return fromChar(...res); -} - -/** @private */ -function char8(bytes) { - return fromChar(...bytes); -} - -/** @private */ -function char32(bytes) { - const res = []; - for (let i = 0; i < bytes.length; i += 4) { - res.push(int32([bytes[i], bytes[i + 1], bytes[i + 2], bytes[i + 3]])); + const code = (bytes[i] << 8) | bytes[i + 1]; + res.push(code); } return fromChar(...res); } @@ -94,7 +81,7 @@ function double(bytes) { // Bytes come in big-endian order, convert to little-endian for typed array const byteArray = new Uint8Array([ bytes[7], bytes[6], bytes[5], bytes[4], - bytes[3], bytes[2], bytes[1], bytes[0] + bytes[3], bytes[2], bytes[1], bytes[0], ]); const doubleArray = new Float64Array(byteArray.buffer); @@ -118,9 +105,6 @@ export default { float, double, string, - char8, - char16: string, - char32, array, object, unsigned, diff --git a/src/encoder.ts b/src/encoder.ts index c3f77de..f850030 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -33,20 +33,11 @@ function unsigned32(val) { } /** @private */ -function char8(val) { +function string(val) { const chars = []; for (let i = 0; i < val.length; i++) { - chars.push(val.charCodeAt(i) % 0xff); - } - - return chars; -} - -/** @private */ -function string(encoding, val) { - const chars = []; - for (let i = 0; i < val.length; i++) { - chars.push(...encoding(val.charCodeAt(i))); + const code = val.charCodeAt(i); + chars.push(code >> 8, code & 0xff); } return chars; @@ -98,7 +89,7 @@ function double(val) { // Return bytes in big-endian order return [ byteArray[7], byteArray[6], byteArray[5], byteArray[4], - byteArray[3], byteArray[2], byteArray[1], byteArray[0] + byteArray[3], byteArray[2], byteArray[1], byteArray[0], ]; } @@ -124,10 +115,7 @@ export default { int64, float, double, - string: string.bind(null, unsigned16), - char8, - char16: string.bind(null, unsigned16), - char32: string.bind(null, unsigned32), + string, array, object, getSize, diff --git a/src/schema.ts b/src/schema.ts index c70bb5e..5372344 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -36,9 +36,6 @@ export default function Schema(schema, options = { keyOrder: false }) { float: 4, double: 8, string: 2, - char8: 1, - char16: 2, - char32: 4, array: 2, object: 1, }; diff --git a/tests/integration/index.ts b/tests/integration/index.ts index dcf1ab6..ef3dd53 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -105,6 +105,14 @@ describe('Data integrity - simple', () => { it('should preserve string value and type', () => { expect(Schema.read(Schema.write({ test: 'hello world' }).buffer())).toEqual({ test: 'hello world' }); }); + + it('should support special characters', () => { + expect(Schema.read(Schema.write({ test: '한자' }).buffer())).toEqual({ test: '한자' }); + }); + + it('should support emojis', () => { + expect(Schema.read(Schema.write({ test: '🚀' }).buffer())).toEqual({ test: '🚀' }); + }); }); describe('Array', () => { diff --git a/types.d.ts b/types.d.ts index 705f914..379efa5 100644 --- a/types.d.ts +++ b/types.d.ts @@ -56,7 +56,7 @@ declare module 'compactr' { } export interface SchemaFieldDefinition { - type: 'boolean' | 'integer' | 'number' | 'string' | 'char8' | 'char16' | 'char32' | 'array' | 'object' + type: 'boolean' | 'integer' | 'number' | 'string' | 'array' | 'object' format?: 'int32' | 'int64' | 'float' | 'double' count?: number size?: number From 5ba4d02a24b40e5a33c46edf88f75cfe35b61f75 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Tue, 28 Oct 2025 17:09:36 -0400 Subject: [PATCH 06/19] added support for nullable --- README.md | 2 +- src/decoder.ts | 4 + src/encoder.ts | 4 + src/reader.ts | 38 +++++++- src/schema.ts | 1 + src/writer.ts | 44 +++++++-- tests/integration/index.ts | 180 +++++++++++++++++++++++++++++++++++++ types.d.ts | 1 + 8 files changed, 265 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 0901cf4..0ea0571 100644 --- a/README.md +++ b/README.md @@ -141,4 +141,4 @@ You are awesome! Open an issue on this project, identifying the feature that you ## License -[Apache 2.0](LICENSE) (c) 2020 Frederic Charette +[Apache 2.0](LICENSE) (c) 2025 Frederic Charette diff --git a/src/decoder.ts b/src/decoder.ts index 3860663..90a2042 100644 --- a/src/decoder.ts +++ b/src/decoder.ts @@ -4,6 +4,10 @@ const fromChar = String.fromCharCode; +// Presence indicators for nullable fields +export const NULL_INDICATOR = 0x00; // Field is null +export const PRESENT_INDICATOR = 0x01; // Field is present (not null) + /* Methods ------------------------------------------------------------------- */ /** @private */ diff --git a/src/encoder.ts b/src/encoder.ts index f850030..9c1307c 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -4,6 +4,10 @@ const intMap = [null, unsigned8, unsigned16, null, unsigned32]; +// Presence indicators for nullable fields +export const NULL_INDICATOR = 0x00; // Field is null +export const PRESENT_INDICATOR = 0x01; // Field is present (not null) + /* Methods ------------------------------------------------------------------- */ /** @private */ diff --git a/src/reader.ts b/src/reader.ts index 063f336..3c5dcc3 100644 --- a/src/reader.ts +++ b/src/reader.ts @@ -2,7 +2,7 @@ /* Requires ------------------------------------------------------------------ */ -import Decoder from './decoder'; +import Decoder, { NULL_INDICATOR } from './decoder'; /* Methods ------------------------------------------------------------------- */ @@ -27,12 +27,38 @@ export default function Reader(scope) { /** @private */ function readKey(bytes, caret, index) { const key = getSchemaDef(bytes[caret]); + caret++; // Move past field index + + let size; + + // Check for presence byte if field is nullable + if (key.nullable) { + const presenceByte = bytes[caret]; + caret++; // Move past presence byte + + if (presenceByte === NULL_INDICATOR) { + // Field is null - no size or content follows + size = -1; // Use -1 as internal null marker + } + else { + // Field is present - read size + const sizeBytes = bytes.slice(caret, caret + key.count); + size = key.size || Decoder.unsigned(sizeBytes); + caret += key.count; // Move past size bytes + } + } + else { + // Non-nullable field - read size directly + const sizeBytes = bytes.slice(caret, caret + key.count); + size = key.size || Decoder.unsigned(sizeBytes); + caret += key.count; // Move past size bytes + } scope.header[index] = { key, - size: key.size || Decoder.unsigned(bytes.slice(caret + 1, caret + key.count + 1)), + size, }; - return caret + key.count + 1; + return caret; } /** @private */ @@ -51,6 +77,12 @@ export default function Reader(scope) { } } for (let i = 0; i < scope.header.length; i++) { + // Handle nullable fields with size -1 (null marker detected) + if (scope.header[i].key.nullable && scope.header[i].size === -1) { + ret[scope.header[i].key.name] = null; + continue; + } + ret[scope.header[i].key.name] = scope.header[i].key.transformOut(bytes.slice(caret, caret + scope.header[i].size)); caret += scope.header[i].size; } diff --git a/src/schema.ts b/src/schema.ts index 5372344..e562249 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -80,6 +80,7 @@ export default function Schema(schema, options = { keyOrder: false }) { name: key, index, type: internalType, + nullable: schema[key].nullable || false, transformIn: (childSchema !== undefined) ? Encoder[internalType].bind(null, childSchema) : Encoder[internalType], transformOut: (childSchema !== undefined) ? Decoder[internalType].bind(null, childSchema) : Decoder[internalType], coerse: Converter[internalType], diff --git a/src/writer.ts b/src/writer.ts index e1bcce5..f5510aa 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -1,5 +1,9 @@ /** Data writer component */ +/* Requires ------------------------------------------------------------------ */ + +import { NULL_INDICATOR, PRESENT_INDICATOR } from './encoder'; + /* Methods ------------------------------------------------------------------- */ export default function Writer(scope) { @@ -11,23 +15,42 @@ export default function Writer(scope) { scope.headerBytes[0] = keys.length; for (let i = 0; i < keys.length; i++) { let keyData = data[keys[i]]; + + // Handle nullable fields with null values + if (scope.indices[keys[i]].nullable && keyData === null) { + // Write header with NULL_INDICATOR (no size or content follows) + scope.headerBytes.push(scope.indices[keys[i]].index, NULL_INDICATOR); + continue; + } + + // For nullable fields that are not null, add PRESENT_INDICATOR + if (scope.indices[keys[i]].nullable) { + scope.headerBytes.push(scope.indices[keys[i]].index, PRESENT_INDICATOR); + } + else { + scope.headerBytes.push(scope.indices[keys[i]].index); + } + if (options !== undefined) { if (options.coerse === true) keyData = scope.indices[keys[i]].coerse(keyData); if (options.validate === true) scope.indices[keys[i]].validate(keyData); } - splitBytes(scope.indices[keys[i]].transformIn(keyData), keys[i]); + + // Add size and content + const encoded = scope.indices[keys[i]].transformIn(keyData); + addSizeAndContent(encoded, keys[i]); } return this; } /** @private */ - function splitBytes(encoded, key) { + function addSizeAndContent(encoded, key) { if (scope.indices[key].fixedSize !== null) { - scope.headerBytes.push(scope.indices[key].index, ...scope.indices[key].fixedSize); + scope.headerBytes.push(...scope.indices[key].fixedSize); } else { - scope.headerBytes.push(scope.indices[key].index, ...scope.indices[key].getSize(encoded.length)); + scope.headerBytes.push(...scope.indices[key].getSize(encoded.length)); if (scope.indices[key].size !== encoded.length && scope.indices[key].size !== null) { const fixedSize = new Array(scope.indices[key].size).fill(0); const smallestSize = Math.min(encoded.length, fixedSize.length); @@ -55,7 +78,18 @@ export default function Writer(scope) { function filterKeys(data) { const res = []; for (const key in data) { - if (scope.items.indexOf(key) !== -1 && data[key] !== null && data[key] !== undefined) res.push(key); + if (scope.items.indexOf(key) === -1) continue; + + // Include nullable fields even when null + if (scope.indices[key].nullable && data[key] === null) { + res.push(key); + continue; + } + + // Skip non-nullable fields that are null or undefined + if (data[key] !== null && data[key] !== undefined) { + res.push(key); + } } return res; } diff --git a/tests/integration/index.ts b/tests/integration/index.ts index ef3dd53..9790002 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -345,3 +345,183 @@ describe('Format size differences', () => { }); }); }); + +/* Nullable properties tests ------------------------------------------------- */ + +describe('Nullable properties', () => { + describe('Nullable string', () => { + const Schema = schema({ test: { type: 'string', nullable: true } }); + + it('should preserve null value', () => { + expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({ test: null }); + }); + + it('should preserve non-null string value', () => { + expect(Schema.read(Schema.write({ test: 'hello' }).buffer())).toEqual({ test: 'hello' }); + }); + + it('should encode null with minimal bytes (header only)', () => { + const buffer = Schema.write({ test: null }).buffer(); + const nonNullBuffer = Schema.write({ test: 'a' }).buffer(); + expect(buffer.length).toBeLessThan(nonNullBuffer.length); + }); + }); + + describe('Nullable number', () => { + const Schema = schema({ test: { type: 'number', format: 'double', nullable: true } }); + + it('should preserve null value', () => { + expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({ test: null }); + }); + + it('should preserve non-null number value', () => { + expect(Schema.read(Schema.write({ test: 42.5 }).buffer())).toEqual({ test: 42.5 }); + }); + }); + + describe('Nullable integer', () => { + const Schema = schema({ test: { type: 'integer', format: 'int32', nullable: true } }); + + it('should preserve null value', () => { + expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({ test: null }); + }); + + it('should preserve non-null integer value', () => { + expect(Schema.read(Schema.write({ test: 123 }).buffer())).toEqual({ test: 123 }); + }); + }); + + describe('Nullable boolean', () => { + const Schema = schema({ test: { type: 'boolean', nullable: true } }); + + it('should preserve null value', () => { + expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({ test: null }); + }); + + it('should preserve false value (not confused with null)', () => { + expect(Schema.read(Schema.write({ test: false }).buffer())).toEqual({ test: false }); + }); + + it('should preserve true value', () => { + expect(Schema.read(Schema.write({ test: true }).buffer())).toEqual({ test: true }); + }); + }); + + describe('Mixed nullable and non-nullable', () => { + const Schema = schema({ + nullableField: { type: 'string', nullable: true }, + regularField: { type: 'string' }, + anotherNullable: { type: 'integer', format: 'int32', nullable: true }, + }); + + it('should handle mix of null and non-null values', () => { + const data = { nullableField: null, regularField: 'hello', anotherNullable: 42 }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should skip non-nullable fields when null', () => { + const data = { nullableField: 'test', regularField: null, anotherNullable: null }; + const result = Schema.read(Schema.write(data).buffer()); + expect(result).toEqual({ nullableField: 'test', anotherNullable: null }); + expect(result.regularField).toBeUndefined(); + }); + + it('should preserve all null values in nullable fields', () => { + const data = { nullableField: null, regularField: 'value', anotherNullable: null }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Nullable object', () => { + const Schema = schema({ + test: { + type: 'object', + nullable: true, + schema: { name: { type: 'string' } }, + }, + }); + + it('should preserve null value', () => { + expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({ test: null }); + }); + + it('should preserve non-null object value', () => { + expect(Schema.read(Schema.write({ test: { name: 'John' } }).buffer())).toEqual({ test: { name: 'John' } }); + }); + }); + + describe('Nullable array', () => { + const Schema = schema({ + test: { + type: 'array', + nullable: true, + items: { type: 'string' }, + }, + }); + + it('should preserve null value', () => { + expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({ test: null }); + }); + + it('should preserve non-null array value', () => { + expect(Schema.read(Schema.write({ test: ['a', 'b', 'c'] }).buffer())).toEqual({ test: ['a', 'b', 'c'] }); + }); + + it('should preserve empty array (different from null)', () => { + expect(Schema.read(Schema.write({ test: [] }).buffer())).toEqual({ test: [] }); + }); + + it('empty array should have different encoding than null', () => { + const emptyArrayBuffer = Schema.write({ test: [] }).buffer(); + const nullBuffer = Schema.write({ test: null }).buffer(); + expect(emptyArrayBuffer).not.toEqual(nullBuffer); + }); + }); + + describe('Empty vs null distinction', () => { + describe('Empty string vs null', () => { + const Schema = schema({ test: { type: 'string', nullable: true } }); + + it('should distinguish empty string from null', () => { + const emptyString = Schema.read(Schema.write({ test: '' }).buffer()); + const nullValue = Schema.read(Schema.write({ test: null }).buffer()); + + expect(emptyString).toEqual({ test: '' }); + expect(nullValue).toEqual({ test: null }); + expect(emptyString.test).not.toBe(nullValue.test); + }); + + it('should have different byte encodings', () => { + const emptyStringBuffer = Schema.write({ test: '' }).buffer(); + const nullBuffer = Schema.write({ test: null }).buffer(); + expect(emptyStringBuffer).not.toEqual(nullBuffer); + }); + }); + + describe('Zero vs null for numbers', () => { + const Schema = schema({ test: { type: 'number', format: 'double', nullable: true } }); + + it('should distinguish zero from null', () => { + const zero = Schema.read(Schema.write({ test: 0 }).buffer()); + const nullValue = Schema.read(Schema.write({ test: null }).buffer()); + + expect(zero).toEqual({ test: 0 }); + expect(nullValue).toEqual({ test: null }); + expect(zero.test).not.toBe(nullValue.test); + }); + }); + + describe('False vs null for booleans', () => { + const Schema = schema({ test: { type: 'boolean', nullable: true } }); + + it('should distinguish false from null', () => { + const falseValue = Schema.read(Schema.write({ test: false }).buffer()); + const nullValue = Schema.read(Schema.write({ test: null }).buffer()); + + expect(falseValue).toEqual({ test: false }); + expect(nullValue).toEqual({ test: null }); + expect(falseValue.test).not.toBe(nullValue.test); + }); + }); + }); +}); diff --git a/types.d.ts b/types.d.ts index 379efa5..0899c89 100644 --- a/types.d.ts +++ b/types.d.ts @@ -58,6 +58,7 @@ declare module 'compactr' { export interface SchemaFieldDefinition { type: 'boolean' | 'integer' | 'number' | 'string' | 'array' | 'object' format?: 'int32' | 'int64' | 'float' | 'double' + nullable?: boolean count?: number size?: number schema?: SchemaDefinition From 6b2abf93718855be2bea29d13972a087a739415c Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Tue, 28 Oct 2025 17:31:11 -0400 Subject: [PATCH 07/19] added string optimizations for common formats --- src/converter.ts | 72 ++++++++++++++++ src/decoder.ts | 164 +++++++++++++++++++++++++++++++++++++ src/encoder.ts | 126 ++++++++++++++++++++++++++++ src/schema.ts | 44 +++++++--- tests/integration/index.ts | 158 +++++++++++++++++++++++++++++++++++ types.d.ts | 2 +- 6 files changed, 552 insertions(+), 14 deletions(-) diff --git a/src/converter.ts b/src/converter.ts index 13b90af..c378d13 100644 --- a/src/converter.ts +++ b/src/converter.ts @@ -32,6 +32,73 @@ function string(value) { return '' + value; } +/** @private */ +function uuid(value) { + // Validate and normalize UUID string + const str = '' + value; + const normalized = str.toLowerCase().replace(/-/g, ''); + if (!/^[0-9a-f]{32}$/.test(normalized)) { + throw new Error('Invalid UUID format'); + } + // Return in standard UUID format + return [ + normalized.substr(0, 8), + normalized.substr(8, 4), + normalized.substr(12, 4), + normalized.substr(16, 4), + normalized.substr(20, 12), + ].join('-'); +} + +/** @private */ +function ipv4(value) { + const str = '' + value; + const parts = str.split('.'); + if (parts.length !== 4) { + throw new Error('Invalid IPv4 format'); + } + for (let i = 0; i < 4; i++) { + const num = parseInt(parts[i], 10); + if (isNaN(num) || num < 0 || num > 255) { + throw new Error('Invalid IPv4 format'); + } + } + return str; +} + +/** @private */ +function ipv6(value) { + const str = '' + value; + // Basic IPv6 validation - accepts both compressed and full formats + if (!/^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$/.test(str) && !/^::$/.test(str)) { + throw new Error('Invalid IPv6 format'); + } + return str.toLowerCase(); +} + +/** @private */ +function date(value) { + const str = '' + value; + if (!/^\d{4}-\d{2}-\d{2}$/.test(str)) { + throw new Error('Invalid date format, expected YYYY-MM-DD'); + } + const parsed = new Date(str + 'T00:00:00Z'); + if (isNaN(parsed.getTime())) { + throw new Error('Invalid date format'); + } + return str; +} + +/** @private */ +function dateTime(value) { + const str = '' + value; + const parsed = new Date(str); + if (isNaN(parsed.getTime())) { + throw new Error('Invalid date-time format'); + } + return parsed.toISOString(); +} + /** @private */ function boolean(value) { return !!value; @@ -55,6 +122,11 @@ export default { float, double, string, + uuid, + ipv4, + ipv6, + date, + 'date-time': dateTime, boolean, array, object, diff --git a/src/decoder.ts b/src/decoder.ts index 90a2042..cf31bfb 100644 --- a/src/decoder.ts +++ b/src/decoder.ts @@ -45,6 +45,165 @@ function string(bytes) { return fromChar(...res); } +/** + * UUID decoder - converts 16 bytes to UUID string + * Binary format: 16 bytes (128 bits) + * UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx (36 chars) + * @private + */ +function uuid(bytes) { + if (bytes.length !== 16) { + throw new Error('Invalid UUID byte length'); + } + + // Convert bytes to hex string + const hex = []; + for (let i = 0; i < 16; i++) { + const byte = bytes[i].toString(16).padStart(2, '0'); + hex.push(byte); + } + + // Insert hyphens at proper positions: 8-4-4-4-12 + return [ + hex.slice(0, 4).join(''), + hex.slice(4, 6).join(''), + hex.slice(6, 8).join(''), + hex.slice(8, 10).join(''), + hex.slice(10, 16).join(''), + ].join('-'); +} + +/** + * IPv4 decoder - converts 4 bytes to IPv4 string + * Binary format: 4 bytes + * IPv4 format: "192.168.1.1" + * @private + */ +function ipv4(bytes) { + if (bytes.length !== 4) { + throw new Error('Invalid IPv4 byte length'); + } + + return bytes.join('.'); +} + +/** + * IPv6 decoder - converts 16 bytes to IPv6 string + * Binary format: 16 bytes + * IPv6 format: "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + * @private + */ +function ipv6(bytes) { + if (bytes.length !== 16) { + throw new Error('Invalid IPv6 byte length'); + } + + const parts = []; + for (let i = 0; i < 16; i += 2) { + const value = (bytes[i] << 8) | bytes[i + 1]; + parts.push(value.toString(16).padStart(4, '0')); + } + + // Find longest sequence of consecutive '0000' groups for compression + let longestZeroStart = -1; + let longestZeroLength = 0; + let currentZeroStart = -1; + let currentZeroLength = 0; + + for (let i = 0; i < parts.length; i++) { + if (parts[i] === '0000') { + if (currentZeroStart === -1) { + currentZeroStart = i; + currentZeroLength = 1; + } + else { + currentZeroLength++; + } + } + else { + if (currentZeroLength > longestZeroLength) { + longestZeroStart = currentZeroStart; + longestZeroLength = currentZeroLength; + } + currentZeroStart = -1; + currentZeroLength = 0; + } + } + + // Check final sequence + if (currentZeroLength > longestZeroLength) { + longestZeroStart = currentZeroStart; + longestZeroLength = currentZeroLength; + } + + // Remove leading zeros from each part (except if it's all zeros) + const strippedParts = parts.map((p) => { + const stripped = p.replace(/^0+/, ''); + return stripped === '' ? '0' : stripped; + }); + + // Apply compression if we found at least one zero group + if (longestZeroLength > 0) { + const before = strippedParts.slice(0, longestZeroStart); + const after = strippedParts.slice(longestZeroStart + longestZeroLength); + + // Build result with :: compression + let result = ''; + if (before.length > 0) { + result = before.join(':'); + } + result += '::'; + if (after.length > 0) { + result += after.join(':'); + } + + return result; + } + + // No zero sequences, return with leading zeros stripped + return strippedParts.join(':'); +} + +/** + * Date decoder - converts 4 bytes (days since epoch) to YYYY-MM-DD + * Binary format: 4 bytes (signed int32, days since Jan 1, 1970) + * Date format: "2025-10-28" + * @private + */ +function date(bytes) { + if (bytes.length !== 4) { + throw new Error('Invalid date byte length'); + } + + const days = int32(bytes); + const epochMs = days * 86400000; + const dateObj = new Date(epochMs); + + // Format as YYYY-MM-DD + const year = dateObj.getUTCFullYear(); + const month = String(dateObj.getUTCMonth() + 1).padStart(2, '0'); + const day = String(dateObj.getUTCDate()).padStart(2, '0'); + + return `${year}-${month}-${day}`; +} + +/** + * Date-time decoder - converts 8 bytes to ISO 8601 string + * Binary format: 8 bytes (int64, milliseconds since Jan 1, 1970) + * DateTime format: "2025-10-28T14:30:00.000Z" + * @private + */ +function dateTime(bytes) { + if (bytes.length !== 8) { + throw new Error('Invalid date-time byte length'); + } + + const ms = double(bytes); + const dateObj = new Date(ms); + + return dateObj.toISOString(); +} + /** @private */ function array(schema, bytes) { const ret = []; @@ -109,6 +268,11 @@ export default { float, double, string, + uuid, + ipv4, + ipv6, + date, + 'date-time': dateTime, array, object, unsigned, diff --git a/src/encoder.ts b/src/encoder.ts index 9c1307c..22b4917 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -47,6 +47,127 @@ function string(val) { return chars; } +/** + * UUID encoder - converts UUID string to 16 bytes + * UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx (36 chars) + * Binary format: 16 bytes (128 bits) + * Saves 56 bytes per UUID compared to string encoding + * @private + */ +function uuid(val) { + // Remove hyphens and validate format + const hex = val.replace(/-/g, ''); + if (hex.length !== 32) { + throw new Error('Invalid UUID format'); + } + + // Convert hex string to 16 bytes + const bytes = []; + for (let i = 0; i < 32; i += 2) { + bytes.push(parseInt(hex.substr(i, 2), 16)); + } + + return bytes; +} + +/** + * IPv4 encoder - converts IPv4 string to 4 bytes + * IPv4 format: "192.168.1.1" (max 15 chars = 30 bytes as string) + * Binary format: 4 bytes + * Saves up to 26 bytes + * @private + */ +function ipv4(val) { + const parts = val.split('.'); + if (parts.length !== 4) { + throw new Error('Invalid IPv4 format'); + } + + const bytes = []; + for (let i = 0; i < 4; i++) { + const num = parseInt(parts[i], 10); + if (isNaN(num) || num < 0 || num > 255) { + throw new Error('Invalid IPv4 format'); + } + bytes.push(num); + } + + return bytes; +} + +/** + * IPv6 encoder - converts IPv6 string to 16 bytes + * IPv6 format: "2001:0db8:85a3::8a2e:0370:7334" (max 39 chars = 78 bytes as string) + * Binary format: 16 bytes + * Saves up to 62 bytes + * @private + */ +function ipv6(val) { + // Expand :: notation + let expanded = val; + if (expanded.includes('::')) { + const parts = expanded.split('::'); + const leftParts = parts[0] ? parts[0].split(':') : []; + const rightParts = parts[1] ? parts[1].split(':') : []; + const missingParts = 8 - leftParts.length - rightParts.length; + const zeros = Array(missingParts).fill('0'); + expanded = [...leftParts, ...zeros, ...rightParts].join(':'); + } + + const parts = expanded.split(':'); + if (parts.length !== 8) { + throw new Error('Invalid IPv6 format'); + } + + const bytes = []; + for (let i = 0; i < 8; i++) { + const num = parseInt(parts[i] || '0', 16); + if (isNaN(num) || num < 0 || num > 0xffff) { + throw new Error('Invalid IPv6 format'); + } + bytes.push(num >> 8, num & 0xff); + } + + return bytes; +} + +/** + * Date encoder - converts YYYY-MM-DD to 4 bytes (days since epoch) + * Date format: "2025-10-28" (10 chars = 20 bytes as string) + * Binary format: 4 bytes (signed int32, days since Jan 1, 1970) + * Saves 16 bytes + * @private + */ +function date(val) { + const parsed = new Date(val + 'T00:00:00Z'); + if (isNaN(parsed.getTime())) { + throw new Error('Invalid date format'); + } + + // Calculate days since epoch + const epochMs = parsed.getTime(); + const days = Math.floor(epochMs / 86400000); + + return int32(days); +} + +/** + * Date-time encoder - converts ISO 8601 to 8 bytes (milliseconds since epoch) + * DateTime format: "2025-10-28T14:30:00Z" (20+ chars = 40+ bytes as string) + * Binary format: 8 bytes (int64, milliseconds since Jan 1, 1970) + * Saves 32+ bytes + * @private + */ +function dateTime(val) { + const parsed = new Date(val); + if (isNaN(parsed.getTime())) { + throw new Error('Invalid date-time format'); + } + + // Store as milliseconds since epoch using double precision + return double(parsed.getTime()); +} + /** @private */ function array(schema, val) { const ret = []; @@ -120,6 +241,11 @@ export default { float, double, string, + uuid, + ipv4, + ipv6, + date, + 'date-time': dateTime, array, object, getSize, diff --git a/src/schema.ts b/src/schema.ts index e562249..46356eb 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -25,27 +25,45 @@ function resolveType(type, format) { return fmt === 'float' ? 'float' : 'double'; } + if (type === 'string') { + if (format === 'uuid') return 'uuid'; + if (format === 'ipv4') return 'ipv4'; + if (format === 'ipv6') return 'ipv6'; + if (format === 'date') return 'date'; + if (format === 'date-time') return 'date-time'; + } + return type; } export default function Schema(schema, options = { keyOrder: false }) { const sizeRef = { - boolean: 1, - int32: 4, - int64: 8, - float: 4, - double: 8, - string: 2, - array: 2, - object: 1, + 'boolean': 1, + 'int32': 4, + 'int64': 8, + 'float': 4, + 'double': 8, + 'string': 2, + 'uuid': 1, + 'ipv4': 1, + 'ipv6': 1, + 'date': 1, + 'date-time': 1, + 'array': 2, + 'object': 1, }; const defaultSizes = { - boolean: 1, - int32: 4, - int64: 8, - float: 4, - double: 8, + 'boolean': 1, + 'int32': 4, + 'int64': 8, + 'float': 4, + 'double': 8, + 'uuid': 16, + 'ipv4': 4, + 'ipv6': 16, + 'date': 4, + 'date-time': 8, }; const scope = { diff --git a/tests/integration/index.ts b/tests/integration/index.ts index 9790002..c887f86 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -115,6 +115,164 @@ describe('Data integrity - simple', () => { }); }); + describe('UUID', () => { + const Schema = schema({ test: { type: 'string', format: 'uuid' } }); + + it('should preserve UUID value', () => { + const uuid = '550e8400-e29b-4d4e-a7d4-426614174000'; + expect(Schema.read(Schema.write({ test: uuid }).buffer())).toEqual({ test: uuid }); + }); + + it('should compress UUID to 16 bytes instead of 72', () => { + const uuid = '550e8400-e29b-4d4e-a7d4-426614174000'; + const buffer = Schema.write({ test: uuid }).buffer(); + // Header: 1 byte (field count) + 1 byte (field index) + 1 byte (size) = 3 bytes + // Content: 16 bytes (UUID binary) + // Total: 19 bytes (vs 75 bytes for string encoding: 3 header + 72 content) + expect(buffer.length).toBe(19); + }); + + it('should handle uppercase UUIDs', () => { + const uuid = '550E8400-E29B-4D4E-A7D4-426614174000'; + const result = Schema.read(Schema.write({ test: uuid }).buffer()); + // UUID should be normalized to lowercase + expect(result.test).toBe('550e8400-e29b-4d4e-a7d4-426614174000'); + }); + + it('should handle nil UUID', () => { + const uuid = '00000000-0000-0000-0000-000000000000'; + expect(Schema.read(Schema.write({ test: uuid }).buffer())).toEqual({ test: uuid }); + }); + }); + + describe('IPv4', () => { + const Schema = schema({ test: { type: 'string', format: 'ipv4' } }); + + it('should preserve IPv4 value', () => { + const ip = '192.168.1.1'; + expect(Schema.read(Schema.write({ test: ip }).buffer())).toEqual({ test: ip }); + }); + + it('should compress IPv4 to 4 bytes instead of 30', () => { + const ip = '192.168.1.1'; + const buffer = Schema.write({ test: ip }).buffer(); + // Header: 1 byte (field count) + 1 byte (field index) + 1 byte (size) = 3 bytes + // Content: 4 bytes (IPv4 binary) + // Total: 7 bytes (vs 33 bytes for string encoding) + expect(buffer.length).toBe(7); + }); + + it('should handle edge cases', () => { + expect(Schema.read(Schema.write({ test: '0.0.0.0' }).buffer())).toEqual({ test: '0.0.0.0' }); + expect(Schema.read(Schema.write({ test: '255.255.255.255' }).buffer())).toEqual({ test: '255.255.255.255' }); + }); + }); + + describe('IPv6', () => { + const Schema = schema({ test: { type: 'string', format: 'ipv6' } }); + + it('should compress IPv6 value', () => { + const ip = '2001:0db8:85a3:0000:0000:8a2e:0370:7334'; + expect(Schema.read(Schema.write({ test: ip }).buffer())).toEqual({ test: '2001:db8:85a3::8a2e:370:7334' }); + }); + + it('should compress IPv6 to 16 bytes instead of 78', () => { + const ip = '2001:0db8:85a3:0000:0000:8a2e:0370:7334'; + const buffer = Schema.write({ test: ip }).buffer(); + // Header: 1 byte (field count) + 1 byte (field index) + 1 byte (size) = 3 bytes + // Content: 16 bytes (IPv6 binary) + // Total: 19 bytes (vs 81 bytes for string encoding) + expect(buffer.length).toBe(19); + }); + + it('should handle compressed IPv6 notation', () => { + const ip = '2001:db8:85a3::8a2e:370:7334'; + const result = Schema.read(Schema.write({ test: ip }).buffer()); + // Should be decoded back with compression + expect(result.test).toBe(ip); + }); + + it('should handle loopback', () => { + const ip = '::1'; + const result = Schema.read(Schema.write({ test: ip }).buffer()); + expect(result.test).toBe('::1'); + }); + + it('should handle all zeros', () => { + const ip = '::'; + const result = Schema.read(Schema.write({ test: ip }).buffer()); + expect(result.test).toBe('::'); + }); + }); + + describe('Date', () => { + const Schema = schema({ test: { type: 'string', format: 'date' } }); + + it('should preserve date value', () => { + const date = '2025-10-28'; + expect(Schema.read(Schema.write({ test: date }).buffer())).toEqual({ test: date }); + }); + + it('should compress date to 4 bytes instead of 20', () => { + const date = '2025-10-28'; + const buffer = Schema.write({ test: date }).buffer(); + // Header: 1 byte (field count) + 1 byte (field index) + 1 byte (size) = 3 bytes + // Content: 4 bytes (days since epoch) + // Total: 7 bytes (vs 23 bytes for string encoding) + expect(buffer.length).toBe(7); + }); + + it('should handle epoch date', () => { + const date = '1970-01-01'; + expect(Schema.read(Schema.write({ test: date }).buffer())).toEqual({ test: date }); + }); + + it('should handle dates before epoch', () => { + const date = '1969-12-31'; + expect(Schema.read(Schema.write({ test: date }).buffer())).toEqual({ test: date }); + }); + + it('should handle far future dates', () => { + const date = '2099-12-31'; + expect(Schema.read(Schema.write({ test: date }).buffer())).toEqual({ test: date }); + }); + }); + + describe('DateTime', () => { + const Schema = schema({ test: { type: 'string', format: 'date-time' } }); + + it('should preserve date-time value', () => { + const datetime = '2025-10-28T14:30:00.000Z'; + expect(Schema.read(Schema.write({ test: datetime }).buffer())).toEqual({ test: datetime }); + }); + + it('should compress date-time to 8 bytes instead of 40+', () => { + const datetime = '2025-10-28T14:30:00.000Z'; + const buffer = Schema.write({ test: datetime }).buffer(); + // Header: 1 byte (field count) + 1 byte (field index) + 1 byte (size) = 3 bytes + // Content: 8 bytes (milliseconds since epoch) + // Total: 11 bytes (vs 43+ bytes for string encoding) + expect(buffer.length).toBe(11); + }); + + it('should handle epoch datetime', () => { + const datetime = '1970-01-01T00:00:00.000Z'; + expect(Schema.read(Schema.write({ test: datetime }).buffer())).toEqual({ test: datetime }); + }); + + it('should handle millisecond precision', () => { + const datetime = '2025-10-28T14:30:00.123Z'; + expect(Schema.read(Schema.write({ test: datetime }).buffer())).toEqual({ test: datetime }); + }); + + it('should normalize various ISO 8601 formats', () => { + // Input without milliseconds, output should have .000Z + const input = '2025-10-28T14:30:00Z'; + const result = Schema.read(Schema.write({ test: input }).buffer()); + expect(result.test).toBe('2025-10-28T14:30:00.000Z'); + }); + }); + describe('Array', () => { const Schema = schema({ test: { type: 'array', items: { type: 'string' } } }); diff --git a/types.d.ts b/types.d.ts index 0899c89..d71580c 100644 --- a/types.d.ts +++ b/types.d.ts @@ -57,7 +57,7 @@ declare module 'compactr' { export interface SchemaFieldDefinition { type: 'boolean' | 'integer' | 'number' | 'string' | 'array' | 'object' - format?: 'int32' | 'int64' | 'float' | 'double' + format?: 'int32' | 'int64' | 'float' | 'double' | 'uuid' | 'ipv4' | 'ipv6' | 'date' | 'date-time' nullable?: boolean count?: number size?: number From 9d486ccd5035cd5b989a3de132bb0efce55e91fe Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Tue, 28 Oct 2025 18:44:20 -0400 Subject: [PATCH 08/19] added binary type, oneOf and anyOf support --- src/converter.ts | 22 +++++ src/decoder.ts | 16 +++- src/encoder.ts | 31 ++++++- src/reader.ts | 51 ++++++++--- src/schema.ts | 74 +++++++++++++++- src/writer.ts | 123 ++++++++++++++++++++++++--- tests/integration/index.ts | 169 +++++++++++++++++++++++++++++++++++++ types.d.ts | 6 +- 8 files changed, 461 insertions(+), 31 deletions(-) diff --git a/src/converter.ts b/src/converter.ts index c378d13..937ac3f 100644 --- a/src/converter.ts +++ b/src/converter.ts @@ -99,6 +99,27 @@ function dateTime(value) { return parsed.toISOString(); } +/** @private */ +function binary(value) { + // Accept Buffer, Uint8Array, or base64 string + if (Buffer.isBuffer(value)) { + return value.toString('base64'); + } + + if (value instanceof Uint8Array) { + return Buffer.from(value).toString('base64'); + } + + // Validate base64 string + if (typeof value === 'string') { + // Try to decode and re-encode to validate + const buffer = Buffer.from(value, 'base64'); + return buffer.toString('base64'); + } + + throw new Error('Invalid binary format: expected Buffer, Uint8Array, or base64 string'); +} + /** @private */ function boolean(value) { return !!value; @@ -127,6 +148,7 @@ export default { ipv6, date, 'date-time': dateTime, + binary, boolean, array, object, diff --git a/src/decoder.ts b/src/decoder.ts index cf31bfb..a35c144 100644 --- a/src/decoder.ts +++ b/src/decoder.ts @@ -4,9 +4,9 @@ const fromChar = String.fromCharCode; -// Presence indicators for nullable fields +// Discriminator byte for nullable/oneOf/anyOf fields export const NULL_INDICATOR = 0x00; // Field is null -export const PRESENT_INDICATOR = 0x01; // Field is present (not null) +export const VARIANT_BASE = 0x01; // First variant (or present for simple nullable) /* Methods ------------------------------------------------------------------- */ @@ -204,6 +204,17 @@ function dateTime(bytes) { return dateObj.toISOString(); } +/** + * Binary decoder - converts raw bytes to base64 string + * Binary format: raw bytes + * Base64 format: "SGVsbG8gV29ybGQ=" + * @private + */ +function binary(bytes) { + const buffer = Buffer.from(bytes); + return buffer.toString('base64'); +} + /** @private */ function array(schema, bytes) { const ret = []; @@ -273,6 +284,7 @@ export default { ipv6, date, 'date-time': dateTime, + binary, array, object, unsigned, diff --git a/src/encoder.ts b/src/encoder.ts index 22b4917..3fc29d9 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -4,9 +4,9 @@ const intMap = [null, unsigned8, unsigned16, null, unsigned32]; -// Presence indicators for nullable fields +// Discriminator byte for nullable/oneOf/anyOf fields export const NULL_INDICATOR = 0x00; // Field is null -export const PRESENT_INDICATOR = 0x01; // Field is present (not null) +export const VARIANT_BASE = 0x01; // First variant (or present for simple nullable) /* Methods ------------------------------------------------------------------- */ @@ -168,6 +168,32 @@ function dateTime(val) { return double(parsed.getTime()); } +/** + * Binary encoder - converts base64 string or Buffer to raw bytes + * Base64 format: "SGVsbG8gV29ybGQ=" (4 chars per 3 bytes) + * Binary format: raw bytes (1 byte per byte) + * Saves ~33% compared to storing base64 as string (2 bytes per char) + * @private + */ +function binary(val) { + // Accept Buffer, Uint8Array, or base64 string + if (Buffer.isBuffer(val)) { + return Array.from(val); + } + + if (val instanceof Uint8Array) { + return Array.from(val); + } + + // Assume base64 encoded string + if (typeof val === 'string') { + const buffer = Buffer.from(val, 'base64'); + return Array.from(buffer); + } + + throw new Error('Binary format requires Buffer, Uint8Array, or base64 string'); +} + /** @private */ function array(schema, val) { const ret = []; @@ -246,6 +272,7 @@ export default { ipv6, date, 'date-time': dateTime, + binary, array, object, getSize, diff --git a/src/reader.ts b/src/reader.ts index 3c5dcc3..90ec42a 100644 --- a/src/reader.ts +++ b/src/reader.ts @@ -2,7 +2,7 @@ /* Requires ------------------------------------------------------------------ */ -import Decoder, { NULL_INDICATOR } from './decoder'; +import Decoder, { NULL_INDICATOR, VARIANT_BASE } from './decoder'; /* Methods ------------------------------------------------------------------- */ @@ -30,25 +30,40 @@ export default function Reader(scope) { caret++; // Move past field index let size; + let variantIndex = null; - // Check for presence byte if field is nullable - if (key.nullable) { - const presenceByte = bytes[caret]; - caret++; // Move past presence byte + // Check for discriminator byte if field is nullable or has variants + if (key.nullable || key.variants) { + const discriminatorByte = bytes[caret]; + caret++; // Move past discriminator byte - if (presenceByte === NULL_INDICATOR) { + if (discriminatorByte === NULL_INDICATOR) { // Field is null - no size or content follows size = -1; // Use -1 as internal null marker } else { - // Field is present - read size - const sizeBytes = bytes.slice(caret, caret + key.count); - size = key.size || Decoder.unsigned(sizeBytes); - caret += key.count; // Move past size bytes + // For variants, extract which variant to use (0-indexed internally) + if (key.variants) { + variantIndex = discriminatorByte - VARIANT_BASE; + if (variantIndex < 0 || variantIndex >= key.variants.length) { + throw new Error(`Invalid variant discriminator: ${discriminatorByte}`); + } + // Use the variant's metadata for size reading + const variant = key.variants[variantIndex]; + const sizeBytes = bytes.slice(caret, caret + variant.count); + size = variant.size || Decoder.unsigned(sizeBytes); + caret += variant.count; // Move past size bytes + } + else { + // Regular nullable field is present - read size + const sizeBytes = bytes.slice(caret, caret + key.count); + size = key.size || Decoder.unsigned(sizeBytes); + caret += key.count; // Move past size bytes + } } } else { - // Non-nullable field - read size directly + // Non-nullable, non-variant field - read size directly const sizeBytes = bytes.slice(caret, caret + key.count); size = key.size || Decoder.unsigned(sizeBytes); caret += key.count; // Move past size bytes @@ -57,6 +72,7 @@ export default function Reader(scope) { scope.header[index] = { key, size, + variantIndex, }; return caret; } @@ -77,12 +93,21 @@ export default function Reader(scope) { } } for (let i = 0; i < scope.header.length; i++) { - // Handle nullable fields with size -1 (null marker detected) - if (scope.header[i].key.nullable && scope.header[i].size === -1) { + // Handle null values (size -1 indicates null) + if (scope.header[i].size === -1) { ret[scope.header[i].key.name] = null; continue; } + // Handle variant fields + if (scope.header[i].variantIndex !== null && scope.header[i].key.variants) { + const variant = scope.header[i].key.variants[scope.header[i].variantIndex]; + ret[scope.header[i].key.name] = variant.transformOut(bytes.slice(caret, caret + scope.header[i].size)); + caret += scope.header[i].size; + continue; + } + + // Handle regular fields ret[scope.header[i].key.name] = scope.header[i].key.transformOut(bytes.slice(caret, caret + scope.header[i].size)); caret += scope.header[i].size; } diff --git a/src/schema.ts b/src/schema.ts index 46356eb..118fc1e 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -31,6 +31,7 @@ function resolveType(type, format) { if (format === 'ipv6') return 'ipv6'; if (format === 'date') return 'date'; if (format === 'date-time') return 'date-time'; + if (format === 'binary') return 'binary'; } return type; @@ -49,6 +50,7 @@ export default function Schema(schema, options = { keyOrder: false }) { 'ipv6': 1, 'date': 1, 'date-time': 1, + 'binary': 4, 'array': 2, 'object': 1, }; @@ -88,10 +90,49 @@ export default function Schema(schema, options = { keyOrder: false }) { Object.keys(schema) .sort() .forEach((key, index) => { + // Handle oneOf/anyOf fields + if (schema[key].oneOf || schema[key].anyOf) { + const variantDefs = schema[key].oneOf || schema[key].anyOf; + const variants = variantDefs.map((variantDef) => { + const variantType = variantDef.type; + const variantFormat = variantDef.format; + const variantInternalType = resolveType(variantType, variantFormat); + // Binary fields need 4-byte counter by default to support large data + const variantCount = variantDef.count || (variantInternalType === 'binary' ? 4 : 1); + const variantChildSchema = computeNestedVariant(variantDef); + + return { + type: variantInternalType, + transformIn: (variantChildSchema !== undefined) + ? Encoder[variantInternalType].bind(null, variantChildSchema) + : Encoder[variantInternalType], + transformOut: (variantChildSchema !== undefined) + ? Decoder[variantInternalType].bind(null, variantChildSchema) + : Decoder[variantInternalType], + coerse: Converter[variantInternalType], + getSize: Encoder.getSize.bind(null, variantCount), + fixedSize: (defaultSizes[variantInternalType] && Encoder.getSize(variantCount, defaultSizes[variantInternalType])) || null, + size: variantDef.size || defaultSizes[variantInternalType] || null, + count: variantCount, + nested: variantChildSchema, + }; + }); + + ret[key] = { + name: key, + index, + nullable: schema[key].nullable || false, + variants, + }; + return; + } + + // Handle regular fields const fieldType = schema[key].type; const fieldFormat = schema[key].format; const internalType = resolveType(fieldType, fieldFormat); - const count = schema[key].count || 1; + // Binary fields need 4-byte counter by default to support large data + const count = schema[key].count || (internalType === 'binary' ? 4 : 1); const childSchema = computeNested(schema, key); ret[key] = { @@ -116,6 +157,10 @@ export default function Schema(schema, options = { keyOrder: false }) { /** @private */ function applyBlank() { for (const key in scope.schema) { + // Skip variant fields in applyBlank as their size depends on runtime variant + if (scope.indices[key].variants) { + continue; + } scope.header.push({ key: scope.indices[key], size: scope.indices[key].size || sizeRef[scope.indices[key].type], @@ -150,5 +195,32 @@ export default function Schema(schema, options = { keyOrder: false }) { return childSchema; } + /** @private */ + function computeNestedVariant(variantDef) { + const variantType = variantDef.type; + const isObject = (variantType === 'object'); + const isArray = (variantType === 'array'); + let childSchema; + + if (isObject === true || isArray === true) { + if (isObject === true) childSchema = Schema(variantDef.schema, options); + if (isArray === true) { + const itemChildSchema = variantDef.items?.schema ? Schema(variantDef.items.schema, options) : undefined; + const itemType = variantDef.items.type; + const itemFormat = variantDef.items.format; + const internalItemType = resolveType(itemType, itemFormat); + + childSchema = { + count: variantDef.items.count || 1, + getSize: Encoder.getSize.bind(null, variantDef.items.count || 1), + transformIn: (itemChildSchema !== undefined) ? Encoder[internalItemType].bind(null, itemChildSchema) : Encoder[internalItemType], + transformOut: (itemChildSchema !== undefined) ? Decoder[internalItemType].bind(null, itemChildSchema) : Decoder[internalItemType], + }; + } + } + + return childSchema; + } + return Object.assign({}, writer, reader); } diff --git a/src/writer.ts b/src/writer.ts index f5510aa..eb8d657 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -2,7 +2,7 @@ /* Requires ------------------------------------------------------------------ */ -import { NULL_INDICATOR, PRESENT_INDICATOR } from './encoder'; +import { NULL_INDICATOR, VARIANT_BASE } from './encoder'; /* Methods ------------------------------------------------------------------- */ @@ -15,35 +15,119 @@ export default function Writer(scope) { scope.headerBytes[0] = keys.length; for (let i = 0; i < keys.length; i++) { let keyData = data[keys[i]]; + const field = scope.indices[keys[i]]; // Handle nullable fields with null values - if (scope.indices[keys[i]].nullable && keyData === null) { + if (field.nullable && keyData === null) { // Write header with NULL_INDICATOR (no size or content follows) - scope.headerBytes.push(scope.indices[keys[i]].index, NULL_INDICATOR); + scope.headerBytes.push(field.index, NULL_INDICATOR); continue; } - // For nullable fields that are not null, add PRESENT_INDICATOR - if (scope.indices[keys[i]].nullable) { - scope.headerBytes.push(scope.indices[keys[i]].index, PRESENT_INDICATOR); + // Handle oneOf/anyOf fields + if (field.variants) { + // Determine which variant matches the data + let variantIndex = -1; + let variantField = null; + + for (let v = 0; v < field.variants.length; v++) { + if (matchesVariant(keyData, field.variants[v])) { + variantIndex = v; + variantField = field.variants[v]; + break; + } + } + + if (variantIndex === -1) { + throw new Error(`Data does not match any variant for field: ${keys[i]}`); + } + + // Write field index and variant discriminator (1-indexed) + scope.headerBytes.push(field.index, VARIANT_BASE + variantIndex); + + // Apply coercion/validation if needed + if (options !== undefined) { + if (options.coerse === true && variantField.coerse) { + keyData = variantField.coerse(keyData); + } + if (options.validate === true && variantField.validate) { + variantField.validate(keyData); + } + } + + // Encode using the matched variant's transformer + const encoded = variantField.transformIn(keyData); + addSizeAndContentForVariant(encoded, variantField); + continue; + } + + // Handle regular fields with discriminator byte if nullable + if (field.nullable) { + scope.headerBytes.push(field.index, VARIANT_BASE); } else { - scope.headerBytes.push(scope.indices[keys[i]].index); + scope.headerBytes.push(field.index); } if (options !== undefined) { - if (options.coerse === true) keyData = scope.indices[keys[i]].coerse(keyData); - if (options.validate === true) scope.indices[keys[i]].validate(keyData); + if (options.coerse === true) keyData = field.coerse(keyData); + if (options.validate === true) field.validate(keyData); } // Add size and content - const encoded = scope.indices[keys[i]].transformIn(keyData); + const encoded = field.transformIn(keyData); addSizeAndContent(encoded, keys[i]); } return this; } + /** @private */ + function matchesVariant(data, variant) { + const dataType = Array.isArray(data) + ? 'array' + : data === null + ? 'null' + : typeof data === 'boolean' + ? 'boolean' + : typeof data === 'number' + ? 'number' + : typeof data === 'string' + ? 'string' + : typeof data === 'object' + ? 'object' + : 'unknown'; + + // Map internal types to JavaScript types + // Number types: int32, int64, float, double + if (variant.type === 'int32' || variant.type === 'int64' + || variant.type === 'float' || variant.type === 'double') { + return dataType === 'number'; + } + + // String types: string, uuid, ipv4, ipv6, date, date-time, binary + if (variant.type === 'string' || variant.type === 'uuid' + || variant.type === 'ipv4' || variant.type === 'ipv6' + || variant.type === 'date' || variant.type === 'date-time' + || variant.type === 'binary') { + return dataType === 'string'; + } + + if (variant.type === 'boolean') { + return dataType === 'boolean'; + } + + if (variant.type === 'array') { + return dataType === 'array'; + } + + if (variant.type === 'object') { + return dataType === 'object'; + } + + return false; + } + /** @private */ function addSizeAndContent(encoded, key) { if (scope.indices[key].fixedSize !== null) { @@ -61,6 +145,23 @@ export default function Writer(scope) { scope.contentBytes.push(...encoded); } + /** @private */ + function addSizeAndContentForVariant(encoded, variant) { + if (variant.fixedSize !== null) { + scope.headerBytes.push(...variant.fixedSize); + } + else { + scope.headerBytes.push(...variant.getSize(encoded.length)); + if (variant.size !== encoded.length && variant.size !== null) { + const fixedSize = new Array(variant.size).fill(0); + const smallestSize = Math.min(encoded.length, fixedSize.length); + fixedSize.splice(0, smallestSize, ...encoded.slice(0, smallestSize)); + return scope.contentBytes.push(...fixedSize); + } + } + scope.contentBytes.push(...encoded); + } + function sizes(data) { const s: any = {}; for (const key in data) { @@ -111,5 +212,5 @@ export default function Writer(scope) { return Buffer.from(typedArray()); } - return { write, headerBuffer, contentBuffer, buffer, typedArray, sizes }; + return { write, headerBuffer, contentBuffer, buffer, typedArray, sizes, matchesVariant }; } diff --git a/tests/integration/index.ts b/tests/integration/index.ts index c887f86..e4bad49 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -273,6 +273,51 @@ describe('Data integrity - simple', () => { }); }); + describe('Binary', () => { + const Schema = schema({ test: { type: 'string', format: 'binary' } }); + + it('should preserve binary data via base64', () => { + const base64 = 'SGVsbG8gV29ybGQh'; // "Hello World!" + expect(Schema.read(Schema.write({ test: base64 }).buffer())).toEqual({ test: base64 }); + }); + + it('should compress binary data efficiently', () => { + const base64 = 'SGVsbG8gV29ybGQh'; // 16 chars = 32 bytes as string + const buffer = Schema.write({ test: base64 }).buffer(); + // Header: 1 byte (field count) + 1 byte (field index) + 4 bytes (size) = 6 bytes + // Content: 12 bytes (raw binary data decoded from base64) + // Total: 18 bytes (vs 35 bytes for string encoding) + expect(buffer.length).toBe(18); + }); + + it('should handle Buffer input', () => { + const data = Buffer.from('Hello World!', 'utf8'); + const result = Schema.read(Schema.write({ test: data }).buffer()); + expect(result.test).toBe('SGVsbG8gV29ybGQh'); + }); + + it('should handle Uint8Array input', () => { + const data = new Uint8Array([72, 101, 108, 108, 111]); + const result = Schema.read(Schema.write({ test: data }).buffer()); + expect(result.test).toBe('SGVsbG8='); // "Hello" in base64 + }); + + it('should handle empty binary data', () => { + const base64 = ''; // Empty + expect(Schema.read(Schema.write({ test: base64 }).buffer())).toEqual({ test: base64 }); + }); + + it('should handle large binary data', () => { + // Create 256 bytes of data + const bytes = new Uint8Array(256); + for (let i = 0; i < 256; i++) { + bytes[i] = i; + } + const base64 = Buffer.from(bytes).toString('base64'); + expect(Schema.read(Schema.write({ test: base64 }).buffer())).toEqual({ test: base64 }); + }); + }); + describe('Array', () => { const Schema = schema({ test: { type: 'array', items: { type: 'string' } } }); @@ -288,6 +333,130 @@ describe('Data integrity - simple', () => { expect(Schema.read(Schema.write({ test: { test: 23.23 } }).buffer())).toEqual({ test: { test: 23.23 } }); }); }); + + describe('OneOf', () => { + const Schema = schema({ + value: { + oneOf: [ + { type: 'string' }, + { type: 'integer', format: 'int32' }, + { type: 'boolean' }, + ], + }, + }); + + it('should handle string variant (first)', () => { + const data = { value: 'hello' }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle integer variant (second)', () => { + const data = { value: 42 }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle boolean variant (third)', () => { + const data = { value: true }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should use correct discriminator for each variant', () => { + // String (first variant) should have discriminator 0x01 + const stringBuffer = Schema.write({ value: 'test' }).buffer(); + expect(stringBuffer[2]).toBe(0x01); // discriminator byte + + // Integer (second variant) should have discriminator 0x02 + const intBuffer = Schema.write({ value: 42 }).buffer(); + expect(intBuffer[2]).toBe(0x02); // discriminator byte + + // Boolean (third variant) should have discriminator 0x03 + const boolBuffer = Schema.write({ value: true }).buffer(); + expect(boolBuffer[2]).toBe(0x03); // discriminator byte + }); + }); + + describe('AnyOf', () => { + const Schema = schema({ + data: { + anyOf: [ + { type: 'number', format: 'double' }, + { type: 'string' }, + ], + }, + }); + + it('should handle number variant', () => { + const data = { data: 3.14 }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle string variant', () => { + const data = { data: 'hello' }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('OneOf with nullable', () => { + const Schema = schema({ + value: { + nullable: true, + oneOf: [ + { type: 'string' }, + { type: 'integer', format: 'int32' }, + ], + }, + }); + + it('should handle null value', () => { + const data = { value: null }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle string variant when not null', () => { + const data = { value: 'test' }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle integer variant when not null', () => { + const data = { value: 123 }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should use 0x00 for null, 0x01+ for variants', () => { + // Null should use 0x00 + const nullBuffer = Schema.write({ value: null }).buffer(); + expect(nullBuffer[2]).toBe(0x00); + + // String variant should use 0x01 + const stringBuffer = Schema.write({ value: 'test' }).buffer(); + expect(stringBuffer[2]).toBe(0x01); + + // Integer variant should use 0x02 + const intBuffer = Schema.write({ value: 42 }).buffer(); + expect(intBuffer[2]).toBe(0x02); + }); + }); + + describe('OneOf with complex types', () => { + const Schema = schema({ + item: { + oneOf: [ + { type: 'array', items: { type: 'string' } }, + { type: 'object', schema: { x: { type: 'integer', format: 'int32' }, y: { type: 'integer', format: 'int32' } } }, + ], + }, + }); + + it('should handle array variant', () => { + const data = { item: ['a', 'b', 'c'] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle object variant', () => { + const data = { item: { x: 10, y: 20 } }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); }); describe('Data integrity - multi simple', () => { diff --git a/types.d.ts b/types.d.ts index d71580c..cdf9cd0 100644 --- a/types.d.ts +++ b/types.d.ts @@ -56,8 +56,8 @@ declare module 'compactr' { } export interface SchemaFieldDefinition { - type: 'boolean' | 'integer' | 'number' | 'string' | 'array' | 'object' - format?: 'int32' | 'int64' | 'float' | 'double' | 'uuid' | 'ipv4' | 'ipv6' | 'date' | 'date-time' + type?: 'boolean' | 'integer' | 'number' | 'string' | 'array' | 'object' + format?: 'int32' | 'int64' | 'float' | 'double' | 'uuid' | 'ipv4' | 'ipv6' | 'date' | 'date-time' | 'binary' nullable?: boolean count?: number size?: number @@ -68,6 +68,8 @@ declare module 'compactr' { count?: number schema?: SchemaDefinition } + oneOf?: SchemaFieldDefinition[] + anyOf?: SchemaFieldDefinition[] } export interface SchemaDefinition { From 7d5ac8a1e51f8838521350166d8c82826fa85fe6 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Thu, 30 Oct 2025 14:31:37 -0400 Subject: [PATCH 09/19] migrated arrays --- src/decoder.ts | 37 ++++- src/encoder.ts | 82 +++++++++- src/schema.ts | 94 ++++++++--- tests/integration/index.ts | 318 +++++++++++++++++++++++++++++++++++++ types.d.ts | 7 +- 5 files changed, 507 insertions(+), 31 deletions(-) diff --git a/src/decoder.ts b/src/decoder.ts index a35c144..88e5f20 100644 --- a/src/decoder.ts +++ b/src/decoder.ts @@ -218,11 +218,44 @@ function binary(bytes) { /** @private */ function array(schema, bytes) { const ret = []; + for (let i = 0; i < bytes.length;) { + // Handle nullable or variant array items + if (schema.nullable || schema.variants) { + const discriminator = bytes[i]; + i++; + + // Handle null value + if (discriminator === 0x00) { + ret.push(null); + continue; + } + + // Handle variant items + if (schema.variants) { + const variantIndex = discriminator - 0x01; + const variantField = schema.variants[variantIndex]; + + if (!variantField) { + throw new Error(`Invalid variant discriminator: ${discriminator}`); + } + + const size = unsigned(bytes.slice(i, i + variantField.count)); + i += variantField.count; + ret.push(variantField.transformOut(bytes.slice(i, i + size))); + i += size; + continue; + } + + // For nullable non-variant items, discriminator 0x01 means value is present + // Continue to decode the value normally below + } + + // Handle regular array items const size = unsigned(bytes.slice(i, i + schema.count)); - i = (i + schema.count); + i += schema.count; ret.push(schema.transformOut(bytes.slice(i, i + size))); - i = (i + size); + i += size; } return ret; diff --git a/src/encoder.ts b/src/encoder.ts index 3fc29d9..67ecab0 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -197,13 +197,93 @@ function binary(val) { /** @private */ function array(schema, val) { const ret = []; + for (let i = 0; i < val.length; i++) { - const encoded = schema.transformIn(val[i]); + const item = val[i]; + + // Handle nullable array items + if (schema.nullable && item === null) { + ret.push(NULL_INDICATOR); + continue; + } + + // Handle variant array items (oneOf/anyOf) + if (schema.variants) { + let variantIndex = -1; + let variantField = null; + + // Find matching variant + for (let v = 0; v < schema.variants.length; v++) { + if (matchesVariantItem(item, schema.variants[v])) { + variantIndex = v; + variantField = schema.variants[v]; + break; + } + } + + if (variantIndex === -1) { + throw new Error(`Array item does not match any variant`); + } + + // Encode discriminator (1-indexed for nullable support) + const discriminator = schema.nullable ? VARIANT_BASE + variantIndex : VARIANT_BASE + variantIndex; + ret.push(discriminator); + + // Encode the item using the matched variant + const encoded = variantField.transformIn(item); + ret.push(...variantField.getSize(encoded.length), ...encoded); + continue; + } + + // Handle nullable non-null items (add presence indicator) + if (schema.nullable) { + ret.push(VARIANT_BASE); + } + + // Handle regular array items + const encoded = schema.transformIn(item); ret.push(...schema.getSize(encoded.length), ...encoded); } + return ret; } +/** @private */ +function matchesVariantItem(data, variant) { + const dataType = Array.isArray(data) + ? 'array' + : data === null + ? 'null' + : typeof data; + + // Map internal types to JavaScript types + if (variant.type === 'int32' || variant.type === 'int64' + || variant.type === 'float' || variant.type === 'double') { + return dataType === 'number'; + } + + if (variant.type === 'string' || variant.type === 'uuid' + || variant.type === 'ipv4' || variant.type === 'ipv6' + || variant.type === 'date' || variant.type === 'date-time' + || variant.type === 'binary') { + return dataType === 'string' || data instanceof Buffer || data instanceof Uint8Array; + } + + if (variant.type === 'boolean') { + return dataType === 'boolean'; + } + + if (variant.type === 'array') { + return dataType === 'array'; + } + + if (variant.type === 'object') { + return dataType === 'object'; + } + + return false; +} + /** @private */ function object(schema, val) { return schema.write(val).typedArray(); diff --git a/src/schema.ts b/src/schema.ts index 118fc1e..3130eaf 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -178,23 +178,83 @@ export default function Schema(schema, options = { keyOrder: false }) { if (isObject === true || isArray === true) { if (isObject === true) childSchema = Schema(schema[key].schema, options); if (isArray === true) { - const itemChildSchema = computeNested(schema[key], 'items'); - const itemType = schema[key].items.type; - const itemFormat = schema[key].items.format; - const internalItemType = resolveType(itemType, itemFormat); - - childSchema = { - count: schema[key].items.count || 1, - getSize: Encoder.getSize.bind(null, schema[key].items.count || 1), - transformIn: (itemChildSchema !== undefined) ? Encoder[internalItemType].bind(null, itemChildSchema) : Encoder[internalItemType], - transformOut: (itemChildSchema !== undefined) ? Decoder[internalItemType].bind(null, itemChildSchema) : Decoder[internalItemType], - }; + childSchema = processArrayItems(schema[key].items); } } return childSchema; } + /** @private */ + function processArrayItems(itemDef) { + // Handle oneOf/anyOf in array items + if (itemDef.oneOf || itemDef.anyOf) { + const variantDefs = itemDef.oneOf || itemDef.anyOf; + const variants = variantDefs.map((variantDef) => { + const variantType = variantDef.type; + const variantFormat = variantDef.format; + const variantInternalType = resolveType(variantType, variantFormat); + const variantCount = variantDef.count || (variantInternalType === 'binary' ? 4 : 1); + const variantChildSchema = processArrayItemsNested(variantDef); + + return { + type: variantInternalType, + transformIn: (variantChildSchema !== undefined) + ? Encoder[variantInternalType].bind(null, variantChildSchema) + : Encoder[variantInternalType], + transformOut: (variantChildSchema !== undefined) + ? Decoder[variantInternalType].bind(null, variantChildSchema) + : Decoder[variantInternalType], + coerse: Converter[variantInternalType], + getSize: Encoder.getSize.bind(null, variantCount), + fixedSize: (defaultSizes[variantInternalType] && Encoder.getSize(variantCount, defaultSizes[variantInternalType])) || null, + size: variantDef.size || defaultSizes[variantInternalType] || null, + count: variantCount, + nested: variantChildSchema, + }; + }); + + return { + nullable: itemDef.nullable || false, + variants, + count: 1, + getSize: Encoder.getSize.bind(null, 1), + }; + } + + // Handle regular array items + const itemType = itemDef.type; + const itemFormat = itemDef.format; + const internalItemType = resolveType(itemType, itemFormat); + const itemCount = itemDef.count || (internalItemType === 'binary' ? 4 : 1); + const itemChildSchema = processArrayItemsNested(itemDef); + + return { + nullable: itemDef.nullable || false, + count: itemCount, + getSize: Encoder.getSize.bind(null, itemCount), + transformIn: (itemChildSchema !== undefined) ? Encoder[internalItemType].bind(null, itemChildSchema) : Encoder[internalItemType], + transformOut: (itemChildSchema !== undefined) ? Decoder[internalItemType].bind(null, itemChildSchema) : Decoder[internalItemType], + }; + } + + /** @private */ + function processArrayItemsNested(itemDef) { + const itemType = itemDef.type; + const isObject = (itemType === 'object'); + const isArray = (itemType === 'array'); + + if (isObject === true) { + return Schema(itemDef.schema, options); + } + + if (isArray === true) { + return processArrayItems(itemDef.items); + } + + return undefined; + } + /** @private */ function computeNestedVariant(variantDef) { const variantType = variantDef.type; @@ -205,17 +265,7 @@ export default function Schema(schema, options = { keyOrder: false }) { if (isObject === true || isArray === true) { if (isObject === true) childSchema = Schema(variantDef.schema, options); if (isArray === true) { - const itemChildSchema = variantDef.items?.schema ? Schema(variantDef.items.schema, options) : undefined; - const itemType = variantDef.items.type; - const itemFormat = variantDef.items.format; - const internalItemType = resolveType(itemType, itemFormat); - - childSchema = { - count: variantDef.items.count || 1, - getSize: Encoder.getSize.bind(null, variantDef.items.count || 1), - transformIn: (itemChildSchema !== undefined) ? Encoder[internalItemType].bind(null, itemChildSchema) : Encoder[internalItemType], - transformOut: (itemChildSchema !== undefined) ? Decoder[internalItemType].bind(null, itemChildSchema) : Decoder[internalItemType], - }; + childSchema = processArrayItems(variantDef.items); } } diff --git a/tests/integration/index.ts b/tests/integration/index.ts index e4bad49..e458a4f 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -852,3 +852,321 @@ describe('Nullable properties', () => { }); }); }); + +/* OpenAPI Array Format Tests ----------------------------------------------- */ + +describe('OpenAPI-compatible array formats', () => { + describe('Array of integers (int32)', () => { + const Schema = schema({ test: { type: 'array', items: { type: 'integer', format: 'int32' } } }); + + it('should preserve array of integers', () => { + expect(Schema.read(Schema.write({ test: [1, 2, 3, 4, 5] }).buffer())).toEqual({ test: [1, 2, 3, 4, 5] }); + }); + + it('should handle negative integers', () => { + expect(Schema.read(Schema.write({ test: [-100, 0, 100] }).buffer())).toEqual({ test: [-100, 0, 100] }); + }); + }); + + describe('Array of integers (int64)', () => { + const Schema = schema({ test: { type: 'array', items: { type: 'integer', format: 'int64' } } }); + + it('should preserve array of int64 values', () => { + const data = { test: [9007199254740991, -9007199254740991, 0] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Array of floats', () => { + const Schema = schema({ test: { type: 'array', items: { type: 'number', format: 'float' } } }); + + it('should preserve array of floats with precision loss', () => { + const result = Schema.read(Schema.write({ test: [3.14, 2.71, 1.41] }).buffer()); + expect(result.test[0]).toBeCloseTo(3.14, 5); + expect(result.test[1]).toBeCloseTo(2.71, 5); + expect(result.test[2]).toBeCloseTo(1.41, 5); + }); + }); + + describe('Array of doubles', () => { + const Schema = schema({ test: { type: 'array', items: { type: 'number', format: 'double' } } }); + + it('should preserve array of doubles', () => { + expect(Schema.read(Schema.write({ test: [3.141592653589793, 2.718281828459045] }).buffer())).toEqual({ test: [3.141592653589793, 2.718281828459045] }); + }); + }); + + describe('Array of UUIDs', () => { + const Schema = schema({ test: { type: 'array', items: { type: 'string', format: 'uuid' } } }); + + it('should preserve array of UUIDs', () => { + const data = { + test: [ + '550e8400-e29b-4d4e-a7d4-426614174000', + '6ba7b810-9dad-11d1-80b4-00c04fd430c8', + '00000000-0000-0000-0000-000000000000', + ], + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should compress UUIDs efficiently', () => { + const data = { + test: ['550e8400-e29b-4d4e-a7d4-426614174000', '6ba7b810-9dad-11d1-80b4-00c04fd430c8'], + }; + const buffer = Schema.write(data).buffer(); + // Each UUID is 16 bytes + 1 byte size = 17 bytes per UUID + // Plus array overhead + expect(buffer.length).toBeLessThan(100); // Much less than string encoding + }); + }); + + describe('Array of IPv4 addresses', () => { + const Schema = schema({ test: { type: 'array', items: { type: 'string', format: 'ipv4' } } }); + + it('should preserve array of IPv4 addresses', () => { + const data = { test: ['192.168.1.1', '10.0.0.1', '172.16.0.1'] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Array of IPv6 addresses', () => { + const Schema = schema({ test: { type: 'array', items: { type: 'string', format: 'ipv6' } } }); + + it('should preserve array of IPv6 addresses', () => { + const data = { + test: ['2001:db8:85a3::8a2e:370:7334', '::1', '::'], + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Array of dates', () => { + const Schema = schema({ test: { type: 'array', items: { type: 'string', format: 'date' } } }); + + it('should preserve array of dates', () => { + const data = { test: ['2025-10-28', '2024-01-01', '1970-01-01'] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Array of date-times', () => { + const Schema = schema({ test: { type: 'array', items: { type: 'string', format: 'date-time' } } }); + + it('should preserve array of date-times', () => { + const data = { + test: ['2025-10-28T14:30:00.000Z', '2024-01-01T00:00:00.000Z'], + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Array of binary data', () => { + const Schema = schema({ test: { type: 'array', items: { type: 'string', format: 'binary' } } }); + + it('should preserve array of binary data', () => { + const data = { test: ['SGVsbG8=', 'V29ybGQ=', 'Zm9vYmFy'] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle Buffer inputs', () => { + const input = { test: [Buffer.from('Hello'), Buffer.from('World')] }; + const result = Schema.read(Schema.write(input).buffer()); + expect(result.test).toEqual(['SGVsbG8=', 'V29ybGQ=']); + }); + }); + + describe('Array with nullable items', () => { + const Schema = schema({ test: { type: 'array', items: { type: 'string', nullable: true } } }); + + it('should preserve null values in array', () => { + const data = { test: ['a', null, 'b', null, 'c'] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should distinguish empty string from null', () => { + const data = { test: ['', null, 'text'] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle all null array', () => { + const data = { test: [null, null, null] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Array with nullable integer items', () => { + const Schema = schema({ test: { type: 'array', items: { type: 'integer', format: 'int32', nullable: true } } }); + + it('should preserve null values with integers', () => { + const data = { test: [1, null, 2, null, 3] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should distinguish zero from null', () => { + const data = { test: [0, null, -1] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Array with oneOf items', () => { + const Schema = schema({ + test: { + type: 'array', + items: { + oneOf: [ + { type: 'string' }, + { type: 'integer', format: 'int32' }, + { type: 'boolean' }, + ], + }, + }, + }); + + it('should handle mixed types in array', () => { + const data = { test: ['hello', 42, true, 'world', false, 123] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle all strings', () => { + const data = { test: ['a', 'b', 'c'] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle all integers', () => { + const data = { test: [1, 2, 3] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle all booleans', () => { + const data = { test: [true, false, true] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Array with anyOf items', () => { + const Schema = schema({ + test: { + type: 'array', + items: { + anyOf: [ + { type: 'number', format: 'double' }, + { type: 'string' }, + ], + }, + }, + }); + + it('should handle mixed numbers and strings', () => { + const data = { test: [3.14, 'pi', 2.71, 'e'] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Array with oneOf and nullable items', () => { + const Schema = schema({ + test: { + type: 'array', + items: { + nullable: true, + oneOf: [ + { type: 'string' }, + { type: 'integer', format: 'int32' }, + ], + }, + }, + }); + + it('should handle null with oneOf variants', () => { + const data = { test: ['hello', null, 42, null, 'world'] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Nested arrays', () => { + const Schema = schema({ + test: { + type: 'array', + items: { + type: 'array', + items: { type: 'integer', format: 'int32' }, + }, + }, + }); + + it('should handle 2D arrays', () => { + const data = { test: [[1, 2, 3], [4, 5, 6], [7, 8, 9]] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle empty nested arrays', () => { + const data = { test: [[], [1], []] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Nested arrays with strings', () => { + const Schema = schema({ + test: { + type: 'array', + items: { + type: 'array', + items: { type: 'string' }, + }, + }, + }); + + it('should handle 2D string arrays', () => { + const data = { test: [['a', 'b'], ['c', 'd', 'e'], ['f']] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Arrays of objects', () => { + const Schema = schema({ + test: { + type: 'array', + items: { + type: 'object', + schema: { + x: { type: 'integer', format: 'int32' }, + y: { type: 'integer', format: 'int32' }, + }, + }, + }, + }); + + it('should handle array of objects', () => { + const data = { test: [{ x: 1, y: 2 }, { x: 3, y: 4 }, { x: 5, y: 6 }] }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Complex nested structure', () => { + const Schema = schema({ + test: { + type: 'array', + items: { + oneOf: [ + { type: 'string' }, + { type: 'array', items: { type: 'integer', format: 'int32' } }, + { type: 'object', schema: { name: { type: 'string' } } }, + ], + }, + }, + }); + + it('should handle complex nested structures with oneOf', () => { + const data = { + test: [ + 'hello', + [1, 2, 3], + { name: 'test' }, + 'world', + [4, 5], + ], + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); +}); diff --git a/types.d.ts b/types.d.ts index cdf9cd0..f63457e 100644 --- a/types.d.ts +++ b/types.d.ts @@ -62,12 +62,7 @@ declare module 'compactr' { count?: number size?: number schema?: SchemaDefinition - items?: { - type: string - format?: string - count?: number - schema?: SchemaDefinition - } + items?: SchemaFieldDefinition oneOf?: SchemaFieldDefinition[] anyOf?: SchemaFieldDefinition[] } From 0bdb721adbce4b7c2cd13904eecff2fef82a13a2 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Thu, 30 Oct 2025 15:16:53 -0400 Subject: [PATCH 10/19] added support for basic objects --- src/writer.ts | 21 +- tests/integration/index.ts | 508 +++++++++++++++++++++++++++++++++++++ 2 files changed, 525 insertions(+), 4 deletions(-) diff --git a/src/writer.ts b/src/writer.ts index eb8d657..900e13d 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -11,7 +11,7 @@ export default function Writer(scope) { scope.headerBytes = [0]; scope.contentBytes = []; - const keys = filterKeys(data); + const keys = filterKeys(data, options); scope.headerBytes[0] = keys.length; for (let i = 0; i < keys.length; i++) { let keyData = data[keys[i]]; @@ -70,8 +70,8 @@ export default function Writer(scope) { } if (options !== undefined) { - if (options.coerse === true) keyData = field.coerse(keyData); - if (options.validate === true) field.validate(keyData); + if (options.coerse === true && field.coerse) keyData = field.coerse(keyData); + if (options.validate === true && field.validate) field.validate(keyData); } // Add size and content @@ -178,8 +178,13 @@ export default function Writer(scope) { /** @private */ function filterKeys(data) { const res = []; + const undeclaredKeys = []; + for (const key in data) { - if (scope.items.indexOf(key) === -1) continue; + if (scope.items.indexOf(key) === -1) { + undeclaredKeys.push(key); + continue; + } // Include nullable fields even when null if (scope.indices[key].nullable && data[key] === null) { @@ -192,6 +197,14 @@ export default function Writer(scope) { res.push(key); } } + + // Always warn about undeclared properties + if (undeclaredKeys.length > 0) { + console.warn( + `Schema validation warning: Object contains undeclared properties that will not be serialized: ${undeclaredKeys.join(', ')}`, + ); + } + return res; } diff --git a/tests/integration/index.ts b/tests/integration/index.ts index e458a4f..1385eb0 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -1170,3 +1170,511 @@ describe('OpenAPI-compatible array formats', () => { }); }); }); + +/* OpenAPI Object Format Tests ---------------------------------------------- */ + +describe('OpenAPI-compatible object formats', () => { + describe('Object with various format types', () => { + const Schema = schema({ + user: { + type: 'object', + schema: { + id: { type: 'string', format: 'uuid' }, + name: { type: 'string' }, + age: { type: 'integer', format: 'int32' }, + balance: { type: 'number', format: 'double' }, + active: { type: 'boolean' }, + created: { type: 'string', format: 'date-time' }, + ip: { type: 'string', format: 'ipv4' }, + }, + }, + }); + + it('should preserve object with mixed format types', () => { + const data = { + user: { + id: '550e8400-e29b-4d4e-a7d4-426614174000', + name: 'John Doe', + age: 30, + balance: 1234.56, + active: true, + created: '2025-10-28T14:30:00.000Z', + ip: '192.168.1.1', + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Object with nullable properties', () => { + const Schema = schema({ + data: { + type: 'object', + schema: { + required: { type: 'string' }, + optional: { type: 'string', nullable: true }, + number: { type: 'integer', format: 'int32', nullable: true }, + }, + }, + }); + + it('should preserve null values in object properties', () => { + const data = { + data: { + required: 'value', + optional: null, + number: null, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should preserve non-null values', () => { + const data = { + data: { + required: 'value', + optional: 'text', + number: 42, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle mix of null and non-null', () => { + const data = { + data: { + required: 'value', + optional: 'text', + number: null, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Object with oneOf properties', () => { + const Schema = schema({ + response: { + type: 'object', + schema: { + status: { type: 'integer', format: 'int32' }, + data: { + oneOf: [ + { type: 'string' }, + { type: 'integer', format: 'int32' }, + { type: 'object', schema: { message: { type: 'string' } } }, + ], + }, + }, + }, + }); + + it('should handle string variant', () => { + const data = { + response: { + status: 200, + data: 'success', + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle integer variant', () => { + const data = { + response: { + status: 200, + data: 42, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle object variant', () => { + const data = { + response: { + status: 200, + data: { message: 'Operation completed' }, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Object with anyOf properties', () => { + const Schema = schema({ + item: { + type: 'object', + schema: { + id: { type: 'integer', format: 'int32' }, + value: { + anyOf: [ + { type: 'number', format: 'double' }, + { type: 'string' }, + ], + }, + }, + }, + }); + + it('should handle number variant', () => { + const data = { + item: { + id: 1, + value: 3.14159, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle string variant', () => { + const data = { + item: { + id: 1, + value: 'text value', + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Deeply nested objects', () => { + const Schema = schema({ + root: { + type: 'object', + schema: { + level1: { + type: 'object', + schema: { + level2: { + type: 'object', + schema: { + level3: { + type: 'object', + schema: { + value: { type: 'string' }, + }, + }, + }, + }, + }, + }, + }, + }, + }); + + it('should handle deeply nested objects', () => { + const data = { + root: { + level1: { + level2: { + level3: { + value: 'deep', + }, + }, + }, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Object with array properties', () => { + const Schema = schema({ + data: { + type: 'object', + schema: { + tags: { type: 'array', items: { type: 'string' } }, + scores: { type: 'array', items: { type: 'integer', format: 'int32' } }, + metadata: { + type: 'array', + items: { + type: 'object', + schema: { + key: { type: 'string' }, + value: { type: 'string' }, + }, + }, + }, + }, + }, + }); + + it('should handle objects with array properties', () => { + const data = { + data: { + tags: ['typescript', 'serialization'], + scores: [10, 20, 30], + metadata: [ + { key: 'author', value: 'John' }, + { key: 'version', value: '1.0' }, + ], + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Object with nullable oneOf properties', () => { + const Schema = schema({ + record: { + type: 'object', + schema: { + id: { type: 'integer', format: 'int32' }, + value: { + nullable: true, + oneOf: [ + { type: 'string' }, + { type: 'integer', format: 'int32' }, + ], + }, + }, + }, + }); + + it('should handle null in nullable oneOf', () => { + const data = { + record: { + id: 1, + value: null, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle string variant', () => { + const data = { + record: { + id: 1, + value: 'text', + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle integer variant', () => { + const data = { + record: { + id: 1, + value: 42, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Complex object with all OpenAPI features', () => { + const Schema = schema({ + entity: { + type: 'object', + schema: { + id: { type: 'string', format: 'uuid' }, + type: { type: 'string' }, + attributes: { + type: 'object', + schema: { + name: { type: 'string' }, + age: { type: 'integer', format: 'int32', nullable: true }, + balance: { type: 'number', format: 'double' }, + }, + }, + tags: { type: 'array', items: { type: 'string' } }, + metadata: { + oneOf: [ + { type: 'string' }, + { type: 'object', schema: { key: { type: 'string' }, value: { type: 'string' } } }, + ], + }, + created: { type: 'string', format: 'date-time' }, + }, + }, + }); + + it('should handle complex object with all features', () => { + const data = { + entity: { + id: '550e8400-e29b-4d4e-a7d4-426614174000', + type: 'user', + attributes: { + name: 'John Doe', + age: null, + balance: 1234.56, + }, + tags: ['active', 'premium'], + metadata: { key: 'region', value: 'us-east-1' }, + created: '2025-10-28T14:30:00.000Z', + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle with string metadata variant', () => { + const data = { + entity: { + id: '550e8400-e29b-4d4e-a7d4-426614174000', + type: 'user', + attributes: { + name: 'Jane Doe', + age: 25, + balance: 5678.90, + }, + tags: ['new'], + metadata: 'simple string metadata', + created: '2025-10-28T15:00:00.000Z', + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Property order independence', () => { + const Schema = schema({ + data: { + type: 'object', + schema: { + first: { type: 'string' }, + second: { type: 'integer', format: 'int32' }, + third: { type: 'boolean' }, + fourth: { type: 'number', format: 'double' }, + }, + }, + }); + + it('should handle properties in schema order', () => { + const data = { + data: { + first: 'value1', + second: 42, + third: true, + fourth: 3.14, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should handle properties in different order than schema', () => { + const data = { + data: { + fourth: 3.14, + first: 'value1', + third: true, + second: 42, + }, + }; + const expected = { + data: { + first: 'value1', + second: 42, + third: true, + fourth: 3.14, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(expected); + }); + + it('should handle properties in reverse order', () => { + const data = { + data: { + fourth: 2.71, + third: false, + second: 100, + first: 'reversed', + }, + }; + const expected = { + data: { + first: 'reversed', + second: 100, + third: false, + fourth: 2.71, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(expected); + }); + + it('should handle properties in random order', () => { + const data = { + data: { + third: true, + first: 'random', + fourth: 1.41, + second: 7, + }, + }; + const expected = { + data: { + first: 'random', + second: 7, + third: true, + fourth: 1.41, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(expected); + }); + }); + + describe('Undeclared properties validation', () => { + const Schema = schema({ + data: { + type: 'object', + schema: { + declared: { type: 'string' }, + }, + }, + }); + + it('should ignore undeclared properties', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(); + + const input = { + data: { + declared: 'value', + undeclared: 'should be ignored', + }, + }; + const result = Schema.read(Schema.write(input).buffer()); + expect(result).toEqual({ data: { declared: 'value' } }); + expect(result.data).not.toHaveProperty('undeclared'); + + warnSpy.mockRestore(); + }); + + it('should always warn about undeclared properties', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(); + + const input = { + data: { + declared: 'value', + extra1: 'ignored', + extra2: 'also ignored', + }, + }; + + Schema.write(input); + + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('undeclared properties'), + ); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('extra1, extra2'), + ); + + warnSpy.mockRestore(); + }); + + it('should not warn when all properties are declared', () => { + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(); + + const input = { + data: { + declared: 'value', + }, + }; + + Schema.write(input); + + expect(warnSpy).not.toHaveBeenCalled(); + + warnSpy.mockRestore(); + }); + }); +}); From a800581fc21788ba4ab7d575814224e9f75ca0c0 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Thu, 30 Oct 2025 21:01:30 -0400 Subject: [PATCH 11/19] added missing openapi features, modernized benchmark suite --- benchmarks/array.ts | 57 ++-- benchmarks/boolean.ts | 75 +++-- benchmarks/double.ts | 72 ---- benchmarks/index.ts | 39 ++- benchmarks/integer.ts | 78 +++-- benchmarks/jsonapiresponse.ts | 85 +++++ benchmarks/object.ts | 78 ----- benchmarks/schema.ts | 90 +++++ benchmarks/string.ts | 79 +++-- benchmarks/utils.ts | 18 + eslint.config.mjs | 1 + src/encoder.ts | 21 +- src/schema.ts | 122 ++++++- src/writer.ts | 22 +- static/exampleSpec.ts | 614 ++++++++++++++++++++++++++++++++++ tests/integration/index.ts | 460 ++++++++++++++++++++++++- tests/integration/openapi.ts | 40 +++ types.d.ts | 3 + 18 files changed, 1640 insertions(+), 314 deletions(-) delete mode 100644 benchmarks/double.ts create mode 100644 benchmarks/jsonapiresponse.ts delete mode 100644 benchmarks/object.ts create mode 100644 benchmarks/schema.ts create mode 100644 benchmarks/utils.ts create mode 100644 static/exampleSpec.ts create mode 100644 tests/integration/openapi.ts diff --git a/benchmarks/array.ts b/benchmarks/array.ts index 96b5112..24a672f 100644 --- a/benchmarks/array.ts +++ b/benchmarks/array.ts @@ -2,15 +2,16 @@ /* Requires ------------------------------------------------------------------*/ -const Benchmark = require('benchmark'); -const Compactr = require('../'); +import Benchmark from 'benchmark'; +import {schema} from '../dist/compactr.js'; +import {deferred} from './utils.ts'; /* Local variables -----------------------------------------------------------*/ -let User = Compactr.schema({ - id: { type: 'integer', format: 'int32', size: 4 }, - arr: { type: 'array', size: 6, items: { type: 'string', size: 1 }}, +let User = schema({ + id: { type: 'integer', format: 'int32' }, + arr: { type: 'array', items: { type: 'string' }}, }); const mult = 32; @@ -18,30 +19,36 @@ const sizes = { json: 0, compactr: 0 }; const arraySuite = new Benchmark.Suite(); -/* Float suite ---------------------------------------------------------------*/ +/* Array suite ---------------------------------------------------------------*/ -arraySuite.add('[Array] JSON', arrJSON) - .add('[Array] Compactr', arrCompactr) - .on('cycle', e => console.log(String(e.target))) - .run({ 'async': true }) - .on('complete', _ => console.log(sizes)); +export function init() { + const {promise, resolve} = deferred(); -function arrJSON() { - let packed, unpacked; + arraySuite.add('[Array] JSON', arrJSON) + .add('[Array] Compactr', arrCompactr) + .on('cycle', e => console.log(String(e.target))) + .run({ 'async': true }) + .on('complete', _ => resolve(sizes)); - for(let i = 0; i sizes.json) sizes.json = packed.length; + function arrJSON() { + let packed, unpacked; + + for(let i = 0; i sizes.json) sizes.json = packed.length; + } } -} -function arrCompactr() { - let packed, unpacked; + function arrCompactr() { + let packed, unpacked; - for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; + for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; + } } -} \ No newline at end of file + + return promise; +} diff --git a/benchmarks/boolean.ts b/benchmarks/boolean.ts index 405efd6..b9ca5a8 100644 --- a/benchmarks/boolean.ts +++ b/benchmarks/boolean.ts @@ -2,15 +2,16 @@ /* Requires ------------------------------------------------------------------*/ -const Benchmark = require('benchmark'); -const Compactr = require('../'); -const protobuf = require('protobufjs'); +import Benchmark from 'benchmark'; +import {schema} from '../dist/compactr.js'; +import protobuf from 'protobufjs'; +import {deferred} from './utils.ts'; /* Local variables -----------------------------------------------------------*/ -let User = Compactr.schema({ - id: { type: 'int32', size: 4 }, +let User = schema({ + id: { type: 'int32' }, bool: { type: 'boolean' }, }); @@ -31,42 +32,48 @@ var BoolBenchTest = root.lookupType('BoolBenchTest'); const boolSuite = new Benchmark.Suite(); -/* Float suite ---------------------------------------------------------------*/ +/* Boolean suite ---------------------------------------------------------------*/ -boolSuite.add('[Boolean] JSON', boolJSON) - .add('[Boolean] Compactr', boolCompactr) - .add('[Boolean] Protobuf', boolProtobuf) - .on('cycle', e => console.log(String(e.target))) - .run({ 'async': true }) - .on('complete', _ => console.log(sizes)); +export function init() { + const {promise, resolve} = deferred(); -function boolJSON(e) { - let packed, unpacked; + boolSuite.add('[Boolean] JSON', boolJSON) + .add('[Boolean] Compactr', boolCompactr) + .add('[Boolean] Protobuf', boolProtobuf) + .on('cycle', e => console.log(String(e.target))) + .run({ 'async': true }) + .on('complete', _ => resolve(sizes)); - for(let i = 0; i sizes.json) sizes.json = packed.length; + function boolJSON(e) { + let packed, unpacked; + + for(let i = 0; i sizes.json) sizes.json = packed.length; + } } -} -function boolCompactr() { - let packed, unpacked; + function boolCompactr() { + let packed, unpacked; - for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; + for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; + } } -} -function boolProtobuf() { - let packed, unpacked; + function boolProtobuf() { + let packed, unpacked; - for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; + for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; + } } -} \ No newline at end of file + + return promise; +} diff --git a/benchmarks/double.ts b/benchmarks/double.ts deleted file mode 100644 index b7bc1ce..0000000 --- a/benchmarks/double.ts +++ /dev/null @@ -1,72 +0,0 @@ -/** Benchmarks */ - -/* Requires ------------------------------------------------------------------*/ - -const Benchmark = require('benchmark'); -const Compactr = require('../'); -const protobuf = require('protobufjs'); - -/* Local variables -----------------------------------------------------------*/ - - -let User = Compactr.schema({ - id: { type: 'int32', size: 4 }, - int: { type: 'double', size: 8 }, -}); - -const mult = 32; -const sizes = { json: 0, compactr: 0, protobuf: 0 }; - -let root = protobuf.Root.fromJSON({ - nested: { - DoubleBenchTest: { - fields: { - id: { type: 'uint32', id: 1 }, - int: { type: 'double', id: 2 }, - }, - }, - }, -}); -var DoubleBenchTest = root.lookupType('DoubleBenchTest'); - -const floatSuite = new Benchmark.Suite(); - -/* Float suite ---------------------------------------------------------------*/ - -floatSuite.add('[Float] JSON', floatJSON) - .add('[Float] Compactr', floatCompactr) - .add('[Float] Protobuf', floatProtobuf) - .on('cycle', e => console.log(String(e.target))) - .run({ 'async': true }) - .on('complete', _ => console.log(sizes)); - -function floatJSON() { - let packed, unpacked; - - for(let i = 0; i sizes.json) sizes.json = packed.length; - } -} - -function floatCompactr() { - let packed, unpacked; - - for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; - } -} - -function floatProtobuf() { - let packed, unpacked; - - for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; - } -} \ No newline at end of file diff --git a/benchmarks/index.ts b/benchmarks/index.ts index 840a258..87b9992 100644 --- a/benchmarks/index.ts +++ b/benchmarks/index.ts @@ -1,27 +1,26 @@ -#!/usr/bin/env node -/** Benchmark runner script */ +// Synthetic +import { init as array } from './array.ts'; +import { init as boolean } from './boolean.ts'; +import { init as integer } from './integer.ts'; +import { init as schema } from './schema.ts'; +import { init as string } from './string.ts'; -import { execSync } from 'child_process'; +// Realistic +import { init as jsonapiresponse } from './jsonapiresponse.ts'; + +import { sequence } from './utils.ts'; const benchmarks = [ - 'array', - 'boolean', - 'double', - 'integer', - 'object', - 'string', + array, + boolean, + integer, + schema, + string, + jsonapiresponse, ]; console.log('Running Compactr benchmarks...\n'); -for (const benchmark of benchmarks) { - console.log(`\n=== ${benchmark.toUpperCase()} BENCHMARK ===\n`); - try { - execSync(`node ./${benchmark}.ts`, { stdio: 'inherit' }); - } - catch (error) { - console.error(`Failed to run ${benchmark} benchmark:`, error); - } -} - -console.log('\nAll benchmarks completed!'); +sequence(benchmarks, (i) => i().then((sizes) => console.log(sizes))).then(() => { + console.log('\nAll benchmarks completed!'); +}); diff --git a/benchmarks/integer.ts b/benchmarks/integer.ts index beeee55..145036a 100644 --- a/benchmarks/integer.ts +++ b/benchmarks/integer.ts @@ -2,16 +2,16 @@ /* Requires ------------------------------------------------------------------*/ -const Benchmark = require('benchmark'); -const Compactr = require('../'); -const protobuf = require('protobufjs'); +import Benchmark from 'benchmark'; +import {schema} from '../dist/compactr.js'; +import protobuf from 'protobufjs'; +import {deferred} from './utils.ts'; /* Local variables -----------------------------------------------------------*/ - -let User = Compactr.schema({ - id: { type: 'int32', size: 4 }, - int: { type: 'int32', size: 8 }, +let User = schema({ + id: { type: 'int32' }, + int: { type: 'int32' }, }); const mult = 32; @@ -31,42 +31,48 @@ var IntBenchTest = root.lookupType('IntBenchTest'); const intSuite = new Benchmark.Suite(); -/* Float suite ---------------------------------------------------------------*/ +/* Integer suite ---------------------------------------------------------------*/ + +export function init() { + const {promise, resolve} = deferred(); -intSuite.add('[Integer] JSON', intJSON) - .add('[Integer] Compactr', intCompactr) - .add('[Integer] Protobuf', intProtobuf) - .on('cycle', e => console.log(String(e.target))) - .run({ 'async': true }) - .on('complete', _ => console.log(sizes)); + intSuite.add('[Integer] JSON', intJSON) + .add('[Integer] Compactr', intCompactr) + .add('[Integer] Protobuf', intProtobuf) + .on('cycle', e => console.log(String(e.target))) + .run({ 'async': true }) + .on('complete', _ => resolve(sizes)); -function intJSON() { - let packed, unpacked; + function intJSON() { + let packed, unpacked; - for(let i = 0; i sizes.json) sizes.json = packed.length; + for(let i = 0; i sizes.json) sizes.json = packed.length; + } } -} -function intCompactr() { - let packed, unpacked; + function intCompactr() { + let packed, unpacked; - for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; + for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; + } } -} -function intProtobuf() { - let packed, unpacked; + function intProtobuf() { + let packed, unpacked; - for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; + for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; + } } -} \ No newline at end of file + + return promise; +} diff --git a/benchmarks/jsonapiresponse.ts b/benchmarks/jsonapiresponse.ts new file mode 100644 index 0000000..cf7ffe4 --- /dev/null +++ b/benchmarks/jsonapiresponse.ts @@ -0,0 +1,85 @@ +/** Benchmarks */ + +/* Requires ------------------------------------------------------------------*/ + +import Benchmark from 'benchmark'; +import {schema} from '../dist/compactr.js'; +import protobuf from 'protobufjs'; +import {deferred} from './utils.ts'; + +/* Local variables -----------------------------------------------------------*/ + + +let User = schema({ + id: { type: 'int32', size: 4 }, + obj: { type: 'object', size: 9, schema: { str: { type: 'string', size: 6 } } }, +}); + +const mult = 32; +const sizes = { json: 0, compactr: 0, protobuf: 0 }; + +let root = protobuf.Root.fromJSON({ + nested: { + StringBenchTest: { + fields: { + str: { type: 'string', id: 2 }, + }, + }, + ObjectBenchTest: { + fields: { + id: { type: 'uint32', id: 1 }, + obj: { type: 'StringBenchTest', id: 2}, + }, + }, + }, +}); +var ObjectBenchTest = root.lookupType('ObjectBenchTest'); + +const objectSuite = new Benchmark.Suite(); + +/* JSON-API Reponse suite ---------------------------------------------------------------*/ + +export function init() { + const {promise, resolve} = deferred(); + + objectSuite.add('[JSON-API Reponse] JSON', objJSON) + .add('[JSON-API Reponse] Compactr', objCompactr) + .add('[JSON-API Reponse] Protobuf', objProtobuf) + .on('cycle', e => console.log(String(e.target))) + .run({ 'async': true }) + .on('complete', _ => resolve(sizes)); + + + function objJSON() { + let packed, unpacked; + + for(let i = 0; i sizes.json) sizes.json = packed.length; + } + } + + function objCompactr() { + let packed, unpacked; + + for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; + } + } + + function objProtobuf() { + let packed, unpacked; + + for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; + } + } + + return promise; +} diff --git a/benchmarks/object.ts b/benchmarks/object.ts deleted file mode 100644 index 92c07de..0000000 --- a/benchmarks/object.ts +++ /dev/null @@ -1,78 +0,0 @@ -/** Benchmarks */ - -/* Requires ------------------------------------------------------------------*/ - -const Benchmark = require('benchmark'); -const Compactr = require('../'); -const protobuf = require('protobufjs'); - -/* Local variables -----------------------------------------------------------*/ - - -let User = Compactr.schema({ - id: { type: 'int32', size: 4 }, - obj: { type: 'object', size: 9, schema: { str: { type: 'string', size: 6 } } }, -}); - -const mult = 32; -const sizes = { json: 0, compactr: 0, protobuf: 0 }; - -let root = protobuf.Root.fromJSON({ - nested: { - StringBenchTest: { - fields: { - str: { type: 'string', id: 2 }, - }, - }, - ObjectBenchTest: { - fields: { - id: { type: 'uint32', id: 1 }, - obj: { type: 'StringBenchTest', id: 2}, - }, - }, - }, -}); -var ObjectBenchTest = root.lookupType('ObjectBenchTest'); - -const objectSuite = new Benchmark.Suite(); - -/* Float suite ---------------------------------------------------------------*/ - -objectSuite.add('[Object] JSON', objJSON) - .add('[Object] Compactr', objCompactr) - .add('[Object] Protobuf', objProtobuf) - .on('cycle', e => console.log(String(e.target))) - .run({ 'async': true }) - .on('complete', _ => console.log(sizes)); - - -function objJSON() { - let packed, unpacked; - - for(let i = 0; i sizes.json) sizes.json = packed.length; - } -} - -function objCompactr() { - let packed, unpacked; - - for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; - } -} - -function objProtobuf() { - let packed, unpacked; - - for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; - } -} \ No newline at end of file diff --git a/benchmarks/schema.ts b/benchmarks/schema.ts new file mode 100644 index 0000000..19890ea --- /dev/null +++ b/benchmarks/schema.ts @@ -0,0 +1,90 @@ +/** Benchmarks */ + +/* Requires ------------------------------------------------------------------*/ + +import Benchmark from 'benchmark'; +import {schema} from '../dist/compactr.js'; +import protobuf from 'protobufjs'; +import {deferred} from './utils.ts'; + +/* Local variables -----------------------------------------------------------*/ + + +let User = schema({ + id: { type: 'int32' }, + obj: { + type: 'object', + properties: { + str: { type: 'string' } + }, + }, +}); + +const mult = 32; +const sizes = { json: 0, compactr: 0, protobuf: 0 }; + +let root = protobuf.Root.fromJSON({ + nested: { + StringBenchTest: { + fields: { + str: { type: 'string', id: 2 }, + }, + }, + ObjectBenchTest: { + fields: { + id: { type: 'uint32', id: 1 }, + obj: { type: 'StringBenchTest', id: 2}, + }, + }, + }, +}); +var ObjectBenchTest = root.lookupType('ObjectBenchTest'); + +const objectSuite = new Benchmark.Suite(); + +/* Schema suite ---------------------------------------------------------------*/ + +export function init() { + const {promise, resolve} = deferred(); + + objectSuite.add('[Schema] JSON', objJSON) + .add('[Schema] Compactr', objCompactr) + .add('[Schema] Protobuf', objProtobuf) + .on('cycle', e => console.log(String(e.target))) + .run({ 'async': true }) + .on('complete', _ => resolve(sizes)); + + + function objJSON() { + let packed, unpacked; + + for(let i = 0; i sizes.json) sizes.json = packed.length; + } + } + + function objCompactr() { + let packed, unpacked; + + for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; + } + } + + function objProtobuf() { + let packed, unpacked; + + for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; + } + } + + return promise; +} diff --git a/benchmarks/string.ts b/benchmarks/string.ts index f638fe9..0d13101 100644 --- a/benchmarks/string.ts +++ b/benchmarks/string.ts @@ -2,17 +2,18 @@ /* Requires ------------------------------------------------------------------*/ -const Benchmark = require('benchmark'); -const Compactr = require('../'); -const protobuf = require('protobufjs'); +import Benchmark from 'benchmark'; +import {schema} from '../dist/compactr.js'; +import protobuf from 'protobufjs'; +import {deferred} from './utils.ts'; /* Local variables -----------------------------------------------------------*/ -let User = Compactr.schema({ - id: { type: 'integer', format: 'int32', size: 4 }, - str: { type: 'string', size: 6 }, - special: { type: 'string', size: 4 }, +let User = schema({ + id: { type: 'integer', format: 'int32'}, + str: { type: 'string' }, + special: { type: 'string' }, }); let root = protobuf.Root.fromJSON({ @@ -33,43 +34,49 @@ const sizes = { json: 0, compactr: 0, protobuf: 0 }; const stringSuite = new Benchmark.Suite(); -/* Float suite ---------------------------------------------------------------*/ +/* String suite ---------------------------------------------------------------*/ -stringSuite.add('[String] JSON', strJSON) - .add('[String] Compactr', strCompactr) - .add('[String] Protobuf', strProtobuf) - .on('cycle', e => console.log(String(e.target))) - .run({ 'async': true }) - .on('complete', _ => console.log(sizes)); +export function init() { + const {promise, resolve} = deferred(); + stringSuite.add('[String] JSON', strJSON) + .add('[String] Compactr', strCompactr) + .add('[String] Protobuf', strProtobuf) + .on('cycle', e => console.log(String(e.target))) + .run({ 'async': true }) + .on('complete', _ => resolve(sizes)); -function strJSON() { - let packed, unpacked; - for(let i = 0; i sizes.json) sizes.json = packed.length; + function strJSON() { + let packed, unpacked; + + for(let i = 0; i sizes.json) sizes.json = packed.length; + } } -} -function strCompactr() { - let packed, unpacked; + function strCompactr() { + let packed, unpacked; - for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; + for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; + } } -} -function strProtobuf() { - let packed, unpacked; + function strProtobuf() { + let packed, unpacked; - for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; + for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; + } } -} \ No newline at end of file + + return promise; +} diff --git a/benchmarks/utils.ts b/benchmarks/utils.ts new file mode 100644 index 0000000..03a1740 --- /dev/null +++ b/benchmarks/utils.ts @@ -0,0 +1,18 @@ +export function deferred() { + let resolve: (value?: T | PromiseLike) => void; + let reject: (reason?: any) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + + return { promise, resolve, reject }; + } + + export function sequence(array: Array, operation: (item: K, index: number) => Promise): Promise> { + return array.reduce((promiseChain: Promise>, item, index) => { + return promiseChain.then((chainResults: Array) => { + return operation(item, index).then((currentResult) => [...chainResults, currentResult]); + }); + }, Promise.resolve([])); + } \ No newline at end of file diff --git a/eslint.config.mjs b/eslint.config.mjs index 7ed04c6..ba3eafd 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -15,6 +15,7 @@ export default tseslint.config( '@typescript-eslint/no-require-imports': 1, 'jest/no-done-callback': 0, 'jest/no-conditional-expect': 0, + 'no-prototype-builtins': 'warn', }, }, { diff --git a/src/encoder.ts b/src/encoder.ts index 67ecab0..6ea2437 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -278,7 +278,26 @@ function matchesVariantItem(data, variant) { } if (variant.type === 'object') { - return dataType === 'object'; + if (dataType !== 'object') return false; + + // For objects with schema keys, check if the data properties match + if (variant.schemaKeys && variant.schemaKeys.length > 0) { + const schemaKeys = variant.schemaKeys; + const dataKeys = Object.keys(data); + + // Check if data keys match schema keys + let matchCount = 0; + for (const key of schemaKeys) { + if (data.hasOwnProperty(key)) { + matchCount++; + } + } + + // Require at least one matching key and 50% overlap + return matchCount > 0 && matchCount >= Math.min(schemaKeys.length, dataKeys.length) * 0.5; + } + + return true; } return false; diff --git a/src/schema.ts b/src/schema.ts index 3130eaf..8404fe4 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -38,6 +38,16 @@ function resolveType(type, format) { } export default function Schema(schema, options = { keyOrder: false }) { + // Handle top-level schema object (OpenAPI format) + // If schema has type: 'object' and properties, unwrap it + let unwrappedSchema = schema; + if (schema.type === 'object' && schema.properties) { + unwrappedSchema = schema.properties; + } + + // Normalize OpenAPI schema format to internal format + const normalizedSchema = normalizeSchema(unwrappedSchema, options); + const sizeRef = { 'boolean': 1, 'int32': 4, @@ -69,16 +79,99 @@ export default function Schema(schema, options = { keyOrder: false }) { }; const scope = { - schema, + schema: normalizedSchema, indices: {}, - items: Object.keys(schema), + items: Object.keys(normalizedSchema), headerBytes: [0], contentBytes: [0], header: [], contentBegins: 0, options, }; - scope.indices = preformat(schema); + scope.indices = preformat(normalizedSchema); + + /** @private */ + function resolveRef(ref, options) { + if (!ref || !ref.startsWith('#/')) { + throw new Error(`Invalid $ref format: ${ref}. Only internal references (#/...) are supported.`); + } + + const parts = ref.split('/').slice(1); // Remove leading '#' + let resolved = options.schemas; + + for (const part of parts) { + if (!resolved || typeof resolved !== 'object') { + throw new Error(`Cannot resolve $ref: ${ref}`); + } + resolved = resolved[part]; + } + + if (!resolved) { + throw new Error(`$ref not found: ${ref}`); + } + + return resolved; + } + + /** @private */ + function normalizeFieldDefinition(fieldDef, options) { + // Handle $ref + if (fieldDef.$ref) { + if (!options.schemas) { + throw new Error(`$ref "${fieldDef.$ref}" found but no schemas provided in options`); + } + // Resolve the reference + let resolved = resolveRef(fieldDef.$ref, options); + + // Unwrap if the component is a wrapped object schema + if (resolved.type === 'object' && resolved.properties && !resolved.schema) { + resolved = { ...resolved }; + resolved.schema = resolved.properties; + delete resolved.properties; + } + + // Normalize the resolved component + return normalizeFieldDefinition(resolved, options); + } + + // Create a copy to avoid mutating the original + const normalized = { ...fieldDef }; + + // Transform OpenAPI 'properties' to internal 'schema' + if (normalized.properties && !normalized.schema) { + normalized.schema = normalizeSchema(normalized.properties, options); + delete normalized.properties; + } + + // Normalize nested items + if (normalized.items) { + normalized.items = normalizeFieldDefinition(normalized.items, options); + } + + // Normalize oneOf/anyOf variants + if (normalized.oneOf) { + normalized.oneOf = normalized.oneOf.map(v => normalizeFieldDefinition(v, options)); + } + if (normalized.anyOf) { + normalized.anyOf = normalized.anyOf.map(v => normalizeFieldDefinition(v, options)); + } + + // Normalize nested schema + if (normalized.schema && typeof normalized.schema === 'object') { + normalized.schema = normalizeSchema(normalized.schema, options); + } + + return normalized; + } + + /** @private */ + function normalizeSchema(schema, options) { + const normalized = {}; + for (const key in schema) { + normalized[key] = normalizeFieldDefinition(schema[key], options); + } + return normalized; + } const writer = Writer(scope); const reader = Reader(scope); @@ -101,6 +194,11 @@ export default function Schema(schema, options = { keyOrder: false }) { const variantCount = variantDef.count || (variantInternalType === 'binary' ? 4 : 1); const variantChildSchema = computeNestedVariant(variantDef); + // For object variants, extract schema keys for variant matching + const schemaKeys = (variantInternalType === 'object' && variantDef.schema) + ? Object.keys(variantDef.schema) + : null; + return { type: variantInternalType, transformIn: (variantChildSchema !== undefined) @@ -115,6 +213,7 @@ export default function Schema(schema, options = { keyOrder: false }) { size: variantDef.size || defaultSizes[variantInternalType] || null, count: variantCount, nested: variantChildSchema, + schemaKeys, }; }); @@ -176,7 +275,10 @@ export default function Schema(schema, options = { keyOrder: false }) { let childSchema; if (isObject === true || isArray === true) { - if (isObject === true) childSchema = Schema(schema[key].schema, options); + if (isObject === true) { + // After normalization, schema should always be present for objects + childSchema = Schema(schema[key].schema, options); + } if (isArray === true) { childSchema = processArrayItems(schema[key].items); } @@ -197,6 +299,11 @@ export default function Schema(schema, options = { keyOrder: false }) { const variantCount = variantDef.count || (variantInternalType === 'binary' ? 4 : 1); const variantChildSchema = processArrayItemsNested(variantDef); + // For object variants, extract schema keys for variant matching + const schemaKeys = (variantInternalType === 'object' && variantDef.schema) + ? Object.keys(variantDef.schema) + : null; + return { type: variantInternalType, transformIn: (variantChildSchema !== undefined) @@ -211,6 +318,7 @@ export default function Schema(schema, options = { keyOrder: false }) { size: variantDef.size || defaultSizes[variantInternalType] || null, count: variantCount, nested: variantChildSchema, + schemaKeys, }; }); @@ -245,6 +353,7 @@ export default function Schema(schema, options = { keyOrder: false }) { const isArray = (itemType === 'array'); if (isObject === true) { + // After normalization, schema should always be present for objects return Schema(itemDef.schema, options); } @@ -263,7 +372,10 @@ export default function Schema(schema, options = { keyOrder: false }) { let childSchema; if (isObject === true || isArray === true) { - if (isObject === true) childSchema = Schema(variantDef.schema, options); + if (isObject === true) { + // After normalization, schema should always be present for objects + childSchema = Schema(variantDef.schema, options); + } if (isArray === true) { childSchema = processArrayItems(variantDef.items); } diff --git a/src/writer.ts b/src/writer.ts index 900e13d..f28c907 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -122,7 +122,27 @@ export default function Writer(scope) { } if (variant.type === 'object') { - return dataType === 'object'; + if (dataType !== 'object') return false; + + // For objects with schema keys, check if the data properties match + if (variant.schemaKeys && variant.schemaKeys.length > 0) { + const schemaKeys = variant.schemaKeys; + const dataKeys = Object.keys(data); + + // Check if data keys match schema keys + let matchCount = 0; + for (const key of schemaKeys) { + if (data.hasOwnProperty(key)) { + matchCount++; + } + } + + // Require at least one matching key and 50% overlap + // This helps distinguish between different object variants + return matchCount > 0 && matchCount >= Math.min(schemaKeys.length, dataKeys.length) * 0.5; + } + + return true; } return false; diff --git a/static/exampleSpec.ts b/static/exampleSpec.ts new file mode 100644 index 0000000..62692eb --- /dev/null +++ b/static/exampleSpec.ts @@ -0,0 +1,614 @@ +// Pulling an OpenAPI spec from https://apis.guru/ + +export default +{ + openapi: '3.0.0', + servers: [ + { + url: 'https://www.googleapis.com/siteVerification/v1', + }, + ], + info: { + 'contact': { + 'name': 'Google', + 'url': 'https://google.com', + 'x-twitter': 'youtube', + }, + 'description': 'Verifies ownership of websites or domains with Google.', + 'license': { + name: 'Creative Commons Attribution 3.0', + url: 'http://creativecommons.org/licenses/by/3.0/', + }, + 'termsOfService': 'https://developers.google.com/terms/', + 'title': 'Google Site Verification API', + 'version': 'v1', + 'x-apiClientRegistration': { + url: 'https://console.developers.google.com', + }, + 'x-apisguru-categories': [ + 'analytics', + 'media', + ], + 'x-logo': { + url: 'https://api.apis.guru/v2/cache/logo/https_www.google.com_images_branding_googlelogo_2x_googlelogo_color_272x92dp.png', + }, + 'x-origin': [ + { + format: 'google', + url: 'https://siteverification.googleapis.com/$discovery/rest?version=v1', + version: 'v1', + }, + ], + 'x-providerName': 'googleapis.com', + 'x-serviceName': 'siteVerification', + }, + externalDocs: { + url: 'https://developers.google.com/site-verification/', + }, + tags: [ + { + name: 'webResource', + }, + ], + paths: { + '/token': { + parameters: [ + { + $ref: '#/components/parameters/alt', + }, + { + $ref: '#/components/parameters/fields', + }, + { + $ref: '#/components/parameters/key', + }, + { + $ref: '#/components/parameters/oauth_token', + }, + { + $ref: '#/components/parameters/prettyPrint', + }, + { + $ref: '#/components/parameters/quotaUser', + }, + { + $ref: '#/components/parameters/userIp', + }, + ], + post: { + description: 'Get a verification token for placing on a website or domain.', + operationId: 'siteVerification.webResource.getToken', + requestBody: { + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/SiteVerificationWebResourceGettokenRequest', + }, + }, + }, + }, + responses: { + 200: { + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/SiteVerificationWebResourceGettokenResponse', + }, + }, + }, + description: 'Successful response', + }, + }, + security: [ + { + Oauth2: [ + 'https://www.googleapis.com/auth/siteverification', + ], + Oauth2c: [ + 'https://www.googleapis.com/auth/siteverification', + ], + }, + { + Oauth2: [ + 'https://www.googleapis.com/auth/siteverification.verify_only', + ], + Oauth2c: [ + 'https://www.googleapis.com/auth/siteverification.verify_only', + ], + }, + ], + tags: [ + 'webResource', + ], + }, + }, + '/webResource': { + get: { + description: 'Get the list of your verified websites and domains.', + operationId: 'siteVerification.webResource.list', + responses: { + 200: { + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/SiteVerificationWebResourceListResponse', + }, + }, + }, + description: 'Successful response', + }, + }, + security: [ + { + Oauth2: [ + 'https://www.googleapis.com/auth/siteverification', + ], + Oauth2c: [ + 'https://www.googleapis.com/auth/siteverification', + ], + }, + ], + tags: [ + 'webResource', + ], + }, + parameters: [ + { + $ref: '#/components/parameters/alt', + }, + { + $ref: '#/components/parameters/fields', + }, + { + $ref: '#/components/parameters/key', + }, + { + $ref: '#/components/parameters/oauth_token', + }, + { + $ref: '#/components/parameters/prettyPrint', + }, + { + $ref: '#/components/parameters/quotaUser', + }, + { + $ref: '#/components/parameters/userIp', + }, + ], + post: { + description: 'Attempt verification of a website or domain.', + operationId: 'siteVerification.webResource.insert', + parameters: [ + { + description: 'The method to use for verifying a site or domain.', + in: 'query', + name: 'verificationMethod', + required: true, + schema: { + type: 'string', + }, + }, + ], + requestBody: { + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/SiteVerificationWebResourceResource', + }, + }, + }, + }, + responses: { + 200: { + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/SiteVerificationWebResourceResource', + }, + }, + }, + description: 'Successful response', + }, + }, + security: [ + { + Oauth2: [ + 'https://www.googleapis.com/auth/siteverification', + ], + Oauth2c: [ + 'https://www.googleapis.com/auth/siteverification', + ], + }, + { + Oauth2: [ + 'https://www.googleapis.com/auth/siteverification.verify_only', + ], + Oauth2c: [ + 'https://www.googleapis.com/auth/siteverification.verify_only', + ], + }, + ], + tags: [ + 'webResource', + ], + }, + }, + '/webResource/{id}': { + delete: { + description: 'Relinquish ownership of a website or domain.', + operationId: 'siteVerification.webResource.delete', + parameters: [ + { + description: 'The id of a verified site or domain.', + in: 'path', + name: 'id', + required: true, + schema: { + type: 'string', + }, + }, + ], + responses: { + 200: { + description: 'Successful response', + }, + }, + security: [ + { + Oauth2: [ + 'https://www.googleapis.com/auth/siteverification', + ], + Oauth2c: [ + 'https://www.googleapis.com/auth/siteverification', + ], + }, + ], + tags: [ + 'webResource', + ], + }, + get: { + description: 'Get the most current data for a website or domain.', + operationId: 'siteVerification.webResource.get', + parameters: [ + { + description: 'The id of a verified site or domain.', + in: 'path', + name: 'id', + required: true, + schema: { + type: 'string', + }, + }, + ], + responses: { + 200: { + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/SiteVerificationWebResourceResource', + }, + }, + }, + description: 'Successful response', + }, + }, + security: [ + { + Oauth2: [ + 'https://www.googleapis.com/auth/siteverification', + ], + Oauth2c: [ + 'https://www.googleapis.com/auth/siteverification', + ], + }, + ], + tags: [ + 'webResource', + ], + }, + parameters: [ + { + $ref: '#/components/parameters/alt', + }, + { + $ref: '#/components/parameters/fields', + }, + { + $ref: '#/components/parameters/key', + }, + { + $ref: '#/components/parameters/oauth_token', + }, + { + $ref: '#/components/parameters/prettyPrint', + }, + { + $ref: '#/components/parameters/quotaUser', + }, + { + $ref: '#/components/parameters/userIp', + }, + ], + patch: { + description: 'Modify the list of owners for your website or domain. This method supports patch semantics.', + operationId: 'siteVerification.webResource.patch', + parameters: [ + { + description: 'The id of a verified site or domain.', + in: 'path', + name: 'id', + required: true, + schema: { + type: 'string', + }, + }, + ], + requestBody: { + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/SiteVerificationWebResourceResource', + }, + }, + }, + }, + responses: { + 200: { + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/SiteVerificationWebResourceResource', + }, + }, + }, + description: 'Successful response', + }, + }, + security: [ + { + Oauth2: [ + 'https://www.googleapis.com/auth/siteverification', + ], + Oauth2c: [ + 'https://www.googleapis.com/auth/siteverification', + ], + }, + ], + tags: [ + 'webResource', + ], + }, + put: { + description: 'Modify the list of owners for your website or domain.', + operationId: 'siteVerification.webResource.update', + parameters: [ + { + description: 'The id of a verified site or domain.', + in: 'path', + name: 'id', + required: true, + schema: { + type: 'string', + }, + }, + ], + requestBody: { + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/SiteVerificationWebResourceResource', + }, + }, + }, + }, + responses: { + 200: { + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/SiteVerificationWebResourceResource', + }, + }, + }, + description: 'Successful response', + }, + }, + security: [ + { + Oauth2: [ + 'https://www.googleapis.com/auth/siteverification', + ], + Oauth2c: [ + 'https://www.googleapis.com/auth/siteverification', + ], + }, + ], + tags: [ + 'webResource', + ], + }, + }, + }, + components: { + user: { + description: 'User model', + type: 'object', + properties: { + id: { type: 'string', required: true }, + name: { type: 'string' }, + age: { type: 'integer' }, + }, + }, + parameters: { + alt: { + description: 'Data format for the response.', + in: 'query', + name: 'alt', + schema: { + enum: [ + 'json', + ], + type: 'string', + }, + }, + fields: { + description: 'Selector specifying which fields to include in a partial response.', + in: 'query', + name: 'fields', + schema: { + type: 'string', + }, + }, + key: { + description: 'API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.', + in: 'query', + name: 'key', + schema: { + type: 'string', + }, + }, + oauth_token: { + description: 'OAuth 2.0 token for the current user.', + in: 'query', + name: 'oauth_token', + schema: { + type: 'string', + }, + }, + prettyPrint: { + description: 'Returns response with indentations and line breaks.', + in: 'query', + name: 'prettyPrint', + schema: { + type: 'boolean', + }, + }, + quotaUser: { + description: 'An opaque string that represents a user for quota purposes. Must not exceed 40 characters.', + in: 'query', + name: 'quotaUser', + schema: { + type: 'string', + }, + }, + userIp: { + description: 'Deprecated. Please use quotaUser instead.', + in: 'query', + name: 'userIp', + schema: { + type: 'string', + }, + }, + }, + schemas: { + SiteVerificationWebResourceGettokenRequest: { + properties: { + site: { + description: 'The site for which a verification token will be generated.', + properties: { + identifier: { + description: 'The site identifier. If the type is set to SITE, the identifier is a URL. If the type is set to INET_DOMAIN, the site identifier is a domain name.', + type: 'string', + }, + type: { + description: 'The type of resource to be verified. Can be SITE or INET_DOMAIN (domain name).', + type: 'string', + }, + }, + type: 'object', + }, + verificationMethod: { + description: 'The verification method that will be used to verify this site. For sites, \'FILE\' or \'META\' methods may be used. For domains, only \'DNS\' may be used.', + type: 'string', + }, + }, + type: 'object', + }, + SiteVerificationWebResourceGettokenResponse: { + properties: { + method: { + description: 'The verification method to use in conjunction with this token. For FILE, the token should be placed in the top-level directory of the site, stored inside a file of the same name. For META, the token should be placed in the HEAD tag of the default page that is loaded for the site. For DNS, the token should be placed in a TXT record of the domain.', + type: 'string', + }, + token: { + description: 'The verification token. The token must be placed appropriately in order for verification to succeed.', + type: 'string', + }, + }, + type: 'object', + }, + SiteVerificationWebResourceListResponse: { + properties: { + items: { + description: 'The list of sites that are owned by the authenticated user.', + items: { + $ref: '#/components/schemas/SiteVerificationWebResourceResource', + }, + type: 'array', + }, + }, + type: 'object', + }, + SiteVerificationWebResourceResource: { + properties: { + id: { + description: 'The string used to identify this site. This value should be used in the "id" portion of the REST URL for the Get, Update, and Delete operations.', + type: 'string', + }, + owners: { + description: 'The email addresses of all verified owners.', + items: { + type: 'string', + }, + type: 'array', + }, + site: { + description: 'The address and type of a site that is verified or will be verified.', + properties: { + identifier: { + description: 'The site identifier. If the type is set to SITE, the identifier is a URL. If the type is set to INET_DOMAIN, the site identifier is a domain name.', + type: 'string', + }, + type: { + description: 'The site type. Can be SITE or INET_DOMAIN (domain name).', + type: 'string', + }, + }, + type: 'object', + }, + }, + type: 'object', + }, + }, + securitySchemes: { + Oauth2: { + description: 'Oauth 2.0 implicit authentication', + flows: { + implicit: { + authorizationUrl: 'https://accounts.google.com/o/oauth2/auth', + scopes: { + 'https://www.googleapis.com/auth/siteverification': 'Manage the list of sites and domains you control', + 'https://www.googleapis.com/auth/siteverification.verify_only': 'Manage your new site verifications with Google', + }, + }, + }, + type: 'oauth2', + }, + Oauth2c: { + description: 'Oauth 2.0 authorizationCode authentication', + flows: { + authorizationCode: { + authorizationUrl: 'https://accounts.google.com/o/oauth2/auth', + scopes: { + 'https://www.googleapis.com/auth/siteverification': 'Manage the list of sites and domains you control', + 'https://www.googleapis.com/auth/siteverification.verify_only': 'Manage your new site verifications with Google', + }, + tokenUrl: 'https://accounts.google.com/o/oauth2/token', + }, + }, + type: 'oauth2', + }, + }, + }, +}; diff --git a/tests/integration/index.ts b/tests/integration/index.ts index 1385eb0..0c589b9 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -1,9 +1,3 @@ -/** - * Unit test suite - */ - -/* Requires ------------------------------------------------------------------ */ - import { schema } from '../../src'; /* Tests --------------------------------------------------------------------- */ @@ -1678,3 +1672,457 @@ describe('OpenAPI-compatible object formats', () => { }); }); }); + +/* OpenAPI Native Format Tests ---------------------------------------------- */ + +describe('OpenAPI native format support', () => { + describe('Using properties instead of schema', () => { + const Schema = schema({ + user: { + type: 'object', + properties: { + id: { type: 'string', format: 'uuid' }, + name: { type: 'string' }, + age: { type: 'integer', format: 'int32' }, + }, + }, + }); + + it('should handle properties field (OpenAPI format)', () => { + const data = { + user: { + id: '550e8400-e29b-4d4e-a7d4-426614174000', + name: 'John Doe', + age: 30, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Using $ref for schema references', () => { + const Schema = schema( + { + user: { + $ref: '#/User', + }, + }, + { + schemas: { + User: { + type: 'object', + properties: { + id: { type: 'string', format: 'uuid' }, + name: { type: 'string' }, + email: { type: 'string' }, + }, + }, + }, + }, + ); + + it('should resolve $ref to schema definition', () => { + const data = { + user: { + id: '550e8400-e29b-4d4e-a7d4-426614174000', + name: 'John Doe', + email: 'john@example.com', + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Nested $ref usage', () => { + const Schema = schema( + { + order: { + type: 'object', + properties: { + id: { type: 'integer', format: 'int32' }, + customer: { $ref: '#/Customer' }, + items: { + type: 'array', + items: { $ref: '#/Product' }, + }, + }, + }, + }, + { + schemas: { + Customer: { + type: 'object', + properties: { + name: { type: 'string' }, + email: { type: 'string' }, + }, + }, + Product: { + type: 'object', + properties: { + sku: { type: 'string' }, + price: { type: 'number', format: 'double' }, + }, + }, + }, + }, + ); + + it('should resolve nested $ref in objects and arrays', () => { + const data = { + order: { + id: 12345, + customer: { + name: 'Jane Doe', + email: 'jane@example.com', + }, + items: [ + { sku: 'PROD-001', price: 29.99 }, + { sku: 'PROD-002', price: 49.99 }, + ], + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('$ref in oneOf/anyOf', () => { + const Schema = schema( + { + payment: { + type: 'object', + properties: { + id: { type: 'integer', format: 'int32' }, + method: { + oneOf: [{ $ref: '#/CreditCard' }, { $ref: '#/BankAccount' }], + }, + }, + }, + }, + { + schemas: { + CreditCard: { + type: 'object', + properties: { + cardNumber: { type: 'string' }, + expiry: { type: 'string' }, + }, + }, + BankAccount: { + type: 'object', + properties: { + accountNumber: { type: 'string' }, + routingNumber: { type: 'string' }, + }, + }, + }, + }, + ); + + it('should resolve $ref in oneOf (credit card)', () => { + const data = { + payment: { + id: 1, + method: { + cardNumber: '4111111111111111', + expiry: '12/25', + }, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + + it('should resolve $ref in oneOf (bank account)', () => { + const data = { + payment: { + id: 1, + method: { + accountNumber: '123456789', + routingNumber: '987654321', + }, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Complex OpenAPI schema', () => { + const Schema = schema( + { + response: { + type: 'object', + properties: { + status: { type: 'integer', format: 'int32' }, + data: { $ref: '#/UserResponse' }, + metadata: { + type: 'object', + properties: { + timestamp: { type: 'string', format: 'date-time' }, + requestId: { type: 'string', format: 'uuid' }, + }, + }, + }, + }, + }, + { + schemas: { + UserResponse: { + type: 'object', + properties: { + user: { $ref: '#/User' }, + permissions: { + type: 'array', + items: { type: 'string' }, + }, + }, + }, + User: { + type: 'object', + properties: { + id: { type: 'string', format: 'uuid' }, + name: { type: 'string' }, + email: { type: 'string' }, + profile: { $ref: '#/Profile' }, + }, + }, + Profile: { + type: 'object', + properties: { + bio: { type: 'string', nullable: true }, + avatar: { type: 'string', format: 'binary' }, + settings: { + type: 'object', + properties: { + theme: { type: 'string' }, + notifications: { type: 'boolean' }, + }, + }, + }, + }, + }, + }, + ); + + it('should handle complex nested OpenAPI schema', () => { + const data = { + response: { + status: 200, + data: { + user: { + id: '550e8400-e29b-4d4e-a7d4-426614174000', + name: 'John Doe', + email: 'john@example.com', + profile: { + bio: null, + avatar: 'SGVsbG8gV29ybGQh', + settings: { + theme: 'dark', + notifications: true, + }, + }, + }, + permissions: ['read', 'write', 'admin'], + }, + metadata: { + timestamp: '2025-10-28T14:30:00.000Z', + requestId: '6ba7b810-9dad-11d1-80b4-00c04fd430c8', + }, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Mixed format (properties and schema)', () => { + const Schema = schema( + { + // Using properties (OpenAPI format) + user: { + type: 'object', + properties: { + id: { type: 'string' }, + name: { type: 'string' }, + }, + }, + // Using schema (internal format) + product: { + type: 'object', + schema: { + sku: { type: 'string' }, + price: { type: 'number', format: 'double' }, + }, + }, + // Using $ref + order: { + $ref: '#/Order', + }, + }, + { + schemas: { + Order: { + type: 'object', + properties: { + id: { type: 'integer', format: 'int32' }, + total: { type: 'number', format: 'double' }, + }, + }, + }, + }, + ); + + it('should handle mixed schema formats', () => { + const data = { + user: { + id: 'user-123', + name: 'John', + }, + product: { + sku: 'PROD-001', + price: 29.99, + }, + order: { + id: 12345, + total: 99.99, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Array with $ref items', () => { + const Schema = schema( + { + users: { + type: 'array', + items: { $ref: '#/User' }, + }, + }, + { + schemas: { + User: { + type: 'object', + properties: { + id: { type: 'integer', format: 'int32' }, + name: { type: 'string' }, + }, + }, + }, + }, + ); + + it('should handle array items with $ref', () => { + const data = { + users: [ + { id: 1, name: 'Alice' }, + { id: 2, name: 'Bob' }, + { id: 3, name: 'Charlie' }, + ], + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Top-level schema object (OpenAPI format)', () => { + const Schema = schema({ + type: 'object', + properties: { + id: { type: 'string', format: 'uuid' }, + name: { type: 'string' }, + email: { type: 'string' }, + age: { type: 'integer', format: 'int32' }, + active: { type: 'boolean' }, + }, + }); + + it('should unwrap top-level schema object', () => { + const data = { + id: '550e8400-e29b-4d4e-a7d4-426614174000', + name: 'John Doe', + email: 'john@example.com', + age: 30, + active: true, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Top-level schema with nested objects', () => { + const Schema = schema({ + type: 'object', + properties: { + user: { + type: 'object', + properties: { + id: { type: 'integer', format: 'int32' }, + name: { type: 'string' }, + }, + }, + settings: { + type: 'object', + properties: { + theme: { type: 'string' }, + notifications: { type: 'boolean' }, + }, + }, + }, + }); + + it('should handle nested objects in top-level schema', () => { + const data = { + user: { + id: 123, + name: 'Jane Doe', + }, + settings: { + theme: 'dark', + notifications: true, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); + + describe('Top-level schema with $ref', () => { + const Schema = schema( + { + type: 'object', + properties: { + user: { $ref: '#/User' }, + product: { $ref: '#/Product' }, + }, + }, + { + schemas: { + User: { + type: 'object', + properties: { + id: { type: 'integer', format: 'int32' }, + name: { type: 'string' }, + }, + }, + Product: { + type: 'object', + properties: { + sku: { type: 'string' }, + price: { type: 'number', format: 'double' }, + }, + }, + }, + }, + ); + + it('should handle $ref in top-level schema object', () => { + const data = { + user: { + id: 1, + name: 'Alice', + }, + product: { + sku: 'PROD-001', + price: 29.99, + }, + }; + expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + }); + }); +}); diff --git a/tests/integration/openapi.ts b/tests/integration/openapi.ts new file mode 100644 index 0000000..e10f340 --- /dev/null +++ b/tests/integration/openapi.ts @@ -0,0 +1,40 @@ +import { schema } from '../../src'; +import spec from '../../static/exampleSpec'; + +/* Tests --------------------------------------------------------------------- */ + +describe('OpenAPI spec test', () => { + describe('Parse simple API response', () => { + const Schema = schema(spec.components.schemas.SiteVerificationWebResourceGettokenResponse); + + const response = { + method: 'META', token: 'abc', + }; + + it('should return the response object unchanged', () => { + expect(Schema.read(Schema.write(response).buffer())).toEqual(response); + }); + }); + + describe('Parse API response with local $ref', () => { + const Schema = schema(spec.components.schemas.SiteVerificationWebResourceListResponse, { schemas: spec }); + + const response = { + items: [{ + id: 'compactr', + owners: [ + 'bob', + 'mary', + ], + site: { + identifier: 'compactr.js.org', + type: 'SITE', + }, + }], + }; + + it('should return the response object unchanged', () => { + expect(Schema.read(Schema.write(response).buffer())).toEqual(response); + }); + }); +}); diff --git a/types.d.ts b/types.d.ts index f63457e..3f3cd49 100644 --- a/types.d.ts +++ b/types.d.ts @@ -62,9 +62,11 @@ declare module 'compactr' { count?: number size?: number schema?: SchemaDefinition + properties?: SchemaDefinition items?: SchemaFieldDefinition oneOf?: SchemaFieldDefinition[] anyOf?: SchemaFieldDefinition[] + $ref?: string } export interface SchemaDefinition { @@ -73,6 +75,7 @@ declare module 'compactr' { export interface SchemaOptions { keyOrder?: boolean + schemas?: { [key: string]: SchemaFieldDefinition } } export interface WriteOptions { From f14c1ccc9b7df4deb8fed2139e50d2c785b88479 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Sat, 1 Nov 2025 22:45:44 -0400 Subject: [PATCH 12/19] Removed partial mode, added benchmarks, sequential write and zero-copy reader --- README.md | 18 +- benchmarks/array.ts | 35 ++- benchmarks/boolean.ts | 11 +- benchmarks/index.ts | 6 +- benchmarks/integer.ts | 13 +- benchmarks/jsonapiresponse.ts | 99 ++++++- benchmarks/schema.ts | 18 +- benchmarks/string.ts | 7 +- benchmarks/uuid.ts | 80 +++++ src/decoder.ts | 311 +++++++++++++------- src/encoder.ts | 535 +++++++++++++++++++++------------- src/reader.ts | 218 +++++++++----- src/schema.ts | 66 ++--- src/size-writer.ts | 19 ++ src/variant-matcher.ts | 59 ++++ src/writer.ts | 379 ++++++++++++++++-------- tests/integration/index.ts | 142 +-------- types.d.ts | 21 +- 18 files changed, 1268 insertions(+), 769 deletions(-) create mode 100644 benchmarks/uuid.ts create mode 100644 src/size-writer.ts create mode 100644 src/variant-matcher.ts diff --git a/README.md b/README.md index 0ea0571..56b32fc 100644 --- a/README.md +++ b/README.md @@ -48,23 +48,11 @@ const userSchema = Compactr.schema({ // Encoding userSchema.write({ id: 123, name: 'John' }); -// Get the header bytes -const header = userSchema.headerBuffer(); - -// Get the content bytes -const partial = userSchema.contentBuffer(); - -// Get the full payload (header + content bytes) +// Get the encoded buffer const buffer = userSchema.buffer(); - - - -// Decoding a full payload -const content = userSchema.read(buffer); - -// Decoding a partial payload (content) -const content = userSchema.readContent(partial); +// Decoding +const decoded = userSchema.read(buffer); ``` ## Size comparison diff --git a/benchmarks/array.ts b/benchmarks/array.ts index 24a672f..4a413fc 100644 --- a/benchmarks/array.ts +++ b/benchmarks/array.ts @@ -4,6 +4,7 @@ import Benchmark from 'benchmark'; import {schema} from '../dist/compactr.js'; +import protobuf from 'protobufjs'; import {deferred} from './utils.ts'; /* Local variables -----------------------------------------------------------*/ @@ -14,22 +15,35 @@ let User = schema({ arr: { type: 'array', items: { type: 'string' }}, }); -const mult = 32; -const sizes = { json: 0, compactr: 0 }; +let root = protobuf.Root.fromJSON({ + nested: { + ArrayBenchTest: { + fields: { + id: { type: 'uint32', id: 1 }, + arr: { rule: 'repeated', type: 'string', id: 2 }, + }, + }, + }, +}); +var ArrayBenchTest = root.lookupType('ArrayBenchTest'); + +const sizes = { json: 0, compactr: 0, protobuf: 0 }; const arraySuite = new Benchmark.Suite(); /* Array suite ---------------------------------------------------------------*/ -export function init() { +export function init(mult) { const {promise, resolve} = deferred(); arraySuite.add('[Array] JSON', arrJSON) .add('[Array] Compactr', arrCompactr) + .add('[Array] Protobuf', arrProtobuf) .on('cycle', e => console.log(String(e.target))) .run({ 'async': true }) .on('complete', _ => resolve(sizes)); + function arrJSON() { let packed, unpacked; @@ -44,11 +58,22 @@ export function init() { let packed, unpacked; for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; } } + function arrProtobuf() { + let packed, unpacked; + + for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; + } + } + return promise; } diff --git a/benchmarks/boolean.ts b/benchmarks/boolean.ts index b9ca5a8..cc7660e 100644 --- a/benchmarks/boolean.ts +++ b/benchmarks/boolean.ts @@ -10,12 +10,11 @@ import {deferred} from './utils.ts'; /* Local variables -----------------------------------------------------------*/ -let User = schema({ - id: { type: 'int32' }, +let User = schema({ + id: { type: 'integer', format: 'int32' }, bool: { type: 'boolean' }, }); -const mult = 32; const sizes = { json: 0, compactr: 0, protobuf: 0 }; let root = protobuf.Root.fromJSON({ @@ -34,7 +33,7 @@ const boolSuite = new Benchmark.Suite(); /* Boolean suite ---------------------------------------------------------------*/ -export function init() { +export function init(mult) { const {promise, resolve} = deferred(); boolSuite.add('[Boolean] JSON', boolJSON) @@ -58,8 +57,8 @@ export function init() { let packed, unpacked; for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; } } diff --git a/benchmarks/index.ts b/benchmarks/index.ts index 87b9992..0d66704 100644 --- a/benchmarks/index.ts +++ b/benchmarks/index.ts @@ -4,6 +4,7 @@ import { init as boolean } from './boolean.ts'; import { init as integer } from './integer.ts'; import { init as schema } from './schema.ts'; import { init as string } from './string.ts'; +import { init as uuid } from './uuid.ts'; // Realistic import { init as jsonapiresponse } from './jsonapiresponse.ts'; @@ -16,11 +17,14 @@ const benchmarks = [ integer, schema, string, + uuid, jsonapiresponse, ]; +const mult = 32; + console.log('Running Compactr benchmarks...\n'); -sequence(benchmarks, (i) => i().then((sizes) => console.log(sizes))).then(() => { +sequence(benchmarks, (i) => i(mult).then((sizes) => console.log(sizes))).then(() => { console.log('\nAll benchmarks completed!'); }); diff --git a/benchmarks/integer.ts b/benchmarks/integer.ts index 145036a..c271e0e 100644 --- a/benchmarks/integer.ts +++ b/benchmarks/integer.ts @@ -9,12 +9,11 @@ import {deferred} from './utils.ts'; /* Local variables -----------------------------------------------------------*/ -let User = schema({ - id: { type: 'int32' }, - int: { type: 'int32' }, +let User = schema({ + id: { type: 'integer', format: 'int32' }, + int: { type: 'integer', format: 'int32' }, }); -const mult = 32; const sizes = { json: 0, compactr: 0, protobuf: 0 }; let root = protobuf.Root.fromJSON({ @@ -33,7 +32,7 @@ const intSuite = new Benchmark.Suite(); /* Integer suite ---------------------------------------------------------------*/ -export function init() { +export function init(mult) { const {promise, resolve} = deferred(); intSuite.add('[Integer] JSON', intJSON) @@ -57,8 +56,8 @@ export function init() { let packed, unpacked; for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; } } diff --git a/benchmarks/jsonapiresponse.ts b/benchmarks/jsonapiresponse.ts index cf7ffe4..dda1126 100644 --- a/benchmarks/jsonapiresponse.ts +++ b/benchmarks/jsonapiresponse.ts @@ -6,40 +6,67 @@ import Benchmark from 'benchmark'; import {schema} from '../dist/compactr.js'; import protobuf from 'protobufjs'; import {deferred} from './utils.ts'; +import {randomUUID} from 'crypto'; /* Local variables -----------------------------------------------------------*/ +function generateIPDigit() { + return Math.floor(Math.random() * 255); +} -let User = schema({ - id: { type: 'int32', size: 4 }, - obj: { type: 'object', size: 9, schema: { str: { type: 'string', size: 6 } } }, +let User = schema({ + id: { type: 'string', format: 'uuid' }, + name: { type: 'string' }, + age: { type: 'integer', format: 'int32' }, + last_connected_ip: { type: 'string', format: 'ipv4' }, + date_created: { type: 'string', format: 'date-time' }, + date_updated: { type: 'string', format: 'date-time' }, + user_settings: { + type: 'object', + properties: { + flag_a: { type: 'boolean' }, + flag_b: { type: 'boolean' }, + flag_c: { type: 'boolean' }, + } + }, + user_friends: { + type: 'array', + items: { type: 'string', format: 'uuid' } + } }); -const mult = 32; const sizes = { json: 0, compactr: 0, protobuf: 0 }; let root = protobuf.Root.fromJSON({ nested: { - StringBenchTest: { + SettingsBenchTest: { fields: { - str: { type: 'string', id: 2 }, + flag_a: { type: 'bool', id: 1 }, + flag_b: { type: 'bool', id: 2 }, + flag_c: { type: 'bool', id: 3 }, }, }, - ObjectBenchTest: { + JsonAPIBenchTest: { fields: { - id: { type: 'uint32', id: 1 }, - obj: { type: 'StringBenchTest', id: 2}, + id: { type: 'string', id: 1 }, + name: { type: 'string', id: 2}, + age: { type: 'int32', id: 3 }, + last_connected_ip: { type: 'string', id: 4}, + date_created: { type: 'string', id: 5 }, + date_updated: { type: 'string', id: 6 }, + user_settings: { type: 'SettingsBenchTest', id: 7}, + user_friends: { rule: 'repeated', type: 'string', id: 8 } }, }, }, }); -var ObjectBenchTest = root.lookupType('ObjectBenchTest'); +var ObjectBenchTest = root.lookupType('JsonAPIBenchTest'); const objectSuite = new Benchmark.Suite(); /* JSON-API Reponse suite ---------------------------------------------------------------*/ -export function init() { +export function init(mult) { const {promise, resolve} = deferred(); objectSuite.add('[JSON-API Reponse] JSON', objJSON) @@ -52,9 +79,23 @@ export function init() { function objJSON() { let packed, unpacked; + let now = (new Date()).toISOString(); for(let i = 0; i sizes.json) sizes.json = packed.length; } @@ -62,19 +103,47 @@ export function init() { function objCompactr() { let packed, unpacked; + let now = (new Date()).toISOString(); for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; } } function objProtobuf() { let packed, unpacked; + let now = (new Date()).toISOString(); for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; diff --git a/benchmarks/schema.ts b/benchmarks/schema.ts index 19890ea..b82613b 100644 --- a/benchmarks/schema.ts +++ b/benchmarks/schema.ts @@ -6,21 +6,21 @@ import Benchmark from 'benchmark'; import {schema} from '../dist/compactr.js'; import protobuf from 'protobufjs'; import {deferred} from './utils.ts'; +import {randomUUID} from 'crypto'; /* Local variables -----------------------------------------------------------*/ -let User = schema({ - id: { type: 'int32' }, +let User = schema({ + id: { type: 'integer', format: 'int32' }, obj: { type: 'object', properties: { str: { type: 'string' } }, - }, + }, }); -const mult = 32; const sizes = { json: 0, compactr: 0, protobuf: 0 }; let root = protobuf.Root.fromJSON({ @@ -44,7 +44,7 @@ const objectSuite = new Benchmark.Suite(); /* Schema suite ---------------------------------------------------------------*/ -export function init() { +export function init(mult) { const {promise, resolve} = deferred(); objectSuite.add('[Schema] JSON', objJSON) @@ -59,7 +59,7 @@ export function init() { let packed, unpacked; for(let i = 0; i sizes.json) sizes.json = packed.length; } @@ -69,8 +69,8 @@ export function init() { let packed, unpacked; for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; } } @@ -79,7 +79,7 @@ export function init() { let packed, unpacked; for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; diff --git a/benchmarks/string.ts b/benchmarks/string.ts index 0d13101..a408e79 100644 --- a/benchmarks/string.ts +++ b/benchmarks/string.ts @@ -29,14 +29,13 @@ let root = protobuf.Root.fromJSON({ }); var StringBenchTest = root.lookupType('StringBenchTest'); -const mult = 32; const sizes = { json: 0, compactr: 0, protobuf: 0 }; const stringSuite = new Benchmark.Suite(); /* String suite ---------------------------------------------------------------*/ -export function init() { +export function init(mult) { const {promise, resolve} = deferred(); stringSuite.add('[String] JSON', strJSON) @@ -61,8 +60,8 @@ export function init() { let packed, unpacked; for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; } } diff --git a/benchmarks/uuid.ts b/benchmarks/uuid.ts new file mode 100644 index 0000000..160eab8 --- /dev/null +++ b/benchmarks/uuid.ts @@ -0,0 +1,80 @@ +/** Benchmarks */ + +/* Requires ------------------------------------------------------------------*/ + +import Benchmark from 'benchmark'; +import {schema} from '../dist/compactr.js'; +import protobuf from 'protobufjs'; +import {deferred} from './utils.ts'; +import {randomUUID} from 'crypto'; + +/* Local variables -----------------------------------------------------------*/ + + +let User = schema({ + id: { type: 'integer', format: 'int32'}, + uid: { type: 'string', format: 'uuid' }, +}); + +let root = protobuf.Root.fromJSON({ + nested: { + StringBenchTest: { + fields: { + id: { type: 'uint32', id: 1 }, + uid: { type: 'string', id: 2 }, + }, + }, + }, +}); +var StringBenchTest = root.lookupType('StringBenchTest'); + +const sizes = { json: 0, compactr: 0, protobuf: 0 }; + +const stringSuite = new Benchmark.Suite(); + +/* UUID suite ---------------------------------------------------------------*/ + +export function init(mult) { + const {promise, resolve} = deferred(); + + stringSuite.add('[UUID] JSON', strJSON) + .add('[UUID] Compactr', strCompactr) + .add('[UUID] Protobuf', strProtobuf) + .on('cycle', e => console.log(String(e.target))) + .run({ 'async': true }) + .on('complete', _ => resolve(sizes)); + + + function strJSON() { + let packed, unpacked; + + for(let i = 0; i sizes.json) sizes.json = packed.length; + } + } + + function strCompactr() { + let packed, unpacked; + + for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; + } + } + + function strProtobuf() { + let packed, unpacked; + + for(let i = 0; i sizes.protobuf) sizes.protobuf = packed.length; + } + } + + return promise; +} diff --git a/src/decoder.ts b/src/decoder.ts index 88e5f20..ebe2168 100644 --- a/src/decoder.ts +++ b/src/decoder.ts @@ -1,76 +1,118 @@ /** Decoding utilities */ -/* Local variables ----------------------------------------------------------- */ - -const fromChar = String.fromCharCode; - // Discriminator byte for nullable/oneOf/anyOf fields export const NULL_INDICATOR = 0x00; // Field is null export const VARIANT_BASE = 0x01; // First variant (or present for simple nullable) +/* Performance: Reusable typed array buffers --------------------------------- */ + +// Module-level reusable buffers for float/double decoding +// This eliminates allocation overhead (300-500% performance improvement) +const floatBuffer = new Float32Array(1); +const floatBytes = new Uint8Array(floatBuffer.buffer); +const doubleBuffer = new Float64Array(1); +const doubleBytes = new Uint8Array(doubleBuffer.buffer); + /* Methods ------------------------------------------------------------------- */ /** @private */ -function boolean(bytes) { - return !!bytes[0]; +function boolean(bytes, offset = 0) { + return !!bytes[offset]; } /** @private */ -function int32(bytes) { - return (bytes[0] << 24) | (bytes[1] << 16) | (bytes[2] << 8) | (bytes[3]); +function int32(bytes, offset = 0) { + return (bytes[offset] << 24) | (bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | (bytes[offset + 3]); } -function uint8(bytes) { - return bytes[0]; +function uint8(bytes, offset = 0) { + return bytes[offset]; } -function uint16(bytes) { - return bytes[0] << 8 | bytes[1]; +function uint16(bytes, offset = 0) { + return bytes[offset] << 8 | bytes[offset + 1]; } /** @private */ -function unsigned(bytes) { - if (bytes.length === 1) return uint8(bytes); - if (bytes.length === 2) return uint16(bytes); - return int32(bytes); +function unsigned(bytes, offset = 0, length?) { + const len = length !== undefined ? length : bytes.length - offset; + if (len === 1) return uint8(bytes, offset); + if (len === 2) return uint16(bytes, offset); + return int32(bytes, offset); } -/** @private */ -function string(bytes) { - const res = []; - for (let i = 0; i < bytes.length; i += 2) { - const code = (bytes[i] << 8) | bytes[i + 1]; - res.push(code); +/** + * Manual UTF-8 decoder - faster than Buffer for typical API strings + * Assumes mostly ASCII content (field names, english text, numbers, URLs) + * @private + */ +function string(bytes, offset = 0, length?) { + const len = length !== undefined ? length : bytes.length - offset; + const chars = []; + const end = offset + len; + + for (let i = offset; i < end;) { + const byte1 = bytes[i++]; + + // ASCII (0-127): 1 byte + if (byte1 < 0x80) { + chars.push(byte1); + } + // 2-byte character + else if ((byte1 & 0xE0) === 0xC0) { + const byte2 = bytes[i++]; + chars.push(((byte1 & 0x1F) << 6) | (byte2 & 0x3F)); + } + // 3-byte character + else if ((byte1 & 0xF0) === 0xE0) { + const byte2 = bytes[i++]; + const byte3 = bytes[i++]; + chars.push(((byte1 & 0x0F) << 12) | ((byte2 & 0x3F) << 6) | (byte3 & 0x3F)); + } + // 4-byte character (surrogate pair) + else if ((byte1 & 0xF8) === 0xF0) { + const byte2 = bytes[i++]; + const byte3 = bytes[i++]; + const byte4 = bytes[i++]; + let code = ((byte1 & 0x07) << 18) | ((byte2 & 0x3F) << 12) | ((byte3 & 0x3F) << 6) | (byte4 & 0x3F); + // Convert to surrogate pair + code -= 0x10000; + chars.push(0xD800 | (code >> 10)); + chars.push(0xDC00 | (code & 0x3FF)); + } } - return fromChar(...res); + + return String.fromCharCode.apply(null, chars); } /** * UUID decoder - converts 16 bytes to UUID string * Binary format: 16 bytes (128 bits) * UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx (36 chars) + * Performance: 100-150% faster using direct hex lookup * @private */ -function uuid(bytes) { - if (bytes.length !== 16) { +function uuid(bytes, offset = 0, length?) { + const len = length !== undefined ? length : bytes.length - offset; + if (len !== 16) { throw new Error('Invalid UUID byte length'); } - // Convert bytes to hex string - const hex = []; + // Performance: Direct hex lookup instead of toString(16) + padStart + const hex = '0123456789abcdef'; + let result = ''; + for (let i = 0; i < 16; i++) { - const byte = bytes[i].toString(16).padStart(2, '0'); - hex.push(byte); + const byte = bytes[offset + i]; + result += hex[byte >> 4] + hex[byte & 0x0f]; + + // Add hyphens at positions 4, 6, 8, 10 (after bytes 3, 5, 7, 9) + if (i === 3 || i === 5 || i === 7 || i === 9) { + result += '-'; + } } - // Insert hyphens at proper positions: 8-4-4-4-12 - return [ - hex.slice(0, 4).join(''), - hex.slice(4, 6).join(''), - hex.slice(6, 8).join(''), - hex.slice(8, 10).join(''), - hex.slice(10, 16).join(''), - ].join('-'); + return result; } /** @@ -79,39 +121,41 @@ function uuid(bytes) { * IPv4 format: "192.168.1.1" * @private */ -function ipv4(bytes) { - if (bytes.length !== 4) { +function ipv4(bytes, offset = 0, length?) { + const len = length !== undefined ? length : bytes.length - offset; + if (len !== 4) { throw new Error('Invalid IPv4 byte length'); } - return bytes.join('.'); + return `${bytes[offset]}.${bytes[offset + 1]}.${bytes[offset + 2]}.${bytes[offset + 3]}`; } /** * IPv6 decoder - converts 16 bytes to IPv6 string * Binary format: 16 bytes * IPv6 format: "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + * Performance: Optimized with single-pass algorithm and direct hex conversion * @private */ -function ipv6(bytes) { - if (bytes.length !== 16) { +function ipv6(bytes, offset = 0, length?) { + const len = length !== undefined ? length : bytes.length - offset; + if (len !== 16) { throw new Error('Invalid IPv6 byte length'); } - const parts = []; - for (let i = 0; i < 16; i += 2) { - const value = (bytes[i] << 8) | bytes[i + 1]; - parts.push(value.toString(16).padStart(4, '0')); - } - - // Find longest sequence of consecutive '0000' groups for compression + // Parse groups and find longest zero sequence in single pass + const groups = new Array(8); let longestZeroStart = -1; let longestZeroLength = 0; let currentZeroStart = -1; let currentZeroLength = 0; - for (let i = 0; i < parts.length; i++) { - if (parts[i] === '0000') { + for (let i = 0; i < 8; i++) { + const value = (bytes[offset + i * 2] << 8) | bytes[offset + i * 2 + 1]; + groups[i] = value; + + // Track zero sequences + if (value === 0) { if (currentZeroStart === -1) { currentZeroStart = i; currentZeroLength = 1; @@ -136,32 +180,36 @@ function ipv6(bytes) { longestZeroLength = currentZeroLength; } - // Remove leading zeros from each part (except if it's all zeros) - const strippedParts = parts.map((p) => { - const stripped = p.replace(/^0+/, ''); - return stripped === '' ? '0' : stripped; - }); + // Build result string + let result = ''; - // Apply compression if we found at least one zero group + // Apply :: compression if we found at least one zero group if (longestZeroLength > 0) { - const before = strippedParts.slice(0, longestZeroStart); - const after = strippedParts.slice(longestZeroStart + longestZeroLength); - - // Build result with :: compression - let result = ''; - if (before.length > 0) { - result = before.join(':'); + // Before compressed section + for (let i = 0; i < longestZeroStart; i++) { + if (i > 0) result += ':'; + result += groups[i].toString(16); } + + // Compressed section result += '::'; - if (after.length > 0) { - result += after.join(':'); - } - return result; + // After compressed section + const afterStart = longestZeroStart + longestZeroLength; + for (let i = afterStart; i < 8; i++) { + if (i > afterStart) result += ':'; + result += groups[i].toString(16); + } + } + else { + // No compression, output all groups + for (let i = 0; i < 8; i++) { + if (i > 0) result += ':'; + result += groups[i].toString(16); + } } - // No zero sequences, return with leading zeros stripped - return strippedParts.join(':'); + return result; } /** @@ -170,12 +218,13 @@ function ipv6(bytes) { * Date format: "2025-10-28" * @private */ -function date(bytes) { - if (bytes.length !== 4) { +function date(bytes, offset = 0, length?) { + const len = length !== undefined ? length : bytes.length - offset; + if (len !== 4) { throw new Error('Invalid date byte length'); } - const days = int32(bytes); + const days = int32(bytes, offset); const epochMs = days * 86400000; const dateObj = new Date(epochMs); @@ -193,33 +242,41 @@ function date(bytes) { * DateTime format: "2025-10-28T14:30:00.000Z" * @private */ -function dateTime(bytes) { - if (bytes.length !== 8) { +function dateTime(bytes, offset = 0, length?) { + const len = length !== undefined ? length : bytes.length - offset; + if (len !== 8) { throw new Error('Invalid date-time byte length'); } - const ms = double(bytes); + const ms = double(bytes, offset); const dateObj = new Date(ms); return dateObj.toISOString(); } /** - * Binary decoder - converts raw bytes to base64 string - * Binary format: raw bytes - * Base64 format: "SGVsbG8gV29ybGQ=" + * Binary decoder - converts raw bytes to base64 string (zero-copy) + * Uses Buffer.toString with offset and length instead of slicing * @private */ -function binary(bytes) { - const buffer = Buffer.from(bytes); - return buffer.toString('base64'); +function binary(bytes, offset = 0, length?) { + const len = length !== undefined ? length : bytes.length - offset; + // Zero-copy: use Buffer.toString with offset and length + // Create Buffer view of the data without copying + if (offset === 0 && len === bytes.length) { + return Buffer.from(bytes).toString('base64'); + } + // Use subarray (zero-copy view) instead of slice (copy) + return Buffer.from(bytes.subarray(offset, offset + len)).toString('base64'); } /** @private */ -function array(schema, bytes) { +function array(schema, bytes, offset = 0, length?) { + const len = length !== undefined ? length : bytes.length - offset; const ret = []; + const end = offset + len; - for (let i = 0; i < bytes.length;) { + for (let i = offset; i < end;) { // Handle nullable or variant array items if (schema.nullable || schema.variants) { const discriminator = bytes[i]; @@ -240,9 +297,12 @@ function array(schema, bytes) { throw new Error(`Invalid variant discriminator: ${discriminator}`); } - const size = unsigned(bytes.slice(i, i + variantField.count)); + // Zero-copy: pass offset and length instead of slicing + const size = unsigned(bytes, i, variantField.count); i += variantField.count; - ret.push(variantField.transformOut(bytes.slice(i, i + size))); + + // Zero-copy for all types: pass offset and length + ret.push(variantField.transformOut(bytes, i, size)); i += size; continue; } @@ -252,55 +312,86 @@ function array(schema, bytes) { } // Handle regular array items - const size = unsigned(bytes.slice(i, i + schema.count)); + // Zero-copy: pass offset and length instead of slicing + const size = unsigned(bytes, i, schema.count); i += schema.count; - ret.push(schema.transformOut(bytes.slice(i, i + size))); + + // Zero-copy for all types: pass offset and length + ret.push(schema.transformOut(bytes, i, size)); i += size; } return ret; } -/** @private */ -function object(schema, bytes) { - return schema.read(bytes); +/** + * Object decoder - zero-copy nested object reading + * Uses readFromOffset if available, otherwise creates zero-copy subarray view + * @private + */ +function object(schema, bytes, offset = 0, length?) { + // Zero-copy: use readFromOffset if schema supports it, otherwise subarray + if (offset === 0 && length === undefined) { + return schema.read(bytes); + } + + const len = length !== undefined ? length : bytes.length - offset; + + // Use readFromOffset for zero-copy if available + if (schema.readFromOffset) { + return schema.readFromOffset(bytes, offset, len); + } + + // Fallback: use subarray (zero-copy view) instead of slice (copy) + // Most TypedArrays support subarray which creates a view, not a copy + if (bytes.subarray) { + return schema.read(bytes.subarray(offset, offset + len)); + } + + // Last resort: slice (creates copy) + return schema.read(bytes.slice(offset, offset + len)); } /** * IEEE 754 single precision (32-bit float) decoder - * Simplified implementation using JavaScript's Float32Array + * Performance: Uses reusable module-level buffer (300-500% faster) * @private */ -function float(bytes) { - // Bytes come in big-endian order, convert to little-endian for typed array - const byteArray = new Uint8Array([bytes[3], bytes[2], bytes[1], bytes[0]]); - const floatArray = new Float32Array(byteArray.buffer); - - return floatArray[0]; +function float(bytes, offset = 0) { + // Bytes come in big-endian order, convert to little-endian for reusable buffer + floatBytes[0] = bytes[offset + 3]; + floatBytes[1] = bytes[offset + 2]; + floatBytes[2] = bytes[offset + 1]; + floatBytes[3] = bytes[offset]; + + return floatBuffer[0]; } /** * IEEE 754 double precision (64-bit float) decoder - * Simplified implementation using JavaScript's Float64Array + * Performance: Uses reusable module-level buffer (300-500% faster) * @private */ -function double(bytes) { - // Bytes come in big-endian order, convert to little-endian for typed array - const byteArray = new Uint8Array([ - bytes[7], bytes[6], bytes[5], bytes[4], - bytes[3], bytes[2], bytes[1], bytes[0], - ]); - const doubleArray = new Float64Array(byteArray.buffer); - - return doubleArray[0]; +function double(bytes, offset = 0) { + // Bytes come in big-endian order, convert to little-endian for reusable buffer + doubleBytes[0] = bytes[offset + 7]; + doubleBytes[1] = bytes[offset + 6]; + doubleBytes[2] = bytes[offset + 5]; + doubleBytes[3] = bytes[offset + 4]; + doubleBytes[4] = bytes[offset + 3]; + doubleBytes[5] = bytes[offset + 2]; + doubleBytes[6] = bytes[offset + 1]; + doubleBytes[7] = bytes[offset]; + + return doubleBuffer[0]; } /** * 64-bit integer decoder (uses double for JavaScript compatibility) * @private */ -function int64(bytes) { - return double(bytes); +function int64(bytes, offset = 0) { + return double(bytes, offset); } /* Exports ------------------------------------------------------------------- */ diff --git a/src/encoder.ts b/src/encoder.ts index 6ea2437..aed8fd9 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -1,98 +1,123 @@ /** Encoding utilities */ -/* Local variables ----------------------------------------------------------- */ +/* Requires ------------------------------------------------------------------ */ + +import { matchesVariant } from './variant-matcher'; -const intMap = [null, unsigned8, unsigned16, null, unsigned32]; +/* Local variables ----------------------------------------------------------- */ // Discriminator byte for nullable/oneOf/anyOf fields export const NULL_INDICATOR = 0x00; // Field is null export const VARIANT_BASE = 0x01; // First variant (or present for simple nullable) -/* Methods ------------------------------------------------------------------- */ +/* Performance: Reusable typed array buffers --------------------------------- */ -/** @private */ -function boolean(val) { - return [val ? 1 : 0]; -} +// Module-level reusable buffers for float/double encoding +// This eliminates allocation overhead (300-500% performance improvement) +const floatBuffer = new Float32Array(1); +const floatBytes = new Uint8Array(floatBuffer.buffer); +const doubleBuffer = new Float64Array(1); +const doubleBytes = new Uint8Array(doubleBuffer.buffer); -/** @private */ -function int32(val) { - if (val < 0) val = 0xffffffff + val + 1; - return [val >> 24, val >> 16, val >> 8, val & 0xff]; -} +/* Methods ------------------------------------------------------------------- */ -/** @private */ -function unsigned8(val) { - return [val & 0xff]; +/** + * Boolean encoder - writes 1 byte directly to buffer + * @returns new position + */ +function boolean(val, buffer, pos) { + buffer[pos] = val ? 1 : 0; + return pos + 1; } -/** @private */ -function unsigned16(val) { - return [val >> 8, val & 0xff]; +/** + * int32 encoder - writes 4 bytes directly to buffer (big-endian) + * @returns new position + */ +function int32(val, buffer, pos) { + if (val < 0) val = 0xffffffff + val + 1; + buffer[pos] = val >> 24; + buffer[pos + 1] = val >> 16; + buffer[pos + 2] = val >> 8; + buffer[pos + 3] = val & 0xff; + return pos + 4; } -/** @private */ -function unsigned32(val) { - return [val >> 24, val >> 16, val >> 8, val & 0xff]; +/** + * UTF-8 string encoder - writes directly to buffer using native code + * Uses Buffer.write() for zero-copy C++ UTF-8 encoding + * @returns new position + */ +function string(val, buffer, pos) { + // Write UTF-8 bytes directly into target buffer using native C++ code + // Buffer.prototype.write is much faster than manual UTF-8 encoding + const bytesWritten = Buffer.prototype.write.call(buffer, val, pos, undefined, 'utf8'); + return pos + bytesWritten; } -/** @private */ -function string(val) { - const chars = []; - for (let i = 0; i < val.length; i++) { - const code = val.charCodeAt(i); - chars.push(code >> 8, code & 0xff); - } - - return chars; +/** + * Performance: Fast hex character to number conversion + * @private + */ +function hexCharToNum(char) { + const code = char.charCodeAt(0); + if (code >= 48 && code <= 57) return code - 48; // 0-9 + if (code >= 97 && code <= 102) return code - 87; // a-f + if (code >= 65 && code <= 70) return code - 55; // A-F + throw new Error('Invalid hex character: ' + char); } /** - * UUID encoder - converts UUID string to 16 bytes + * UUID encoder - writes 16 bytes directly to buffer * UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx (36 chars) - * Binary format: 16 bytes (128 bits) - * Saves 56 bytes per UUID compared to string encoding - * @private + * @returns new position */ -function uuid(val) { - // Remove hyphens and validate format - const hex = val.replace(/-/g, ''); - if (hex.length !== 32) { - throw new Error('Invalid UUID format'); +function uuid(val, buffer, pos) { + // Validate length + if (val.length !== 36) { + throw new Error('Invalid UUID format: expected 36 characters'); + } + + let byteIdx = pos; + + // Parse hex pairs directly and write to buffer + for (let i = 0; i < val.length; i++) { + const char = val[i]; + if (char === '-') continue; // Skip hyphens + + const high = hexCharToNum(val[i]); + const low = hexCharToNum(val[i + 1]); + buffer[byteIdx++] = (high << 4) | low; + i++; // Skip next char since we processed it } - // Convert hex string to 16 bytes - const bytes = []; - for (let i = 0; i < 32; i += 2) { - bytes.push(parseInt(hex.substr(i, 2), 16)); + if (byteIdx - pos !== 16) { + throw new Error('Invalid UUID format: incorrect number of hex digits'); } - return bytes; + return byteIdx; } /** - * IPv4 encoder - converts IPv4 string to 4 bytes - * IPv4 format: "192.168.1.1" (max 15 chars = 30 bytes as string) - * Binary format: 4 bytes - * Saves up to 26 bytes - * @private + * IPv4 encoder - writes 4 bytes directly to buffer + * IPv4 format: "192.168.1.1" + * @returns new position */ -function ipv4(val) { +function ipv4(val, buffer, pos) { const parts = val.split('.'); if (parts.length !== 4) { throw new Error('Invalid IPv4 format'); } - const bytes = []; for (let i = 0; i < 4; i++) { const num = parseInt(parts[i], 10); if (isNaN(num) || num < 0 || num > 255) { throw new Error('Invalid IPv4 format'); } - bytes.push(num); + buffer[pos + i] = num; } - return bytes; + return pos + 4; } /** @@ -100,45 +125,113 @@ function ipv4(val) { * IPv6 format: "2001:0db8:85a3::8a2e:0370:7334" (max 39 chars = 78 bytes as string) * Binary format: 16 bytes * Saves up to 62 bytes + * Performance: Optimized to avoid multiple string operations * @private */ -function ipv6(val) { - // Expand :: notation - let expanded = val; - if (expanded.includes('::')) { - const parts = expanded.split('::'); - const leftParts = parts[0] ? parts[0].split(':') : []; - const rightParts = parts[1] ? parts[1].split(':') : []; - const missingParts = 8 - leftParts.length - rightParts.length; - const zeros = Array(missingParts).fill('0'); - expanded = [...leftParts, ...zeros, ...rightParts].join(':'); - } +/** + * IPv6 encoder - writes 16 bytes directly to buffer + * @returns new position + */ +function ipv6(val, buffer, pos) { + let byteIdx = 0; + + // Handle :: expansion inline + const doubleColonPos = val.indexOf('::'); + + if (doubleColonPos !== -1) { + // Parse left side of :: + let i = 0; + while (i < doubleColonPos) { + let hexStr = ''; + while (i < doubleColonPos && val[i] !== ':') { + hexStr += val[i]; + i++; + } + if (hexStr) { + const num = parseInt(hexStr, 16); + if (isNaN(num) || num < 0 || num > 0xffff) { + throw new Error('Invalid IPv6 format'); + } + buffer[pos + byteIdx++] = num >> 8; + buffer[pos + byteIdx++] = num & 0xff; + } + i++; // Skip colon + } + + // Calculate how many zero groups to insert + const leftGroups = byteIdx / 2; + + // Parse right side of :: + i = doubleColonPos + 2; // Skip :: + const rightStart = []; + while (i < val.length) { + let hexStr = ''; + while (i < val.length && val[i] !== ':') { + hexStr += val[i]; + i++; + } + if (hexStr) { + const num = parseInt(hexStr, 16); + if (isNaN(num) || num < 0 || num > 0xffff) { + throw new Error('Invalid IPv6 format'); + } + rightStart.push(num >> 8, num & 0xff); + } + i++; // Skip colon + } + + const rightGroups = rightStart.length / 2; + const zeroGroups = 8 - leftGroups - rightGroups; + + // Fill zeros + for (let z = 0; z < zeroGroups * 2; z++) { + buffer[pos + byteIdx++] = 0; + } - const parts = expanded.split(':'); - if (parts.length !== 8) { - throw new Error('Invalid IPv6 format'); + // Add right side + for (let r = 0; r < rightStart.length; r++) { + buffer[pos + byteIdx++] = rightStart[r]; + } } + else { + // No :: expansion needed, parse directly + let i = 0; + let groupCount = 0; + while (i < val.length) { + let hexStr = ''; + while (i < val.length && val[i] !== ':') { + hexStr += val[i]; + i++; + } + if (hexStr) { + const num = parseInt(hexStr || '0', 16); + if (isNaN(num) || num < 0 || num > 0xffff) { + throw new Error('Invalid IPv6 format'); + } + buffer[pos + byteIdx++] = num >> 8; + buffer[pos + byteIdx++] = num & 0xff; + groupCount++; + } + i++; // Skip colon + } - const bytes = []; - for (let i = 0; i < 8; i++) { - const num = parseInt(parts[i] || '0', 16); - if (isNaN(num) || num < 0 || num > 0xffff) { - throw new Error('Invalid IPv6 format'); + if (groupCount !== 8) { + throw new Error('Invalid IPv6 format: expected 8 groups'); } - bytes.push(num >> 8, num & 0xff); } - return bytes; + if (byteIdx !== 16) { + throw new Error('Invalid IPv6 format: incorrect byte count'); + } + + return pos + 16; } /** - * Date encoder - converts YYYY-MM-DD to 4 bytes (days since epoch) - * Date format: "2025-10-28" (10 chars = 20 bytes as string) - * Binary format: 4 bytes (signed int32, days since Jan 1, 1970) - * Saves 16 bytes - * @private + * Date encoder - writes 4 bytes directly to buffer (days since epoch) + * @returns new position */ -function date(val) { +function date(val, buffer, pos) { const parsed = new Date(val + 'T00:00:00Z'); if (isNaN(parsed.getTime())) { throw new Error('Invalid date format'); @@ -148,62 +241,63 @@ function date(val) { const epochMs = parsed.getTime(); const days = Math.floor(epochMs / 86400000); - return int32(days); + return int32(days, buffer, pos); } /** - * Date-time encoder - converts ISO 8601 to 8 bytes (milliseconds since epoch) - * DateTime format: "2025-10-28T14:30:00Z" (20+ chars = 40+ bytes as string) - * Binary format: 8 bytes (int64, milliseconds since Jan 1, 1970) - * Saves 32+ bytes - * @private + * Date-time encoder - writes 8 bytes directly to buffer (milliseconds since epoch) + * @returns new position */ -function dateTime(val) { +function dateTime(val, buffer, pos) { const parsed = new Date(val); if (isNaN(parsed.getTime())) { throw new Error('Invalid date-time format'); } // Store as milliseconds since epoch using double precision - return double(parsed.getTime()); + return double(parsed.getTime(), buffer, pos); } /** - * Binary encoder - converts base64 string or Buffer to raw bytes - * Base64 format: "SGVsbG8gV29ybGQ=" (4 chars per 3 bytes) - * Binary format: raw bytes (1 byte per byte) - * Saves ~33% compared to storing base64 as string (2 bytes per char) - * @private + * Binary encoder - writes bytes directly to buffer + * Accepts Buffer, Uint8Array, or base64 string + * @returns new position */ -function binary(val) { - // Accept Buffer, Uint8Array, or base64 string +function binary(val, buffer, pos) { if (Buffer.isBuffer(val)) { - return Array.from(val); + // Copy buffer bytes directly + val.copy(buffer, pos); + return pos + val.length; } if (val instanceof Uint8Array) { - return Array.from(val); + // Copy Uint8Array bytes directly + buffer.set(val, pos); + return pos + val.length; } // Assume base64 encoded string if (typeof val === 'string') { - const buffer = Buffer.from(val, 'base64'); - return Array.from(buffer); + // Decode base64 directly into target buffer + const bytesWritten = Buffer.from(val, 'base64').copy(buffer, pos); + return pos + bytesWritten; } throw new Error('Binary format requires Buffer, Uint8Array, or base64 string'); } -/** @private */ -function array(schema, val) { - const ret = []; - +/** + * Array encoder - writes array items directly to buffer + * Uses reserve-and-fill strategy for variable-length items + * @returns new position + */ +function array(schema, val, buffer, pos) { for (let i = 0; i < val.length; i++) { const item = val[i]; // Handle nullable array items if (schema.nullable && item === null) { - ret.push(NULL_INDICATOR); + buffer[pos++] = NULL_INDICATOR; continue; } @@ -214,7 +308,7 @@ function array(schema, val) { // Find matching variant for (let v = 0; v < schema.variants.length; v++) { - if (matchesVariantItem(item, schema.variants[v])) { + if (matchesVariant(item, schema.variants[v])) { variantIndex = v; variantField = schema.variants[v]; break; @@ -225,122 +319,145 @@ function array(schema, val) { throw new Error(`Array item does not match any variant`); } - // Encode discriminator (1-indexed for nullable support) - const discriminator = schema.nullable ? VARIANT_BASE + variantIndex : VARIANT_BASE + variantIndex; - ret.push(discriminator); - - // Encode the item using the matched variant - const encoded = variantField.transformIn(item); - ret.push(...variantField.getSize(encoded.length), ...encoded); + // Write discriminator + buffer[pos++] = schema.nullable ? VARIANT_BASE + variantIndex : VARIANT_BASE + variantIndex; + + // For fixed-size types, write size first, then data + // For variable-size, reserve space, write data, fill in size + if (variantField.size) { + // Fixed size - write size directly + const count = variantField.count; + if (count === 1) buffer[pos++] = variantField.size; + else if (count === 2) { + buffer[pos++] = variantField.size >> 8; + buffer[pos++] = variantField.size & 0xff; + } + else if (count === 4) { + buffer[pos++] = variantField.size >> 24; + buffer[pos++] = variantField.size >> 16; + buffer[pos++] = variantField.size >> 8; + buffer[pos++] = variantField.size & 0xff; + } + pos = variantField.transformIn(item, buffer, pos); + } + else { + // Variable size - reserve space for size, write data, fill in size + const sizePos = pos; + pos += variantField.count; // Reserve space + const dataStart = pos; + pos = variantField.transformIn(item, buffer, pos); + const size = pos - dataStart; + + // Fill in size + if (variantField.count === 1) buffer[sizePos] = size & 0xff; + else if (variantField.count === 2) { + buffer[sizePos] = size >> 8; + buffer[sizePos + 1] = size & 0xff; + } + else if (variantField.count === 4) { + buffer[sizePos] = size >> 24; + buffer[sizePos + 1] = size >> 16; + buffer[sizePos + 2] = size >> 8; + buffer[sizePos + 3] = size & 0xff; + } + } continue; } // Handle nullable non-null items (add presence indicator) if (schema.nullable) { - ret.push(VARIANT_BASE); + buffer[pos++] = VARIANT_BASE; } // Handle regular array items - const encoded = schema.transformIn(item); - ret.push(...schema.getSize(encoded.length), ...encoded); - } - - return ret; -} - -/** @private */ -function matchesVariantItem(data, variant) { - const dataType = Array.isArray(data) - ? 'array' - : data === null - ? 'null' - : typeof data; - - // Map internal types to JavaScript types - if (variant.type === 'int32' || variant.type === 'int64' - || variant.type === 'float' || variant.type === 'double') { - return dataType === 'number'; - } - - if (variant.type === 'string' || variant.type === 'uuid' - || variant.type === 'ipv4' || variant.type === 'ipv6' - || variant.type === 'date' || variant.type === 'date-time' - || variant.type === 'binary') { - return dataType === 'string' || data instanceof Buffer || data instanceof Uint8Array; - } - - if (variant.type === 'boolean') { - return dataType === 'boolean'; - } - - if (variant.type === 'array') { - return dataType === 'array'; - } - - if (variant.type === 'object') { - if (dataType !== 'object') return false; - - // For objects with schema keys, check if the data properties match - if (variant.schemaKeys && variant.schemaKeys.length > 0) { - const schemaKeys = variant.schemaKeys; - const dataKeys = Object.keys(data); - - // Check if data keys match schema keys - let matchCount = 0; - for (const key of schemaKeys) { - if (data.hasOwnProperty(key)) { - matchCount++; - } + if (schema.size) { + // Fixed size - write size first, then data + const count = schema.count; + if (count === 1) buffer[pos++] = schema.size; + else if (count === 2) { + buffer[pos++] = schema.size >> 8; + buffer[pos++] = schema.size & 0xff; + } + else if (count === 4) { + buffer[pos++] = schema.size >> 24; + buffer[pos++] = schema.size >> 16; + buffer[pos++] = schema.size >> 8; + buffer[pos++] = schema.size & 0xff; + } + pos = schema.transformIn(item, buffer, pos); + } + else { + // Variable size - reserve space for size, write data, fill in size + const sizePos = pos; + pos += schema.count; // Reserve space + const dataStart = pos; + pos = schema.transformIn(item, buffer, pos); + const size = pos - dataStart; + + // Fill in size + if (schema.count === 1) buffer[sizePos] = size & 0xff; + else if (schema.count === 2) { + buffer[sizePos] = size >> 8; + buffer[sizePos + 1] = size & 0xff; + } + else if (schema.count === 4) { + buffer[sizePos] = size >> 24; + buffer[sizePos + 1] = size >> 16; + buffer[sizePos + 2] = size >> 8; + buffer[sizePos + 3] = size & 0xff; } - - // Require at least one matching key and 50% overlap - return matchCount > 0 && matchCount >= Math.min(schemaKeys.length, dataKeys.length) * 0.5; } - - return true; } - return false; + return pos; } -/** @private */ -function object(schema, val) { - return schema.write(val).typedArray(); +/** + * Object encoder - writes nested object directly to buffer + * Delegates to nested schema's write function + * @returns new position + */ +function object(schema, val, buffer, pos) { + // Call nested schema's writeToBuffer method (will be added to writer) + return schema.writeToBuffer(val, buffer, pos); } /** - * IEEE 754 single precision (32-bit float) - * Simplified implementation using JavaScript's Float32Array - * @private + * IEEE 754 single precision (32-bit float) - writes 4 bytes directly to buffer + * Uses reusable module-level buffer for conversion + * @returns new position */ -function float(val) { - // Use Float32Array to get proper IEEE 754 single precision encoding - const floatArray = new Float32Array(1); - const byteArray = new Uint8Array(floatArray.buffer); - - floatArray[0] = val; - - // Return bytes in big-endian order to match double implementation - return [byteArray[3], byteArray[2], byteArray[1], byteArray[0]]; +function float(val, buffer, pos) { + // Reuse module-level buffer to avoid allocation overhead + floatBuffer[0] = val; + + // Write bytes in big-endian order + buffer[pos] = floatBytes[3]; + buffer[pos + 1] = floatBytes[2]; + buffer[pos + 2] = floatBytes[1]; + buffer[pos + 3] = floatBytes[0]; + return pos + 4; } /** - * IEEE 754 double precision (64-bit float) - * Simplified implementation using JavaScript's Float64Array - * @private + * IEEE 754 double precision (64-bit float) - writes 8 bytes directly to buffer + * Uses reusable module-level buffer for conversion + * @returns new position */ -function double(val) { - // Use Float64Array to get proper IEEE 754 double precision encoding - const doubleArray = new Float64Array(1); - const byteArray = new Uint8Array(doubleArray.buffer); - - doubleArray[0] = val; - - // Return bytes in big-endian order - return [ - byteArray[7], byteArray[6], byteArray[5], byteArray[4], - byteArray[3], byteArray[2], byteArray[1], byteArray[0], - ]; +function double(val, buffer, pos) { + // Reuse module-level buffer to avoid allocation overhead + doubleBuffer[0] = val; + + // Write bytes in big-endian order + buffer[pos] = doubleBytes[7]; + buffer[pos + 1] = doubleBytes[6]; + buffer[pos + 2] = doubleBytes[5]; + buffer[pos + 3] = doubleBytes[4]; + buffer[pos + 4] = doubleBytes[3]; + buffer[pos + 5] = doubleBytes[2]; + buffer[pos + 6] = doubleBytes[1]; + buffer[pos + 7] = doubleBytes[0]; + return pos + 8; } /** @@ -348,13 +465,27 @@ function double(val) { * JavaScript's Number type can safely represent integers up to 2^53-1 * @private */ -function int64(val) { - return double(val); +function int64(val, buffer, pos) { + return double(val, buffer, pos); } /** @private */ function getSize(count, byteLength) { - return intMap[count](byteLength); + if (count === 1) { + return Buffer.from([byteLength & 0xff]); + } + if (count === 2) { + return Buffer.from([byteLength >> 8, byteLength & 0xff]); + } + if (count === 4) { + return Buffer.from([ + byteLength >> 24, + (byteLength >> 16) & 0xff, + (byteLength >> 8) & 0xff, + byteLength & 0xff, + ]); + } + return Buffer.from([]); } /* Exports ------------------------------------------------------------------- */ diff --git a/src/reader.ts b/src/reader.ts index 90ec42a..b07509f 100644 --- a/src/reader.ts +++ b/src/reader.ts @@ -7,112 +7,170 @@ import Decoder, { NULL_INDICATOR, VARIANT_BASE } from './decoder'; /* Methods ------------------------------------------------------------------- */ export default function Reader(scope) { - function read(bytes) { - readHeader(bytes); - return readContent(bytes, scope.contentBegins); - } - - function readHeader(bytes) { - scope.header = []; - let caret = 1; - const keys = bytes[0]; - for (let i = 0; i < keys; i++) { - caret = readKey(bytes, caret, i); + /** + * Read from buffer at specific offset (zero-copy for nested objects) + * @private + */ + function readFromOffset(bytes, offset, length) { + const ret = {}; + if (scope.options.keyOrder === true) { + for (let i = 0; i < scope.items.length; i++) { + ret[scope.items[i]] = undefined; + } } - scope.contentBegins = caret; - - return this; - } - /** @private */ - function readKey(bytes, caret, index) { - const key = getSchemaDef(bytes[caret]); - caret++; // Move past field index + let caret = offset + 1; // Start after field count + const fieldCount = bytes[offset]; + const end = offset + length; - let size; - let variantIndex = null; + for (let i = 0; i < fieldCount; i++) { + if (caret >= end) break; - // Check for discriminator byte if field is nullable or has variants - if (key.nullable || key.variants) { - const discriminatorByte = bytes[caret]; - caret++; // Move past discriminator byte + // Read field index + const fieldIndex = bytes[caret]; + caret++; - if (discriminatorByte === NULL_INDICATOR) { - // Field is null - no size or content follows - size = -1; // Use -1 as internal null marker + const field = scope.indexToField[fieldIndex]; + if (!field) { + throw new Error(`Unknown field index: ${fieldIndex}`); } - else { - // For variants, extract which variant to use (0-indexed internally) - if (key.variants) { - variantIndex = discriminatorByte - VARIANT_BASE; - if (variantIndex < 0 || variantIndex >= key.variants.length) { + + // Check for discriminator byte if field is nullable or has variants + if (field.nullable || field.variants) { + const discriminatorByte = bytes[caret]; + caret++; + + if (discriminatorByte === NULL_INDICATOR) { + // Field is null - no size or content follows + ret[field.name] = null; + continue; + } + + // Handle variant fields + if (field.variants) { + const variantIndex = discriminatorByte - VARIANT_BASE; + if (variantIndex < 0 || variantIndex >= field.variants.length) { throw new Error(`Invalid variant discriminator: ${discriminatorByte}`); } - // Use the variant's metadata for size reading - const variant = key.variants[variantIndex]; - const sizeBytes = bytes.slice(caret, caret + variant.count); - size = variant.size || Decoder.unsigned(sizeBytes); - caret += variant.count; // Move past size bytes - } - else { - // Regular nullable field is present - read size - const sizeBytes = bytes.slice(caret, caret + key.count); - size = key.size || Decoder.unsigned(sizeBytes); - caret += key.count; // Move past size bytes + + const variant = field.variants[variantIndex]; + const size = variant.size || readSize(bytes, caret, variant.count); + caret += variant.count; + + // Zero-copy decode: pass offset and length for all types + ret[field.name] = variant.transformOut(bytes, caret, size); + caret += size; + continue; } - } - } - else { - // Non-nullable, non-variant field - read size directly - const sizeBytes = bytes.slice(caret, caret + key.count); - size = key.size || Decoder.unsigned(sizeBytes); - caret += key.count; // Move past size bytes - } - scope.header[index] = { - key, - size, - variantIndex, - }; - return caret; - } + // Regular nullable field is present + const size = field.size || readSize(bytes, caret, field.count); + caret += field.count; + + // Zero-copy decode: pass offset and length for all types + ret[field.name] = field.transformOut(bytes, caret, size); + caret += size; + } + else { + // Non-nullable, non-variant field + const size = field.size || readSize(bytes, caret, field.count); + caret += field.count; - /** @private */ - function getSchemaDef(index) { - for (let i = 0; i < scope.items.length; i++) { - if (scope.indices[scope.items[i]].index === index) return scope.indices[scope.items[i]]; + // Zero-copy decode: pass offset and length for all types + ret[field.name] = field.transformOut(bytes, caret, size); + caret += size; + } } + + return ret; } - function readContent(bytes, caret?) { - caret = caret || 0; + function read(bytes) { const ret = {}; if (scope.options.keyOrder === true) { for (let i = 0; i < scope.items.length; i++) { ret[scope.items[i]] = undefined; } } - for (let i = 0; i < scope.header.length; i++) { - // Handle null values (size -1 indicates null) - if (scope.header[i].size === -1) { - ret[scope.header[i].key.name] = null; - continue; + + let caret = 1; // Start after field count + const fieldCount = bytes[0]; + + for (let i = 0; i < fieldCount; i++) { + // Read field index + const fieldIndex = bytes[caret]; + caret++; + + const field = scope.indexToField[fieldIndex]; + if (!field) { + throw new Error(`Unknown field index: ${fieldIndex}`); } - // Handle variant fields - if (scope.header[i].variantIndex !== null && scope.header[i].key.variants) { - const variant = scope.header[i].key.variants[scope.header[i].variantIndex]; - ret[scope.header[i].key.name] = variant.transformOut(bytes.slice(caret, caret + scope.header[i].size)); - caret += scope.header[i].size; - continue; + // Check for discriminator byte if field is nullable or has variants + if (field.nullable || field.variants) { + const discriminatorByte = bytes[caret]; + caret++; + + if (discriminatorByte === NULL_INDICATOR) { + // Field is null - no size or content follows + ret[field.name] = null; + continue; + } + + // Handle variant fields + if (field.variants) { + const variantIndex = discriminatorByte - VARIANT_BASE; + if (variantIndex < 0 || variantIndex >= field.variants.length) { + throw new Error(`Invalid variant discriminator: ${discriminatorByte}`); + } + + const variant = field.variants[variantIndex]; + const size = variant.size || readSize(bytes, caret, variant.count); + caret += variant.count; + + // Zero-copy decode: pass offset and length for all types + ret[field.name] = variant.transformOut(bytes, caret, size); + caret += size; + continue; + } + + // Regular nullable field is present + const size = field.size || readSize(bytes, caret, field.count); + caret += field.count; + + // Zero-copy decode: pass offset and length for all types + ret[field.name] = field.transformOut(bytes, caret, size); + caret += size; } + else { + // Non-nullable, non-variant field + const size = field.size || readSize(bytes, caret, field.count); + caret += field.count; - // Handle regular fields - ret[scope.header[i].key.name] = scope.header[i].key.transformOut(bytes.slice(caret, caret + scope.header[i].size)); - caret += scope.header[i].size; + // Zero-copy decode: pass offset and length for all types + ret[field.name] = field.transformOut(bytes, caret, size); + caret += size; + } } + return ret; } - return { read, readHeader, readContent }; + /** + * Performance: Fast size reading without array slicing + * Reads 1-4 bytes directly from buffer instead of creating slice + * @private + */ + function readSize(bytes, offset, count) { + if (count === 1) return bytes[offset]; + if (count === 2) return (bytes[offset] << 8) | bytes[offset + 1]; + if (count === 4) { + return (bytes[offset] << 24) | (bytes[offset + 1] << 16) + | (bytes[offset + 2] << 8) | bytes[offset + 3]; + } + // Fallback for unexpected sizes (should not happen in practice) + return Decoder.unsigned(bytes.slice(offset, offset + count)); + } + + return { read, readFromOffset }; } diff --git a/src/schema.ts b/src/schema.ts index 8404fe4..8e1d63c 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -48,23 +48,6 @@ export default function Schema(schema, options = { keyOrder: false }) { // Normalize OpenAPI schema format to internal format const normalizedSchema = normalizeSchema(unwrappedSchema, options); - const sizeRef = { - 'boolean': 1, - 'int32': 4, - 'int64': 8, - 'float': 4, - 'double': 8, - 'string': 2, - 'uuid': 1, - 'ipv4': 1, - 'ipv6': 1, - 'date': 1, - 'date-time': 1, - 'binary': 4, - 'array': 2, - 'object': 1, - }; - const defaultSizes = { 'boolean': 1, 'int32': 4, @@ -82,14 +65,21 @@ export default function Schema(schema, options = { keyOrder: false }) { schema: normalizedSchema, indices: {}, items: Object.keys(normalizedSchema), - headerBytes: [0], - contentBytes: [0], - header: [], - contentBegins: 0, + buffer: [], options, + indexToField: {}, // Performance: O(1) reverse lookup from index to field + itemsSet: null, // Performance: O(1) field validation in writer }; scope.indices = preformat(normalizedSchema); + // Build reverse index map for O(1) lookups during deserialization + for (const fieldName of scope.items) { + scope.indexToField[scope.indices[fieldName].index] = scope.indices[fieldName]; + } + + // Build Set for O(1) field validation during serialization + scope.itemsSet = new Set(scope.items); + /** @private */ function resolveRef(ref, options) { if (!ref || !ref.startsWith('#/')) { @@ -175,8 +165,6 @@ export default function Schema(schema, options = { keyOrder: false }) { const writer = Writer(scope); const reader = Reader(scope); - applyBlank(); // Pre-load header for easy streaming - /** @private */ function preformat(schema) { const ret = {}; @@ -190,8 +178,8 @@ export default function Schema(schema, options = { keyOrder: false }) { const variantType = variantDef.type; const variantFormat = variantDef.format; const variantInternalType = resolveType(variantType, variantFormat); - // Binary fields need 4-byte counter by default to support large data - const variantCount = variantDef.count || (variantInternalType === 'binary' ? 4 : 1); + // Binary fields need 4 bytes, arrays/objects need 2 bytes for size counters + const variantCount = variantDef.count || (variantInternalType === 'binary' ? 4 : (variantInternalType === 'array' || variantInternalType === 'object' ? 2 : 1)); const variantChildSchema = computeNestedVariant(variantDef); // For object variants, extract schema keys for variant matching @@ -210,7 +198,7 @@ export default function Schema(schema, options = { keyOrder: false }) { coerse: Converter[variantInternalType], getSize: Encoder.getSize.bind(null, variantCount), fixedSize: (defaultSizes[variantInternalType] && Encoder.getSize(variantCount, defaultSizes[variantInternalType])) || null, - size: variantDef.size || defaultSizes[variantInternalType] || null, + size: defaultSizes[variantInternalType] || null, count: variantCount, nested: variantChildSchema, schemaKeys, @@ -230,8 +218,8 @@ export default function Schema(schema, options = { keyOrder: false }) { const fieldType = schema[key].type; const fieldFormat = schema[key].format; const internalType = resolveType(fieldType, fieldFormat); - // Binary fields need 4-byte counter by default to support large data - const count = schema[key].count || (internalType === 'binary' ? 4 : 1); + // Binary fields need 4 bytes, arrays/objects need 2 bytes for size counters + const count = schema[key].count || (internalType === 'binary' ? 4 : (internalType === 'array' || internalType === 'object' ? 2 : 1)); const childSchema = computeNested(schema, key); ret[key] = { @@ -244,7 +232,7 @@ export default function Schema(schema, options = { keyOrder: false }) { coerse: Converter[internalType], getSize: Encoder.getSize.bind(null, count), fixedSize: (defaultSizes[internalType] && Encoder.getSize(count, defaultSizes[internalType])) || null, - size: schema[key].size || defaultSizes[internalType] || null, + size: defaultSizes[internalType] || null, count, nested: childSchema, }; @@ -253,20 +241,6 @@ export default function Schema(schema, options = { keyOrder: false }) { return ret; } - /** @private */ - function applyBlank() { - for (const key in scope.schema) { - // Skip variant fields in applyBlank as their size depends on runtime variant - if (scope.indices[key].variants) { - continue; - } - scope.header.push({ - key: scope.indices[key], - size: scope.indices[key].size || sizeRef[scope.indices[key].type], - }); - } - } - /** @private */ function computeNested(schema, key) { const keyType = schema[key].type; @@ -296,7 +270,7 @@ export default function Schema(schema, options = { keyOrder: false }) { const variantType = variantDef.type; const variantFormat = variantDef.format; const variantInternalType = resolveType(variantType, variantFormat); - const variantCount = variantDef.count || (variantInternalType === 'binary' ? 4 : 1); + const variantCount = variantDef.count || (variantInternalType === 'binary' ? 4 : (variantInternalType === 'array' || variantInternalType === 'object' ? 2 : 1)); const variantChildSchema = processArrayItemsNested(variantDef); // For object variants, extract schema keys for variant matching @@ -315,7 +289,7 @@ export default function Schema(schema, options = { keyOrder: false }) { coerse: Converter[variantInternalType], getSize: Encoder.getSize.bind(null, variantCount), fixedSize: (defaultSizes[variantInternalType] && Encoder.getSize(variantCount, defaultSizes[variantInternalType])) || null, - size: variantDef.size || defaultSizes[variantInternalType] || null, + size: defaultSizes[variantInternalType] || null, count: variantCount, nested: variantChildSchema, schemaKeys, @@ -334,7 +308,7 @@ export default function Schema(schema, options = { keyOrder: false }) { const itemType = itemDef.type; const itemFormat = itemDef.format; const internalItemType = resolveType(itemType, itemFormat); - const itemCount = itemDef.count || (internalItemType === 'binary' ? 4 : 1); + const itemCount = itemDef.count || (internalItemType === 'binary' ? 4 : (internalItemType === 'array' || internalItemType === 'object' ? 2 : 1)); const itemChildSchema = processArrayItemsNested(itemDef); return { diff --git a/src/size-writer.ts b/src/size-writer.ts new file mode 100644 index 0000000..1710850 --- /dev/null +++ b/src/size-writer.ts @@ -0,0 +1,19 @@ +/** Shared size writing utilities */ + +/** + * Write size bytes directly to array in big-endian format + * @param target - Array to write to + * @param size - Size value to encode + * @param count - Number of bytes (1, 2, or 4) + */ +export function writeSizeBytes(target: number[], size: number, count: number): void { + if (count === 1) { + target.push(size & 0xff); + } + else if (count === 2) { + target.push(size >> 8, size & 0xff); + } + else if (count === 4) { + target.push(size >> 24, size >> 16, size >> 8, size & 0xff); + } +} diff --git a/src/variant-matcher.ts b/src/variant-matcher.ts new file mode 100644 index 0000000..8671ac1 --- /dev/null +++ b/src/variant-matcher.ts @@ -0,0 +1,59 @@ +/** Shared variant matching utilities */ + +/* Type compatibility lookup for variant matching */ +export const TYPE_COMPAT_MAP = { + number: new Set(['int32', 'int64', 'float', 'double']), + string: new Set(['string', 'uuid', 'ipv4', 'ipv6', 'date', 'date-time', 'binary']), + boolean: new Set(['boolean']), + array: new Set(['array']), + object: new Set(['object']), +}; + +/** + * Check if data matches a variant schema definition + * @private + */ +export function matchesVariant(data, variant) { + // Performance: Optimized type detection + let dataType; + if (data === null) { + dataType = 'null'; + } + else if (Array.isArray(data)) { + dataType = 'array'; + } + else { + dataType = typeof data; + } + + // Special handling for binary types (Buffer/Uint8Array) + if (variant.type === 'binary' && (data instanceof Buffer || data instanceof Uint8Array)) { + return true; + } + + // Fast path: Use type compatibility map for quick lookup + const compatibleTypes = TYPE_COMPAT_MAP[dataType]; + if (compatibleTypes && compatibleTypes.has(variant.type)) { + // For objects, additional validation needed + if (variant.type === 'object' && variant.schemaKeys && variant.schemaKeys.length > 0) { + const schemaKeys = variant.schemaKeys; + const dataKeys = Object.keys(data); + + // Check if data keys match schema keys + let matchCount = 0; + for (const key of schemaKeys) { + if (data.hasOwnProperty(key)) { + matchCount++; + } + } + + // Require at least one matching key and 50% overlap + // This helps distinguish between different object variants + return matchCount > 0 && matchCount >= Math.min(schemaKeys.length, dataKeys.length) * 0.5; + } + + return true; + } + + return false; +} diff --git a/src/writer.ts b/src/writer.ts index f28c907..0a219c2 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -3,24 +3,86 @@ /* Requires ------------------------------------------------------------------ */ import { NULL_INDICATOR, VARIANT_BASE } from './encoder'; +import { matchesVariant } from './variant-matcher'; /* Methods ------------------------------------------------------------------- */ export default function Writer(scope) { - function write(data, options?) { - scope.headerBytes = [0]; - scope.contentBytes = []; + /** + * Estimate buffer size for pre-allocation + * Conservative estimate to minimize reallocation + */ + function estimateBufferSize(keys) { + let size = 1; // Field count byte - const keys = filterKeys(data, options); - scope.headerBytes[0] = keys.length; for (let i = 0; i < keys.length; i++) { - let keyData = data[keys[i]]; const field = scope.indices[keys[i]]; + // Field index (1 byte) + optional discriminator (1 byte) + size += field.nullable || field.variants ? 2 : 1; + + // Size marker bytes + if (field.fixedSize) { + size += field.fixedSize.length; + } + else { + size += field.count || 1; + } + + // Estimated data size + if (field.size) { + // Fixed size field (int32, float, etc.) + size += field.size; + } + else { + // Variable size field - estimate conservatively + // For typical API fields: strings ~20 bytes, arrays ~50 bytes, objects ~100 bytes + if (field.type === 'string') size += 32; + else if (field.type === 'array') size += 64; + else if (field.type === 'object') size += 128; + else size += 16; // Other variable types + } + } + + // Add 50% buffer for safety + return Math.max(Math.floor(size * 1.5), 256); + } + + /** + * Ensure buffer has capacity and grow if needed + */ + function ensureCapacity(needed) { + if (scope.position + needed > scope.buffer.length) { + const newSize = Math.max(scope.buffer.length * 2, scope.position + needed); + const newBuffer = Buffer.allocUnsafe(newSize); + scope.buffer.copy(newBuffer, 0, 0, scope.position); + scope.buffer = newBuffer; + } + } + + function write(data, options?) { + const keys = filterKeys(data, options); + + // Pre-allocate buffer based on estimation + // Use Buffer.allocUnsafe for performance and compatibility with Buffer.write() + const estimatedSize = estimateBufferSize(keys); + scope.buffer = Buffer.allocUnsafe(estimatedSize); + scope.position = 0; + + // Write field count + scope.buffer[scope.position++] = keys.length; + + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + let keyData = data[key]; + const field = scope.indices[key]; // Performance: Single lookup, reused throughout + // Handle nullable fields with null values if (field.nullable && keyData === null) { - // Write header with NULL_INDICATOR (no size or content follows) - scope.headerBytes.push(field.index, NULL_INDICATOR); + // Write field index and NULL_INDICATOR (no size or content follows) + ensureCapacity(2); + scope.buffer[scope.position++] = field.index; + scope.buffer[scope.position++] = NULL_INDICATOR; continue; } @@ -43,7 +105,9 @@ export default function Writer(scope) { } // Write field index and variant discriminator (1-indexed) - scope.headerBytes.push(field.index, VARIANT_BASE + variantIndex); + ensureCapacity(2); + scope.buffer[scope.position++] = field.index; + scope.buffer[scope.position++] = VARIANT_BASE + variantIndex; // Apply coercion/validation if needed if (options !== undefined) { @@ -55,18 +119,20 @@ export default function Writer(scope) { } } - // Encode using the matched variant's transformer - const encoded = variantField.transformIn(keyData); - addSizeAndContentForVariant(encoded, variantField); + // NEW API: Write size and value directly to buffer + scope.position = writeFieldValue(keyData, variantField); continue; } // Handle regular fields with discriminator byte if nullable if (field.nullable) { - scope.headerBytes.push(field.index, VARIANT_BASE); + ensureCapacity(2); + scope.buffer[scope.position++] = field.index; + scope.buffer[scope.position++] = VARIANT_BASE; } else { - scope.headerBytes.push(field.index); + ensureCapacity(1); + scope.buffer[scope.position++] = field.index; } if (options !== undefined) { @@ -74,112 +140,66 @@ export default function Writer(scope) { if (options.validate === true && field.validate) field.validate(keyData); } - // Add size and content - const encoded = field.transformIn(keyData); - addSizeAndContent(encoded, keys[i]); + // NEW API: Write size and value directly to buffer + scope.position = writeFieldValue(keyData, field); } return this; } - /** @private */ - function matchesVariant(data, variant) { - const dataType = Array.isArray(data) - ? 'array' - : data === null - ? 'null' - : typeof data === 'boolean' - ? 'boolean' - : typeof data === 'number' - ? 'number' - : typeof data === 'string' - ? 'string' - : typeof data === 'object' - ? 'object' - : 'unknown'; - - // Map internal types to JavaScript types - // Number types: int32, int64, float, double - if (variant.type === 'int32' || variant.type === 'int64' - || variant.type === 'float' || variant.type === 'double') { - return dataType === 'number'; - } - - // String types: string, uuid, ipv4, ipv6, date, date-time, binary - if (variant.type === 'string' || variant.type === 'uuid' - || variant.type === 'ipv4' || variant.type === 'ipv6' - || variant.type === 'date' || variant.type === 'date-time' - || variant.type === 'binary') { - return dataType === 'string'; - } - - if (variant.type === 'boolean') { - return dataType === 'boolean'; - } - - if (variant.type === 'array') { - return dataType === 'array'; + /** + * Write field value using new encoder API + * Encoders now write directly to buffer and return new position + * For fixed-size fields, writes size then data + * For variable-size fields, reserves space for size, writes data, fills in size + * @returns new position + */ + function writeFieldValue(value, fieldOrVariant) { + ensureCapacity(1024); // Ensure some headroom (will grow if needed) + + // Fixed-size fields: write size then data directly + if (fieldOrVariant.size) { + const count = fieldOrVariant.count; + if (count === 1) scope.buffer[scope.position++] = fieldOrVariant.size; + else if (count === 2) { + scope.buffer[scope.position++] = fieldOrVariant.size >> 8; + scope.buffer[scope.position++] = fieldOrVariant.size & 0xff; + } + else if (count === 4) { + scope.buffer[scope.position++] = fieldOrVariant.size >> 24; + scope.buffer[scope.position++] = fieldOrVariant.size >> 16; + scope.buffer[scope.position++] = fieldOrVariant.size >> 8; + scope.buffer[scope.position++] = fieldOrVariant.size & 0xff; + } + // Call encoder with new API: (val, buffer, pos) => newPos + return fieldOrVariant.transformIn(value, scope.buffer, scope.position); } - if (variant.type === 'object') { - if (dataType !== 'object') return false; - - // For objects with schema keys, check if the data properties match - if (variant.schemaKeys && variant.schemaKeys.length > 0) { - const schemaKeys = variant.schemaKeys; - const dataKeys = Object.keys(data); - - // Check if data keys match schema keys - let matchCount = 0; - for (const key of schemaKeys) { - if (data.hasOwnProperty(key)) { - matchCount++; - } - } + // Variable-size fields: reserve space for size, write data, fill in size + const sizePos = scope.position; + scope.position += fieldOrVariant.count; // Reserve space for size + const dataStart = scope.position; - // Require at least one matching key and 50% overlap - // This helps distinguish between different object variants - return matchCount > 0 && matchCount >= Math.min(schemaKeys.length, dataKeys.length) * 0.5; - } + // Call encoder with new API: (val, buffer, pos) => newPos + const newPos = fieldOrVariant.transformIn(value, scope.buffer, scope.position); + const size = newPos - dataStart; - return true; + // Fill in size at reserved position + if (fieldOrVariant.count === 1) { + scope.buffer[sizePos] = size & 0xff; } - - return false; - } - - /** @private */ - function addSizeAndContent(encoded, key) { - if (scope.indices[key].fixedSize !== null) { - scope.headerBytes.push(...scope.indices[key].fixedSize); + else if (fieldOrVariant.count === 2) { + scope.buffer[sizePos] = size >> 8; + scope.buffer[sizePos + 1] = size & 0xff; } - else { - scope.headerBytes.push(...scope.indices[key].getSize(encoded.length)); - if (scope.indices[key].size !== encoded.length && scope.indices[key].size !== null) { - const fixedSize = new Array(scope.indices[key].size).fill(0); - const smallestSize = Math.min(encoded.length, fixedSize.length); - fixedSize.splice(0, smallestSize, ...encoded.slice(0, smallestSize)); - return scope.contentBytes.push(...fixedSize); - } + else if (fieldOrVariant.count === 4) { + scope.buffer[sizePos] = size >> 24; + scope.buffer[sizePos + 1] = size >> 16; + scope.buffer[sizePos + 2] = size >> 8; + scope.buffer[sizePos + 3] = size & 0xff; } - scope.contentBytes.push(...encoded); - } - /** @private */ - function addSizeAndContentForVariant(encoded, variant) { - if (variant.fixedSize !== null) { - scope.headerBytes.push(...variant.fixedSize); - } - else { - scope.headerBytes.push(...variant.getSize(encoded.length)); - if (variant.size !== encoded.length && variant.size !== null) { - const fixedSize = new Array(variant.size).fill(0); - const smallestSize = Math.min(encoded.length, fixedSize.length); - fixedSize.splice(0, smallestSize, ...encoded.slice(0, smallestSize)); - return scope.contentBytes.push(...fixedSize); - } - } - scope.contentBytes.push(...encoded); + return newPos; } function sizes(data) { @@ -201,7 +221,8 @@ export default function Writer(scope) { const undeclaredKeys = []; for (const key in data) { - if (scope.items.indexOf(key) === -1) { + // Performance: O(1) Set lookup instead of O(n) indexOf + if (!scope.itemsSet.has(key)) { undeclaredKeys.push(key); continue; } @@ -228,22 +249,142 @@ export default function Writer(scope) { return res; } - /** @private */ - function typedArray() { - return [...scope.headerBytes, ...scope.contentBytes]; - } + /** + * Write to buffer at specific position (for nested objects) + * Used by object encoder when writing nested schemas + * @returns new position + */ + function writeToBuffer(data, buffer, pos) { + const keys = filterKeys(data); - function headerBuffer() { - return Buffer.from(scope.headerBytes); - } + // Write field count + buffer[pos++] = keys.length; + + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + const keyData = data[key]; + const field = scope.indices[key]; + + // Write field index + buffer[pos++] = field.index; + + // Handle nullable null values + if (field.nullable && keyData === null) { + buffer[pos++] = NULL_INDICATOR; + continue; + } + + // Handle oneOf/anyOf fields + if (field.variants) { + // Determine which variant matches the data + let variantIndex = -1; + let variantField = null; + + for (let v = 0; v < field.variants.length; v++) { + if (matchesVariant(keyData, field.variants[v])) { + variantIndex = v; + variantField = field.variants[v]; + break; + } + } + + if (variantIndex === -1) { + throw new Error(`Data does not match any variant for field: ${key}`); + } + + // Write variant discriminator + buffer[pos++] = VARIANT_BASE + variantIndex; + + // For fixed-size types, write size then data + if (variantField.size) { + const count = variantField.count; + if (count === 1) buffer[pos++] = variantField.size; + else if (count === 2) { + buffer[pos++] = variantField.size >> 8; + buffer[pos++] = variantField.size & 0xff; + } + else if (count === 4) { + buffer[pos++] = variantField.size >> 24; + buffer[pos++] = variantField.size >> 16; + buffer[pos++] = variantField.size >> 8; + buffer[pos++] = variantField.size & 0xff; + } + pos = variantField.transformIn(keyData, buffer, pos); + } + else { + // Variable size - reserve space for size, write data, fill in size + const sizePos = pos; + pos += variantField.count; // Reserve space + const dataStart = pos; + pos = variantField.transformIn(keyData, buffer, pos); + const size = pos - dataStart; + + // Fill in size + if (variantField.count === 1) buffer[sizePos] = size & 0xff; + else if (variantField.count === 2) { + buffer[sizePos] = size >> 8; + buffer[sizePos + 1] = size & 0xff; + } + else if (variantField.count === 4) { + buffer[sizePos] = size >> 24; + buffer[sizePos + 1] = size >> 16; + buffer[sizePos + 2] = size >> 8; + buffer[sizePos + 3] = size & 0xff; + } + } + continue; + } + + // Add presence indicator for nullable non-null + if (field.nullable) { + buffer[pos++] = VARIANT_BASE; + } + + // For fixed-size, write size then data + if (field.size) { + const count = field.count; + if (count === 1) buffer[pos++] = field.size; + else if (count === 2) { + buffer[pos++] = field.size >> 8; + buffer[pos++] = field.size & 0xff; + } + else if (count === 4) { + buffer[pos++] = field.size >> 24; + buffer[pos++] = field.size >> 16; + buffer[pos++] = field.size >> 8; + buffer[pos++] = field.size & 0xff; + } + pos = field.transformIn(keyData, buffer, pos); + } + else { + // Variable size - reserve, write, fill + const sizePos = pos; + pos += field.count; + const dataStart = pos; + pos = field.transformIn(keyData, buffer, pos); + const size = pos - dataStart; + + if (field.count === 1) buffer[sizePos] = size & 0xff; + else if (field.count === 2) { + buffer[sizePos] = size >> 8; + buffer[sizePos + 1] = size & 0xff; + } + else if (field.count === 4) { + buffer[sizePos] = size >> 24; + buffer[sizePos + 1] = size >> 16; + buffer[sizePos + 2] = size >> 8; + buffer[sizePos + 3] = size & 0xff; + } + } + } - function contentBuffer() { - return Buffer.from(scope.contentBytes); + return pos; } function buffer() { - return Buffer.from(typedArray()); + // Return trimmed buffer (only the bytes that were written) + return Buffer.from(scope.buffer.subarray(0, scope.position)); } - return { write, headerBuffer, contentBuffer, buffer, typedArray, sizes, matchesVariant }; + return { write, buffer, sizes, writeToBuffer }; } diff --git a/tests/integration/index.ts b/tests/integration/index.ts index 0c589b9..d240a12 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -515,124 +515,6 @@ describe('Data integrity - multi mixed', () => { }); }); -/* Partial ------------------------------------------------------------------- */ - -describe('Data integrity - partial - simple', () => { - describe('Boolean', () => { - const Schema = schema({ test: { type: 'boolean' } }); - - it('should preserve boolean value and type - true', () => { - expect(Schema.readContent(Schema.write({ test: true }).contentBuffer())).toEqual({ test: true }); - }); - - it('should preserve boolean value and type - false', () => { - expect(Schema.readContent(Schema.write({ test: false }).contentBuffer())).toEqual({ test: false }); - }); - - it('should still send one 0 byte in case of null (coersed)', () => { - expect(Schema.readContent(Schema.write({ test: null }).contentBuffer())).toEqual({ test: false }); - }); - }); - - describe('Number', () => { - const Schema = schema({ test: { type: 'number', format: 'double' } }); - - it('should preserve number value and type', () => { - expect(Schema.readContent(Schema.write({ test: 23.23 }).contentBuffer())).toEqual({ test: 23.23 }); - }); - - it('should preserve number value and type for negative values', () => { - expect(Schema.readContent(Schema.write({ test: -23.23 }).contentBuffer())).toEqual({ test: -23.23 }); - }); - }); - - describe('String', () => { - const Schema = schema({ test: { type: 'string', size: 22 } }); - - it('should preserve string value and type', () => { - expect(Schema.readContent(Schema.write({ test: 'hello world' }).contentBuffer())).toEqual({ test: 'hello world' }); - }); - }); - - describe('Array', () => { - const Schema = schema({ test: { type: 'array', size: 12, items: { type: 'string' } } }); - - it('should preserve array values and types', () => { - expect(Schema.readContent(Schema.write({ test: ['a', 'b', 'c'] }).contentBuffer())).toEqual({ test: ['a', 'b', 'c', '', '', ''] }); - }); - }); - - describe('Schema', () => { - const Schema = schema({ test: { type: 'object', size: 20, schema: { test: { type: 'number', format: 'double' } } } }); - - it('should preserve object values and types', () => { - expect(Schema.readContent(Schema.write({ test: { test: 23.23 } }).contentBuffer())).toEqual({ test: { test: 23.23 } }); - }); - }); -}); - -describe('Data integrity - partial - multi simple', () => { - describe('Booleans', () => { - const Schema = schema({ test: { type: 'boolean' }, test2: { type: 'boolean' } }); - - it('should preserve boolean value and type - false', () => { - expect(Schema.readContent(Schema.write({ test: false, test2: true }).contentBuffer())).toEqual({ test: false, test2: true }); - }); - - it('should skip null or undefined values', () => { - expect(Schema.readContent(Schema.write({ test: null, test2: false }).contentBuffer())).toEqual({ test: false, test2: false }); - }); - }); - - describe('Numbers', () => { - const Schema = schema({ test: { type: 'number', format: 'double' }, test2: { type: 'number', format: 'double' } }); - - it('should preserve number value and type', () => { - expect(Schema.readContent(Schema.write({ test: 23.23, test2: -97.7 }).contentBuffer())).toEqual({ test: 23.23, test2: -97.7 }); - }); - }); - - describe('Strings', () => { - const Schema = schema({ test: { type: 'string', size: 22 }, test2: { type: 'string', size: 8 } }); - - it('should preserve string value and type', () => { - expect(Schema.readContent(Schema.write({ test: 'hello world', test2: 'écho' }).contentBuffer())).toEqual({ test: 'hello world', test2: 'écho' }); - }); - }); - - describe('Arrays', () => { - const Schema = schema({ test: { type: 'array', size: 9, items: { type: 'string' } }, test2: { type: 'array', size: 9, items: { type: 'string' } } }); - - it('should preserve array values and types', () => { - expect(Schema.readContent(Schema.write({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }).contentBuffer())).toEqual({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }); - }); - }); - - describe('Schemas', () => { - const Schema = schema({ test: { type: 'object', size: 11, schema: { test: { type: 'number', format: 'double' } } }, test2: { type: 'object', size: 11, schema: { test: { type: 'number', format: 'double' } } } }); - - it('should preserve object values and types', () => { - expect(Schema.readContent(Schema.write({ test: { test: 23.23 }, test2: { test: -97.7 } }).contentBuffer())).toEqual({ test: { test: 23.23 }, test2: { test: -97.7 } }); - }); - }); -}); - -describe('Data integrity - partial - multi mixed', () => { - describe('Boolean + number + string + array + object', () => { - const Schema = schema({ - bool: { type: 'boolean' }, - num: { type: 'number', format: 'double' }, - str: { type: 'string', size: 22 }, - arr: { type: 'array', items: { type: 'string' }, size: 9 }, - obj: { type: 'object', size: 9, schema: { sub: { type: 'string' } } }, - }); - - it('should preserve values and types', () => { - expect(Schema.readContent(Schema.write({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }).contentBuffer())).toEqual({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }); - }); - }); -}); - /* Size comparison tests ----------------------------------------------------- */ describe('Format size differences', () => { @@ -640,14 +522,14 @@ describe('Format size differences', () => { const FloatSchema = schema({ value: { type: 'number', format: 'float' } }); const DoubleSchema = schema({ value: { type: 'number', format: 'double' } }); - it('float should use 4 bytes for content', () => { - const buffer = FloatSchema.write({ value: 3.14 }).contentBuffer(); - expect(buffer.length).toBe(4); + it('float should use 4 bytes for value (7 bytes total with metadata)', () => { + const buffer = FloatSchema.write({ value: 3.14 }).buffer(); + expect(buffer.length).toBe(7); // 1 field count + 1 field index + 1 size + 4 value }); - it('double should use 8 bytes for content', () => { - const buffer = DoubleSchema.write({ value: 3.14 }).contentBuffer(); - expect(buffer.length).toBe(8); + it('double should use 8 bytes for value (11 bytes total with metadata)', () => { + const buffer = DoubleSchema.write({ value: 3.14 }).buffer(); + expect(buffer.length).toBe(11); // 1 field count + 1 field index + 1 size + 8 value }); }); @@ -655,14 +537,14 @@ describe('Format size differences', () => { const Int32Schema = schema({ value: { type: 'integer', format: 'int32' } }); const Int64Schema = schema({ value: { type: 'integer', format: 'int64' } }); - it('int32 should use 4 bytes for content', () => { - const buffer = Int32Schema.write({ value: 12345 }).contentBuffer(); - expect(buffer.length).toBe(4); + it('int32 should use 4 bytes for value (7 bytes total with metadata)', () => { + const buffer = Int32Schema.write({ value: 12345 }).buffer(); + expect(buffer.length).toBe(7); // 1 field count + 1 field index + 1 size + 4 value }); - it('int64 should use 8 bytes for content', () => { - const buffer = Int64Schema.write({ value: 12345 }).contentBuffer(); - expect(buffer.length).toBe(8); + it('int64 should use 8 bytes for value (11 bytes total with metadata)', () => { + const buffer = Int64Schema.write({ value: 12345 }).buffer(); + expect(buffer.length).toBe(11); // 1 field count + 1 field index + 1 size + 8 value }); }); }); diff --git a/types.d.ts b/types.d.ts index 3f3cd49..ed3c198 100644 --- a/types.d.ts +++ b/types.d.ts @@ -15,30 +15,11 @@ declare module 'compactr' { write(data: any, options?: WriteOptions): this /** - * Returns the bytes from the header of the encoded data buffer. - * A fresh schema with no written data will return a blank, usable for partial encodings. - * @returns The header buffer - */ - headerBuffer(): Buffer - - /** - * Returns the bytes from the content of the encoded data buffer. - * @returns The content buffer - */ - contentBuffer(): Buffer - - /** - * Returns the bytes from the header AND content of the encoded data buffer. + * Returns the bytes from the encoded data buffer. * @returns The data buffer */ buffer(): Buffer - /** - * Returns the typedArray from the header AND content of the encoded data buffer. - * @returns The typed array - */ - typedArray(): number[] - /** * Returns the byte sizes of a data object, for insight or troubleshooting * @param data The data to extract size information of From 4f9df41b4d7170d55441852d0e77987d59109bf9 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Sun, 2 Nov 2025 20:23:45 -0500 Subject: [PATCH 13/19] removed old code, comments, updated readme --- README.md | 216 ++++++++++++++++++++++++----------------- docs/ABOUT.md | 19 ---- package.json | 9 +- src/converter.ts | 28 ------ src/decoder.ts | 118 +--------------------- src/encoder.ts | 149 ++-------------------------- src/index.ts | 6 -- src/reader.ts | 38 +------- src/schema.ts | 49 +--------- src/size-writer.ts | 19 ---- src/variant-matcher.ts | 14 --- src/writer.ts | 79 ++------------- 12 files changed, 156 insertions(+), 588 deletions(-) delete mode 100644 docs/ABOUT.md delete mode 100644 src/size-writer.ts diff --git a/README.md b/README.md index 56b32fc..7175278 100644 --- a/README.md +++ b/README.md @@ -1,132 +1,174 @@

- + Compactr

Compactr

- Schema based serialization made easy + OpenAPI schema serialization



-[![Compactr](https://img.shields.io/npm/v/compactr.svg)](https://www.npmjs.com/package/compactr) -[![Build Status](https://travis-ci.org/compactr/compactr.js.svg?branch=master)](https://travis-ci.org/compactr/compactr.js) -[![Gitter](https://img.shields.io/gitter/room/compactr/compactr.svg)](https://gitter.im/compactr/compactr) - ---- - -## What is this and why does it matter? - -Protocol Buffers are awesome. Having schemas to deflate and inflate data while maintaining some kind of validation is a great concept. Compactr's goal is to build on that to better suit the Javascript ecosystem. - -[More](docs/ABOUT.md) - +- **OpenAPI 3.x Compatible** Use your existing OpenAPI schemas, no extra definitions required +- **Cuts bandwidth in half** Average API responses are multiple times smaller than JSON +- **Built for actual APIs** Specialty field types to optimize common items, ex: UUID, date-time, ivp4 +- **Zero dependencies** and can be bundled down to ~5kb! ## Install +```bash +npm install compactr ``` - npm install compactr -``` - -## Implementation +## Usage -```node +### Basic Example -const Compactr = require('compactr'); +```typescript +import { schema } from 'compactr'; -// Defining a schema -const userSchema = Compactr.schema({ - id: { type: 'number' }, - name: { type: 'string' } +const userSchema = schema({ + type: 'object', + properties: { + id: { type: 'string', format: 'uuid' }, + name: { type: 'string' }, + age: { type: 'integer', format: 'int32' }, + balance: { type: 'number', format: 'double' }, + created: { type: 'string', format: 'date-time' }, + tags: { type: 'array', items: { type: 'string' } } + } }); +const data = { + id: '550e8400-e29b-4d4e-a7d4-426614174000', + name: 'Alice', + age: 30, + balance: 1234.56, + created: '2025-10-28T14:30:00.000Z', + tags: ['premium', 'verified'] +}; - -// Encoding -userSchema.write({ id: 123, name: 'John' }); - -// Get the encoded buffer -const buffer = userSchema.buffer(); - -// Decoding +const buffer = userSchema.write(data).buffer(); const decoded = userSchema.read(buffer); ``` -## Size comparison +Compactr also supports component references ($ref). + +* Only local references are allowed * + +**Component References** + +```typescript +const schema = schema( + { + user: { $ref: '#/User' }, + product: { $ref: '#/Product' } + }, + { + schemas: { + User: { + type: 'object', + properties: { + id: { type: 'integer', format: 'int32' }, + name: { type: 'string' } + } + }, + Product: { + type: 'object', + properties: { + sku: { type: 'string' }, + price: { type: 'number', format: 'double' } + } + } + } + } +); +``` -JSON: `{"id":123,"name":"John"}`: 24 bytes +## Supported Types + +Compactr supports the following OpenAPI types and formats: + +| Type | Format | Bytes | Description | +| --- | --- | --- | --- | +| boolean | - | 1 | Boolean value | +| integer | int32 | 4 | 32-bit integer | +| integer | int64 | 8 | 64-bit integer | +| number | float | 4 | 32-bit floating point | +| number | double | 8 | 64-bit floating point | +| string | - | 2/char | UTF-8 string | +| string | uuid | 16 | UUID (compressed) | +| string | ipv4 | 4 | IPv4 address | +| string | ipv6 | 16 | IPv6 address | +| string | date | 4 | Date (YYYY-MM-DD) | +| string | date-time | 8 | ISO 8601 date-time | +| string | binary | variable | Base64 binary data | +| array | - | variable | Array of items | +| object | - | variable | Nested object | + + +## Size Comparison + +**Input:** +```javascript +{ + id: 123, + name: 'John', + email: 'john@example.com', + active: true +} +``` -Compactr (full): ``: 10 bytes +**JSON:** `{"id":123,"name":"John","email":"john@example.com","active":true}` - 61 bytes -Compactr (partial): ``: 5 bytes +**Compactr:** `` - 32 bytes +**Savings:** 48% smaller -## Protocol details +## Performance -### Data types +I realistic scenarios, compactr performs a bit slower than JSON.stringify/ JSON.parse as well as other schema-based protocols such as `protobuf`, but can yield a byte reduction of 3.5x. -Type | Count bytes | Byte size ---- | --- | --- -boolean | 0 | 1 -number | 0 | 8 -integer | 0 | 8 -int32 | 0 | 4 -int64 | 0 | 8 -double | 0 | 8 -float | 0 | 8 -string | 1 | 2/char -char8 | 1 | 1/char -char16 | 1 | 2/char -char32 | 1 | 4/char -array | 1 | (x)/entry -object | 1 | (x) +``` +[JSON-API Reponse] JSON x 289 ops/sec ±1.10% (83 runs sampled) +[JSON-API Reponse] Compactr x 115 ops/sec ±0.93% (74 runs sampled) +[JSON-API Reponse] Protobuf x 272 ops/sec ±1.06% (82 runs sampled) +Buffer size (bytes): { json: 277, compactr: 78, protobuf: 129 } +``` -* Count bytes range can be specified per-item in the schema* +## Testing -See the full [Compactr protocol](https://github.com/compactr/protocol) +Run the test suite: -## Benchmarks +```bash +npm test +``` + +Run benchmarks: +```bash +npm run bench ``` -[Array] JSON x 737 ops/sec ±1.04% (91 runs sampled) -[Array] Compactr x 536 ops/sec ±1.25% (82 runs sampled) -[Array] size: { json: 0, compactr: 10 } - -[Boolean] JSON x 996 ops/sec ±0.84% (93 runs sampled) -[Boolean] Compactr x 1,439 ops/sec ±1.24% (83 runs sampled) -[Boolean] Protobuf x 2,585 ops/sec ±1.32% (91 runs sampled) -[Boolean] size: { json: 23, compactr: 5, protobuf: 5 } - -[Float] JSON x 713 ops/sec ±1.29% (88 runs sampled) -[Float] Compactr x 1,056 ops/sec ±1.38% (86 runs sampled) -[Float] Protobuf x 2,432 ops/sec ±1.45% (88 runs sampled) -[Float] size: { json: 41, compactr: 12, protobuf: 12 } - -[Integer] JSON x 954 ops/sec ±0.77% (93 runs sampled) -[Integer] Compactr x 1,520 ops/sec ±1.21% (89 runs sampled) -[Integer] Protobuf x 2,676 ops/sec ±1.22% (90 runs sampled) -[Integer] size: { json: 24, compactr: 8, protobuf: 7 } - -[Object] JSON x 512 ops/sec ±0.62% (90 runs sampled) -[Object] Compactr x 412 ops/sec ±1.46% (81 runs sampled) -[Object] Protobuf x 898 ops/sec ±0.92% (93 runs sampled) -[Object] size: { json: 46, compactr: 13, protobuf: 25 } - -[String] JSON x 432 ops/sec ±0.79% (88 runs sampled) -[String] Compactr x 428 ops/sec ±0.87% (87 runs sampled) -[String] Protobuf x 738 ops/sec ±0.71% (90 runs sampled) -[String] size: { json: 57, compactr: 14, protobuf: 28 } + +Build the project: + +```bash +npm run build ``` -## Want to help ? +## Contributing -You are awesome! Open an issue on this project, identifying the feature that you want to tackle and we'll take the discussion there! +Contributions are welcome! Please feel free to submit a Pull Request. +1. Fork the repository +2. Create your feature branch (`git checkout -b feature/amazing-feature`) +3. Commit your changes (`git commit -m 'Add some amazing feature'`) +4. Push to the branch (`git push origin feature/amazing-feature`) +5. Open a Pull Request -## License +## License [Apache 2.0](LICENSE) (c) 2025 Frederic Charette diff --git a/docs/ABOUT.md b/docs/ABOUT.md deleted file mode 100644 index 456e6b1..0000000 --- a/docs/ABOUT.md +++ /dev/null @@ -1,19 +0,0 @@ -# About Compactr.js - -## Why choose Compactr for serialization - -### Who is this for? - -The Compactr protocol is mainly geared to empower high-debit realtime applications such as online games. That said, there's no reason why you couldn't use this in any type of application where serialization occurs like web servers transactions, database gossipping or even data storage. - -### How does this compare? - -Compactr is designed for easier integration with Javascript/Typescript applications. It's performances are very much similar to those of JSON serialization, but the output compares in size with Protobuf. - -It would be possible for compactr to be closer in speed to protobuf if we manage to implement codegen on schema initialization, like protobufjs does. - -Though the area where compactr really shines is in it's ability to handle nested objects and object collections. - -You can store information of any size or with no penalty. - -It is also extremely light in terms of package size, has zero dependencies, goes easy on the CPU and RAM. diff --git a/package.json b/package.json index 8034cca..3d5e03d 100644 --- a/package.json +++ b/package.json @@ -25,12 +25,11 @@ }, "keywords": [ "serialize", - "serializing", - "buffer", - "byte", + "openapi", + "swagger", "compress", - "protobuff", - "snappy", + "protobuf", + "encode", "msgpack" ], "files": [ diff --git a/src/converter.ts b/src/converter.ts index 937ac3f..fb450c4 100644 --- a/src/converter.ts +++ b/src/converter.ts @@ -1,46 +1,32 @@ -/** Type Coersion utilities */ - -/* Methods ------------------------------------------------------------------- */ - -/** @private */ - -/** @private */ function int32(value) { return Number(value) & 0xffffffff; } -/** @private */ function float(value) { const ret = Number(value); return (Number.isFinite(ret)) ? ret : 0; } -/** @private */ function double(value) { const ret = Number(value); return (Number.isFinite(ret)) ? ret : 0; } -/** @private */ function int64(value) { const ret = Number(value); return (Number.isFinite(ret)) ? Math.trunc(ret) : 0; } -/** @private */ function string(value) { return '' + value; } -/** @private */ function uuid(value) { - // Validate and normalize UUID string const str = '' + value; const normalized = str.toLowerCase().replace(/-/g, ''); if (!/^[0-9a-f]{32}$/.test(normalized)) { throw new Error('Invalid UUID format'); } - // Return in standard UUID format return [ normalized.substr(0, 8), normalized.substr(8, 4), @@ -50,7 +36,6 @@ function uuid(value) { ].join('-'); } -/** @private */ function ipv4(value) { const str = '' + value; const parts = str.split('.'); @@ -66,17 +51,14 @@ function ipv4(value) { return str; } -/** @private */ function ipv6(value) { const str = '' + value; - // Basic IPv6 validation - accepts both compressed and full formats if (!/^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$/.test(str) && !/^::$/.test(str)) { throw new Error('Invalid IPv6 format'); } return str.toLowerCase(); } -/** @private */ function date(value) { const str = '' + value; if (!/^\d{4}-\d{2}-\d{2}$/.test(str)) { @@ -89,7 +71,6 @@ function date(value) { return str; } -/** @private */ function dateTime(value) { const str = '' + value; const parsed = new Date(str); @@ -99,9 +80,7 @@ function dateTime(value) { return parsed.toISOString(); } -/** @private */ function binary(value) { - // Accept Buffer, Uint8Array, or base64 string if (Buffer.isBuffer(value)) { return value.toString('base64'); } @@ -110,9 +89,7 @@ function binary(value) { return Buffer.from(value).toString('base64'); } - // Validate base64 string if (typeof value === 'string') { - // Try to decode and re-encode to validate const buffer = Buffer.from(value, 'base64'); return buffer.toString('base64'); } @@ -120,23 +97,18 @@ function binary(value) { throw new Error('Invalid binary format: expected Buffer, Uint8Array, or base64 string'); } -/** @private */ function boolean(value) { return !!value; } -/** @private */ function object(value) { return (value.constructor === Object) ? value : {}; } -/** @private */ function array(value) { return (value.concat !== undefined) ? value : [value]; } -/* Exports ------------------------------------------------------------------- */ - export default { int32, int64, diff --git a/src/decoder.ts b/src/decoder.ts index ebe2168..776383a 100644 --- a/src/decoder.ts +++ b/src/decoder.ts @@ -1,26 +1,15 @@ -/** Decoding utilities */ +export const NULL_INDICATOR = 0x00; +export const VARIANT_BASE = 0x01; -// Discriminator byte for nullable/oneOf/anyOf fields -export const NULL_INDICATOR = 0x00; // Field is null -export const VARIANT_BASE = 0x01; // First variant (or present for simple nullable) - -/* Performance: Reusable typed array buffers --------------------------------- */ - -// Module-level reusable buffers for float/double decoding -// This eliminates allocation overhead (300-500% performance improvement) const floatBuffer = new Float32Array(1); const floatBytes = new Uint8Array(floatBuffer.buffer); const doubleBuffer = new Float64Array(1); const doubleBytes = new Uint8Array(doubleBuffer.buffer); -/* Methods ------------------------------------------------------------------- */ - -/** @private */ function boolean(bytes, offset = 0) { return !!bytes[offset]; } -/** @private */ function int32(bytes, offset = 0) { return (bytes[offset] << 24) | (bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | (bytes[offset + 3]); } @@ -33,7 +22,6 @@ function uint16(bytes, offset = 0) { return bytes[offset] << 8 | bytes[offset + 1]; } -/** @private */ function unsigned(bytes, offset = 0, length?) { const len = length !== undefined ? length : bytes.length - offset; if (len === 1) return uint8(bytes, offset); @@ -41,11 +29,6 @@ function unsigned(bytes, offset = 0, length?) { return int32(bytes, offset); } -/** - * Manual UTF-8 decoder - faster than Buffer for typical API strings - * Assumes mostly ASCII content (field names, english text, numbers, URLs) - * @private - */ function string(bytes, offset = 0, length?) { const len = length !== undefined ? length : bytes.length - offset; const chars = []; @@ -54,28 +37,23 @@ function string(bytes, offset = 0, length?) { for (let i = offset; i < end;) { const byte1 = bytes[i++]; - // ASCII (0-127): 1 byte if (byte1 < 0x80) { chars.push(byte1); } - // 2-byte character else if ((byte1 & 0xE0) === 0xC0) { const byte2 = bytes[i++]; chars.push(((byte1 & 0x1F) << 6) | (byte2 & 0x3F)); } - // 3-byte character else if ((byte1 & 0xF0) === 0xE0) { const byte2 = bytes[i++]; const byte3 = bytes[i++]; chars.push(((byte1 & 0x0F) << 12) | ((byte2 & 0x3F) << 6) | (byte3 & 0x3F)); } - // 4-byte character (surrogate pair) else if ((byte1 & 0xF8) === 0xF0) { const byte2 = bytes[i++]; const byte3 = bytes[i++]; const byte4 = bytes[i++]; let code = ((byte1 & 0x07) << 18) | ((byte2 & 0x3F) << 12) | ((byte3 & 0x3F) << 6) | (byte4 & 0x3F); - // Convert to surrogate pair code -= 0x10000; chars.push(0xD800 | (code >> 10)); chars.push(0xDC00 | (code & 0x3FF)); @@ -85,20 +63,12 @@ function string(bytes, offset = 0, length?) { return String.fromCharCode.apply(null, chars); } -/** - * UUID decoder - converts 16 bytes to UUID string - * Binary format: 16 bytes (128 bits) - * UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx (36 chars) - * Performance: 100-150% faster using direct hex lookup - * @private - */ function uuid(bytes, offset = 0, length?) { const len = length !== undefined ? length : bytes.length - offset; if (len !== 16) { throw new Error('Invalid UUID byte length'); } - // Performance: Direct hex lookup instead of toString(16) + padStart const hex = '0123456789abcdef'; let result = ''; @@ -106,7 +76,6 @@ function uuid(bytes, offset = 0, length?) { const byte = bytes[offset + i]; result += hex[byte >> 4] + hex[byte & 0x0f]; - // Add hyphens at positions 4, 6, 8, 10 (after bytes 3, 5, 7, 9) if (i === 3 || i === 5 || i === 7 || i === 9) { result += '-'; } @@ -115,12 +84,6 @@ function uuid(bytes, offset = 0, length?) { return result; } -/** - * IPv4 decoder - converts 4 bytes to IPv4 string - * Binary format: 4 bytes - * IPv4 format: "192.168.1.1" - * @private - */ function ipv4(bytes, offset = 0, length?) { const len = length !== undefined ? length : bytes.length - offset; if (len !== 4) { @@ -130,20 +93,12 @@ function ipv4(bytes, offset = 0, length?) { return `${bytes[offset]}.${bytes[offset + 1]}.${bytes[offset + 2]}.${bytes[offset + 3]}`; } -/** - * IPv6 decoder - converts 16 bytes to IPv6 string - * Binary format: 16 bytes - * IPv6 format: "2001:0db8:85a3:0000:0000:8a2e:0370:7334" - * Performance: Optimized with single-pass algorithm and direct hex conversion - * @private - */ function ipv6(bytes, offset = 0, length?) { const len = length !== undefined ? length : bytes.length - offset; if (len !== 16) { throw new Error('Invalid IPv6 byte length'); } - // Parse groups and find longest zero sequence in single pass const groups = new Array(8); let longestZeroStart = -1; let longestZeroLength = 0; @@ -154,7 +109,6 @@ function ipv6(bytes, offset = 0, length?) { const value = (bytes[offset + i * 2] << 8) | bytes[offset + i * 2 + 1]; groups[i] = value; - // Track zero sequences if (value === 0) { if (currentZeroStart === -1) { currentZeroStart = i; @@ -174,27 +128,21 @@ function ipv6(bytes, offset = 0, length?) { } } - // Check final sequence if (currentZeroLength > longestZeroLength) { longestZeroStart = currentZeroStart; longestZeroLength = currentZeroLength; } - // Build result string let result = ''; - // Apply :: compression if we found at least one zero group if (longestZeroLength > 0) { - // Before compressed section for (let i = 0; i < longestZeroStart; i++) { if (i > 0) result += ':'; result += groups[i].toString(16); } - // Compressed section result += '::'; - // After compressed section const afterStart = longestZeroStart + longestZeroLength; for (let i = afterStart; i < 8; i++) { if (i > afterStart) result += ':'; @@ -202,7 +150,6 @@ function ipv6(bytes, offset = 0, length?) { } } else { - // No compression, output all groups for (let i = 0; i < 8; i++) { if (i > 0) result += ':'; result += groups[i].toString(16); @@ -212,12 +159,6 @@ function ipv6(bytes, offset = 0, length?) { return result; } -/** - * Date decoder - converts 4 bytes (days since epoch) to YYYY-MM-DD - * Binary format: 4 bytes (signed int32, days since Jan 1, 1970) - * Date format: "2025-10-28" - * @private - */ function date(bytes, offset = 0, length?) { const len = length !== undefined ? length : bytes.length - offset; if (len !== 4) { @@ -228,7 +169,6 @@ function date(bytes, offset = 0, length?) { const epochMs = days * 86400000; const dateObj = new Date(epochMs); - // Format as YYYY-MM-DD const year = dateObj.getUTCFullYear(); const month = String(dateObj.getUTCMonth() + 1).padStart(2, '0'); const day = String(dateObj.getUTCDate()).padStart(2, '0'); @@ -236,12 +176,6 @@ function date(bytes, offset = 0, length?) { return `${year}-${month}-${day}`; } -/** - * Date-time decoder - converts 8 bytes to ISO 8601 string - * Binary format: 8 bytes (int64, milliseconds since Jan 1, 1970) - * DateTime format: "2025-10-28T14:30:00.000Z" - * @private - */ function dateTime(bytes, offset = 0, length?) { const len = length !== undefined ? length : bytes.length - offset; if (len !== 8) { @@ -254,41 +188,29 @@ function dateTime(bytes, offset = 0, length?) { return dateObj.toISOString(); } -/** - * Binary decoder - converts raw bytes to base64 string (zero-copy) - * Uses Buffer.toString with offset and length instead of slicing - * @private - */ function binary(bytes, offset = 0, length?) { const len = length !== undefined ? length : bytes.length - offset; - // Zero-copy: use Buffer.toString with offset and length - // Create Buffer view of the data without copying if (offset === 0 && len === bytes.length) { return Buffer.from(bytes).toString('base64'); } - // Use subarray (zero-copy view) instead of slice (copy) return Buffer.from(bytes.subarray(offset, offset + len)).toString('base64'); } -/** @private */ function array(schema, bytes, offset = 0, length?) { const len = length !== undefined ? length : bytes.length - offset; const ret = []; const end = offset + len; for (let i = offset; i < end;) { - // Handle nullable or variant array items if (schema.nullable || schema.variants) { const discriminator = bytes[i]; i++; - // Handle null value if (discriminator === 0x00) { ret.push(null); continue; } - // Handle variant items if (schema.variants) { const variantIndex = discriminator - 0x01; const variantField = schema.variants[variantIndex]; @@ -297,26 +219,18 @@ function array(schema, bytes, offset = 0, length?) { throw new Error(`Invalid variant discriminator: ${discriminator}`); } - // Zero-copy: pass offset and length instead of slicing const size = unsigned(bytes, i, variantField.count); i += variantField.count; - // Zero-copy for all types: pass offset and length ret.push(variantField.transformOut(bytes, i, size)); i += size; continue; } - - // For nullable non-variant items, discriminator 0x01 means value is present - // Continue to decode the value normally below } - // Handle regular array items - // Zero-copy: pass offset and length instead of slicing const size = unsigned(bytes, i, schema.count); i += schema.count; - // Zero-copy for all types: pass offset and length ret.push(schema.transformOut(bytes, i, size)); i += size; } @@ -324,41 +238,25 @@ function array(schema, bytes, offset = 0, length?) { return ret; } -/** - * Object decoder - zero-copy nested object reading - * Uses readFromOffset if available, otherwise creates zero-copy subarray view - * @private - */ function object(schema, bytes, offset = 0, length?) { - // Zero-copy: use readFromOffset if schema supports it, otherwise subarray if (offset === 0 && length === undefined) { return schema.read(bytes); } const len = length !== undefined ? length : bytes.length - offset; - // Use readFromOffset for zero-copy if available if (schema.readFromOffset) { return schema.readFromOffset(bytes, offset, len); } - // Fallback: use subarray (zero-copy view) instead of slice (copy) - // Most TypedArrays support subarray which creates a view, not a copy if (bytes.subarray) { return schema.read(bytes.subarray(offset, offset + len)); } - // Last resort: slice (creates copy) return schema.read(bytes.slice(offset, offset + len)); } -/** - * IEEE 754 single precision (32-bit float) decoder - * Performance: Uses reusable module-level buffer (300-500% faster) - * @private - */ function float(bytes, offset = 0) { - // Bytes come in big-endian order, convert to little-endian for reusable buffer floatBytes[0] = bytes[offset + 3]; floatBytes[1] = bytes[offset + 2]; floatBytes[2] = bytes[offset + 1]; @@ -367,13 +265,7 @@ function float(bytes, offset = 0) { return floatBuffer[0]; } -/** - * IEEE 754 double precision (64-bit float) decoder - * Performance: Uses reusable module-level buffer (300-500% faster) - * @private - */ function double(bytes, offset = 0) { - // Bytes come in big-endian order, convert to little-endian for reusable buffer doubleBytes[0] = bytes[offset + 7]; doubleBytes[1] = bytes[offset + 6]; doubleBytes[2] = bytes[offset + 5]; @@ -386,16 +278,10 @@ function double(bytes, offset = 0) { return doubleBuffer[0]; } -/** - * 64-bit integer decoder (uses double for JavaScript compatibility) - * @private - */ function int64(bytes, offset = 0) { return double(bytes, offset); } -/* Exports ------------------------------------------------------------------- */ - export default { boolean, int32, diff --git a/src/encoder.ts b/src/encoder.ts index aed8fd9..949fcb3 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -1,39 +1,18 @@ -/** Encoding utilities */ - -/* Requires ------------------------------------------------------------------ */ - import { matchesVariant } from './variant-matcher'; -/* Local variables ----------------------------------------------------------- */ - -// Discriminator byte for nullable/oneOf/anyOf fields -export const NULL_INDICATOR = 0x00; // Field is null -export const VARIANT_BASE = 0x01; // First variant (or present for simple nullable) +export const NULL_INDICATOR = 0x00; +export const VARIANT_BASE = 0x01; -/* Performance: Reusable typed array buffers --------------------------------- */ - -// Module-level reusable buffers for float/double encoding -// This eliminates allocation overhead (300-500% performance improvement) const floatBuffer = new Float32Array(1); const floatBytes = new Uint8Array(floatBuffer.buffer); const doubleBuffer = new Float64Array(1); const doubleBytes = new Uint8Array(doubleBuffer.buffer); -/* Methods ------------------------------------------------------------------- */ - -/** - * Boolean encoder - writes 1 byte directly to buffer - * @returns new position - */ function boolean(val, buffer, pos) { buffer[pos] = val ? 1 : 0; return pos + 1; } -/** - * int32 encoder - writes 4 bytes directly to buffer (big-endian) - * @returns new position - */ function int32(val, buffer, pos) { if (val < 0) val = 0xffffffff + val + 1; buffer[pos] = val >> 24; @@ -43,22 +22,11 @@ function int32(val, buffer, pos) { return pos + 4; } -/** - * UTF-8 string encoder - writes directly to buffer using native code - * Uses Buffer.write() for zero-copy C++ UTF-8 encoding - * @returns new position - */ function string(val, buffer, pos) { - // Write UTF-8 bytes directly into target buffer using native C++ code - // Buffer.prototype.write is much faster than manual UTF-8 encoding const bytesWritten = Buffer.prototype.write.call(buffer, val, pos, undefined, 'utf8'); return pos + bytesWritten; } -/** - * Performance: Fast hex character to number conversion - * @private - */ function hexCharToNum(char) { const code = char.charCodeAt(0); if (code >= 48 && code <= 57) return code - 48; // 0-9 @@ -67,28 +35,21 @@ function hexCharToNum(char) { throw new Error('Invalid hex character: ' + char); } -/** - * UUID encoder - writes 16 bytes directly to buffer - * UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx (36 chars) - * @returns new position - */ function uuid(val, buffer, pos) { - // Validate length if (val.length !== 36) { throw new Error('Invalid UUID format: expected 36 characters'); } let byteIdx = pos; - // Parse hex pairs directly and write to buffer for (let i = 0; i < val.length; i++) { const char = val[i]; - if (char === '-') continue; // Skip hyphens + if (char === '-') continue; const high = hexCharToNum(val[i]); const low = hexCharToNum(val[i + 1]); buffer[byteIdx++] = (high << 4) | low; - i++; // Skip next char since we processed it + i++; } if (byteIdx - pos !== 16) { @@ -98,11 +59,6 @@ function uuid(val, buffer, pos) { return byteIdx; } -/** - * IPv4 encoder - writes 4 bytes directly to buffer - * IPv4 format: "192.168.1.1" - * @returns new position - */ function ipv4(val, buffer, pos) { const parts = val.split('.'); if (parts.length !== 4) { @@ -120,26 +76,12 @@ function ipv4(val, buffer, pos) { return pos + 4; } -/** - * IPv6 encoder - converts IPv6 string to 16 bytes - * IPv6 format: "2001:0db8:85a3::8a2e:0370:7334" (max 39 chars = 78 bytes as string) - * Binary format: 16 bytes - * Saves up to 62 bytes - * Performance: Optimized to avoid multiple string operations - * @private - */ -/** - * IPv6 encoder - writes 16 bytes directly to buffer - * @returns new position - */ function ipv6(val, buffer, pos) { let byteIdx = 0; - // Handle :: expansion inline const doubleColonPos = val.indexOf('::'); if (doubleColonPos !== -1) { - // Parse left side of :: let i = 0; while (i < doubleColonPos) { let hexStr = ''; @@ -155,14 +97,12 @@ function ipv6(val, buffer, pos) { buffer[pos + byteIdx++] = num >> 8; buffer[pos + byteIdx++] = num & 0xff; } - i++; // Skip colon + i++; } - // Calculate how many zero groups to insert const leftGroups = byteIdx / 2; - // Parse right side of :: - i = doubleColonPos + 2; // Skip :: + i = doubleColonPos + 2; const rightStart = []; while (i < val.length) { let hexStr = ''; @@ -177,24 +117,21 @@ function ipv6(val, buffer, pos) { } rightStart.push(num >> 8, num & 0xff); } - i++; // Skip colon + i++; } const rightGroups = rightStart.length / 2; const zeroGroups = 8 - leftGroups - rightGroups; - // Fill zeros for (let z = 0; z < zeroGroups * 2; z++) { buffer[pos + byteIdx++] = 0; } - // Add right side for (let r = 0; r < rightStart.length; r++) { buffer[pos + byteIdx++] = rightStart[r]; } } else { - // No :: expansion needed, parse directly let i = 0; let groupCount = 0; while (i < val.length) { @@ -212,7 +149,7 @@ function ipv6(val, buffer, pos) { buffer[pos + byteIdx++] = num & 0xff; groupCount++; } - i++; // Skip colon + i++; } if (groupCount !== 8) { @@ -227,58 +164,39 @@ function ipv6(val, buffer, pos) { return pos + 16; } -/** - * Date encoder - writes 4 bytes directly to buffer (days since epoch) - * @returns new position - */ function date(val, buffer, pos) { const parsed = new Date(val + 'T00:00:00Z'); if (isNaN(parsed.getTime())) { throw new Error('Invalid date format'); } - // Calculate days since epoch const epochMs = parsed.getTime(); const days = Math.floor(epochMs / 86400000); return int32(days, buffer, pos); } -/** - * Date-time encoder - writes 8 bytes directly to buffer (milliseconds since epoch) - * @returns new position - */ function dateTime(val, buffer, pos) { const parsed = new Date(val); if (isNaN(parsed.getTime())) { throw new Error('Invalid date-time format'); } - // Store as milliseconds since epoch using double precision return double(parsed.getTime(), buffer, pos); } -/** - * Binary encoder - writes bytes directly to buffer - * Accepts Buffer, Uint8Array, or base64 string - * @returns new position - */ function binary(val, buffer, pos) { if (Buffer.isBuffer(val)) { - // Copy buffer bytes directly val.copy(buffer, pos); return pos + val.length; } if (val instanceof Uint8Array) { - // Copy Uint8Array bytes directly buffer.set(val, pos); return pos + val.length; } - // Assume base64 encoded string if (typeof val === 'string') { - // Decode base64 directly into target buffer const bytesWritten = Buffer.from(val, 'base64').copy(buffer, pos); return pos + bytesWritten; } @@ -286,27 +204,19 @@ function binary(val, buffer, pos) { throw new Error('Binary format requires Buffer, Uint8Array, or base64 string'); } -/** - * Array encoder - writes array items directly to buffer - * Uses reserve-and-fill strategy for variable-length items - * @returns new position - */ function array(schema, val, buffer, pos) { for (let i = 0; i < val.length; i++) { const item = val[i]; - // Handle nullable array items if (schema.nullable && item === null) { buffer[pos++] = NULL_INDICATOR; continue; } - // Handle variant array items (oneOf/anyOf) if (schema.variants) { let variantIndex = -1; let variantField = null; - // Find matching variant for (let v = 0; v < schema.variants.length; v++) { if (matchesVariant(item, schema.variants[v])) { variantIndex = v; @@ -319,13 +229,9 @@ function array(schema, val, buffer, pos) { throw new Error(`Array item does not match any variant`); } - // Write discriminator buffer[pos++] = schema.nullable ? VARIANT_BASE + variantIndex : VARIANT_BASE + variantIndex; - // For fixed-size types, write size first, then data - // For variable-size, reserve space, write data, fill in size if (variantField.size) { - // Fixed size - write size directly const count = variantField.count; if (count === 1) buffer[pos++] = variantField.size; else if (count === 2) { @@ -341,14 +247,12 @@ function array(schema, val, buffer, pos) { pos = variantField.transformIn(item, buffer, pos); } else { - // Variable size - reserve space for size, write data, fill in size const sizePos = pos; - pos += variantField.count; // Reserve space + pos += variantField.count; const dataStart = pos; pos = variantField.transformIn(item, buffer, pos); const size = pos - dataStart; - // Fill in size if (variantField.count === 1) buffer[sizePos] = size & 0xff; else if (variantField.count === 2) { buffer[sizePos] = size >> 8; @@ -364,14 +268,11 @@ function array(schema, val, buffer, pos) { continue; } - // Handle nullable non-null items (add presence indicator) if (schema.nullable) { buffer[pos++] = VARIANT_BASE; } - // Handle regular array items if (schema.size) { - // Fixed size - write size first, then data const count = schema.count; if (count === 1) buffer[pos++] = schema.size; else if (count === 2) { @@ -387,14 +288,12 @@ function array(schema, val, buffer, pos) { pos = schema.transformIn(item, buffer, pos); } else { - // Variable size - reserve space for size, write data, fill in size const sizePos = pos; - pos += schema.count; // Reserve space + pos += schema.count; const dataStart = pos; pos = schema.transformIn(item, buffer, pos); const size = pos - dataStart; - // Fill in size if (schema.count === 1) buffer[sizePos] = size & 0xff; else if (schema.count === 2) { buffer[sizePos] = size >> 8; @@ -412,26 +311,13 @@ function array(schema, val, buffer, pos) { return pos; } -/** - * Object encoder - writes nested object directly to buffer - * Delegates to nested schema's write function - * @returns new position - */ function object(schema, val, buffer, pos) { - // Call nested schema's writeToBuffer method (will be added to writer) return schema.writeToBuffer(val, buffer, pos); } -/** - * IEEE 754 single precision (32-bit float) - writes 4 bytes directly to buffer - * Uses reusable module-level buffer for conversion - * @returns new position - */ function float(val, buffer, pos) { - // Reuse module-level buffer to avoid allocation overhead floatBuffer[0] = val; - // Write bytes in big-endian order buffer[pos] = floatBytes[3]; buffer[pos + 1] = floatBytes[2]; buffer[pos + 2] = floatBytes[1]; @@ -439,16 +325,9 @@ function float(val, buffer, pos) { return pos + 4; } -/** - * IEEE 754 double precision (64-bit float) - writes 8 bytes directly to buffer - * Uses reusable module-level buffer for conversion - * @returns new position - */ function double(val, buffer, pos) { - // Reuse module-level buffer to avoid allocation overhead doubleBuffer[0] = val; - // Write bytes in big-endian order buffer[pos] = doubleBytes[7]; buffer[pos + 1] = doubleBytes[6]; buffer[pos + 2] = doubleBytes[5]; @@ -460,16 +339,10 @@ function double(val, buffer, pos) { return pos + 8; } -/** - * 64-bit integer encoding (uses double for JavaScript compatibility) - * JavaScript's Number type can safely represent integers up to 2^53-1 - * @private - */ function int64(val, buffer, pos) { return double(val, buffer, pos); } -/** @private */ function getSize(count, byteLength) { if (count === 1) { return Buffer.from([byteLength & 0xff]); @@ -488,8 +361,6 @@ function getSize(count, byteLength) { return Buffer.from([]); } -/* Exports ------------------------------------------------------------------- */ - export default { boolean, int32, diff --git a/src/index.ts b/src/index.ts index 4ed873d..bd6fa05 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,9 +1,3 @@ -/** Entry point */ - -/* Requires ------------------------------------------------------------------ */ - import schema from './schema'; -/* Exports ------------------------------------------------------------------- */ - export { schema }; diff --git a/src/reader.ts b/src/reader.ts index b07509f..18a502b 100644 --- a/src/reader.ts +++ b/src/reader.ts @@ -1,16 +1,6 @@ -/** Data reader component */ - -/* Requires ------------------------------------------------------------------ */ - import Decoder, { NULL_INDICATOR, VARIANT_BASE } from './decoder'; -/* Methods ------------------------------------------------------------------- */ - export default function Reader(scope) { - /** - * Read from buffer at specific offset (zero-copy for nested objects) - * @private - */ function readFromOffset(bytes, offset, length) { const ret = {}; if (scope.options.keyOrder === true) { @@ -19,14 +9,13 @@ export default function Reader(scope) { } } - let caret = offset + 1; // Start after field count + let caret = offset + 1; const fieldCount = bytes[offset]; const end = offset + length; for (let i = 0; i < fieldCount; i++) { if (caret >= end) break; - // Read field index const fieldIndex = bytes[caret]; caret++; @@ -35,18 +24,15 @@ export default function Reader(scope) { throw new Error(`Unknown field index: ${fieldIndex}`); } - // Check for discriminator byte if field is nullable or has variants if (field.nullable || field.variants) { const discriminatorByte = bytes[caret]; caret++; if (discriminatorByte === NULL_INDICATOR) { - // Field is null - no size or content follows ret[field.name] = null; continue; } - // Handle variant fields if (field.variants) { const variantIndex = discriminatorByte - VARIANT_BASE; if (variantIndex < 0 || variantIndex >= field.variants.length) { @@ -57,26 +43,21 @@ export default function Reader(scope) { const size = variant.size || readSize(bytes, caret, variant.count); caret += variant.count; - // Zero-copy decode: pass offset and length for all types ret[field.name] = variant.transformOut(bytes, caret, size); caret += size; continue; } - // Regular nullable field is present const size = field.size || readSize(bytes, caret, field.count); caret += field.count; - // Zero-copy decode: pass offset and length for all types ret[field.name] = field.transformOut(bytes, caret, size); caret += size; } else { - // Non-nullable, non-variant field const size = field.size || readSize(bytes, caret, field.count); caret += field.count; - // Zero-copy decode: pass offset and length for all types ret[field.name] = field.transformOut(bytes, caret, size); caret += size; } @@ -93,11 +74,10 @@ export default function Reader(scope) { } } - let caret = 1; // Start after field count + let caret = 1; const fieldCount = bytes[0]; for (let i = 0; i < fieldCount; i++) { - // Read field index const fieldIndex = bytes[caret]; caret++; @@ -106,18 +86,15 @@ export default function Reader(scope) { throw new Error(`Unknown field index: ${fieldIndex}`); } - // Check for discriminator byte if field is nullable or has variants if (field.nullable || field.variants) { const discriminatorByte = bytes[caret]; caret++; if (discriminatorByte === NULL_INDICATOR) { - // Field is null - no size or content follows ret[field.name] = null; continue; } - // Handle variant fields if (field.variants) { const variantIndex = discriminatorByte - VARIANT_BASE; if (variantIndex < 0 || variantIndex >= field.variants.length) { @@ -128,26 +105,21 @@ export default function Reader(scope) { const size = variant.size || readSize(bytes, caret, variant.count); caret += variant.count; - // Zero-copy decode: pass offset and length for all types ret[field.name] = variant.transformOut(bytes, caret, size); caret += size; continue; } - // Regular nullable field is present const size = field.size || readSize(bytes, caret, field.count); caret += field.count; - // Zero-copy decode: pass offset and length for all types ret[field.name] = field.transformOut(bytes, caret, size); caret += size; } else { - // Non-nullable, non-variant field const size = field.size || readSize(bytes, caret, field.count); caret += field.count; - // Zero-copy decode: pass offset and length for all types ret[field.name] = field.transformOut(bytes, caret, size); caret += size; } @@ -156,11 +128,6 @@ export default function Reader(scope) { return ret; } - /** - * Performance: Fast size reading without array slicing - * Reads 1-4 bytes directly from buffer instead of creating slice - * @private - */ function readSize(bytes, offset, count) { if (count === 1) return bytes[offset]; if (count === 2) return (bytes[offset] << 8) | bytes[offset + 1]; @@ -168,7 +135,6 @@ export default function Reader(scope) { return (bytes[offset] << 24) | (bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3]; } - // Fallback for unexpected sizes (should not happen in practice) return Decoder.unsigned(bytes.slice(offset, offset + count)); } diff --git a/src/schema.ts b/src/schema.ts index 8e1d63c..c569035 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -1,19 +1,9 @@ -/** Schema parsing component */ - -/* Requires ------------------------------------------------------------------ */ - import Encoder from './encoder'; import Decoder from './decoder'; import Reader from './reader'; import Writer from './writer'; import Converter from './converter'; -/* Methods ------------------------------------------------------------------- */ - -/** - * Resolves the internal type based on OpenAPI type and format - * @private - */ function resolveType(type, format) { if (type === 'integer') { const fmt = format || 'int32'; @@ -38,14 +28,11 @@ function resolveType(type, format) { } export default function Schema(schema, options = { keyOrder: false }) { - // Handle top-level schema object (OpenAPI format) - // If schema has type: 'object' and properties, unwrap it let unwrappedSchema = schema; if (schema.type === 'object' && schema.properties) { unwrappedSchema = schema.properties; } - // Normalize OpenAPI schema format to internal format const normalizedSchema = normalizeSchema(unwrappedSchema, options); const defaultSizes = { @@ -67,26 +54,23 @@ export default function Schema(schema, options = { keyOrder: false }) { items: Object.keys(normalizedSchema), buffer: [], options, - indexToField: {}, // Performance: O(1) reverse lookup from index to field - itemsSet: null, // Performance: O(1) field validation in writer + indexToField: {}, + itemsSet: null, }; scope.indices = preformat(normalizedSchema); - // Build reverse index map for O(1) lookups during deserialization for (const fieldName of scope.items) { scope.indexToField[scope.indices[fieldName].index] = scope.indices[fieldName]; } - // Build Set for O(1) field validation during serialization scope.itemsSet = new Set(scope.items); - /** @private */ function resolveRef(ref, options) { if (!ref || !ref.startsWith('#/')) { throw new Error(`Invalid $ref format: ${ref}. Only internal references (#/...) are supported.`); } - const parts = ref.split('/').slice(1); // Remove leading '#' + const parts = ref.split('/').slice(1); let resolved = options.schemas; for (const part of parts) { @@ -103,42 +87,33 @@ export default function Schema(schema, options = { keyOrder: false }) { return resolved; } - /** @private */ function normalizeFieldDefinition(fieldDef, options) { - // Handle $ref if (fieldDef.$ref) { if (!options.schemas) { throw new Error(`$ref "${fieldDef.$ref}" found but no schemas provided in options`); } - // Resolve the reference let resolved = resolveRef(fieldDef.$ref, options); - // Unwrap if the component is a wrapped object schema if (resolved.type === 'object' && resolved.properties && !resolved.schema) { resolved = { ...resolved }; resolved.schema = resolved.properties; delete resolved.properties; } - // Normalize the resolved component return normalizeFieldDefinition(resolved, options); } - // Create a copy to avoid mutating the original const normalized = { ...fieldDef }; - // Transform OpenAPI 'properties' to internal 'schema' if (normalized.properties && !normalized.schema) { normalized.schema = normalizeSchema(normalized.properties, options); delete normalized.properties; } - // Normalize nested items if (normalized.items) { normalized.items = normalizeFieldDefinition(normalized.items, options); } - // Normalize oneOf/anyOf variants if (normalized.oneOf) { normalized.oneOf = normalized.oneOf.map(v => normalizeFieldDefinition(v, options)); } @@ -146,7 +121,6 @@ export default function Schema(schema, options = { keyOrder: false }) { normalized.anyOf = normalized.anyOf.map(v => normalizeFieldDefinition(v, options)); } - // Normalize nested schema if (normalized.schema && typeof normalized.schema === 'object') { normalized.schema = normalizeSchema(normalized.schema, options); } @@ -154,7 +128,6 @@ export default function Schema(schema, options = { keyOrder: false }) { return normalized; } - /** @private */ function normalizeSchema(schema, options) { const normalized = {}; for (const key in schema) { @@ -165,24 +138,20 @@ export default function Schema(schema, options = { keyOrder: false }) { const writer = Writer(scope); const reader = Reader(scope); - /** @private */ function preformat(schema) { const ret = {}; Object.keys(schema) .sort() .forEach((key, index) => { - // Handle oneOf/anyOf fields if (schema[key].oneOf || schema[key].anyOf) { const variantDefs = schema[key].oneOf || schema[key].anyOf; const variants = variantDefs.map((variantDef) => { const variantType = variantDef.type; const variantFormat = variantDef.format; const variantInternalType = resolveType(variantType, variantFormat); - // Binary fields need 4 bytes, arrays/objects need 2 bytes for size counters const variantCount = variantDef.count || (variantInternalType === 'binary' ? 4 : (variantInternalType === 'array' || variantInternalType === 'object' ? 2 : 1)); const variantChildSchema = computeNestedVariant(variantDef); - // For object variants, extract schema keys for variant matching const schemaKeys = (variantInternalType === 'object' && variantDef.schema) ? Object.keys(variantDef.schema) : null; @@ -214,11 +183,9 @@ export default function Schema(schema, options = { keyOrder: false }) { return; } - // Handle regular fields const fieldType = schema[key].type; const fieldFormat = schema[key].format; const internalType = resolveType(fieldType, fieldFormat); - // Binary fields need 4 bytes, arrays/objects need 2 bytes for size counters const count = schema[key].count || (internalType === 'binary' ? 4 : (internalType === 'array' || internalType === 'object' ? 2 : 1)); const childSchema = computeNested(schema, key); @@ -241,7 +208,6 @@ export default function Schema(schema, options = { keyOrder: false }) { return ret; } - /** @private */ function computeNested(schema, key) { const keyType = schema[key].type; const isObject = (keyType === 'object'); @@ -250,7 +216,6 @@ export default function Schema(schema, options = { keyOrder: false }) { if (isObject === true || isArray === true) { if (isObject === true) { - // After normalization, schema should always be present for objects childSchema = Schema(schema[key].schema, options); } if (isArray === true) { @@ -261,9 +226,7 @@ export default function Schema(schema, options = { keyOrder: false }) { return childSchema; } - /** @private */ function processArrayItems(itemDef) { - // Handle oneOf/anyOf in array items if (itemDef.oneOf || itemDef.anyOf) { const variantDefs = itemDef.oneOf || itemDef.anyOf; const variants = variantDefs.map((variantDef) => { @@ -273,7 +236,6 @@ export default function Schema(schema, options = { keyOrder: false }) { const variantCount = variantDef.count || (variantInternalType === 'binary' ? 4 : (variantInternalType === 'array' || variantInternalType === 'object' ? 2 : 1)); const variantChildSchema = processArrayItemsNested(variantDef); - // For object variants, extract schema keys for variant matching const schemaKeys = (variantInternalType === 'object' && variantDef.schema) ? Object.keys(variantDef.schema) : null; @@ -304,7 +266,6 @@ export default function Schema(schema, options = { keyOrder: false }) { }; } - // Handle regular array items const itemType = itemDef.type; const itemFormat = itemDef.format; const internalItemType = resolveType(itemType, itemFormat); @@ -320,14 +281,12 @@ export default function Schema(schema, options = { keyOrder: false }) { }; } - /** @private */ function processArrayItemsNested(itemDef) { const itemType = itemDef.type; const isObject = (itemType === 'object'); const isArray = (itemType === 'array'); if (isObject === true) { - // After normalization, schema should always be present for objects return Schema(itemDef.schema, options); } @@ -338,7 +297,6 @@ export default function Schema(schema, options = { keyOrder: false }) { return undefined; } - /** @private */ function computeNestedVariant(variantDef) { const variantType = variantDef.type; const isObject = (variantType === 'object'); @@ -347,7 +305,6 @@ export default function Schema(schema, options = { keyOrder: false }) { if (isObject === true || isArray === true) { if (isObject === true) { - // After normalization, schema should always be present for objects childSchema = Schema(variantDef.schema, options); } if (isArray === true) { diff --git a/src/size-writer.ts b/src/size-writer.ts deleted file mode 100644 index 1710850..0000000 --- a/src/size-writer.ts +++ /dev/null @@ -1,19 +0,0 @@ -/** Shared size writing utilities */ - -/** - * Write size bytes directly to array in big-endian format - * @param target - Array to write to - * @param size - Size value to encode - * @param count - Number of bytes (1, 2, or 4) - */ -export function writeSizeBytes(target: number[], size: number, count: number): void { - if (count === 1) { - target.push(size & 0xff); - } - else if (count === 2) { - target.push(size >> 8, size & 0xff); - } - else if (count === 4) { - target.push(size >> 24, size >> 16, size >> 8, size & 0xff); - } -} diff --git a/src/variant-matcher.ts b/src/variant-matcher.ts index 8671ac1..809f763 100644 --- a/src/variant-matcher.ts +++ b/src/variant-matcher.ts @@ -1,6 +1,3 @@ -/** Shared variant matching utilities */ - -/* Type compatibility lookup for variant matching */ export const TYPE_COMPAT_MAP = { number: new Set(['int32', 'int64', 'float', 'double']), string: new Set(['string', 'uuid', 'ipv4', 'ipv6', 'date', 'date-time', 'binary']), @@ -9,12 +6,7 @@ export const TYPE_COMPAT_MAP = { object: new Set(['object']), }; -/** - * Check if data matches a variant schema definition - * @private - */ export function matchesVariant(data, variant) { - // Performance: Optimized type detection let dataType; if (data === null) { dataType = 'null'; @@ -26,20 +18,16 @@ export function matchesVariant(data, variant) { dataType = typeof data; } - // Special handling for binary types (Buffer/Uint8Array) if (variant.type === 'binary' && (data instanceof Buffer || data instanceof Uint8Array)) { return true; } - // Fast path: Use type compatibility map for quick lookup const compatibleTypes = TYPE_COMPAT_MAP[dataType]; if (compatibleTypes && compatibleTypes.has(variant.type)) { - // For objects, additional validation needed if (variant.type === 'object' && variant.schemaKeys && variant.schemaKeys.length > 0) { const schemaKeys = variant.schemaKeys; const dataKeys = Object.keys(data); - // Check if data keys match schema keys let matchCount = 0; for (const key of schemaKeys) { if (data.hasOwnProperty(key)) { @@ -47,8 +35,6 @@ export function matchesVariant(data, variant) { } } - // Require at least one matching key and 50% overlap - // This helps distinguish between different object variants return matchCount > 0 && matchCount >= Math.min(schemaKeys.length, dataKeys.length) * 0.5; } diff --git a/src/writer.ts b/src/writer.ts index 0a219c2..5a2989c 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -1,27 +1,15 @@ -/** Data writer component */ - -/* Requires ------------------------------------------------------------------ */ - import { NULL_INDICATOR, VARIANT_BASE } from './encoder'; import { matchesVariant } from './variant-matcher'; -/* Methods ------------------------------------------------------------------- */ - export default function Writer(scope) { - /** - * Estimate buffer size for pre-allocation - * Conservative estimate to minimize reallocation - */ function estimateBufferSize(keys) { - let size = 1; // Field count byte + let size = 1; for (let i = 0; i < keys.length; i++) { const field = scope.indices[keys[i]]; - // Field index (1 byte) + optional discriminator (1 byte) size += field.nullable || field.variants ? 2 : 1; - // Size marker bytes if (field.fixedSize) { size += field.fixedSize.length; } @@ -29,28 +17,20 @@ export default function Writer(scope) { size += field.count || 1; } - // Estimated data size if (field.size) { - // Fixed size field (int32, float, etc.) size += field.size; } else { - // Variable size field - estimate conservatively - // For typical API fields: strings ~20 bytes, arrays ~50 bytes, objects ~100 bytes if (field.type === 'string') size += 32; else if (field.type === 'array') size += 64; else if (field.type === 'object') size += 128; - else size += 16; // Other variable types + else size += 16; } } - // Add 50% buffer for safety return Math.max(Math.floor(size * 1.5), 256); } - /** - * Ensure buffer has capacity and grow if needed - */ function ensureCapacity(needed) { if (scope.position + needed > scope.buffer.length) { const newSize = Math.max(scope.buffer.length * 2, scope.position + needed); @@ -63,32 +43,25 @@ export default function Writer(scope) { function write(data, options?) { const keys = filterKeys(data, options); - // Pre-allocate buffer based on estimation - // Use Buffer.allocUnsafe for performance and compatibility with Buffer.write() const estimatedSize = estimateBufferSize(keys); scope.buffer = Buffer.allocUnsafe(estimatedSize); scope.position = 0; - // Write field count scope.buffer[scope.position++] = keys.length; for (let i = 0; i < keys.length; i++) { const key = keys[i]; let keyData = data[key]; - const field = scope.indices[key]; // Performance: Single lookup, reused throughout + const field = scope.indices[key]; - // Handle nullable fields with null values if (field.nullable && keyData === null) { - // Write field index and NULL_INDICATOR (no size or content follows) ensureCapacity(2); scope.buffer[scope.position++] = field.index; scope.buffer[scope.position++] = NULL_INDICATOR; continue; } - // Handle oneOf/anyOf fields if (field.variants) { - // Determine which variant matches the data let variantIndex = -1; let variantField = null; @@ -104,12 +77,10 @@ export default function Writer(scope) { throw new Error(`Data does not match any variant for field: ${keys[i]}`); } - // Write field index and variant discriminator (1-indexed) ensureCapacity(2); scope.buffer[scope.position++] = field.index; scope.buffer[scope.position++] = VARIANT_BASE + variantIndex; - // Apply coercion/validation if needed if (options !== undefined) { if (options.coerse === true && variantField.coerse) { keyData = variantField.coerse(keyData); @@ -119,12 +90,10 @@ export default function Writer(scope) { } } - // NEW API: Write size and value directly to buffer scope.position = writeFieldValue(keyData, variantField); continue; } - // Handle regular fields with discriminator byte if nullable if (field.nullable) { ensureCapacity(2); scope.buffer[scope.position++] = field.index; @@ -140,24 +109,15 @@ export default function Writer(scope) { if (options.validate === true && field.validate) field.validate(keyData); } - // NEW API: Write size and value directly to buffer scope.position = writeFieldValue(keyData, field); } return this; } - /** - * Write field value using new encoder API - * Encoders now write directly to buffer and return new position - * For fixed-size fields, writes size then data - * For variable-size fields, reserves space for size, writes data, fills in size - * @returns new position - */ function writeFieldValue(value, fieldOrVariant) { - ensureCapacity(1024); // Ensure some headroom (will grow if needed) + ensureCapacity(1024); - // Fixed-size fields: write size then data directly if (fieldOrVariant.size) { const count = fieldOrVariant.count; if (count === 1) scope.buffer[scope.position++] = fieldOrVariant.size; @@ -171,20 +131,16 @@ export default function Writer(scope) { scope.buffer[scope.position++] = fieldOrVariant.size >> 8; scope.buffer[scope.position++] = fieldOrVariant.size & 0xff; } - // Call encoder with new API: (val, buffer, pos) => newPos return fieldOrVariant.transformIn(value, scope.buffer, scope.position); } - // Variable-size fields: reserve space for size, write data, fill in size const sizePos = scope.position; - scope.position += fieldOrVariant.count; // Reserve space for size + scope.position += fieldOrVariant.count; const dataStart = scope.position; - // Call encoder with new API: (val, buffer, pos) => newPos const newPos = fieldOrVariant.transformIn(value, scope.buffer, scope.position); const size = newPos - dataStart; - // Fill in size at reserved position if (fieldOrVariant.count === 1) { scope.buffer[sizePos] = size & 0xff; } @@ -215,31 +171,26 @@ export default function Writer(scope) { return s; } - /** @private */ function filterKeys(data) { const res = []; const undeclaredKeys = []; for (const key in data) { - // Performance: O(1) Set lookup instead of O(n) indexOf if (!scope.itemsSet.has(key)) { undeclaredKeys.push(key); continue; } - // Include nullable fields even when null if (scope.indices[key].nullable && data[key] === null) { res.push(key); continue; } - // Skip non-nullable fields that are null or undefined if (data[key] !== null && data[key] !== undefined) { res.push(key); } } - // Always warn about undeclared properties if (undeclaredKeys.length > 0) { console.warn( `Schema validation warning: Object contains undeclared properties that will not be serialized: ${undeclaredKeys.join(', ')}`, @@ -249,15 +200,9 @@ export default function Writer(scope) { return res; } - /** - * Write to buffer at specific position (for nested objects) - * Used by object encoder when writing nested schemas - * @returns new position - */ function writeToBuffer(data, buffer, pos) { const keys = filterKeys(data); - // Write field count buffer[pos++] = keys.length; for (let i = 0; i < keys.length; i++) { @@ -265,18 +210,14 @@ export default function Writer(scope) { const keyData = data[key]; const field = scope.indices[key]; - // Write field index buffer[pos++] = field.index; - // Handle nullable null values if (field.nullable && keyData === null) { buffer[pos++] = NULL_INDICATOR; continue; } - // Handle oneOf/anyOf fields if (field.variants) { - // Determine which variant matches the data let variantIndex = -1; let variantField = null; @@ -292,10 +233,8 @@ export default function Writer(scope) { throw new Error(`Data does not match any variant for field: ${key}`); } - // Write variant discriminator buffer[pos++] = VARIANT_BASE + variantIndex; - // For fixed-size types, write size then data if (variantField.size) { const count = variantField.count; if (count === 1) buffer[pos++] = variantField.size; @@ -312,14 +251,12 @@ export default function Writer(scope) { pos = variantField.transformIn(keyData, buffer, pos); } else { - // Variable size - reserve space for size, write data, fill in size const sizePos = pos; - pos += variantField.count; // Reserve space + pos += variantField.count; const dataStart = pos; pos = variantField.transformIn(keyData, buffer, pos); const size = pos - dataStart; - // Fill in size if (variantField.count === 1) buffer[sizePos] = size & 0xff; else if (variantField.count === 2) { buffer[sizePos] = size >> 8; @@ -335,12 +272,10 @@ export default function Writer(scope) { continue; } - // Add presence indicator for nullable non-null if (field.nullable) { buffer[pos++] = VARIANT_BASE; } - // For fixed-size, write size then data if (field.size) { const count = field.count; if (count === 1) buffer[pos++] = field.size; @@ -357,7 +292,6 @@ export default function Writer(scope) { pos = field.transformIn(keyData, buffer, pos); } else { - // Variable size - reserve, write, fill const sizePos = pos; pos += field.count; const dataStart = pos; @@ -382,7 +316,6 @@ export default function Writer(scope) { } function buffer() { - // Return trimmed buffer (only the bytes that were written) return Buffer.from(scope.buffer.subarray(0, scope.position)); } From 339b84fd8a219db345e628646c4248349d87addf Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Sun, 2 Nov 2025 20:56:38 -0500 Subject: [PATCH 14/19] added logger, added msgpack comparison --- README.md | 9 +++++---- benchmarks/array.ts | 14 +++++++++++++- benchmarks/boolean.ts | 14 +++++++++++++- benchmarks/integer.ts | 14 +++++++++++++- benchmarks/jsonapiresponse.ts | 28 +++++++++++++++++++++++++++- benchmarks/schema.ts | 14 +++++++++++++- benchmarks/string.ts | 14 +++++++++++++- benchmarks/uuid.ts | 14 +++++++++++++- package.json | 1 + src/logger.ts | 15 +++++++++++++++ src/writer.ts | 4 +++- tests/integration/index.ts | 7 ++++--- 12 files changed, 133 insertions(+), 15 deletions(-) create mode 100644 src/logger.ts diff --git a/README.md b/README.md index 7175278..6be04a3 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@

- + Compactr

Compactr

- OpenAPI schema serialization + OpenAPI serialization



@@ -56,7 +56,7 @@ const decoded = userSchema.read(buffer); Compactr also supports component references ($ref). -* Only local references are allowed * +*Only local references are allowed* **Component References** @@ -135,8 +135,9 @@ I realistic scenarios, compactr performs a bit slower than JSON.stringify/ JSON. [JSON-API Reponse] JSON x 289 ops/sec ±1.10% (83 runs sampled) [JSON-API Reponse] Compactr x 115 ops/sec ±0.93% (74 runs sampled) [JSON-API Reponse] Protobuf x 272 ops/sec ±1.06% (82 runs sampled) +[JSON-API Reponse] MsgPack x 133 ops/sec ±1.38% (76 runs sampled) -Buffer size (bytes): { json: 277, compactr: 78, protobuf: 129 } +Buffer size (bytes): { json: 277, compactr: 78, protobuf: 129, msgpack: 227 } ``` ## Testing diff --git a/benchmarks/array.ts b/benchmarks/array.ts index 4a413fc..d3c1429 100644 --- a/benchmarks/array.ts +++ b/benchmarks/array.ts @@ -5,6 +5,7 @@ import Benchmark from 'benchmark'; import {schema} from '../dist/compactr.js'; import protobuf from 'protobufjs'; +import * as msgpack from '@msgpack/msgpack'; import {deferred} from './utils.ts'; /* Local variables -----------------------------------------------------------*/ @@ -27,7 +28,7 @@ let root = protobuf.Root.fromJSON({ }); var ArrayBenchTest = root.lookupType('ArrayBenchTest'); -const sizes = { json: 0, compactr: 0, protobuf: 0 }; +const sizes = { json: 0, compactr: 0, protobuf: 0, msgpack: 0 }; const arraySuite = new Benchmark.Suite(); @@ -39,6 +40,7 @@ export function init(mult) { arraySuite.add('[Array] JSON', arrJSON) .add('[Array] Compactr', arrCompactr) .add('[Array] Protobuf', arrProtobuf) + .add('[Array] MsgPack', arrMsgPack) .on('cycle', e => console.log(String(e.target))) .run({ 'async': true }) .on('complete', _ => resolve(sizes)); @@ -75,5 +77,15 @@ export function init(mult) { } } + function arrMsgPack() { + let packed, unpacked; + + for(let i = 0; i sizes.msgpack) sizes.msgpack = packed.length; + } + } + return promise; } diff --git a/benchmarks/boolean.ts b/benchmarks/boolean.ts index cc7660e..5fe9e14 100644 --- a/benchmarks/boolean.ts +++ b/benchmarks/boolean.ts @@ -5,6 +5,7 @@ import Benchmark from 'benchmark'; import {schema} from '../dist/compactr.js'; import protobuf from 'protobufjs'; +import * as msgpack from '@msgpack/msgpack'; import {deferred} from './utils.ts'; /* Local variables -----------------------------------------------------------*/ @@ -15,7 +16,7 @@ let User = schema({ bool: { type: 'boolean' }, }); -const sizes = { json: 0, compactr: 0, protobuf: 0 }; +const sizes = { json: 0, compactr: 0, protobuf: 0, msgpack: 0 }; let root = protobuf.Root.fromJSON({ nested: { @@ -39,6 +40,7 @@ export function init(mult) { boolSuite.add('[Boolean] JSON', boolJSON) .add('[Boolean] Compactr', boolCompactr) .add('[Boolean] Protobuf', boolProtobuf) + .add('[Boolean] MsgPack', boolMsgPack) .on('cycle', e => console.log(String(e.target))) .run({ 'async': true }) .on('complete', _ => resolve(sizes)); @@ -74,5 +76,15 @@ export function init(mult) { } } + function boolMsgPack() { + let packed, unpacked; + + for(let i = 0; i sizes.msgpack) sizes.msgpack = packed.length; + } + } + return promise; } diff --git a/benchmarks/integer.ts b/benchmarks/integer.ts index c271e0e..2fd0c12 100644 --- a/benchmarks/integer.ts +++ b/benchmarks/integer.ts @@ -5,6 +5,7 @@ import Benchmark from 'benchmark'; import {schema} from '../dist/compactr.js'; import protobuf from 'protobufjs'; +import * as msgpack from '@msgpack/msgpack'; import {deferred} from './utils.ts'; /* Local variables -----------------------------------------------------------*/ @@ -14,7 +15,7 @@ let User = schema({ int: { type: 'integer', format: 'int32' }, }); -const sizes = { json: 0, compactr: 0, protobuf: 0 }; +const sizes = { json: 0, compactr: 0, protobuf: 0, msgpack: 0 }; let root = protobuf.Root.fromJSON({ nested: { @@ -38,6 +39,7 @@ export function init(mult) { intSuite.add('[Integer] JSON', intJSON) .add('[Integer] Compactr', intCompactr) .add('[Integer] Protobuf', intProtobuf) + .add('[Integer] MsgPack', intMsgPack) .on('cycle', e => console.log(String(e.target))) .run({ 'async': true }) .on('complete', _ => resolve(sizes)); @@ -73,5 +75,15 @@ export function init(mult) { } } + function intMsgPack() { + let packed, unpacked; + + for(let i = 0; i sizes.msgpack) sizes.msgpack = packed.length; + } + } + return promise; } diff --git a/benchmarks/jsonapiresponse.ts b/benchmarks/jsonapiresponse.ts index dda1126..394ba18 100644 --- a/benchmarks/jsonapiresponse.ts +++ b/benchmarks/jsonapiresponse.ts @@ -5,6 +5,7 @@ import Benchmark from 'benchmark'; import {schema} from '../dist/compactr.js'; import protobuf from 'protobufjs'; +import * as msgpack from '@msgpack/msgpack'; import {deferred} from './utils.ts'; import {randomUUID} from 'crypto'; @@ -35,7 +36,7 @@ let User = schema({ } }); -const sizes = { json: 0, compactr: 0, protobuf: 0 }; +const sizes = { json: 0, compactr: 0, protobuf: 0, msgpack: 0 }; let root = protobuf.Root.fromJSON({ nested: { @@ -72,6 +73,7 @@ export function init(mult) { objectSuite.add('[JSON-API Reponse] JSON', objJSON) .add('[JSON-API Reponse] Compactr', objCompactr) .add('[JSON-API Reponse] Protobuf', objProtobuf) + .add('[JSON-API Reponse] MsgPack', objMsgPack) .on('cycle', e => console.log(String(e.target))) .run({ 'async': true }) .on('complete', _ => resolve(sizes)); @@ -150,5 +152,29 @@ export function init(mult) { } } + function objMsgPack() { + let packed, unpacked; + let now = (new Date()).toISOString(); + + for(let i = 0; i sizes.msgpack) sizes.msgpack = packed.length; + } + } + return promise; } diff --git a/benchmarks/schema.ts b/benchmarks/schema.ts index b82613b..2365705 100644 --- a/benchmarks/schema.ts +++ b/benchmarks/schema.ts @@ -5,6 +5,7 @@ import Benchmark from 'benchmark'; import {schema} from '../dist/compactr.js'; import protobuf from 'protobufjs'; +import * as msgpack from '@msgpack/msgpack'; import {deferred} from './utils.ts'; import {randomUUID} from 'crypto'; @@ -21,7 +22,7 @@ let User = schema({ }, }); -const sizes = { json: 0, compactr: 0, protobuf: 0 }; +const sizes = { json: 0, compactr: 0, protobuf: 0, msgpack: 0 }; let root = protobuf.Root.fromJSON({ nested: { @@ -50,6 +51,7 @@ export function init(mult) { objectSuite.add('[Schema] JSON', objJSON) .add('[Schema] Compactr', objCompactr) .add('[Schema] Protobuf', objProtobuf) + .add('[Schema] MsgPack', objMsgPack) .on('cycle', e => console.log(String(e.target))) .run({ 'async': true }) .on('complete', _ => resolve(sizes)); @@ -86,5 +88,15 @@ export function init(mult) { } } + function objMsgPack() { + let packed, unpacked; + + for(let i = 0; i sizes.msgpack) sizes.msgpack = packed.length; + } + } + return promise; } diff --git a/benchmarks/string.ts b/benchmarks/string.ts index a408e79..f13dfb3 100644 --- a/benchmarks/string.ts +++ b/benchmarks/string.ts @@ -5,6 +5,7 @@ import Benchmark from 'benchmark'; import {schema} from '../dist/compactr.js'; import protobuf from 'protobufjs'; +import * as msgpack from '@msgpack/msgpack'; import {deferred} from './utils.ts'; /* Local variables -----------------------------------------------------------*/ @@ -29,7 +30,7 @@ let root = protobuf.Root.fromJSON({ }); var StringBenchTest = root.lookupType('StringBenchTest'); -const sizes = { json: 0, compactr: 0, protobuf: 0 }; +const sizes = { json: 0, compactr: 0, protobuf: 0, msgpack: 0 }; const stringSuite = new Benchmark.Suite(); @@ -41,6 +42,7 @@ export function init(mult) { stringSuite.add('[String] JSON', strJSON) .add('[String] Compactr', strCompactr) .add('[String] Protobuf', strProtobuf) + .add('[String] MsgPack', strMsgPack) .on('cycle', e => console.log(String(e.target))) .run({ 'async': true }) .on('complete', _ => resolve(sizes)); @@ -77,5 +79,15 @@ export function init(mult) { } } + function strMsgPack() { + let packed, unpacked; + + for(let i = 0; i sizes.msgpack) sizes.msgpack = packed.length; + } + } + return promise; } diff --git a/benchmarks/uuid.ts b/benchmarks/uuid.ts index 160eab8..0b9a8a5 100644 --- a/benchmarks/uuid.ts +++ b/benchmarks/uuid.ts @@ -5,6 +5,7 @@ import Benchmark from 'benchmark'; import {schema} from '../dist/compactr.js'; import protobuf from 'protobufjs'; +import * as msgpack from '@msgpack/msgpack'; import {deferred} from './utils.ts'; import {randomUUID} from 'crypto'; @@ -28,7 +29,7 @@ let root = protobuf.Root.fromJSON({ }); var StringBenchTest = root.lookupType('StringBenchTest'); -const sizes = { json: 0, compactr: 0, protobuf: 0 }; +const sizes = { json: 0, compactr: 0, protobuf: 0, msgpack: 0 }; const stringSuite = new Benchmark.Suite(); @@ -40,6 +41,7 @@ export function init(mult) { stringSuite.add('[UUID] JSON', strJSON) .add('[UUID] Compactr', strCompactr) .add('[UUID] Protobuf', strProtobuf) + .add('[UUID] MsgPack', strMsgPack) .on('cycle', e => console.log(String(e.target))) .run({ 'async': true }) .on('complete', _ => resolve(sizes)); @@ -76,5 +78,15 @@ export function init(mult) { } } + function strMsgPack() { + let packed, unpacked; + + for(let i = 0; i sizes.msgpack) sizes.msgpack = packed.length; + } + } + return promise; } diff --git a/package.json b/package.json index 3d5e03d..6f36a7d 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ } }, "devDependencies": { + "@msgpack/msgpack": "^3.0.0", "@rollup/plugin-node-resolve": "^16.0.0", "@rollup/plugin-sucrase": "^5.0.0", "@stylistic/eslint-plugin": "^5.5.0", diff --git a/src/logger.ts b/src/logger.ts new file mode 100644 index 0000000..674eb3e --- /dev/null +++ b/src/logger.ts @@ -0,0 +1,15 @@ +declare const window: any; + +let enabled: boolean = null; +const prefix = `COMPACTR${typeof process === 'object' && ` (pid:${process.pid})`}`; + +export function log(msg: string): void { + if (enabled === null) { + enabled = ( + (typeof process === 'object' && process.env.NODE_DEBUG) + || (typeof window === 'object' && window.DEBUG) + || '' + ).indexOf('compactr') > -1; + } + if (enabled === true) console.log(`${prefix}: ${msg}`); +} diff --git a/src/writer.ts b/src/writer.ts index 5a2989c..8e11150 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -1,6 +1,8 @@ import { NULL_INDICATOR, VARIANT_BASE } from './encoder'; import { matchesVariant } from './variant-matcher'; +import { log } from './logger'; + export default function Writer(scope) { function estimateBufferSize(keys) { let size = 1; @@ -192,7 +194,7 @@ export default function Writer(scope) { } if (undeclaredKeys.length > 0) { - console.warn( + log( `Schema validation warning: Object contains undeclared properties that will not be serialized: ${undeclaredKeys.join(', ')}`, ); } diff --git a/tests/integration/index.ts b/tests/integration/index.ts index d240a12..d08ffcf 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -1,4 +1,5 @@ import { schema } from '../../src'; +import * as logger from '../../src/logger'; /* Tests --------------------------------------------------------------------- */ @@ -1499,7 +1500,7 @@ describe('OpenAPI-compatible object formats', () => { }); it('should ignore undeclared properties', () => { - const warnSpy = jest.spyOn(console, 'warn').mockImplementation(); + const warnSpy = jest.spyOn(logger, 'log').mockImplementation(); const input = { data: { @@ -1515,7 +1516,7 @@ describe('OpenAPI-compatible object formats', () => { }); it('should always warn about undeclared properties', () => { - const warnSpy = jest.spyOn(console, 'warn').mockImplementation(); + const warnSpy = jest.spyOn(logger, 'log').mockImplementation(); const input = { data: { @@ -1538,7 +1539,7 @@ describe('OpenAPI-compatible object formats', () => { }); it('should not warn when all properties are declared', () => { - const warnSpy = jest.spyOn(console, 'warn').mockImplementation(); + const warnSpy = jest.spyOn(logger, 'log').mockImplementation(); const input = { data: { From 60a8451345791cbad85e8c6d51cd5e944a4384ce Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Sun, 2 Nov 2025 21:41:32 -0500 Subject: [PATCH 15/19] removed deprecated keyOrdering, small refactor --- README.md | 20 +----- src/encoder.ts | 89 +-------------------------- src/reader.ts | 76 ++--------------------- src/schema.ts | 2 +- src/writer.ts | 164 +++++-------------------------------------------- types.d.ts | 1 - 6 files changed, 25 insertions(+), 327 deletions(-) diff --git a/README.md b/README.md index 6be04a3..4aef466 100644 --- a/README.md +++ b/README.md @@ -109,27 +109,9 @@ Compactr supports the following OpenAPI types and formats: | object | - | variable | Nested object | -## Size Comparison - -**Input:** -```javascript -{ - id: 123, - name: 'John', - email: 'john@example.com', - active: true -} -``` - -**JSON:** `{"id":123,"name":"John","email":"john@example.com","active":true}` - 61 bytes - -**Compactr:** `` - 32 bytes - -**Savings:** 48% smaller - ## Performance -I realistic scenarios, compactr performs a bit slower than JSON.stringify/ JSON.parse as well as other schema-based protocols such as `protobuf`, but can yield a byte reduction of 3.5x. +I realistic scenarios, compactr performs a bit slower than JSON.stringify/ JSON.parse as well as other schema-based protocols such as `protobuf`, but can yield a byte reduction of 3.5x compared to JSON. ``` [JSON-API Reponse] JSON x 289 ops/sec ±1.10% (83 runs sampled) diff --git a/src/encoder.ts b/src/encoder.ts index 949fcb3..b8a0fb1 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -1,4 +1,4 @@ -import { matchesVariant } from './variant-matcher'; +import { writeFieldWithSize, processVariantWrite } from './buffer-utils'; export const NULL_INDICATOR = 0x00; export const VARIANT_BASE = 0x01; @@ -214,57 +214,7 @@ function array(schema, val, buffer, pos) { } if (schema.variants) { - let variantIndex = -1; - let variantField = null; - - for (let v = 0; v < schema.variants.length; v++) { - if (matchesVariant(item, schema.variants[v])) { - variantIndex = v; - variantField = schema.variants[v]; - break; - } - } - - if (variantIndex === -1) { - throw new Error(`Array item does not match any variant`); - } - - buffer[pos++] = schema.nullable ? VARIANT_BASE + variantIndex : VARIANT_BASE + variantIndex; - - if (variantField.size) { - const count = variantField.count; - if (count === 1) buffer[pos++] = variantField.size; - else if (count === 2) { - buffer[pos++] = variantField.size >> 8; - buffer[pos++] = variantField.size & 0xff; - } - else if (count === 4) { - buffer[pos++] = variantField.size >> 24; - buffer[pos++] = variantField.size >> 16; - buffer[pos++] = variantField.size >> 8; - buffer[pos++] = variantField.size & 0xff; - } - pos = variantField.transformIn(item, buffer, pos); - } - else { - const sizePos = pos; - pos += variantField.count; - const dataStart = pos; - pos = variantField.transformIn(item, buffer, pos); - const size = pos - dataStart; - - if (variantField.count === 1) buffer[sizePos] = size & 0xff; - else if (variantField.count === 2) { - buffer[sizePos] = size >> 8; - buffer[sizePos + 1] = size & 0xff; - } - else if (variantField.count === 4) { - buffer[sizePos] = size >> 24; - buffer[sizePos + 1] = size >> 16; - buffer[sizePos + 2] = size >> 8; - buffer[sizePos + 3] = size & 0xff; - } - } + pos = processVariantWrite(buffer, pos, item, schema, 'Array item'); continue; } @@ -272,40 +222,7 @@ function array(schema, val, buffer, pos) { buffer[pos++] = VARIANT_BASE; } - if (schema.size) { - const count = schema.count; - if (count === 1) buffer[pos++] = schema.size; - else if (count === 2) { - buffer[pos++] = schema.size >> 8; - buffer[pos++] = schema.size & 0xff; - } - else if (count === 4) { - buffer[pos++] = schema.size >> 24; - buffer[pos++] = schema.size >> 16; - buffer[pos++] = schema.size >> 8; - buffer[pos++] = schema.size & 0xff; - } - pos = schema.transformIn(item, buffer, pos); - } - else { - const sizePos = pos; - pos += schema.count; - const dataStart = pos; - pos = schema.transformIn(item, buffer, pos); - const size = pos - dataStart; - - if (schema.count === 1) buffer[sizePos] = size & 0xff; - else if (schema.count === 2) { - buffer[sizePos] = size >> 8; - buffer[sizePos + 1] = size & 0xff; - } - else if (schema.count === 4) { - buffer[sizePos] = size >> 24; - buffer[sizePos + 1] = size >> 16; - buffer[sizePos + 2] = size >> 8; - buffer[sizePos + 3] = size & 0xff; - } - } + pos = writeFieldWithSize(buffer, pos, item, schema); } return pos; diff --git a/src/reader.ts b/src/reader.ts index 18a502b..adb37ff 100644 --- a/src/reader.ts +++ b/src/reader.ts @@ -1,17 +1,11 @@ import Decoder, { NULL_INDICATOR, VARIANT_BASE } from './decoder'; export default function Reader(scope) { - function readFromOffset(bytes, offset, length) { + function read(bytes, offset = 0, length?) { const ret = {}; - if (scope.options.keyOrder === true) { - for (let i = 0; i < scope.items.length; i++) { - ret[scope.items[i]] = undefined; - } - } - + const end = length !== undefined ? offset + length : bytes.length; let caret = offset + 1; const fieldCount = bytes[offset]; - const end = offset + length; for (let i = 0; i < fieldCount; i++) { if (caret >= end) break; @@ -66,68 +60,6 @@ export default function Reader(scope) { return ret; } - function read(bytes) { - const ret = {}; - if (scope.options.keyOrder === true) { - for (let i = 0; i < scope.items.length; i++) { - ret[scope.items[i]] = undefined; - } - } - - let caret = 1; - const fieldCount = bytes[0]; - - for (let i = 0; i < fieldCount; i++) { - const fieldIndex = bytes[caret]; - caret++; - - const field = scope.indexToField[fieldIndex]; - if (!field) { - throw new Error(`Unknown field index: ${fieldIndex}`); - } - - if (field.nullable || field.variants) { - const discriminatorByte = bytes[caret]; - caret++; - - if (discriminatorByte === NULL_INDICATOR) { - ret[field.name] = null; - continue; - } - - if (field.variants) { - const variantIndex = discriminatorByte - VARIANT_BASE; - if (variantIndex < 0 || variantIndex >= field.variants.length) { - throw new Error(`Invalid variant discriminator: ${discriminatorByte}`); - } - - const variant = field.variants[variantIndex]; - const size = variant.size || readSize(bytes, caret, variant.count); - caret += variant.count; - - ret[field.name] = variant.transformOut(bytes, caret, size); - caret += size; - continue; - } - - const size = field.size || readSize(bytes, caret, field.count); - caret += field.count; - - ret[field.name] = field.transformOut(bytes, caret, size); - caret += size; - } - else { - const size = field.size || readSize(bytes, caret, field.count); - caret += field.count; - - ret[field.name] = field.transformOut(bytes, caret, size); - caret += size; - } - } - - return ret; - } - function readSize(bytes, offset, count) { if (count === 1) return bytes[offset]; if (count === 2) return (bytes[offset] << 8) | bytes[offset + 1]; @@ -138,5 +70,9 @@ export default function Reader(scope) { return Decoder.unsigned(bytes.slice(offset, offset + count)); } + function readFromOffset(bytes, offset, length) { + return read(bytes, offset, length); + } + return { read, readFromOffset }; } diff --git a/src/schema.ts b/src/schema.ts index c569035..6300ed5 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -27,7 +27,7 @@ function resolveType(type, format) { return type; } -export default function Schema(schema, options = { keyOrder: false }) { +export default function Schema(schema, options = {}) { let unwrappedSchema = schema; if (schema.type === 'object' && schema.properties) { unwrappedSchema = schema.properties; diff --git a/src/writer.ts b/src/writer.ts index 8e11150..f392b2b 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -1,6 +1,5 @@ import { NULL_INDICATOR, VARIANT_BASE } from './encoder'; -import { matchesVariant } from './variant-matcher'; - +import { writeFieldWithSize, processVariantWrite } from './buffer-utils'; import { log } from './logger'; export default function Writer(scope) { @@ -64,35 +63,21 @@ export default function Writer(scope) { } if (field.variants) { - let variantIndex = -1; - let variantField = null; - - for (let v = 0; v < field.variants.length; v++) { - if (matchesVariant(keyData, field.variants[v])) { - variantIndex = v; - variantField = field.variants[v]; - break; + if (options !== undefined) { + for (let v = 0; v < field.variants.length; v++) { + const variant = field.variants[v]; + if (options.coerse === true && variant.coerse) { + keyData = variant.coerse(keyData); + } + if (options.validate === true && variant.validate) { + variant.validate(keyData); + } } } - if (variantIndex === -1) { - throw new Error(`Data does not match any variant for field: ${keys[i]}`); - } - ensureCapacity(2); scope.buffer[scope.position++] = field.index; - scope.buffer[scope.position++] = VARIANT_BASE + variantIndex; - - if (options !== undefined) { - if (options.coerse === true && variantField.coerse) { - keyData = variantField.coerse(keyData); - } - if (options.validate === true && variantField.validate) { - variantField.validate(keyData); - } - } - - scope.position = writeFieldValue(keyData, variantField); + scope.position = processVariantWrite(scope.buffer, scope.position, keyData, field, `Field: ${keys[i]}`); continue; } @@ -119,45 +104,7 @@ export default function Writer(scope) { function writeFieldValue(value, fieldOrVariant) { ensureCapacity(1024); - - if (fieldOrVariant.size) { - const count = fieldOrVariant.count; - if (count === 1) scope.buffer[scope.position++] = fieldOrVariant.size; - else if (count === 2) { - scope.buffer[scope.position++] = fieldOrVariant.size >> 8; - scope.buffer[scope.position++] = fieldOrVariant.size & 0xff; - } - else if (count === 4) { - scope.buffer[scope.position++] = fieldOrVariant.size >> 24; - scope.buffer[scope.position++] = fieldOrVariant.size >> 16; - scope.buffer[scope.position++] = fieldOrVariant.size >> 8; - scope.buffer[scope.position++] = fieldOrVariant.size & 0xff; - } - return fieldOrVariant.transformIn(value, scope.buffer, scope.position); - } - - const sizePos = scope.position; - scope.position += fieldOrVariant.count; - const dataStart = scope.position; - - const newPos = fieldOrVariant.transformIn(value, scope.buffer, scope.position); - const size = newPos - dataStart; - - if (fieldOrVariant.count === 1) { - scope.buffer[sizePos] = size & 0xff; - } - else if (fieldOrVariant.count === 2) { - scope.buffer[sizePos] = size >> 8; - scope.buffer[sizePos + 1] = size & 0xff; - } - else if (fieldOrVariant.count === 4) { - scope.buffer[sizePos] = size >> 24; - scope.buffer[sizePos + 1] = size >> 16; - scope.buffer[sizePos + 2] = size >> 8; - scope.buffer[sizePos + 3] = size & 0xff; - } - - return newPos; + return writeFieldWithSize(scope.buffer, scope.position, value, fieldOrVariant); } function sizes(data) { @@ -220,57 +167,7 @@ export default function Writer(scope) { } if (field.variants) { - let variantIndex = -1; - let variantField = null; - - for (let v = 0; v < field.variants.length; v++) { - if (matchesVariant(keyData, field.variants[v])) { - variantIndex = v; - variantField = field.variants[v]; - break; - } - } - - if (variantIndex === -1) { - throw new Error(`Data does not match any variant for field: ${key}`); - } - - buffer[pos++] = VARIANT_BASE + variantIndex; - - if (variantField.size) { - const count = variantField.count; - if (count === 1) buffer[pos++] = variantField.size; - else if (count === 2) { - buffer[pos++] = variantField.size >> 8; - buffer[pos++] = variantField.size & 0xff; - } - else if (count === 4) { - buffer[pos++] = variantField.size >> 24; - buffer[pos++] = variantField.size >> 16; - buffer[pos++] = variantField.size >> 8; - buffer[pos++] = variantField.size & 0xff; - } - pos = variantField.transformIn(keyData, buffer, pos); - } - else { - const sizePos = pos; - pos += variantField.count; - const dataStart = pos; - pos = variantField.transformIn(keyData, buffer, pos); - const size = pos - dataStart; - - if (variantField.count === 1) buffer[sizePos] = size & 0xff; - else if (variantField.count === 2) { - buffer[sizePos] = size >> 8; - buffer[sizePos + 1] = size & 0xff; - } - else if (variantField.count === 4) { - buffer[sizePos] = size >> 24; - buffer[sizePos + 1] = size >> 16; - buffer[sizePos + 2] = size >> 8; - buffer[sizePos + 3] = size & 0xff; - } - } + pos = processVariantWrite(buffer, pos, keyData, field, `Field: ${key}`); continue; } @@ -278,40 +175,7 @@ export default function Writer(scope) { buffer[pos++] = VARIANT_BASE; } - if (field.size) { - const count = field.count; - if (count === 1) buffer[pos++] = field.size; - else if (count === 2) { - buffer[pos++] = field.size >> 8; - buffer[pos++] = field.size & 0xff; - } - else if (count === 4) { - buffer[pos++] = field.size >> 24; - buffer[pos++] = field.size >> 16; - buffer[pos++] = field.size >> 8; - buffer[pos++] = field.size & 0xff; - } - pos = field.transformIn(keyData, buffer, pos); - } - else { - const sizePos = pos; - pos += field.count; - const dataStart = pos; - pos = field.transformIn(keyData, buffer, pos); - const size = pos - dataStart; - - if (field.count === 1) buffer[sizePos] = size & 0xff; - else if (field.count === 2) { - buffer[sizePos] = size >> 8; - buffer[sizePos + 1] = size & 0xff; - } - else if (field.count === 4) { - buffer[sizePos] = size >> 24; - buffer[sizePos + 1] = size >> 16; - buffer[sizePos + 2] = size >> 8; - buffer[sizePos + 3] = size & 0xff; - } - } + pos = writeFieldWithSize(buffer, pos, keyData, field); } return pos; diff --git a/types.d.ts b/types.d.ts index ed3c198..1968810 100644 --- a/types.d.ts +++ b/types.d.ts @@ -55,7 +55,6 @@ declare module 'compactr' { } export interface SchemaOptions { - keyOrder?: boolean schemas?: { [key: string]: SchemaFieldDefinition } } From a0e44ad08ec949ccdde16fc89e13d57c5a2c72e2 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Sun, 2 Nov 2025 21:45:56 -0500 Subject: [PATCH 16/19] missing file --- src/buffer-utils.ts | 70 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 src/buffer-utils.ts diff --git a/src/buffer-utils.ts b/src/buffer-utils.ts new file mode 100644 index 0000000..2666bcc --- /dev/null +++ b/src/buffer-utils.ts @@ -0,0 +1,70 @@ +import { matchesVariant } from './variant-matcher'; + +const VARIANT_BASE = 0x01; + +export function writeSize(buffer: Uint8Array, pos: number, size: number, count: number): number { + if (count === 1) { + buffer[pos] = size & 0xff; + return pos + 1; + } + if (count === 2) { + buffer[pos] = size >> 8; + buffer[pos + 1] = size & 0xff; + return pos + 2; + } + if (count === 4) { + buffer[pos] = size >> 24; + buffer[pos + 1] = size >> 16; + buffer[pos + 2] = size >> 8; + buffer[pos + 3] = size & 0xff; + return pos + 4; + } + return pos; +} + +export function writeFieldWithSize( + buffer: Uint8Array, + pos: number, + value: any, + field: any, +): number { + if (field.size) { + pos = writeSize(buffer, pos, field.size, field.count); + return field.transformIn(value, buffer, pos); + } + + const sizePos = pos; + pos += field.count; + const dataStart = pos; + pos = field.transformIn(value, buffer, pos); + const size = pos - dataStart; + writeSize(buffer, sizePos, size, field.count); + return pos; +} + +export function processVariantWrite( + buffer: Uint8Array, + pos: number, + data: any, + field: any, + contextName?: string, +): number { + let variantIndex = -1; + let variantField = null; + + for (let v = 0; v < field.variants.length; v++) { + if (matchesVariant(data, field.variants[v])) { + variantIndex = v; + variantField = field.variants[v]; + break; + } + } + + if (variantIndex === -1) { + const context = contextName || 'Data'; + throw new Error(`${context} does not match any variant`); + } + + buffer[pos++] = VARIANT_BASE + variantIndex; + return writeFieldWithSize(buffer, pos, data, variantField); +} From 0a258462707a8c42f1060e27fc26c7503e686744 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Mon, 3 Nov 2025 01:26:04 -0500 Subject: [PATCH 17/19] deprecated the .buffer() method --- README.md | 2 +- benchmarks/array.ts | 2 +- benchmarks/boolean.ts | 2 +- benchmarks/index.ts | 4 +- benchmarks/integer.ts | 2 +- benchmarks/jsonapiresponse.ts | 2 +- benchmarks/schema.ts | 2 +- benchmarks/string.ts | 2 +- benchmarks/uuid.ts | 2 +- src/writer.ts | 8 +- tests/integration/index.ts | 332 +++++++++++++++++----------------- tests/integration/openapi.ts | 4 +- types.d.ts | 12 +- 13 files changed, 183 insertions(+), 193 deletions(-) diff --git a/README.md b/README.md index 4aef466..55dc48a 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ const data = { tags: ['premium', 'verified'] }; -const buffer = userSchema.write(data).buffer(); +const buffer = userSchema.write(data); const decoded = userSchema.read(buffer); ``` diff --git a/benchmarks/array.ts b/benchmarks/array.ts index d3c1429..bb379ff 100644 --- a/benchmarks/array.ts +++ b/benchmarks/array.ts @@ -60,7 +60,7 @@ export function init(mult) { let packed, unpacked; for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; } diff --git a/benchmarks/boolean.ts b/benchmarks/boolean.ts index 5fe9e14..9ee7420 100644 --- a/benchmarks/boolean.ts +++ b/benchmarks/boolean.ts @@ -59,7 +59,7 @@ export function init(mult) { let packed, unpacked; for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; } diff --git a/benchmarks/index.ts b/benchmarks/index.ts index 0d66704..8e71dcd 100644 --- a/benchmarks/index.ts +++ b/benchmarks/index.ts @@ -12,12 +12,12 @@ import { init as jsonapiresponse } from './jsonapiresponse.ts'; import { sequence } from './utils.ts'; const benchmarks = [ - array, + /*array, boolean, integer, schema, string, - uuid, + uuid,*/ jsonapiresponse, ]; diff --git a/benchmarks/integer.ts b/benchmarks/integer.ts index 2fd0c12..19d6d88 100644 --- a/benchmarks/integer.ts +++ b/benchmarks/integer.ts @@ -58,7 +58,7 @@ export function init(mult) { let packed, unpacked; for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; } diff --git a/benchmarks/jsonapiresponse.ts b/benchmarks/jsonapiresponse.ts index 394ba18..deceee1 100644 --- a/benchmarks/jsonapiresponse.ts +++ b/benchmarks/jsonapiresponse.ts @@ -121,7 +121,7 @@ export function init(mult) { flag_c: false, }, user_friends: [] - }).buffer(); + }); unpacked = User.read(packed); if (packed.length > sizes.compactr) sizes.compactr = packed.length; } diff --git a/benchmarks/schema.ts b/benchmarks/schema.ts index 2365705..84e6ae5 100644 --- a/benchmarks/schema.ts +++ b/benchmarks/schema.ts @@ -71,7 +71,7 @@ export function init(mult) { let packed, unpacked; for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; } diff --git a/benchmarks/string.ts b/benchmarks/string.ts index f13dfb3..b87bff2 100644 --- a/benchmarks/string.ts +++ b/benchmarks/string.ts @@ -62,7 +62,7 @@ export function init(mult) { let packed, unpacked; for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; } diff --git a/benchmarks/uuid.ts b/benchmarks/uuid.ts index 0b9a8a5..8fe74b7 100644 --- a/benchmarks/uuid.ts +++ b/benchmarks/uuid.ts @@ -61,7 +61,7 @@ export function init(mult) { let packed, unpacked; for(let i = 0; i sizes.compactr) sizes.compactr = packed.length; } diff --git a/src/writer.ts b/src/writer.ts index f392b2b..fdac965 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -99,7 +99,7 @@ export default function Writer(scope) { scope.position = writeFieldValue(keyData, field); } - return this; + return Buffer.from(scope.buffer.subarray(0, scope.position)); } function writeFieldValue(value, fieldOrVariant) { @@ -181,9 +181,5 @@ export default function Writer(scope) { return pos; } - function buffer() { - return Buffer.from(scope.buffer.subarray(0, scope.position)); - } - - return { write, buffer, sizes, writeToBuffer }; + return { write, sizes, writeToBuffer }; } diff --git a/tests/integration/index.ts b/tests/integration/index.ts index d08ffcf..1c20b0c 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -8,15 +8,15 @@ describe('Data integrity - simple', () => { const Schema = schema({ test: { type: 'boolean' } }); it('should preserve boolean value and type - true', () => { - expect(Schema.read(Schema.write({ test: true }).buffer())).toEqual({ test: true }); + expect(Schema.read(Schema.write({ test: true }))).toEqual({ test: true }); }); it('should preserve boolean value and type - false', () => { - expect(Schema.read(Schema.write({ test: false }).buffer())).toEqual({ test: false }); + expect(Schema.read(Schema.write({ test: false }))).toEqual({ test: false }); }); it('should skip null or undefined values', () => { - expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({}); + expect(Schema.read(Schema.write({ test: null }))).toEqual({}); }); }); @@ -24,11 +24,11 @@ describe('Data integrity - simple', () => { const Schema = schema({ test: { type: 'number', format: 'double' } }); it('should preserve number value and type', () => { - expect(Schema.read(Schema.write({ test: 23.23 }).buffer())).toEqual({ test: 23.23 }); + expect(Schema.read(Schema.write({ test: 23.23 }))).toEqual({ test: 23.23 }); }); it('should preserve number value and type for negative values', () => { - expect(Schema.read(Schema.write({ test: -23.23 }).buffer())).toEqual({ test: -23.23 }); + expect(Schema.read(Schema.write({ test: -23.23 }))).toEqual({ test: -23.23 }); }); }); @@ -36,11 +36,11 @@ describe('Data integrity - simple', () => { const Schema = schema({ test: { type: 'integer', format: 'int32' } }); it('should preserve integer value and type', () => { - expect(Schema.read(Schema.write({ test: 123 }).buffer())).toEqual({ test: 123 }); + expect(Schema.read(Schema.write({ test: 123 }))).toEqual({ test: 123 }); }); it('should preserve integer value and type for negative values', () => { - expect(Schema.read(Schema.write({ test: -456 }).buffer())).toEqual({ test: -456 }); + expect(Schema.read(Schema.write({ test: -456 }))).toEqual({ test: -456 }); }); }); @@ -48,11 +48,11 @@ describe('Data integrity - simple', () => { const Schema = schema({ test: { type: 'integer', format: 'int64' } }); it('should preserve int64 value and type', () => { - expect(Schema.read(Schema.write({ test: 9007199254740991 }).buffer())).toEqual({ test: 9007199254740991 }); + expect(Schema.read(Schema.write({ test: 9007199254740991 }))).toEqual({ test: 9007199254740991 }); }); it('should preserve int64 value and type for negative values', () => { - expect(Schema.read(Schema.write({ test: -9007199254740991 }).buffer())).toEqual({ test: -9007199254740991 }); + expect(Schema.read(Schema.write({ test: -9007199254740991 }))).toEqual({ test: -9007199254740991 }); }); }); @@ -60,12 +60,12 @@ describe('Data integrity - simple', () => { const Schema = schema({ test: { type: 'number', format: 'float' } }); it('should preserve float value (with precision loss)', () => { - const result = Schema.read(Schema.write({ test: 3.14159 }).buffer()); + const result = Schema.read(Schema.write({ test: 3.14159 })); expect(result.test).toBeCloseTo(3.14159, 5); }); it('should preserve float value for negative values', () => { - const result = Schema.read(Schema.write({ test: -2.71828 }).buffer()); + const result = Schema.read(Schema.write({ test: -2.71828 })); expect(result.test).toBeCloseTo(-2.71828, 5); }); }); @@ -74,11 +74,11 @@ describe('Data integrity - simple', () => { const Schema = schema({ test: { type: 'integer' } }); it('should default to int32 format', () => { - expect(Schema.read(Schema.write({ test: 42 }).buffer())).toEqual({ test: 42 }); + expect(Schema.read(Schema.write({ test: 42 }))).toEqual({ test: 42 }); }); it('should handle negative values', () => { - expect(Schema.read(Schema.write({ test: -42 }).buffer())).toEqual({ test: -42 }); + expect(Schema.read(Schema.write({ test: -42 }))).toEqual({ test: -42 }); }); }); @@ -86,11 +86,11 @@ describe('Data integrity - simple', () => { const Schema = schema({ test: { type: 'number' } }); it('should default to double format', () => { - expect(Schema.read(Schema.write({ test: 3.141592653589793 }).buffer())).toEqual({ test: 3.141592653589793 }); + expect(Schema.read(Schema.write({ test: 3.141592653589793 }))).toEqual({ test: 3.141592653589793 }); }); it('should handle negative values', () => { - expect(Schema.read(Schema.write({ test: -2.718281828459045 }).buffer())).toEqual({ test: -2.718281828459045 }); + expect(Schema.read(Schema.write({ test: -2.718281828459045 }))).toEqual({ test: -2.718281828459045 }); }); }); @@ -98,15 +98,15 @@ describe('Data integrity - simple', () => { const Schema = schema({ test: { type: 'string' } }); it('should preserve string value and type', () => { - expect(Schema.read(Schema.write({ test: 'hello world' }).buffer())).toEqual({ test: 'hello world' }); + expect(Schema.read(Schema.write({ test: 'hello world' }))).toEqual({ test: 'hello world' }); }); it('should support special characters', () => { - expect(Schema.read(Schema.write({ test: '한자' }).buffer())).toEqual({ test: '한자' }); + expect(Schema.read(Schema.write({ test: '한자' }))).toEqual({ test: '한자' }); }); it('should support emojis', () => { - expect(Schema.read(Schema.write({ test: '🚀' }).buffer())).toEqual({ test: '🚀' }); + expect(Schema.read(Schema.write({ test: '🚀' }))).toEqual({ test: '🚀' }); }); }); @@ -115,12 +115,12 @@ describe('Data integrity - simple', () => { it('should preserve UUID value', () => { const uuid = '550e8400-e29b-4d4e-a7d4-426614174000'; - expect(Schema.read(Schema.write({ test: uuid }).buffer())).toEqual({ test: uuid }); + expect(Schema.read(Schema.write({ test: uuid }))).toEqual({ test: uuid }); }); it('should compress UUID to 16 bytes instead of 72', () => { const uuid = '550e8400-e29b-4d4e-a7d4-426614174000'; - const buffer = Schema.write({ test: uuid }).buffer(); + const buffer = Schema.write({ test: uuid }); // Header: 1 byte (field count) + 1 byte (field index) + 1 byte (size) = 3 bytes // Content: 16 bytes (UUID binary) // Total: 19 bytes (vs 75 bytes for string encoding: 3 header + 72 content) @@ -129,14 +129,14 @@ describe('Data integrity - simple', () => { it('should handle uppercase UUIDs', () => { const uuid = '550E8400-E29B-4D4E-A7D4-426614174000'; - const result = Schema.read(Schema.write({ test: uuid }).buffer()); + const result = Schema.read(Schema.write({ test: uuid })); // UUID should be normalized to lowercase expect(result.test).toBe('550e8400-e29b-4d4e-a7d4-426614174000'); }); it('should handle nil UUID', () => { const uuid = '00000000-0000-0000-0000-000000000000'; - expect(Schema.read(Schema.write({ test: uuid }).buffer())).toEqual({ test: uuid }); + expect(Schema.read(Schema.write({ test: uuid }))).toEqual({ test: uuid }); }); }); @@ -145,12 +145,12 @@ describe('Data integrity - simple', () => { it('should preserve IPv4 value', () => { const ip = '192.168.1.1'; - expect(Schema.read(Schema.write({ test: ip }).buffer())).toEqual({ test: ip }); + expect(Schema.read(Schema.write({ test: ip }))).toEqual({ test: ip }); }); it('should compress IPv4 to 4 bytes instead of 30', () => { const ip = '192.168.1.1'; - const buffer = Schema.write({ test: ip }).buffer(); + const buffer = Schema.write({ test: ip }); // Header: 1 byte (field count) + 1 byte (field index) + 1 byte (size) = 3 bytes // Content: 4 bytes (IPv4 binary) // Total: 7 bytes (vs 33 bytes for string encoding) @@ -158,8 +158,8 @@ describe('Data integrity - simple', () => { }); it('should handle edge cases', () => { - expect(Schema.read(Schema.write({ test: '0.0.0.0' }).buffer())).toEqual({ test: '0.0.0.0' }); - expect(Schema.read(Schema.write({ test: '255.255.255.255' }).buffer())).toEqual({ test: '255.255.255.255' }); + expect(Schema.read(Schema.write({ test: '0.0.0.0' }))).toEqual({ test: '0.0.0.0' }); + expect(Schema.read(Schema.write({ test: '255.255.255.255' }))).toEqual({ test: '255.255.255.255' }); }); }); @@ -168,12 +168,12 @@ describe('Data integrity - simple', () => { it('should compress IPv6 value', () => { const ip = '2001:0db8:85a3:0000:0000:8a2e:0370:7334'; - expect(Schema.read(Schema.write({ test: ip }).buffer())).toEqual({ test: '2001:db8:85a3::8a2e:370:7334' }); + expect(Schema.read(Schema.write({ test: ip }))).toEqual({ test: '2001:db8:85a3::8a2e:370:7334' }); }); it('should compress IPv6 to 16 bytes instead of 78', () => { const ip = '2001:0db8:85a3:0000:0000:8a2e:0370:7334'; - const buffer = Schema.write({ test: ip }).buffer(); + const buffer = Schema.write({ test: ip }); // Header: 1 byte (field count) + 1 byte (field index) + 1 byte (size) = 3 bytes // Content: 16 bytes (IPv6 binary) // Total: 19 bytes (vs 81 bytes for string encoding) @@ -182,20 +182,20 @@ describe('Data integrity - simple', () => { it('should handle compressed IPv6 notation', () => { const ip = '2001:db8:85a3::8a2e:370:7334'; - const result = Schema.read(Schema.write({ test: ip }).buffer()); + const result = Schema.read(Schema.write({ test: ip })); // Should be decoded back with compression expect(result.test).toBe(ip); }); it('should handle loopback', () => { const ip = '::1'; - const result = Schema.read(Schema.write({ test: ip }).buffer()); + const result = Schema.read(Schema.write({ test: ip })); expect(result.test).toBe('::1'); }); it('should handle all zeros', () => { const ip = '::'; - const result = Schema.read(Schema.write({ test: ip }).buffer()); + const result = Schema.read(Schema.write({ test: ip })); expect(result.test).toBe('::'); }); }); @@ -205,12 +205,12 @@ describe('Data integrity - simple', () => { it('should preserve date value', () => { const date = '2025-10-28'; - expect(Schema.read(Schema.write({ test: date }).buffer())).toEqual({ test: date }); + expect(Schema.read(Schema.write({ test: date }))).toEqual({ test: date }); }); it('should compress date to 4 bytes instead of 20', () => { const date = '2025-10-28'; - const buffer = Schema.write({ test: date }).buffer(); + const buffer = Schema.write({ test: date }); // Header: 1 byte (field count) + 1 byte (field index) + 1 byte (size) = 3 bytes // Content: 4 bytes (days since epoch) // Total: 7 bytes (vs 23 bytes for string encoding) @@ -219,17 +219,17 @@ describe('Data integrity - simple', () => { it('should handle epoch date', () => { const date = '1970-01-01'; - expect(Schema.read(Schema.write({ test: date }).buffer())).toEqual({ test: date }); + expect(Schema.read(Schema.write({ test: date }))).toEqual({ test: date }); }); it('should handle dates before epoch', () => { const date = '1969-12-31'; - expect(Schema.read(Schema.write({ test: date }).buffer())).toEqual({ test: date }); + expect(Schema.read(Schema.write({ test: date }))).toEqual({ test: date }); }); it('should handle far future dates', () => { const date = '2099-12-31'; - expect(Schema.read(Schema.write({ test: date }).buffer())).toEqual({ test: date }); + expect(Schema.read(Schema.write({ test: date }))).toEqual({ test: date }); }); }); @@ -238,12 +238,12 @@ describe('Data integrity - simple', () => { it('should preserve date-time value', () => { const datetime = '2025-10-28T14:30:00.000Z'; - expect(Schema.read(Schema.write({ test: datetime }).buffer())).toEqual({ test: datetime }); + expect(Schema.read(Schema.write({ test: datetime }))).toEqual({ test: datetime }); }); it('should compress date-time to 8 bytes instead of 40+', () => { const datetime = '2025-10-28T14:30:00.000Z'; - const buffer = Schema.write({ test: datetime }).buffer(); + const buffer = Schema.write({ test: datetime }); // Header: 1 byte (field count) + 1 byte (field index) + 1 byte (size) = 3 bytes // Content: 8 bytes (milliseconds since epoch) // Total: 11 bytes (vs 43+ bytes for string encoding) @@ -252,18 +252,18 @@ describe('Data integrity - simple', () => { it('should handle epoch datetime', () => { const datetime = '1970-01-01T00:00:00.000Z'; - expect(Schema.read(Schema.write({ test: datetime }).buffer())).toEqual({ test: datetime }); + expect(Schema.read(Schema.write({ test: datetime }))).toEqual({ test: datetime }); }); it('should handle millisecond precision', () => { const datetime = '2025-10-28T14:30:00.123Z'; - expect(Schema.read(Schema.write({ test: datetime }).buffer())).toEqual({ test: datetime }); + expect(Schema.read(Schema.write({ test: datetime }))).toEqual({ test: datetime }); }); it('should normalize various ISO 8601 formats', () => { // Input without milliseconds, output should have .000Z const input = '2025-10-28T14:30:00Z'; - const result = Schema.read(Schema.write({ test: input }).buffer()); + const result = Schema.read(Schema.write({ test: input })); expect(result.test).toBe('2025-10-28T14:30:00.000Z'); }); }); @@ -273,12 +273,12 @@ describe('Data integrity - simple', () => { it('should preserve binary data via base64', () => { const base64 = 'SGVsbG8gV29ybGQh'; // "Hello World!" - expect(Schema.read(Schema.write({ test: base64 }).buffer())).toEqual({ test: base64 }); + expect(Schema.read(Schema.write({ test: base64 }))).toEqual({ test: base64 }); }); it('should compress binary data efficiently', () => { const base64 = 'SGVsbG8gV29ybGQh'; // 16 chars = 32 bytes as string - const buffer = Schema.write({ test: base64 }).buffer(); + const buffer = Schema.write({ test: base64 }); // Header: 1 byte (field count) + 1 byte (field index) + 4 bytes (size) = 6 bytes // Content: 12 bytes (raw binary data decoded from base64) // Total: 18 bytes (vs 35 bytes for string encoding) @@ -287,19 +287,19 @@ describe('Data integrity - simple', () => { it('should handle Buffer input', () => { const data = Buffer.from('Hello World!', 'utf8'); - const result = Schema.read(Schema.write({ test: data }).buffer()); + const result = Schema.read(Schema.write({ test: data })); expect(result.test).toBe('SGVsbG8gV29ybGQh'); }); it('should handle Uint8Array input', () => { const data = new Uint8Array([72, 101, 108, 108, 111]); - const result = Schema.read(Schema.write({ test: data }).buffer()); + const result = Schema.read(Schema.write({ test: data })); expect(result.test).toBe('SGVsbG8='); // "Hello" in base64 }); it('should handle empty binary data', () => { const base64 = ''; // Empty - expect(Schema.read(Schema.write({ test: base64 }).buffer())).toEqual({ test: base64 }); + expect(Schema.read(Schema.write({ test: base64 }))).toEqual({ test: base64 }); }); it('should handle large binary data', () => { @@ -309,7 +309,7 @@ describe('Data integrity - simple', () => { bytes[i] = i; } const base64 = Buffer.from(bytes).toString('base64'); - expect(Schema.read(Schema.write({ test: base64 }).buffer())).toEqual({ test: base64 }); + expect(Schema.read(Schema.write({ test: base64 }))).toEqual({ test: base64 }); }); }); @@ -317,7 +317,7 @@ describe('Data integrity - simple', () => { const Schema = schema({ test: { type: 'array', items: { type: 'string' } } }); it('should preserve array values and types', () => { - expect(Schema.read(Schema.write({ test: ['a', 'b', 'c'] }).buffer())).toEqual({ test: ['a', 'b', 'c'] }); + expect(Schema.read(Schema.write({ test: ['a', 'b', 'c'] }))).toEqual({ test: ['a', 'b', 'c'] }); }); }); @@ -325,7 +325,7 @@ describe('Data integrity - simple', () => { const Schema = schema({ test: { type: 'object', schema: { test: { type: 'number', format: 'double' } } } }); it('should preserve object values and types', () => { - expect(Schema.read(Schema.write({ test: { test: 23.23 } }).buffer())).toEqual({ test: { test: 23.23 } }); + expect(Schema.read(Schema.write({ test: { test: 23.23 } }))).toEqual({ test: { test: 23.23 } }); }); }); @@ -342,30 +342,30 @@ describe('Data integrity - simple', () => { it('should handle string variant (first)', () => { const data = { value: 'hello' }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle integer variant (second)', () => { const data = { value: 42 }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle boolean variant (third)', () => { const data = { value: true }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should use correct discriminator for each variant', () => { // String (first variant) should have discriminator 0x01 - const stringBuffer = Schema.write({ value: 'test' }).buffer(); + const stringBuffer = Schema.write({ value: 'test' }); expect(stringBuffer[2]).toBe(0x01); // discriminator byte // Integer (second variant) should have discriminator 0x02 - const intBuffer = Schema.write({ value: 42 }).buffer(); + const intBuffer = Schema.write({ value: 42 }); expect(intBuffer[2]).toBe(0x02); // discriminator byte // Boolean (third variant) should have discriminator 0x03 - const boolBuffer = Schema.write({ value: true }).buffer(); + const boolBuffer = Schema.write({ value: true }); expect(boolBuffer[2]).toBe(0x03); // discriminator byte }); }); @@ -382,12 +382,12 @@ describe('Data integrity - simple', () => { it('should handle number variant', () => { const data = { data: 3.14 }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle string variant', () => { const data = { data: 'hello' }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -404,30 +404,30 @@ describe('Data integrity - simple', () => { it('should handle null value', () => { const data = { value: null }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle string variant when not null', () => { const data = { value: 'test' }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle integer variant when not null', () => { const data = { value: 123 }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should use 0x00 for null, 0x01+ for variants', () => { // Null should use 0x00 - const nullBuffer = Schema.write({ value: null }).buffer(); + const nullBuffer = Schema.write({ value: null }); expect(nullBuffer[2]).toBe(0x00); // String variant should use 0x01 - const stringBuffer = Schema.write({ value: 'test' }).buffer(); + const stringBuffer = Schema.write({ value: 'test' }); expect(stringBuffer[2]).toBe(0x01); // Integer variant should use 0x02 - const intBuffer = Schema.write({ value: 42 }).buffer(); + const intBuffer = Schema.write({ value: 42 }); expect(intBuffer[2]).toBe(0x02); }); }); @@ -444,12 +444,12 @@ describe('Data integrity - simple', () => { it('should handle array variant', () => { const data = { item: ['a', 'b', 'c'] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle object variant', () => { const data = { item: { x: 10, y: 20 } }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); }); @@ -459,11 +459,11 @@ describe('Data integrity - multi simple', () => { const Schema = schema({ test: { type: 'boolean' }, test2: { type: 'boolean' } }); it('should preserve boolean value and type - false', () => { - expect(Schema.read(Schema.write({ test: false, test2: true }).buffer())).toEqual({ test: false, test2: true }); + expect(Schema.read(Schema.write({ test: false, test2: true }))).toEqual({ test: false, test2: true }); }); it('should skip null or undefined values', () => { - expect(Schema.read(Schema.write({ test: null, test2: false }).buffer())).toEqual({ test2: false }); + expect(Schema.read(Schema.write({ test: null, test2: false }))).toEqual({ test2: false }); }); }); @@ -471,7 +471,7 @@ describe('Data integrity - multi simple', () => { const Schema = schema({ test: { type: 'number', format: 'double' }, test2: { type: 'number', format: 'double' } }); it('should preserve number value and type', () => { - expect(Schema.read(Schema.write({ test: 23.23, test2: -97.7 }).buffer())).toEqual({ test: 23.23, test2: -97.7 }); + expect(Schema.read(Schema.write({ test: 23.23, test2: -97.7 }))).toEqual({ test: 23.23, test2: -97.7 }); }); }); @@ -479,7 +479,7 @@ describe('Data integrity - multi simple', () => { const Schema = schema({ test: { type: 'string' }, test2: { type: 'string' } }); it('should preserve string value and type', () => { - expect(Schema.read(Schema.write({ test: 'hello world', test2: 'écho' }).buffer())).toEqual({ test: 'hello world', test2: 'écho' }); + expect(Schema.read(Schema.write({ test: 'hello world', test2: 'écho' }))).toEqual({ test: 'hello world', test2: 'écho' }); }); }); @@ -487,7 +487,7 @@ describe('Data integrity - multi simple', () => { const Schema = schema({ test: { type: 'array', items: { type: 'string' } }, test2: { type: 'array', items: { type: 'string' } } }); it('should preserve array values and types', () => { - expect(Schema.read(Schema.write({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }).buffer())).toEqual({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }); + expect(Schema.read(Schema.write({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }))).toEqual({ test: ['a', 'b', 'c'], test2: ['d', 'e', 'f'] }); }); }); @@ -495,7 +495,7 @@ describe('Data integrity - multi simple', () => { const Schema = schema({ test: { type: 'object', schema: { test: { type: 'number', format: 'double' } } }, test2: { type: 'object', schema: { test: { type: 'number', format: 'double' } } } }); it('should preserve object values and types', () => { - expect(Schema.read(Schema.write({ test: { test: 23.23 }, test2: { test: -97.7 } }).buffer())).toEqual({ test: { test: 23.23 }, test2: { test: -97.7 } }); + expect(Schema.read(Schema.write({ test: { test: 23.23 }, test2: { test: -97.7 } }))).toEqual({ test: { test: 23.23 }, test2: { test: -97.7 } }); }); }); }); @@ -511,7 +511,7 @@ describe('Data integrity - multi mixed', () => { }); it('should preserve values and types', () => { - expect(Schema.read(Schema.write({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }).buffer())).toEqual({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }); + expect(Schema.read(Schema.write({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }))).toEqual({ bool: true, num: 23.23, str: 'hello world', arr: ['a', 'b', 'c'], obj: { sub: 'way' } }); }); }); }); @@ -524,12 +524,12 @@ describe('Format size differences', () => { const DoubleSchema = schema({ value: { type: 'number', format: 'double' } }); it('float should use 4 bytes for value (7 bytes total with metadata)', () => { - const buffer = FloatSchema.write({ value: 3.14 }).buffer(); + const buffer = FloatSchema.write({ value: 3.14 }); expect(buffer.length).toBe(7); // 1 field count + 1 field index + 1 size + 4 value }); it('double should use 8 bytes for value (11 bytes total with metadata)', () => { - const buffer = DoubleSchema.write({ value: 3.14 }).buffer(); + const buffer = DoubleSchema.write({ value: 3.14 }); expect(buffer.length).toBe(11); // 1 field count + 1 field index + 1 size + 8 value }); }); @@ -539,12 +539,12 @@ describe('Format size differences', () => { const Int64Schema = schema({ value: { type: 'integer', format: 'int64' } }); it('int32 should use 4 bytes for value (7 bytes total with metadata)', () => { - const buffer = Int32Schema.write({ value: 12345 }).buffer(); + const buffer = Int32Schema.write({ value: 12345 }); expect(buffer.length).toBe(7); // 1 field count + 1 field index + 1 size + 4 value }); it('int64 should use 8 bytes for value (11 bytes total with metadata)', () => { - const buffer = Int64Schema.write({ value: 12345 }).buffer(); + const buffer = Int64Schema.write({ value: 12345 }); expect(buffer.length).toBe(11); // 1 field count + 1 field index + 1 size + 8 value }); }); @@ -557,16 +557,16 @@ describe('Nullable properties', () => { const Schema = schema({ test: { type: 'string', nullable: true } }); it('should preserve null value', () => { - expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({ test: null }); + expect(Schema.read(Schema.write({ test: null }))).toEqual({ test: null }); }); it('should preserve non-null string value', () => { - expect(Schema.read(Schema.write({ test: 'hello' }).buffer())).toEqual({ test: 'hello' }); + expect(Schema.read(Schema.write({ test: 'hello' }))).toEqual({ test: 'hello' }); }); it('should encode null with minimal bytes (header only)', () => { - const buffer = Schema.write({ test: null }).buffer(); - const nonNullBuffer = Schema.write({ test: 'a' }).buffer(); + const buffer = Schema.write({ test: null }); + const nonNullBuffer = Schema.write({ test: 'a' }); expect(buffer.length).toBeLessThan(nonNullBuffer.length); }); }); @@ -575,11 +575,11 @@ describe('Nullable properties', () => { const Schema = schema({ test: { type: 'number', format: 'double', nullable: true } }); it('should preserve null value', () => { - expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({ test: null }); + expect(Schema.read(Schema.write({ test: null }))).toEqual({ test: null }); }); it('should preserve non-null number value', () => { - expect(Schema.read(Schema.write({ test: 42.5 }).buffer())).toEqual({ test: 42.5 }); + expect(Schema.read(Schema.write({ test: 42.5 }))).toEqual({ test: 42.5 }); }); }); @@ -587,11 +587,11 @@ describe('Nullable properties', () => { const Schema = schema({ test: { type: 'integer', format: 'int32', nullable: true } }); it('should preserve null value', () => { - expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({ test: null }); + expect(Schema.read(Schema.write({ test: null }))).toEqual({ test: null }); }); it('should preserve non-null integer value', () => { - expect(Schema.read(Schema.write({ test: 123 }).buffer())).toEqual({ test: 123 }); + expect(Schema.read(Schema.write({ test: 123 }))).toEqual({ test: 123 }); }); }); @@ -599,15 +599,15 @@ describe('Nullable properties', () => { const Schema = schema({ test: { type: 'boolean', nullable: true } }); it('should preserve null value', () => { - expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({ test: null }); + expect(Schema.read(Schema.write({ test: null }))).toEqual({ test: null }); }); it('should preserve false value (not confused with null)', () => { - expect(Schema.read(Schema.write({ test: false }).buffer())).toEqual({ test: false }); + expect(Schema.read(Schema.write({ test: false }))).toEqual({ test: false }); }); it('should preserve true value', () => { - expect(Schema.read(Schema.write({ test: true }).buffer())).toEqual({ test: true }); + expect(Schema.read(Schema.write({ test: true }))).toEqual({ test: true }); }); }); @@ -620,19 +620,19 @@ describe('Nullable properties', () => { it('should handle mix of null and non-null values', () => { const data = { nullableField: null, regularField: 'hello', anotherNullable: 42 }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should skip non-nullable fields when null', () => { const data = { nullableField: 'test', regularField: null, anotherNullable: null }; - const result = Schema.read(Schema.write(data).buffer()); + const result = Schema.read(Schema.write(data)); expect(result).toEqual({ nullableField: 'test', anotherNullable: null }); expect(result.regularField).toBeUndefined(); }); it('should preserve all null values in nullable fields', () => { const data = { nullableField: null, regularField: 'value', anotherNullable: null }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -646,11 +646,11 @@ describe('Nullable properties', () => { }); it('should preserve null value', () => { - expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({ test: null }); + expect(Schema.read(Schema.write({ test: null }))).toEqual({ test: null }); }); it('should preserve non-null object value', () => { - expect(Schema.read(Schema.write({ test: { name: 'John' } }).buffer())).toEqual({ test: { name: 'John' } }); + expect(Schema.read(Schema.write({ test: { name: 'John' } }))).toEqual({ test: { name: 'John' } }); }); }); @@ -664,20 +664,20 @@ describe('Nullable properties', () => { }); it('should preserve null value', () => { - expect(Schema.read(Schema.write({ test: null }).buffer())).toEqual({ test: null }); + expect(Schema.read(Schema.write({ test: null }))).toEqual({ test: null }); }); it('should preserve non-null array value', () => { - expect(Schema.read(Schema.write({ test: ['a', 'b', 'c'] }).buffer())).toEqual({ test: ['a', 'b', 'c'] }); + expect(Schema.read(Schema.write({ test: ['a', 'b', 'c'] }))).toEqual({ test: ['a', 'b', 'c'] }); }); it('should preserve empty array (different from null)', () => { - expect(Schema.read(Schema.write({ test: [] }).buffer())).toEqual({ test: [] }); + expect(Schema.read(Schema.write({ test: [] }))).toEqual({ test: [] }); }); it('empty array should have different encoding than null', () => { - const emptyArrayBuffer = Schema.write({ test: [] }).buffer(); - const nullBuffer = Schema.write({ test: null }).buffer(); + const emptyArrayBuffer = Schema.write({ test: [] }); + const nullBuffer = Schema.write({ test: null }); expect(emptyArrayBuffer).not.toEqual(nullBuffer); }); }); @@ -687,8 +687,8 @@ describe('Nullable properties', () => { const Schema = schema({ test: { type: 'string', nullable: true } }); it('should distinguish empty string from null', () => { - const emptyString = Schema.read(Schema.write({ test: '' }).buffer()); - const nullValue = Schema.read(Schema.write({ test: null }).buffer()); + const emptyString = Schema.read(Schema.write({ test: '' })); + const nullValue = Schema.read(Schema.write({ test: null })); expect(emptyString).toEqual({ test: '' }); expect(nullValue).toEqual({ test: null }); @@ -696,8 +696,8 @@ describe('Nullable properties', () => { }); it('should have different byte encodings', () => { - const emptyStringBuffer = Schema.write({ test: '' }).buffer(); - const nullBuffer = Schema.write({ test: null }).buffer(); + const emptyStringBuffer = Schema.write({ test: '' }); + const nullBuffer = Schema.write({ test: null }); expect(emptyStringBuffer).not.toEqual(nullBuffer); }); }); @@ -706,8 +706,8 @@ describe('Nullable properties', () => { const Schema = schema({ test: { type: 'number', format: 'double', nullable: true } }); it('should distinguish zero from null', () => { - const zero = Schema.read(Schema.write({ test: 0 }).buffer()); - const nullValue = Schema.read(Schema.write({ test: null }).buffer()); + const zero = Schema.read(Schema.write({ test: 0 })); + const nullValue = Schema.read(Schema.write({ test: null })); expect(zero).toEqual({ test: 0 }); expect(nullValue).toEqual({ test: null }); @@ -719,8 +719,8 @@ describe('Nullable properties', () => { const Schema = schema({ test: { type: 'boolean', nullable: true } }); it('should distinguish false from null', () => { - const falseValue = Schema.read(Schema.write({ test: false }).buffer()); - const nullValue = Schema.read(Schema.write({ test: null }).buffer()); + const falseValue = Schema.read(Schema.write({ test: false })); + const nullValue = Schema.read(Schema.write({ test: null })); expect(falseValue).toEqual({ test: false }); expect(nullValue).toEqual({ test: null }); @@ -737,11 +737,11 @@ describe('OpenAPI-compatible array formats', () => { const Schema = schema({ test: { type: 'array', items: { type: 'integer', format: 'int32' } } }); it('should preserve array of integers', () => { - expect(Schema.read(Schema.write({ test: [1, 2, 3, 4, 5] }).buffer())).toEqual({ test: [1, 2, 3, 4, 5] }); + expect(Schema.read(Schema.write({ test: [1, 2, 3, 4, 5] }))).toEqual({ test: [1, 2, 3, 4, 5] }); }); it('should handle negative integers', () => { - expect(Schema.read(Schema.write({ test: [-100, 0, 100] }).buffer())).toEqual({ test: [-100, 0, 100] }); + expect(Schema.read(Schema.write({ test: [-100, 0, 100] }))).toEqual({ test: [-100, 0, 100] }); }); }); @@ -750,7 +750,7 @@ describe('OpenAPI-compatible array formats', () => { it('should preserve array of int64 values', () => { const data = { test: [9007199254740991, -9007199254740991, 0] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -758,7 +758,7 @@ describe('OpenAPI-compatible array formats', () => { const Schema = schema({ test: { type: 'array', items: { type: 'number', format: 'float' } } }); it('should preserve array of floats with precision loss', () => { - const result = Schema.read(Schema.write({ test: [3.14, 2.71, 1.41] }).buffer()); + const result = Schema.read(Schema.write({ test: [3.14, 2.71, 1.41] })); expect(result.test[0]).toBeCloseTo(3.14, 5); expect(result.test[1]).toBeCloseTo(2.71, 5); expect(result.test[2]).toBeCloseTo(1.41, 5); @@ -769,7 +769,7 @@ describe('OpenAPI-compatible array formats', () => { const Schema = schema({ test: { type: 'array', items: { type: 'number', format: 'double' } } }); it('should preserve array of doubles', () => { - expect(Schema.read(Schema.write({ test: [3.141592653589793, 2.718281828459045] }).buffer())).toEqual({ test: [3.141592653589793, 2.718281828459045] }); + expect(Schema.read(Schema.write({ test: [3.141592653589793, 2.718281828459045] }))).toEqual({ test: [3.141592653589793, 2.718281828459045] }); }); }); @@ -784,14 +784,14 @@ describe('OpenAPI-compatible array formats', () => { '00000000-0000-0000-0000-000000000000', ], }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should compress UUIDs efficiently', () => { const data = { test: ['550e8400-e29b-4d4e-a7d4-426614174000', '6ba7b810-9dad-11d1-80b4-00c04fd430c8'], }; - const buffer = Schema.write(data).buffer(); + const buffer = Schema.write(data); // Each UUID is 16 bytes + 1 byte size = 17 bytes per UUID // Plus array overhead expect(buffer.length).toBeLessThan(100); // Much less than string encoding @@ -803,7 +803,7 @@ describe('OpenAPI-compatible array formats', () => { it('should preserve array of IPv4 addresses', () => { const data = { test: ['192.168.1.1', '10.0.0.1', '172.16.0.1'] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -814,7 +814,7 @@ describe('OpenAPI-compatible array formats', () => { const data = { test: ['2001:db8:85a3::8a2e:370:7334', '::1', '::'], }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -823,7 +823,7 @@ describe('OpenAPI-compatible array formats', () => { it('should preserve array of dates', () => { const data = { test: ['2025-10-28', '2024-01-01', '1970-01-01'] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -834,7 +834,7 @@ describe('OpenAPI-compatible array formats', () => { const data = { test: ['2025-10-28T14:30:00.000Z', '2024-01-01T00:00:00.000Z'], }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -843,12 +843,12 @@ describe('OpenAPI-compatible array formats', () => { it('should preserve array of binary data', () => { const data = { test: ['SGVsbG8=', 'V29ybGQ=', 'Zm9vYmFy'] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle Buffer inputs', () => { const input = { test: [Buffer.from('Hello'), Buffer.from('World')] }; - const result = Schema.read(Schema.write(input).buffer()); + const result = Schema.read(Schema.write(input)); expect(result.test).toEqual(['SGVsbG8=', 'V29ybGQ=']); }); }); @@ -858,17 +858,17 @@ describe('OpenAPI-compatible array formats', () => { it('should preserve null values in array', () => { const data = { test: ['a', null, 'b', null, 'c'] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should distinguish empty string from null', () => { const data = { test: ['', null, 'text'] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle all null array', () => { const data = { test: [null, null, null] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -877,12 +877,12 @@ describe('OpenAPI-compatible array formats', () => { it('should preserve null values with integers', () => { const data = { test: [1, null, 2, null, 3] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should distinguish zero from null', () => { const data = { test: [0, null, -1] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -902,22 +902,22 @@ describe('OpenAPI-compatible array formats', () => { it('should handle mixed types in array', () => { const data = { test: ['hello', 42, true, 'world', false, 123] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle all strings', () => { const data = { test: ['a', 'b', 'c'] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle all integers', () => { const data = { test: [1, 2, 3] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle all booleans', () => { const data = { test: [true, false, true] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -936,7 +936,7 @@ describe('OpenAPI-compatible array formats', () => { it('should handle mixed numbers and strings', () => { const data = { test: [3.14, 'pi', 2.71, 'e'] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -956,7 +956,7 @@ describe('OpenAPI-compatible array formats', () => { it('should handle null with oneOf variants', () => { const data = { test: ['hello', null, 42, null, 'world'] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -973,12 +973,12 @@ describe('OpenAPI-compatible array formats', () => { it('should handle 2D arrays', () => { const data = { test: [[1, 2, 3], [4, 5, 6], [7, 8, 9]] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle empty nested arrays', () => { const data = { test: [[], [1], []] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -995,7 +995,7 @@ describe('OpenAPI-compatible array formats', () => { it('should handle 2D string arrays', () => { const data = { test: [['a', 'b'], ['c', 'd', 'e'], ['f']] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1015,7 +1015,7 @@ describe('OpenAPI-compatible array formats', () => { it('should handle array of objects', () => { const data = { test: [{ x: 1, y: 2 }, { x: 3, y: 4 }, { x: 5, y: 6 }] }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1043,7 +1043,7 @@ describe('OpenAPI-compatible array formats', () => { [4, 5], ], }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); }); @@ -1079,7 +1079,7 @@ describe('OpenAPI-compatible object formats', () => { ip: '192.168.1.1', }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1103,7 +1103,7 @@ describe('OpenAPI-compatible object formats', () => { number: null, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should preserve non-null values', () => { @@ -1114,7 +1114,7 @@ describe('OpenAPI-compatible object formats', () => { number: 42, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle mix of null and non-null', () => { @@ -1125,7 +1125,7 @@ describe('OpenAPI-compatible object formats', () => { number: null, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1153,7 +1153,7 @@ describe('OpenAPI-compatible object formats', () => { data: 'success', }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle integer variant', () => { @@ -1163,7 +1163,7 @@ describe('OpenAPI-compatible object formats', () => { data: 42, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle object variant', () => { @@ -1173,7 +1173,7 @@ describe('OpenAPI-compatible object formats', () => { data: { message: 'Operation completed' }, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1200,7 +1200,7 @@ describe('OpenAPI-compatible object formats', () => { value: 3.14159, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle string variant', () => { @@ -1210,7 +1210,7 @@ describe('OpenAPI-compatible object formats', () => { value: 'text value', }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1251,7 +1251,7 @@ describe('OpenAPI-compatible object formats', () => { }, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1287,7 +1287,7 @@ describe('OpenAPI-compatible object formats', () => { ], }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1315,7 +1315,7 @@ describe('OpenAPI-compatible object formats', () => { value: null, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle string variant', () => { @@ -1325,7 +1325,7 @@ describe('OpenAPI-compatible object formats', () => { value: 'text', }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle integer variant', () => { @@ -1335,7 +1335,7 @@ describe('OpenAPI-compatible object formats', () => { value: 42, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1381,7 +1381,7 @@ describe('OpenAPI-compatible object formats', () => { created: '2025-10-28T14:30:00.000Z', }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle with string metadata variant', () => { @@ -1399,7 +1399,7 @@ describe('OpenAPI-compatible object formats', () => { created: '2025-10-28T15:00:00.000Z', }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1425,7 +1425,7 @@ describe('OpenAPI-compatible object formats', () => { fourth: 3.14, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should handle properties in different order than schema', () => { @@ -1445,7 +1445,7 @@ describe('OpenAPI-compatible object formats', () => { fourth: 3.14, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(expected); + expect(Schema.read(Schema.write(data))).toEqual(expected); }); it('should handle properties in reverse order', () => { @@ -1465,7 +1465,7 @@ describe('OpenAPI-compatible object formats', () => { fourth: 2.71, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(expected); + expect(Schema.read(Schema.write(data))).toEqual(expected); }); it('should handle properties in random order', () => { @@ -1485,7 +1485,7 @@ describe('OpenAPI-compatible object formats', () => { fourth: 1.41, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(expected); + expect(Schema.read(Schema.write(data))).toEqual(expected); }); }); @@ -1508,7 +1508,7 @@ describe('OpenAPI-compatible object formats', () => { undeclared: 'should be ignored', }, }; - const result = Schema.read(Schema.write(input).buffer()); + const result = Schema.read(Schema.write(input)); expect(result).toEqual({ data: { declared: 'value' } }); expect(result.data).not.toHaveProperty('undeclared'); @@ -1579,7 +1579,7 @@ describe('OpenAPI native format support', () => { age: 30, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1612,7 +1612,7 @@ describe('OpenAPI native format support', () => { email: 'john@example.com', }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1665,7 +1665,7 @@ describe('OpenAPI native format support', () => { ], }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1712,7 +1712,7 @@ describe('OpenAPI native format support', () => { }, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); it('should resolve $ref in oneOf (bank account)', () => { @@ -1725,7 +1725,7 @@ describe('OpenAPI native format support', () => { }, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1812,7 +1812,7 @@ describe('OpenAPI native format support', () => { }, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1868,7 +1868,7 @@ describe('OpenAPI native format support', () => { total: 99.99, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1901,7 +1901,7 @@ describe('OpenAPI native format support', () => { { id: 3, name: 'Charlie' }, ], }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1925,7 +1925,7 @@ describe('OpenAPI native format support', () => { age: 30, active: true, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -1961,7 +1961,7 @@ describe('OpenAPI native format support', () => { notifications: true, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); @@ -2005,7 +2005,7 @@ describe('OpenAPI native format support', () => { price: 29.99, }, }; - expect(Schema.read(Schema.write(data).buffer())).toEqual(data); + expect(Schema.read(Schema.write(data))).toEqual(data); }); }); }); diff --git a/tests/integration/openapi.ts b/tests/integration/openapi.ts index e10f340..a30f2aa 100644 --- a/tests/integration/openapi.ts +++ b/tests/integration/openapi.ts @@ -12,7 +12,7 @@ describe('OpenAPI spec test', () => { }; it('should return the response object unchanged', () => { - expect(Schema.read(Schema.write(response).buffer())).toEqual(response); + expect(Schema.read(Schema.write(response))).toEqual(response); }); }); @@ -34,7 +34,7 @@ describe('OpenAPI spec test', () => { }; it('should return the response object unchanged', () => { - expect(Schema.read(Schema.write(response).buffer())).toEqual(response); + expect(Schema.read(Schema.write(response))).toEqual(response); }); }); }); diff --git a/types.d.ts b/types.d.ts index 1968810..62b3526 100644 --- a/types.d.ts +++ b/types.d.ts @@ -7,18 +7,12 @@ declare module 'compactr' { export interface SchemaInstance { /** - * Start writing some data against a schema + * Encodes data according to the schema * @param data The data to be encoded * @param options The options for the encoding - * @returns Self reference + * @returns The encoded buffer */ - write(data: any, options?: WriteOptions): this - - /** - * Returns the bytes from the encoded data buffer. - * @returns The data buffer - */ - buffer(): Buffer + write(data: any, options?: WriteOptions): Buffer /** * Returns the byte sizes of a data object, for insight or troubleshooting From 38eb1dfb16c12ba9f5b1305cb348c7d20202e436 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Mon, 3 Nov 2025 02:12:58 -0500 Subject: [PATCH 18/19] optimized datetime, increases bytesize --- README.md | 12 +++++------ src/converter.ts | 11 +++------- src/decoder.ts | 23 +++++++++++---------- src/encoder.ts | 42 +++++++++++++++++++++++++++++--------- src/schema.ts | 2 +- tests/integration/index.ts | 6 +++--- 6 files changed, 57 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 55dc48a..9d910ce 100644 --- a/README.md +++ b/README.md @@ -111,15 +111,15 @@ Compactr supports the following OpenAPI types and formats: ## Performance -I realistic scenarios, compactr performs a bit slower than JSON.stringify/ JSON.parse as well as other schema-based protocols such as `protobuf`, but can yield a byte reduction of 3.5x compared to JSON. +I realistic scenarios, compactr performs a bit slower than JSON.stringify/ JSON.parse as well as `protobuf`, but can yield a byte reduction of 3.5x compared to JSON. ``` -[JSON-API Reponse] JSON x 289 ops/sec ±1.10% (83 runs sampled) -[JSON-API Reponse] Compactr x 115 ops/sec ±0.93% (74 runs sampled) -[JSON-API Reponse] Protobuf x 272 ops/sec ±1.06% (82 runs sampled) -[JSON-API Reponse] MsgPack x 133 ops/sec ±1.38% (76 runs sampled) +[JSON-API Reponse] JSON x 379 ops/sec ±0.66% (92 runs sampled) +[JSON-API Reponse] Compactr x 167 ops/sec ±0.75% (85 runs sampled) +[JSON-API Reponse] Protobuf x 358 ops/sec ±1.36% (91 runs sampled) +[JSON-API Reponse] MsgPack x 161 ops/sec ±2.06% (79 runs sampled) -Buffer size (bytes): { json: 277, compactr: 78, protobuf: 129, msgpack: 227 } +Buffer size (bytes): { json: 277, compactr: 80, protobuf: 129, msgpack: 227 } ``` ## Testing diff --git a/src/converter.ts b/src/converter.ts index fb450c4..b89b8de 100644 --- a/src/converter.ts +++ b/src/converter.ts @@ -64,20 +64,15 @@ function date(value) { if (!/^\d{4}-\d{2}-\d{2}$/.test(str)) { throw new Error('Invalid date format, expected YYYY-MM-DD'); } - const parsed = new Date(str + 'T00:00:00Z'); - if (isNaN(parsed.getTime())) { - throw new Error('Invalid date format'); - } return str; } function dateTime(value) { const str = '' + value; - const parsed = new Date(str); - if (isNaN(parsed.getTime())) { - throw new Error('Invalid date-time format'); + if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z?$/.test(str)) { + throw new Error('Invalid date-time format, expected YYYY-MM-DDTHH:mm:ss.sssZ'); } - return parsed.toISOString(); + return str; } function binary(value) { diff --git a/src/decoder.ts b/src/decoder.ts index 776383a..9d6da12 100644 --- a/src/decoder.ts +++ b/src/decoder.ts @@ -165,27 +165,28 @@ function date(bytes, offset = 0, length?) { throw new Error('Invalid date byte length'); } - const days = int32(bytes, offset); - const epochMs = days * 86400000; - const dateObj = new Date(epochMs); - - const year = dateObj.getUTCFullYear(); - const month = String(dateObj.getUTCMonth() + 1).padStart(2, '0'); - const day = String(dateObj.getUTCDate()).padStart(2, '0'); + const year = (bytes[offset] << 8) | bytes[offset + 1]; + const month = String(bytes[offset + 2]).padStart(2, '0'); + const day = String(bytes[offset + 3]).padStart(2, '0'); return `${year}-${month}-${day}`; } function dateTime(bytes, offset = 0, length?) { const len = length !== undefined ? length : bytes.length - offset; - if (len !== 8) { + if (len !== 9) { throw new Error('Invalid date-time byte length'); } - const ms = double(bytes, offset); - const dateObj = new Date(ms); + const year = (bytes[offset] << 8) | bytes[offset + 1]; + const month = String(bytes[offset + 2]).padStart(2, '0'); + const day = String(bytes[offset + 3]).padStart(2, '0'); + const hour = String(bytes[offset + 4]).padStart(2, '0'); + const minute = String(bytes[offset + 5]).padStart(2, '0'); + const second = String(bytes[offset + 6]).padStart(2, '0'); + const millisecond = ((bytes[offset + 7] << 8) | bytes[offset + 8]).toString().padStart(3, '0'); - return dateObj.toISOString(); + return `${year}-${month}-${day}T${hour}:${minute}:${second}.${millisecond}Z`; } function binary(bytes, offset = 0, length?) { diff --git a/src/encoder.ts b/src/encoder.ts index b8a0fb1..1c28f92 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -165,24 +165,46 @@ function ipv6(val, buffer, pos) { } function date(val, buffer, pos) { - const parsed = new Date(val + 'T00:00:00Z'); - if (isNaN(parsed.getTime())) { - throw new Error('Invalid date format'); + const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(val); + if (!match) { + throw new Error('Invalid date format, expected YYYY-MM-DD'); } - const epochMs = parsed.getTime(); - const days = Math.floor(epochMs / 86400000); + const year = parseInt(match[1], 10); + const month = parseInt(match[2], 10); + const day = parseInt(match[3], 10); - return int32(days, buffer, pos); + buffer[pos] = year >> 8; + buffer[pos + 1] = year & 0xff; + buffer[pos + 2] = month; + buffer[pos + 3] = day; + return pos + 4; } function dateTime(val, buffer, pos) { - const parsed = new Date(val); - if (isNaN(parsed.getTime())) { - throw new Error('Invalid date-time format'); + const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{3}))?Z?$/.exec(val); + if (!match) { + throw new Error('Invalid date-time format, expected YYYY-MM-DDTHH:mm:ss.sssZ'); } - return double(parsed.getTime(), buffer, pos); + const year = parseInt(match[1], 10); + const month = parseInt(match[2], 10); + const day = parseInt(match[3], 10); + const hour = parseInt(match[4], 10); + const minute = parseInt(match[5], 10); + const second = parseInt(match[6], 10); + const millisecond = match[7] ? parseInt(match[7], 10) : 0; + + buffer[pos] = year >> 8; + buffer[pos + 1] = year & 0xff; + buffer[pos + 2] = month; + buffer[pos + 3] = day; + buffer[pos + 4] = hour; + buffer[pos + 5] = minute; + buffer[pos + 6] = second; + buffer[pos + 7] = millisecond >> 8; + buffer[pos + 8] = millisecond & 0xff; + return pos + 9; } function binary(val, buffer, pos) { diff --git a/src/schema.ts b/src/schema.ts index 6300ed5..450db6b 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -45,7 +45,7 @@ export default function Schema(schema, options = {}) { 'ipv4': 4, 'ipv6': 16, 'date': 4, - 'date-time': 8, + 'date-time': 9, }; const scope = { diff --git a/tests/integration/index.ts b/tests/integration/index.ts index 1c20b0c..3426ca7 100644 --- a/tests/integration/index.ts +++ b/tests/integration/index.ts @@ -245,9 +245,9 @@ describe('Data integrity - simple', () => { const datetime = '2025-10-28T14:30:00.000Z'; const buffer = Schema.write({ test: datetime }); // Header: 1 byte (field count) + 1 byte (field index) + 1 byte (size) = 3 bytes - // Content: 8 bytes (milliseconds since epoch) - // Total: 11 bytes (vs 43+ bytes for string encoding) - expect(buffer.length).toBe(11); + // Content: 9 bytes (milliseconds since epoch) + // Total: 12 bytes (vs 43+ bytes for string encoding) + expect(buffer.length).toBe(12); }); it('should handle epoch datetime', () => { From 685c8d5ffc1776c4aad3affe7189f14c930fb743 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Mon, 3 Nov 2025 16:50:11 -0500 Subject: [PATCH 19/19] added pr validation pipeline --- .github/workflows/pr-validation.yml | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/pr-validation.yml diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml new file mode 100644 index 0000000..381c93d --- /dev/null +++ b/.github/workflows/pr-validation.yml @@ -0,0 +1,31 @@ +name: pr-validation + +on: + pull_request: + branches: + - '*' + +jobs: + build: + + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [22.x, 24.x] + + steps: + - uses: actions/checkout@v1 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: npm install, build, and test + run: | + npm i + npm run build + npm run lint + npm run test + npm run bench + env: + CI: true \ No newline at end of file