diff --git a/CHANGELOG.md b/CHANGELOG.md index 017b2ac..a81c1e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # ⏩ vNext +- Fix ByteOrder in number serializer - aka support for lsb # v4.2.2 - Fix an issue where the new 53-bit oversize error is emitted when the `float` format is selected, even though it is diff --git a/src/bitstream/reader.test.ts b/src/bitstream/reader.test.ts index 46b8ff9..b6e5878 100644 --- a/src/bitstream/reader.test.ts +++ b/src/bitstream/reader.test.ts @@ -199,7 +199,9 @@ describe('BitstreamReader', it => { 0b11101001, 0b01100100, - 0b10001110 + 0b10001110, + 0b11110000, + 0b10010000 ])); expect(bitstream.readSync(1)).to.equal(0b1); @@ -214,12 +216,15 @@ describe('BitstreamReader', it => { expect(bitstream.readSync(11)).to.equal(0b01100100100); expect(bitstream.readSync(5)).to.equal(0b01110); + + expect(bitstream.readSync(16, 'little-endian')).to.equal( 0b1001000011110000 ); }); it('can correctly deserialize a simple example from multiple buffers', () => { let bitstream = new BitstreamReader(); bitstream.addBuffer(Buffer.from([ 0b11001000, 0b01010100 ])); bitstream.addBuffer(Buffer.from([ 0b11101001, 0b01100100, 0b10001110 ])); + bitstream.addBuffer(Buffer.from([ 0b11110000, 0b10010000 ])); expect(bitstream.readSync(1)).to.equal(0b1); expect(bitstream.readSync(3)).to.equal(0b100); @@ -233,6 +238,8 @@ describe('BitstreamReader', it => { expect(bitstream.readSync(11)).to.equal(0b01100100100); expect(bitstream.readSync(5)).to.equal(0b01110); + + expect(bitstream.readSync(16, 'little-endian')).to.equal(0b1001000011110000); }); it('can read fixed length UTF-8 strings', () => { @@ -336,6 +343,52 @@ describe('BitstreamReader', it => { expect(bitstream.readSync(4)).to.equal(0b0100); }); + it('.readSync() correctly handles unsigned integers', () => { + let bitstream = new BitstreamReader(); + + bitstream.addBuffer(Buffer.from([0xFB])); expect(bitstream.readSync(8)).to.equal(0xFB); + bitstream.addBuffer(Buffer.from([5])); expect(bitstream.readSync(8)).to.equal(5); + bitstream.addBuffer(Buffer.from([0])); expect(bitstream.readSync(8)).to.equal(0); + + bitstream.addBuffer(Buffer.from([0xFC, 0x0A])); expect(bitstream.readSync(16)).to.equal(0xFC0A); + bitstream.addBuffer(Buffer.from([0x03, 0xF6])); expect(bitstream.readSync(16)).to.equal(0x03F6); + bitstream.addBuffer(Buffer.from([0, 0])); expect(bitstream.readSync(16)).to.equal(0); + + bitstream.addBuffer(Buffer.from([0x0A, 0xFC])); expect(bitstream.readSync(16, 'little-endian')).to.equal(0xFC0A); + bitstream.addBuffer(Buffer.from([0xF6, 0x03])); expect(bitstream.readSync(16, 'little-endian')).to.equal(0x03F6); + bitstream.addBuffer(Buffer.from([0, 0])); expect(bitstream.readSync(16, 'little-endian')).to.equal(0); + + bitstream.addBuffer(Buffer.from([0xFF, 0xFE, 0x70, 0x40])); expect(bitstream.readSync(32)).to.equal(0xFFFE7040); + bitstream.addBuffer(Buffer.from([0x00, 0x01, 0x8F, 0xC0])); expect(bitstream.readSync(32)).to.equal(0x00018FC0); + bitstream.addBuffer(Buffer.from([0, 0, 0, 0])); expect(bitstream.readSync(32)).to.equal(0); + + bitstream.addBuffer(Buffer.from([0x40, 0x70, 0xFE, 0xFF])); expect(bitstream.readSync(32, 'little-endian')).to.equal(0xFFFE7040); + bitstream.addBuffer(Buffer.from([0xC0, 0x8F, 0x01, 0x00])); expect(bitstream.readSync(32, 'little-endian')).to.equal(0x00018FC0); + bitstream.addBuffer(Buffer.from([0, 0, 0, 0])); expect(bitstream.readSync(32)).to.equal(0); + }); + it('.read() correctly handles unsigned integers', async () => { + let bitstream = new BitstreamReader(); + bitstream.addBuffer(Buffer.from([0xFB])); expect(await bitstream.read(8)).to.equal(0xFB); + bitstream.addBuffer(Buffer.from([5])); expect(await bitstream.read(8)).to.equal(5); + bitstream.addBuffer(Buffer.from([0])); expect(await bitstream.read(8)).to.equal(0); + + bitstream.addBuffer(Buffer.from([0x0A, 0xFC])); expect(await bitstream.read(16)).to.equal(0x0AFC); + bitstream.addBuffer(Buffer.from([0xF6, 0x03])); expect(await bitstream.read(16)).to.equal(0xF603); + bitstream.addBuffer(Buffer.from([0, 0])); expect(await bitstream.read(16)).to.equal(0); + + bitstream.addBuffer(Buffer.from([0xFC, 0x0A])); expect(await bitstream.read(16, 'little-endian')).to.equal(0x0AFC); + bitstream.addBuffer(Buffer.from([0x03, 0xF6])); expect(await bitstream.read(16, 'little-endian')).to.equal(0xF603); + bitstream.addBuffer(Buffer.from([0, 0])); expect(await bitstream.read(16, 'little-endian')).to.equal(0); + + bitstream.addBuffer(Buffer.from([0xFF, 0xFE, 0x70, 0x40])); expect(await bitstream.read(32)).to.equal(0xFFFE7040); + bitstream.addBuffer(Buffer.from([0x00, 0x01, 0x8F, 0xC0])); expect(await bitstream.read(32)).to.equal(0x00018FC0); + bitstream.addBuffer(Buffer.from([0, 0, 0, 0])); expect(await bitstream.read(32)).to.equal(0); + + bitstream.addBuffer(Buffer.from([0x40, 0x70, 0xFE, 0xFF])); expect(await bitstream.read(32, 'little-endian')).to.equal(0xFFFE7040); + bitstream.addBuffer(Buffer.from([0xC0, 0x8F, 0x01, 0x00])); expect(await bitstream.read(32, 'little-endian')).to.equal(0x00018FC0); + bitstream.addBuffer(Buffer.from([0, 0, 0, 0])); expect(await bitstream.read(32)).to.equal(0); + }); + it('.readSignedSync() correctly handles signed integers', () => { let bitstream = new BitstreamReader(); @@ -347,9 +400,17 @@ describe('BitstreamReader', it => { bitstream.addBuffer(Buffer.from([ 0x03, 0xF6 ])); expect(bitstream.readSignedSync(16)).to.equal(1014); bitstream.addBuffer(Buffer.from([ 0, 0 ])); expect(bitstream.readSignedSync(16)).to.equal(0); + bitstream.addBuffer(Buffer.from([0x0A, 0xFC])); expect(bitstream.readSignedSync(16, 'little-endian')).to.equal(-1014); + bitstream.addBuffer(Buffer.from([0xF6, 0x03])); expect(bitstream.readSignedSync(16, 'little-endian')).to.equal(1014); + bitstream.addBuffer(Buffer.from([0, 0])); expect(bitstream.readSignedSync(16, 'big-endian')).to.equal(0); + bitstream.addBuffer(Buffer.from([ 0xFF, 0xFE, 0x70, 0x40 ])); expect(bitstream.readSignedSync(32)).to.equal(-102336); bitstream.addBuffer(Buffer.from([ 0x00, 0x01, 0x8F, 0xC0 ])); expect(bitstream.readSignedSync(32)).to.equal(102336); bitstream.addBuffer(Buffer.from([ 0, 0, 0, 0 ])); expect(bitstream.readSignedSync(32)).to.equal(0); + + bitstream.addBuffer(Buffer.from([0x40, 0x70, 0xFE, 0xFF])); expect(bitstream.readSignedSync(32, 'little-endian')).to.equal(-102336); + bitstream.addBuffer(Buffer.from([0xC0, 0x8F, 0x01, 0x00])); expect(bitstream.readSignedSync(32, 'little-endian')).to.equal(102336); + bitstream.addBuffer(Buffer.from([0, 0, 0, 0])); expect(bitstream.readSignedSync(32, 'little-endian')).to.equal(0); }); it('.readSigned() correctly handles signed integers', async () => { let bitstream = new BitstreamReader(); @@ -361,9 +422,17 @@ describe('BitstreamReader', it => { bitstream.addBuffer(Buffer.from([ 0x03, 0xF6 ])); expect(await bitstream.readSigned(16)).to.equal(1014); bitstream.addBuffer(Buffer.from([ 0, 0 ])); expect(await bitstream.readSigned(16)).to.equal(0); + bitstream.addBuffer(Buffer.from([ 0x0A, 0xFC ])); expect(await bitstream.readSigned(16, 'little-endian')).to.equal(-1014); + bitstream.addBuffer(Buffer.from([ 0xF6, 0x03 ])); expect(await bitstream.readSigned(16, 'little-endian')).to.equal(1014); + bitstream.addBuffer(Buffer.from([ 0, 0 ])); expect(await bitstream.readSigned(16, 'little-endian')).to.equal(0); + bitstream.addBuffer(Buffer.from([ 0xFF, 0xFE, 0x70, 0x40 ])); expect(await bitstream.readSigned(32)).to.equal(-102336); bitstream.addBuffer(Buffer.from([ 0x00, 0x01, 0x8F, 0xC0 ])); expect(await bitstream.readSigned(32)).to.equal(102336); bitstream.addBuffer(Buffer.from([ 0, 0, 0, 0 ])); expect(await bitstream.readSigned(32)).to.equal(0); + + bitstream.addBuffer(Buffer.from([0x40, 0x70, 0xFE, 0xFF])); expect(await bitstream.readSigned(32, 'little-endian')).to.equal(-102336); + bitstream.addBuffer(Buffer.from([0xC0, 0x8F, 0x01, 0x00])); expect(await bitstream.readSigned(32, 'little-endian')).to.equal(102336); + bitstream.addBuffer(Buffer.from([0, 0, 0, 0])); expect(await bitstream.readSigned(32, 'little-endian')).to.equal(0); }); it('.readSigned() can wait until data is available', async () => { let bitstream = new BitstreamReader(); @@ -385,6 +454,10 @@ describe('BitstreamReader', it => { bitstream.addBuffer(Buffer.from([ 0, 0, 0, 0, 0, 0, 0, 0 ])); expect(bitstream.readFloatSync(64)).to.equal(0); + + bitstream.addBuffer(Buffer.from([ 0xe1, 0x7a, 0x14, 0xae, 0xa4, 0x00, 0x14, 0xc1 ])); + expect(bitstream.readFloatSync(64, 'little-endian')).to.equal(-327721.17); + }); it('.readFloat() correctly handles floats', async () => { @@ -401,6 +474,9 @@ describe('BitstreamReader', it => { bitstream.addBuffer(Buffer.from([ 0, 0, 0, 0, 0, 0, 0, 0 ])); expect(await bitstream.readFloat(64)).to.equal(0); + + bitstream.addBuffer(Buffer.from([0xe1, 0x7a, 0x14, 0xae, 0xa4, 0x00, 0x14, 0xc1])); + expect(await bitstream.readFloat(64, 'little-endian')).to.equal(-327721.17); }); it('.readFloat() can wait until data is available', async () => { @@ -498,6 +574,9 @@ describe('BitstreamReader', it => { bitstream.addBuffer(Buffer.from([ 0x7f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 ])); expect(bitstream.readFloatSync(64)).not.to.be.finite; + + bitstream.addBuffer(Buffer.from([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x7f])); + expect(bitstream.readFloatSync(64, 'little-endian')).not.to.be.finite; }); it('.read() allows only one async read at a time', async () => { let bitstream = new BitstreamReader(); @@ -770,7 +849,7 @@ describe('BitstreamReader', it => { expect(bitstream.bufferIndex).to.equal(2); expect(bitstream.offset).to.equal(16); expect(bitstream.available).to.equal(8); - + bitstream.reset(); expect(bitstream.bufferIndex).to.equal(0); @@ -795,6 +874,18 @@ describe('BitstreamReader (generated)', it => { expect(reader.readSync(size), `Test number #${i} (${num}) should have been read properly`).to.equal(num); } }); + if (size > 8 && (size % 8 == 0)) it(`reads ${size}bit values correctly (LE)`, async () => { + let buf = new ArrayBuffer(8); + let view = new DataView(buf); + + for (let i = 0; i < 10; ++i) { + let num = Math.floor(Math.random() * 2 ** size); + view.setBigUint64(0, BigInt(num), true); + let reader = new BitstreamReader(); + reader.addBuffer(new Uint8Array(buf)); + expect(reader.readSync(size, 'little-endian'), `Test number #${i} (${num}) should have been read properly`).to.equal(num); + } + }); } for (let size = 1; size < 52; ++size) { @@ -812,6 +903,25 @@ describe('BitstreamReader (generated)', it => { expect(reader.readSync(size), `Test number #${i} (${num}) should have been read properly`).to.equal(num); } }); + if (size > 8 && (size % 8 == 0)) it(`reads cross-byte ${size}bit values correctly [1-bit offset] LE`, async () => { + let buf = new ArrayBuffer(8); + let view = new DataView(buf); + + for (let i = 0; i < 10; ++i) { + let num = Math.floor(Math.random() * 2 ** size); + view.setBigUint64(0, BigInt(num), true); + let reader = new BitstreamReader(); + let shifted = new Uint8Array(buf).reduce((prev, cur, index: number, arr) => { + prev[index] = (cur >> 1) | ((index == 0) ? 0x0 : prev[index]); + prev[index + 1] = (cur << 7) & 0b10000000; + return prev; + }, new Uint8Array(buf.byteLength + 1)); + reader.addBuffer(shifted); + reader.readSync(1); + const target = reader.readSync(size, 'little-endian'); + expect(target, `Test number #${i} (${num}) should have been read properly`).to.equal(num); + } + }); } for (let size = 1; size < 49; ++size) { @@ -829,5 +939,23 @@ describe('BitstreamReader (generated)', it => { expect(reader.readSync(size), `Test number #${i} (${num}) should have been read properly`).to.equal(num); } }); + if (size > 8 && (size % 8 == 0)) it(`reads cross-byte ${size}bit values correctly [4-bit offset] LE`, async () => { + let buf = new ArrayBuffer(8); + let view = new DataView(buf); + + for (let i = 0; i < 10; ++i) { + let num = Math.floor(Math.random() * 2 ** size); + view.setBigUint64(0, BigInt(num), true); + let reader = new BitstreamReader(); + let shifted = new Uint8Array(buf).reduce((prev, cur, index, arr) => { + prev[index] = cur >> 4 | prev[index]; + prev[index + 1] = cur << 4; + return prev; + }, new Uint8Array(buf.byteLength + 1)); + reader.addBuffer(shifted); + reader.readSync(4); + expect(reader.readSync(size, 'little-endian'), `Test number #${i} (${num}) should have been read properly`).to.equal(num); + } + }); } }); \ No newline at end of file diff --git a/src/bitstream/reader.ts b/src/bitstream/reader.ts index 1f1df55..cbbf0c3 100644 --- a/src/bitstream/reader.ts +++ b/src/bitstream/reader.ts @@ -260,10 +260,13 @@ export class BitstreamReader { * bits available, an error is thrown. * * @param length The number of bits to read + * @param byteOrder The byte order to use when the length is greater than 8 and is a multiple of 8. + * Defaults to MSB (most significant byte). If the length is not a multiple of 8, + * this is unused * @returns The unsigned integer that was read */ - readSync(length : number): number { - return this.readCoreSync(length, true); + readSync(length: number, byteOrder?: "big-endian" | "little-endian"): number { + return this.readCoreSync(length, true, byteOrder); } /** @@ -376,10 +379,13 @@ export class BitstreamReader { * enough bits available, an error is thrown. * * @param length The number of bits to read + * @param byteOrder The byte order to use when the length is greater than 8 and is a multiple of 8. + * Defaults to MSB (most significant byte). If the length is not a multiple of 8, + * this is unused * @returns The signed integer that was read */ - readSignedSync(length : number): number { - const u = this.readSync(length); + readSignedSync(length: number, byteOrder: "big-endian" | "little-endian" = 'big-endian'): number { + const u = this.readSync(length, byteOrder); const signBit = (2**(length - 1)); const mask = signBit - 1; return (u & signBit) === 0 ? u : -((~(u - 1) & mask) >>> 0); @@ -404,9 +410,12 @@ export class BitstreamReader { * * @param length Must be 32 for 32-bit single-precision or 64 for 64-bit double-precision. All * other values result in TypeError + * @param byteOrder The byte order to use when the length is greater than 8 and is a multiple of 8. + * Defaults to MSB (most significant byte). If the length is not a multiple of 8, + * this is unused * @returns The floating point value that was read */ - readFloatSync(length : number): number { + readFloatSync(length: number, byteOrder: "big-endian" | "little-endian" = 'big-endian'): number { if (length !== 32 && length !== 64) throw new TypeError(`Invalid length (${length} bits) Only 4-byte (32 bit / single-precision) and 8-byte (64 bit / double-precision) IEEE 754 values are supported`); @@ -420,9 +429,9 @@ export class BitstreamReader { view.setUint8(i, this.readSync(8)); if (length === 32) - return view.getFloat32(0, false); + return view.getFloat32(0, byteOrder === 'little-endian'); else if (length === 64) - return view.getFloat64(0, false); + return view.getFloat64(0, byteOrder === 'little-endian'); } private readByteAligned(consume: boolean): number { @@ -460,7 +469,7 @@ export class BitstreamReader { } } - private readShortByteAligned(consume: boolean, byteOrder: 'lsb' | 'msb'): number { + private readShortByteAligned(consume: boolean, byteOrder: "big-endian" | "little-endian"): number { let buffer = this.buffers[this._bufferIndex]; let bufferOffset = this._offsetIntoBuffer / 8; let firstByte = buffer[bufferOffset]; @@ -474,7 +483,7 @@ export class BitstreamReader { if (consume) this.consume(16); - if (byteOrder === 'lsb') { + if (byteOrder === 'little-endian') { let carry = firstByte; firstByte = secondByte; secondByte = carry; @@ -483,62 +492,68 @@ export class BitstreamReader { return firstByte << 8 | secondByte; } - private readLongByteAligned(consume: boolean, byteOrder: 'lsb' | 'msb'): number { + private readLongByteAligned(consume: boolean, byteOrder: "big-endian" | "little-endian"): number { let bufferIndex = this._bufferIndex; let buffer = this.buffers[bufferIndex]; let bufferOffset = this._offsetIntoBuffer / 8; - let firstByte = buffer[bufferOffset++]; - if (bufferOffset >= buffer.length) { - buffer = this.buffers[++bufferIndex]; - bufferOffset = 0; - } - - let secondByte = buffer[bufferOffset++]; - if (bufferOffset >= buffer.length) { - buffer = this.buffers[++bufferIndex]; - bufferOffset = 0; - } - - let thirdByte = buffer[bufferOffset++]; - if (bufferOffset >= buffer.length) { - buffer = this.buffers[++bufferIndex]; - bufferOffset = 0; - } - - let fourthByte = buffer[bufferOffset++]; - if (bufferOffset >= buffer.length) { - buffer = this.buffers[++bufferIndex]; - bufferOffset = 0; - } + const dv = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength); + const value = dv.getUint32(bufferOffset, byteOrder === 'little-endian'); + bufferOffset += 4; + /* + let firstByte = buffer[bufferOffset++]; + if (bufferOffset >= buffer.length) { + buffer = this.buffers[++bufferIndex]; + bufferOffset = 0; + } + + let secondByte = buffer[bufferOffset++]; + if (bufferOffset >= buffer.length) { + buffer = this.buffers[++bufferIndex]; + bufferOffset = 0; + } + + let thirdByte = buffer[bufferOffset++]; + if (bufferOffset >= buffer.length) { + buffer = this.buffers[++bufferIndex]; + bufferOffset = 0; + } + + let fourthByte = buffer[bufferOffset++]; + if (bufferOffset >= buffer.length) { + buffer = this.buffers[++bufferIndex]; + bufferOffset = 0; + } + */ if (consume) this.consume(32); - - let highBit = ((firstByte & 0x80) !== 0); - firstByte &= ~0x80; - - if (byteOrder === 'lsb') { - let b1 = fourthByte; - let b2 = thirdByte; - let b3 = secondByte; - let b4 = firstByte; - - firstByte = b1; - secondByte = b2; - thirdByte = b3; - fourthByte = b4; - } - - let value = firstByte << 24 | secondByte << 16 | thirdByte << 8 | fourthByte; - - if (highBit) - value += 2**31; + /* + let highBit = ((firstByte & 0x80) !== 0); + firstByte &= ~0x80; + + if (byteOrder === 'little-endian') { + let b1 = fourthByte; + let b2 = thirdByte; + let b3 = secondByte; + let b4 = firstByte; + + firstByte = b1; + secondByte = b2; + thirdByte = b3; + fourthByte = b4; + } + + let value = firstByte << 24 | secondByte << 16 | thirdByte << 8 | fourthByte; + + if (highBit) + value += 2 ** 31; + */ return value; } - private read3ByteAligned(consume: boolean, byteOrder: 'lsb' | 'msb'): number { + private read3ByteAligned(consume: boolean, byteOrder: "big-endian" | "little-endian"): number { let bufferIndex = this._bufferIndex; let buffer = this.buffers[bufferIndex]; let bufferOffset = this._offsetIntoBuffer / 8; @@ -564,7 +579,7 @@ export class BitstreamReader { if (consume) this.consume(24); - if (byteOrder === 'lsb') { + if (byteOrder === 'little-endian') { let carry = firstByte; firstByte = thirdByte; thirdByte = carry; @@ -592,12 +607,15 @@ export class BitstreamReader { * this is unused * @returns */ - private readCoreSync(length : number, consume : boolean, byteOrder: 'msb' | 'lsb' = 'msb'): number { + private readCoreSync(length: number, consume: boolean, byteOrder: "big-endian" | "little-endian" = 'big-endian'): number { this.ensureNoReadPending(); if (this.available < length) throw new Error(`underrun: Not enough bits are available (requested=${length}, available=${this.bufferedLength}, buffers=${this.buffers.length})`); + if (byteOrder === 'little-endian' && (length < 8 || length % 8)) + throw new Error(`little-endian is only supported for 16,32 or 64bit length (requested=${length}, available=${this.bufferedLength}, buffers=${this.buffers.length})`); + this.adjustSkip(); let offsetIntoByte = this._offsetIntoBuffer % 8; @@ -669,10 +687,18 @@ export class BitstreamReader { if (consume) this.consume(length); - if (useBigInt) - return Number(bigValue); - else + value = useBigInt ? Number(bigValue) : value; + + if (byteOrder === 'little-endian') { + let nvalue = BigInt(0); + for (let i = 0; i < length; i += 8) { + nvalue = nvalue | ((BigInt(value) >> BigInt(i)) & BigInt(0xFF)) << BigInt(length - i - 8); + } + return Number(nvalue); + } else { return value; + } + } private adjustSkip() { @@ -723,13 +749,16 @@ export class BitstreamReader { * available for the operation. * * @param length The number of bits to read + * @param byteOrder The byte order to use when the length is greater than 8 and is a multiple of 8. + * Defaults to MSB (most significant byte). If the length is not a multiple of 8, + * this is unused * @returns A promise which resolves to the unsigned integer once it is read */ - read(length : number) : Promise { + read(length: number, byteOrder?: "big-endian" | "little-endian"): Promise { this.ensureNoReadPending(); if (this.available >= length) { - return Promise.resolve(this.readSync(length)); + return Promise.resolve(this.readSync(length, byteOrder)); } else { return this.block({ length }); } @@ -740,13 +769,16 @@ export class BitstreamReader { * available for the operation. * * @param length The number of bits to read + * @param byteOrder The byte order to use when the length is greater than 8 and is a multiple of 8. + * Defaults to MSB (most significant byte). If the length is not a multiple of 8, + * this is unused * @returns A promise which resolves to the signed integer value once it is read */ - readSigned(length : number) : Promise { + readSigned(length: number, byteOrder?: "big-endian" | "little-endian"): Promise { this.ensureNoReadPending(); if (this.available >= length) { - return Promise.resolve(this.readSignedSync(length)); + return Promise.resolve(this.readSignedSync(length, byteOrder)); } else { return this.block({ length, signed: true }); } @@ -782,13 +814,16 @@ export class BitstreamReader { * * @param length The number of bits to read (must be 32 for 32-bit single-precision or * 64 for 64-bit double-precision) + * @param byteOrder The byte order to use when the length is greater than 8 and is a multiple of 8. + * Defaults to MSB (most significant byte). If the length is not a multiple of 8, + * this is unused * @returns A promise which resolves to the floating point value once it is read */ - readFloat(length : number) : Promise { + readFloat(length: number, byteOrder?: "big-endian" | "little-endian"): Promise { this.ensureNoReadPending(); if (this.available >= length) { - return Promise.resolve(this.readFloatSync(length)); + return Promise.resolve(this.readFloatSync(length, byteOrder)); } else { return this.block({ length, float: true }); } diff --git a/src/bitstream/writer.test.ts b/src/bitstream/writer.test.ts index f61287f..f400f70 100644 --- a/src/bitstream/writer.test.ts +++ b/src/bitstream/writer.test.ts @@ -74,6 +74,24 @@ describe('BitstreamWriter', it => { expect(bufs[0][0]).to.equal(0b01010101); expect(bufs[1][0]).to.equal(0b10101010); }); + it('works for large writes (1) LE', () => { + let bufs: Buffer[] = []; + let fakeStream: any = { write(buf) { bufs.push(buf); } } + let writer = new BitstreamWriter(fakeStream); + writer.write(16, 0b1111111100000000, 'little-endian'); + expect(bufs.length).to.equal(2); + expect(bufs[0][0]).to.equal(0b00000000); + expect(bufs[1][0]).to.equal(0b11111111); + }); + it('works for large writes (2) LE', () => { + let bufs: Buffer[] = []; + let fakeStream: any = { write(buf) { bufs.push(buf); } } + let writer = new BitstreamWriter(fakeStream); + writer.write(16, 0b0101010110101010, 'little-endian'); + expect(bufs.length).to.equal(2); + expect(bufs[0][0]).to.equal(0b10101010); + expect(bufs[1][0]).to.equal(0b01010101); + }); it('works for offset large writes', () => { let bufs : Buffer[] = []; let fakeStream : any = { write(buf) { bufs.push(buf); } } @@ -87,6 +105,19 @@ describe('BitstreamWriter', it => { expect(bufs[1][0]).to.equal(0b01011010); expect(bufs[2][0]).to.equal(0b10101111); }); + it('works for offset large writes LE', () => { + let bufs: Buffer[] = []; + let fakeStream: any = { write(buf) { bufs.push(buf); } } + let writer = new BitstreamWriter(fakeStream); + writer.write(4, 0b1111); + writer.write(16, 0b0101010110101010, 'little-endian'); + writer.write(4, 0b1111); + + expect(bufs.length).to.equal(3); + expect(bufs[0][0]).to.equal(0b11111010); + expect(bufs[1][0]).to.equal(0b10100101); + expect(bufs[2][0]).to.equal(0b01011111); + }); it('respects configured buffer size', () => { let bufs : Buffer[] = []; let fakeStream : any = { write(buf) { bufs.push(buf); } } @@ -249,6 +280,13 @@ describe('BitstreamWriter', it => { writer.writeSigned(16, 1014); expect(Array.from(bufs[1])).to.eql([0x03, 0xF6]); writer.writeSigned(16, 0); expect(Array.from(bufs[2])).to.eql([0, 0]); + bufs = []; + writer = new BitstreamWriter(fakeStream, 2); + + writer.writeSigned(16, -1014, 'little-endian'); expect(Array.from(bufs[0])).to.eql([0x0A, 0xFC]); + writer.writeSigned(16, 1014, 'little-endian'); expect(Array.from(bufs[1])).to.eql([0xF6, 0x03]); + writer.writeSigned(16, 0, 'little-endian'); expect(Array.from(bufs[2])).to.eql([0, 0]); + bufs = []; writer = new BitstreamWriter(fakeStream, 4); @@ -256,6 +294,13 @@ describe('BitstreamWriter', it => { writer.writeSigned(32, 102336); expect(Array.from(bufs[1])).to.eql([0x00, 0x01, 0x8F, 0xC0]); writer.writeSigned(32, 0); expect(Array.from(bufs[2])).to.eql([0, 0, 0, 0]); + bufs = []; + writer = new BitstreamWriter(fakeStream, 4); + + writer.writeSigned(32, -102336, 'little-endian'); expect(Array.from(bufs[0])).to.eql([0x40, 0x70, 0xFE, 0xFF]); + writer.writeSigned(32, 102336, 'little-endian'); expect(Array.from(bufs[1])).to.eql([0xC0, 0x8F, 0x01, 0x00]); + writer.writeSigned(32, 0, 'little-endian'); expect(Array.from(bufs[2])).to.eql([0, 0, 0, 0]); + }); it('correctly handles floats', () => { let bufs : Buffer[] = []; @@ -266,6 +311,13 @@ describe('BitstreamWriter', it => { writer.writeFloat(32, -436); expect(Array.from(bufs[1])).to.eql([0xC3, 0xDA, 0x00, 0x00]); writer.writeFloat(32, 0); expect(Array.from(bufs[2])).to.eql([0,0,0,0]); + bufs = []; + writer = new BitstreamWriter(fakeStream, 4); + + writer.writeFloat(32, 102.5, 'little-endian'); expect(Array.from(bufs[0])).to.eql([0x00, 0x00, 0xCD, 0x42]); + writer.writeFloat(32, -436, 'little-endian'); expect(Array.from(bufs[1])).to.eql([0x00, 0x00, 0xDA, 0xC3]); + writer.writeFloat(32, 0, 'little-endian'); expect(Array.from(bufs[2])).to.eql([0, 0, 0, 0]); + bufs = []; writer = new BitstreamWriter(fakeStream, 8); @@ -277,6 +329,19 @@ describe('BitstreamWriter', it => { writer.writeFloat(64, 0); expect(Array.from(bufs[2])).to.eql([0, 0, 0, 0, 0, 0, 0, 0]); + + bufs = []; + writer = new BitstreamWriter(fakeStream, 8); + + writer.writeFloat(64, 8745291.56, 'little-endian'); + expect(Array.from(bufs[0])).to.eql([0x1f, 0x85, 0xeb, 0x71, 0x29, 0xae, 0x60, 0x41]); + + writer.writeFloat(64, -327721.17, 'little-endian'); + expect(Array.from(bufs[1])).to.eql([0xe1, 0x7a, 0x14, 0xae, 0xa4, 0x00, 0x14, 0xc1]); + + writer.writeFloat(64, 0, 'little-endian'); + expect(Array.from(bufs[2])).to.eql([0, 0, 0, 0, 0, 0, 0, 0]); + }); it('.writeFloat() throws for lengths other than 32 and 64', () => { diff --git a/src/bitstream/writer.ts b/src/bitstream/writer.ts index 0f90bbc..635f930 100644 --- a/src/bitstream/writer.ts +++ b/src/bitstream/writer.ts @@ -10,13 +10,13 @@ export class BitstreamWriter { * @param stream The writable stream to write to * @param bufferSize The number of bytes to buffer before flushing onto the writable */ - constructor(public stream : Writable, readonly bufferSize = 1) { + constructor(public stream: Writable, readonly bufferSize = 1) { this.buffer = new Uint8Array(bufferSize); } - private pendingByte : bigint = BigInt(0); - private pendingBits : number = 0; - private buffer : Uint8Array; + private pendingByte: bigint = BigInt(0); + private pendingBits: number = 0; + private buffer: Uint8Array; private bufferedBytes = 0; private _offset = 0; @@ -59,7 +59,7 @@ export class BitstreamWriter { this.pendingByte = BigInt(0); } } - + flush() { if (this.bufferedBytes > 0) { this.stream.write(Buffer.from(this.buffer.slice(0, this.bufferedBytes))); @@ -76,7 +76,7 @@ export class BitstreamWriter { * @param value The string to decode and write * @param encoding The encoding to use when writing the string. Defaults to utf-8 */ - writeString(byteCount : number, value : string, encoding : string = 'utf-8') { + writeString(byteCount: number, value: string, encoding: string = 'utf-8') { if (encoding === 'utf-8') { let buffer = new Uint8Array(byteCount); let strBuf = this.textEncoder.encode(value); @@ -100,7 +100,7 @@ export class BitstreamWriter { * @param buffer The buffer to write * @deprecated Use writeBytes() instead */ - writeBuffer(buffer : Uint8Array) { + writeBuffer(buffer: Uint8Array) { this.writeBytes(buffer); } @@ -110,7 +110,7 @@ export class BitstreamWriter { * a set of bytes at a non=zero bit offset if you wish. * @param chunk The buffer to write */ - writeBytes(chunk : Uint8Array, offset = 0, length? : number) { + writeBytes(chunk: Uint8Array, offset = 0, length?: number) { length ??= chunk.length - offset; // Fast path: Byte-aligned @@ -128,11 +128,11 @@ export class BitstreamWriter { return; } - for (let i = offset, max = Math.min(chunk.length, offset+length); i < max; ++i) + for (let i = offset, max = Math.min(chunk.length, offset + length); i < max; ++i) this.write(8, chunk[i]); } - private min(a : bigint, b : bigint) { + private min(a: bigint, b: bigint) { if (a < b) return a; else @@ -144,8 +144,12 @@ export class BitstreamWriter { * number of bits specified, the lower-order bits are written and the higher-order bits are ignored. * @param length The number of bits to write * @param value The number to write + * @param byteOrder The byte order to use when the length is greater than 8 and is a multiple of 8. + * Defaults to MSB (most significant byte). If the length is not a multiple of 8, + * this is unused + * */ - write(length : number, value : number) { + write(length: number, value: number | bigint, byteOrder: "big-endian" | "little-endian" = 'big-endian') { if (value === void 0 || value === null) value = 0; @@ -156,33 +160,51 @@ export class BitstreamWriter { if (!Number.isFinite(value)) throw new Error(`Cannot write to bitstream: Value ${value} must be finite`); - let valueN = BigInt(value % Math.pow(2, length)); - - let remainingLength = length; + if (byteOrder === 'little-endian') { + if (length % 8 && length > 8) throw new Error("Little-endian is only supported for 16,32 or 64bit length"); - while (remainingLength > 0) { - let shift = BigInt(8 - this.pendingBits - remainingLength); - let contribution = (shift >= 0 ? (valueN << shift) : (valueN >> -shift)); - let writtenLength = Number(shift >= 0 ? remainingLength : this.min(-shift, BigInt(8 - this.pendingBits))); + let buf = new ArrayBuffer(length / 8); + let view = new DataView(buf); - this.pendingByte = this.pendingByte | contribution; - this.pendingBits += writtenLength; - this._offset += writtenLength; - - remainingLength -= writtenLength; - valueN = valueN % BigInt(Math.pow(2, remainingLength)); + if (length === 16) + view.setUint16(0, value, byteOrder === 'little-endian'); + else if (length === 32) + view.setUint32(0, value, byteOrder === 'little-endian'); + else if (length === 64) + view.setBigUint64(0, BigInt(value), byteOrder === 'little-endian') - if (this.pendingBits === 8) { - this.finishByte(); + for (let i = 0, max = buf.byteLength; i < max; ++i) + this.write(8, view.getUint8(i)); + } else { - if (this.bufferedBytes >= this.buffer.length) { - this.flush(); + let valueN = BigInt(value % Math.pow(2, length)); + + let remainingLength = length; + + while (remainingLength > 0) { + let shift = BigInt(8 - this.pendingBits - remainingLength); + let contribution = (shift >= 0 ? (valueN << shift) : (valueN >> -shift)); + let writtenLength = Number(shift >= 0 ? remainingLength : this.min(-shift, BigInt(8 - this.pendingBits))); + + this.pendingByte = this.pendingByte | contribution; + this.pendingBits += writtenLength; + this._offset += writtenLength; + + remainingLength -= writtenLength; + valueN = valueN % BigInt(Math.pow(2, remainingLength)); + + if (this.pendingBits === 8) { + this.finishByte(); + + if (this.bufferedBytes >= this.buffer.length) { + this.flush(); + } } } } } - writeSigned(length : number, value : number) { + writeSigned(length: number, value: number | bigint, byteOrder: "big-endian" | "little-endian" = 'big-endian') { if (value === undefined || value === null) value = 0; @@ -200,11 +222,28 @@ export class BitstreamWriter { throw new TypeError(`Cannot represent ${value} in I${length} format: Value too large (min=${min}, max=${max})`); if (value < min) throw new TypeError(`Cannot represent ${value} in I${length} format: Negative value too small (min=${min}, max=${max})`); - - return this.write(length, value >= 0 ? value : (~(-value) + 1) >>> 0); + + if (byteOrder === 'little-endian') { + if (length != 16 && length != 32 && length != 64) throw new Error("Little-endian is only supported for 16,32"); + + let buf = new ArrayBuffer(length / 8); + let view = new DataView(buf); + + if (length === 16) + view.setInt16(0, value, byteOrder === 'little-endian'); + else if (length === 32) + view.setInt32(0, value, byteOrder === 'little-endian'); + else if (length === 64) + view.setBigInt64(0, BigInt(value), byteOrder === 'little-endian'); + + for (let i = 0, max = buf.byteLength; i < max; ++i) + this.write(8, view.getUint8(i)); + } else { + return this.write(length, value >= 0 ? value : (~(-value) + 1) >>> 0); + } } - writeFloat(length : number, value : number) { + writeFloat(length: number, value: number, byteOrder: "big-endian" | "little-endian" = 'big-endian') { if (length !== 32 && length !== 64) throw new TypeError(`Invalid length (${length} bits) Only 4-byte (32 bit / single-precision) and 8-byte (64 bit / double-precision) IEEE 754 values are supported`); @@ -212,9 +251,9 @@ export class BitstreamWriter { let view = new DataView(buf); if (length === 32) - view.setFloat32(0, value); + view.setFloat32(0, value, byteOrder === 'little-endian'); else if (length === 64) - view.setFloat64(0, value); + view.setFloat64(0, value, byteOrder === 'little-endian'); for (let i = 0, max = buf.byteLength; i < max; ++i) this.write(8, view.getUint8(i)); diff --git a/src/elements/number-serializer.ts b/src/elements/number-serializer.ts index 7878597..a214027 100644 --- a/src/elements/number-serializer.ts +++ b/src/elements/number-serializer.ts @@ -23,11 +23,11 @@ export class NumberSerializer implements Serializer { let format = field.options?.number?.format ?? 'unsigned'; if (format === 'unsigned') - return reader.readSync(length); + return reader.readSync(length, field.options?.number?.byteOrder); else if (format === 'signed') - return reader.readSignedSync(length); + return reader.readSignedSync(length, field.options?.number?.byteOrder); else if (format === 'float') - return reader.readFloatSync(length); + return reader.readFloatSync(length, field.options?.number?.byteOrder); else throw new TypeError(`Unsupported number format '${format}'`); } @@ -46,11 +46,11 @@ export class NumberSerializer implements Serializer { let format = field.options?.number?.format ?? 'unsigned'; if (format === 'unsigned') - writer.write(length, value); + writer.write(length, value, field.options?.number?.byteOrder); else if (format === 'signed') - writer.writeSigned(length, value); + writer.writeSigned(length, value, field.options?.number?.byteOrder); else if (format === 'float') - writer.writeFloat(length, value); + writer.writeFloat(length, value, field.options?.number?.byteOrder); else throw new TypeError(`Unsupported number format '${format}'`); }