From 8bb4748c5e8dba93d753baa5112c2327206a4198 Mon Sep 17 00:00:00 2001 From: Leon Zandman Date: Sun, 1 Mar 2026 01:33:51 +0100 Subject: [PATCH 1/8] Added Zstd Compress / Zstd Decompress operations. --- package-lock.json | 7 +++ package.json | 1 + src/core/config/Categories.json | 4 +- src/core/lib/Zstd.mjs | 30 ++++++++++ src/core/operations/ZstdCompress.mjs | 61 ++++++++++++++++++++ src/core/operations/ZstdDecompress.mjs | 59 +++++++++++++++++++ tests/operations/index.mjs | 1 + tests/operations/tests/Zstd.mjs | 80 ++++++++++++++++++++++++++ 8 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 src/core/lib/Zstd.mjs create mode 100644 src/core/operations/ZstdCompress.mjs create mode 100644 src/core/operations/ZstdDecompress.mjs create mode 100644 tests/operations/tests/Zstd.mjs diff --git a/package-lock.json b/package-lock.json index a6cd3a5af6..b04602ce8e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@astronautlabs/amf": "^0.0.6", "@babel/polyfill": "^7.12.1", "@blu3r4y/lzma": "^2.3.3", + "@bokuweb/zstd-wasm": "^0.0.27", "@wavesenterprise/crypto-gost-js": "^2.1.0-RC1", "@xmldom/xmldom": "^0.8.11", "argon2-browser": "^1.18.0", @@ -1838,6 +1839,12 @@ "lzma.js": "bin/lzma.js" } }, + "node_modules/@bokuweb/zstd-wasm": { + "version": "0.0.27", + "resolved": "https://registry.npmjs.org/@bokuweb/zstd-wasm/-/zstd-wasm-0.0.27.tgz", + "integrity": "sha512-GDm2uOTK3ESjnYmSeLQifJnBsRCWajKLvN32D2ZcQaaCIJI/Hse9s74f7APXjHit95S10UImsRGkTsbwHmrtmg==", + "license": "MIT" + }, "node_modules/@codemirror/commands": { "version": "6.10.2", "resolved": "https://registry.npmjs.org/@codemirror/commands/-/commands-6.10.2.tgz", diff --git a/package.json b/package.json index b3d695eaef..676894cd5b 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,7 @@ "@astronautlabs/amf": "^0.0.6", "@babel/polyfill": "^7.12.1", "@blu3r4y/lzma": "^2.3.3", + "@bokuweb/zstd-wasm": "^0.0.27", "@wavesenterprise/crypto-gost-js": "^2.1.0-RC1", "@xmldom/xmldom": "^0.8.11", "argon2-browser": "^1.18.0", diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index aac00ca1ca..0daab37cf3 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -402,7 +402,9 @@ "LZMA Compress", "LZ4 Decompress", "LZ4 Compress", - "LZNT1 Decompress" + "LZNT1 Decompress", + "Zstd Compress", + "Zstd Decompress" ] }, { diff --git a/src/core/lib/Zstd.mjs b/src/core/lib/Zstd.mjs new file mode 100644 index 0000000000..a7e013be39 --- /dev/null +++ b/src/core/lib/Zstd.mjs @@ -0,0 +1,30 @@ +/** + * Zstd shared initialisation. + * + * Both ZstdCompress and ZstdDecompress import from here so that + * WebAssembly.instantiate is called exactly once, regardless of how many + * operations use it or in what order they run. + * + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2024 + * @license Apache-2.0 + */ + +import { init, compress, decompress } from "@bokuweb/zstd-wasm"; + +let initPromise = null; + +/** + * Returns a promise that resolves once the Zstd WASM module is ready. + * Safe to call multiple times — the module is only instantiated once. + * + * @returns {Promise} + */ +export function zstdInit() { + if (!initPromise) { + initPromise = init(); + } + return initPromise; +} + +export { compress, decompress }; diff --git a/src/core/operations/ZstdCompress.mjs b/src/core/operations/ZstdCompress.mjs new file mode 100644 index 0000000000..909f0eb748 --- /dev/null +++ b/src/core/operations/ZstdCompress.mjs @@ -0,0 +1,61 @@ +/** + * @author Leon Zandman [leon@wirwar.com] + * @copyright Crown Copyright 2027 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import { isWorkerEnvironment } from "../Utils.mjs"; +import { zstdInit, compress } from "../lib/Zstd.mjs"; + +/** + * Zstd Compress operation + */ +class ZstdCompress extends Operation { + + /** + * ZstdCompress constructor + */ + constructor() { + super(); + + this.name = "Zstd Compress"; + this.module = "Compression"; + this.description = "Compresses data using the Zstandard (Zstd) algorithm. Zstd offers high compression ratios at fast speeds and is widely used in Linux, databases, container images, and network protocols."; + this.infoURL = "https://wikipedia.org/wiki/Zstandard"; + this.inputType = "ArrayBuffer"; + this.outputType = "ArrayBuffer"; + this.args = [ + { + name: "Compression level", + type: "number", + value: 3, + min: 1, + max: 22 + } + ]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {ArrayBuffer} + */ + async run(input, args) { + const [level] = args; + if (input.byteLength === 0) throw new OperationError("Please provide an input."); + if (isWorkerEnvironment()) self.sendStatusMessage("Loading Zstd..."); + await zstdInit(); + if (isWorkerEnvironment()) self.sendStatusMessage("Compressing data..."); + try { + const result = compress(new Uint8Array(input), level); + return result.buffer.slice(result.byteOffset, result.byteOffset + result.byteLength); + } catch (err) { + throw new OperationError(`Failed to compress: ${err.message}`); + } + } + +} + +export default ZstdCompress; diff --git a/src/core/operations/ZstdDecompress.mjs b/src/core/operations/ZstdDecompress.mjs new file mode 100644 index 0000000000..3e6dd2bc33 --- /dev/null +++ b/src/core/operations/ZstdDecompress.mjs @@ -0,0 +1,59 @@ +/** + * @author Leon Zandman [leon@wirwar.com] + * @copyright Crown Copyright 2027 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import { isWorkerEnvironment } from "../Utils.mjs"; +import { zstdInit, decompress } from "../lib/Zstd.mjs"; + +/** + * Zstd Decompress operation + */ +class ZstdDecompress extends Operation { + + /** + * ZstdDecompress constructor + */ + constructor() { + super(); + + this.name = "Zstd Decompress"; + this.module = "Compression"; + this.description = "Decompresses data compressed with the Zstandard (Zstd) algorithm."; + this.infoURL = "https://wikipedia.org/wiki/Zstandard"; + this.inputType = "ArrayBuffer"; + this.outputType = "ArrayBuffer"; + this.args = []; + this.checks = [ + { + pattern: "^\\x28\\xb5\\x2f\\xfd", + flags: "", + args: [] + } + ]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {ArrayBuffer} + */ + async run(input, args) { + if (input.byteLength === 0) throw new OperationError("Please provide an input."); + if (isWorkerEnvironment()) self.sendStatusMessage("Loading Zstd..."); + await zstdInit(); + if (isWorkerEnvironment()) self.sendStatusMessage("Decompressing data..."); + try { + const result = decompress(new Uint8Array(input)); + return result.buffer.slice(result.byteOffset, result.byteOffset + result.byteLength); + } catch (err) { + throw new OperationError(`Failed to decompress: ${err.message}`); + } + } + +} + +export default ZstdDecompress; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 9fccd0513d..41a95c05dc 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -182,6 +182,7 @@ import "./tests/JSONtoYAML.mjs"; import "./tests/YARA.mjs"; import "./tests/ParseCSR.mjs"; import "./tests/XXTEA.mjs"; +import "./tests/Zstd.mjs"; const testStatus = { allTestsPassing: true, diff --git a/tests/operations/tests/Zstd.mjs b/tests/operations/tests/Zstd.mjs new file mode 100644 index 0000000000..da9577d595 --- /dev/null +++ b/tests/operations/tests/Zstd.mjs @@ -0,0 +1,80 @@ +/** + * Zstd tests. + * + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2024 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + name: "Zstd compress & decompress: string", + input: "The cat sat on the mat.", + expectedOutput: "The cat sat on the mat.", + recipeConfig: [ + { + op: "Zstd Compress", + args: [3] + }, + { + op: "Zstd Decompress", + args: [] + } + ] + }, + { + // Generated using: node --input-type=module -e "import {init,compress} from '@bokuweb/zstd-wasm'; await init(); const r=compress(new TextEncoder().encode('The cat sat on the mat.'),3); console.log(Buffer.from(r).toString('hex'));" + name: "Zstd compress: level 3", + input: "The cat sat on the mat.", + expectedOutput: "28b52ffd2017b900005468652063617420736174206f6e20746865206d61742e", + recipeConfig: [ + { + op: "Zstd Compress", + args: [3] + }, + { + op: "To Hex", + args: ["None", 0] + } + ] + }, + { + // Generated using: node --input-type=module -e "import {init,compress} from '@bokuweb/zstd-wasm'; await init(); const r=compress(new TextEncoder().encode('The cat sat on the mat.'),3); console.log(Buffer.from(r).toString('hex'));" + name: "Zstd decompress: known vector", + input: "28b52ffd2017b900005468652063617420736174206f6e20746865206d61742e", + expectedOutput: "The cat sat on the mat.", + recipeConfig: [ + { + op: "From Hex", + args: ["None"] + }, + { + op: "Zstd Decompress", + args: [] + } + ] + }, + { + name: "Zstd compress: empty input error", + input: "", + expectedOutput: "Please provide an input.", + recipeConfig: [ + { + op: "Zstd Compress", + args: [3] + } + ] + }, + { + name: "Zstd decompress: empty input error", + input: "", + expectedOutput: "Please provide an input.", + recipeConfig: [ + { + op: "Zstd Decompress", + args: [] + } + ] + } +]); From 0d723d3e0514b27285cb36d42a4fd477f4997571 Mon Sep 17 00:00:00 2001 From: Leon Zandman Date: Sun, 10 May 2026 00:26:47 +0200 Subject: [PATCH 2/8] Init fix. --- src/core/lib/Zstd.mjs | 3 ++- webpack.config.js | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/core/lib/Zstd.mjs b/src/core/lib/Zstd.mjs index a7e013be39..6178307bc6 100644 --- a/src/core/lib/Zstd.mjs +++ b/src/core/lib/Zstd.mjs @@ -22,7 +22,8 @@ let initPromise = null; */ export function zstdInit() { if (!initPromise) { - initPromise = init(); + const wasmUrl = `${self.docURL}/assets/zstd.wasm`; + initPromise = init(wasmUrl); } return initPromise; } diff --git a/webpack.config.js b/webpack.config.js index 4c6c00ba29..564040f0ff 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -99,6 +99,10 @@ module.exports = { context: "node_modules/node-forge/dist", from: "prime.worker.min.js", to: "assets/forge/" + }, { + context: "node_modules/@bokuweb/zstd-wasm/dist/web/", + from: "zstd.wasm", + to: "assets/" } ] }), @@ -110,6 +114,17 @@ module.exports = { operations: [ new ReplaceOperation("once", "if (pixelSize < elementMinSize)", "if (false)") ] + }, + { + // @bokuweb/zstd-wasm's init() unconditionally evaluates + // `new URL("./zstd.wasm", import.meta.url)` before checking + // whether a path was passed in. Webpack's `publicPath: ""` + // makes that base URL invalid and the constructor throws. + // Neuter the offending line; we always pass a path from Zstd.mjs. + test: /@bokuweb[\/\\]zstd-wasm[\/\\]dist[\/\\]web[\/\\]index\.web\.js$/, + operations: [ + new ReplaceOperation("once", "new URL(`./zstd.wasm`, import.meta.url).href", "''") + ] } ] }) From 6da5cec354ce633c10cc41c03202fa9f88683b95 Mon Sep 17 00:00:00 2001 From: Leon Zandman Date: Sun, 10 May 2026 00:35:00 +0200 Subject: [PATCH 3/8] Add Zstandard (zstd) file type detection. --- src/core/lib/FileSignatures.mjs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 221328234c..75bb33b05a 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1533,6 +1533,32 @@ export const FILE_SIGNATURES = { }, extractor: extractZlib }, + { + name: "Zstandard (zstd)", + extension: "zst", + mime: "application/zstd", + description: "", + signature: { + 0: 0x28, + 1: 0xb5, + 2: 0x2f, + 3: 0xfd + }, + extractor: null + }, + { + name: "Zstandard skippable frame (zstd)", + extension: "zst", + mime: "application/zstd", + description: "", + signature: { + 0: [0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f], + 1: 0x2a, + 2: 0x4d, + 3: 0x18 + }, + extractor: null + }, { name: "xz compression", extension: "xz", From 4ba952e6e2747b2a745edb9e7887000da23e64bd Mon Sep 17 00:00:00 2001 From: Leon Zandman Date: Sun, 10 May 2026 00:41:23 +0200 Subject: [PATCH 4/8] Fix test. --- src/core/lib/Zstd.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/lib/Zstd.mjs b/src/core/lib/Zstd.mjs index 6178307bc6..0c0b8e9e45 100644 --- a/src/core/lib/Zstd.mjs +++ b/src/core/lib/Zstd.mjs @@ -22,7 +22,9 @@ let initPromise = null; */ export function zstdInit() { if (!initPromise) { - const wasmUrl = `${self.docURL}/assets/zstd.wasm`; + const wasmUrl = typeof self !== "undefined" && self.docURL + ? `${self.docURL}/assets/zstd.wasm` + : undefined; initPromise = init(wasmUrl); } return initPromise; From 5aeb8e61c2b6a3c47e0d721ce33b85b2c4329aba Mon Sep 17 00:00:00 2001 From: Leon Zandman Date: Sun, 10 May 2026 01:04:21 +0200 Subject: [PATCH 5/8] Compression level now dropdown. --- src/core/operations/ZstdCompress.mjs | 15 +++++++++------ tests/operations/tests/Zstd.mjs | 6 +++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/core/operations/ZstdCompress.mjs b/src/core/operations/ZstdCompress.mjs index 909f0eb748..4f03ebf4fe 100644 --- a/src/core/operations/ZstdCompress.mjs +++ b/src/core/operations/ZstdCompress.mjs @@ -22,17 +22,20 @@ class ZstdCompress extends Operation { this.name = "Zstd Compress"; this.module = "Compression"; - this.description = "Compresses data using the Zstandard (Zstd) algorithm. Zstd offers high compression ratios at fast speeds and is widely used in Linux, databases, container images, and network protocols."; + this.description = "Compresses data using the Zstandard (Zstd) algorithm. Zstd offers high compression ratios at fast speeds and is widely used in Linux, databases, container images, and network protocols. Higher compression levels result in better compression but slower speed."; this.infoURL = "https://wikipedia.org/wiki/Zstandard"; this.inputType = "ArrayBuffer"; this.outputType = "ArrayBuffer"; this.args = [ { name: "Compression level", - type: "number", - value: 3, - min: 1, - max: 22 + type: "option", + value: [ + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", + "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", + "21", "22" + ], + defaultIndex: 2 } ]; } @@ -43,7 +46,7 @@ class ZstdCompress extends Operation { * @returns {ArrayBuffer} */ async run(input, args) { - const [level] = args; + const level = parseInt(args[0], 10); if (input.byteLength === 0) throw new OperationError("Please provide an input."); if (isWorkerEnvironment()) self.sendStatusMessage("Loading Zstd..."); await zstdInit(); diff --git a/tests/operations/tests/Zstd.mjs b/tests/operations/tests/Zstd.mjs index da9577d595..33b3c3c4dc 100644 --- a/tests/operations/tests/Zstd.mjs +++ b/tests/operations/tests/Zstd.mjs @@ -15,7 +15,7 @@ TestRegister.addTests([ recipeConfig: [ { op: "Zstd Compress", - args: [3] + args: ["3"] }, { op: "Zstd Decompress", @@ -31,7 +31,7 @@ TestRegister.addTests([ recipeConfig: [ { op: "Zstd Compress", - args: [3] + args: ["3"] }, { op: "To Hex", @@ -62,7 +62,7 @@ TestRegister.addTests([ recipeConfig: [ { op: "Zstd Compress", - args: [3] + args: ["3"] } ] }, From 27706d2e5ad2afbd88118909ea8c946dc4b07f62 Mon Sep 17 00:00:00 2001 From: Leon Zandman Date: Sun, 10 May 2026 01:30:14 +0200 Subject: [PATCH 6/8] Copyright year typo fix. --- src/core/operations/ZstdCompress.mjs | 2 +- src/core/operations/ZstdDecompress.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/operations/ZstdCompress.mjs b/src/core/operations/ZstdCompress.mjs index 4f03ebf4fe..e10cd02d5a 100644 --- a/src/core/operations/ZstdCompress.mjs +++ b/src/core/operations/ZstdCompress.mjs @@ -1,6 +1,6 @@ /** * @author Leon Zandman [leon@wirwar.com] - * @copyright Crown Copyright 2027 + * @copyright Crown Copyright 2026 * @license Apache-2.0 */ diff --git a/src/core/operations/ZstdDecompress.mjs b/src/core/operations/ZstdDecompress.mjs index 3e6dd2bc33..784940ee39 100644 --- a/src/core/operations/ZstdDecompress.mjs +++ b/src/core/operations/ZstdDecompress.mjs @@ -1,6 +1,6 @@ /** * @author Leon Zandman [leon@wirwar.com] - * @copyright Crown Copyright 2027 + * @copyright Crown Copyright 2026 * @license Apache-2.0 */ From 08722589f29d10dd4ba9a0a77abb4e214d1155e1 Mon Sep 17 00:00:00 2001 From: Leon Zandman Date: Sun, 10 May 2026 02:01:53 +0200 Subject: [PATCH 7/8] Fix copy/paste error. Set author to myself. --- src/core/lib/Zstd.mjs | 4 ++-- tests/operations/tests/Zstd.mjs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/lib/Zstd.mjs b/src/core/lib/Zstd.mjs index 0c0b8e9e45..a8dc279e01 100644 --- a/src/core/lib/Zstd.mjs +++ b/src/core/lib/Zstd.mjs @@ -5,8 +5,8 @@ * WebAssembly.instantiate is called exactly once, regardless of how many * operations use it or in what order they run. * - * @author n1474335 [n1474335@gmail.com] - * @copyright Crown Copyright 2024 + * @author Leon Zandman [leon@wirwar.com] + * @copyright Crown Copyright 2026 * @license Apache-2.0 */ diff --git a/tests/operations/tests/Zstd.mjs b/tests/operations/tests/Zstd.mjs index 33b3c3c4dc..d658323246 100644 --- a/tests/operations/tests/Zstd.mjs +++ b/tests/operations/tests/Zstd.mjs @@ -1,8 +1,8 @@ /** * Zstd tests. * - * @author n1474335 [n1474335@gmail.com] - * @copyright Crown Copyright 2024 + * @author Leon Zandman [leon@wirwar.com] + * @copyright Crown Copyright 2026 * @license Apache-2.0 */ import TestRegister from "../../lib/TestRegister.mjs"; From 8982904eb3500bdc963cc04930e188f122bd4a71 Mon Sep 17 00:00:00 2001 From: Leon Zandman Date: Sun, 10 May 2026 02:10:02 +0200 Subject: [PATCH 8/8] Added tests for compression level boundaries (1 and 22). --- tests/operations/tests/Zstd.mjs | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/operations/tests/Zstd.mjs b/tests/operations/tests/Zstd.mjs index d658323246..eb3abed582 100644 --- a/tests/operations/tests/Zstd.mjs +++ b/tests/operations/tests/Zstd.mjs @@ -55,6 +55,36 @@ TestRegister.addTests([ } ] }, + { + name: "Zstd compress & decompress: level 1", + input: "The cat sat on the mat.", + expectedOutput: "The cat sat on the mat.", + recipeConfig: [ + { + op: "Zstd Compress", + args: ["1"] + }, + { + op: "Zstd Decompress", + args: [] + } + ] + }, + { + name: "Zstd compress & decompress: level 22", + input: "The cat sat on the mat.", + expectedOutput: "The cat sat on the mat.", + recipeConfig: [ + { + op: "Zstd Compress", + args: ["22"] + }, + { + op: "Zstd Decompress", + args: [] + } + ] + }, { name: "Zstd compress: empty input error", input: "",