diff --git a/knip.config.ts b/knip.config.ts new file mode 100644 index 00000000..4e3e2824 --- /dev/null +++ b/knip.config.ts @@ -0,0 +1,8 @@ +import type { KnipConfig } from 'knip'; + +const config: KnipConfig = { + ignore: ['examples/**/*', 'src/commands/**/readme.ts'], + ignoreDependencies: ['doctoc'], +}; + +export default config; diff --git a/package.json b/package.json index 06a9394f..493a9bbc 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,8 @@ "test": "vitest run", "script:transcend-json-schema": "tsx scripts/buildTranscendJsonSchema.ts && prettier ./transcend-yml-schema-*.json --write", "script:pathfinder-json-schema": "tsx scripts/buildPathfinderJsonSchema.ts && prettier ./pathfinder-policy-yml-schema.json --write", - "docgen": "tsx scripts/buildReadmeDocs.ts" + "docgen": "tsx scripts/buildReadme.ts", + "knip": "knip" }, "tsup": { "entry": [ @@ -105,7 +106,6 @@ "@transcend-io/privacy-types": "^4.124.1", "@transcend-io/secret-value": "^1.2.0", "@transcend-io/type-utils": "^1.8.0", - "JSONStream": "^1.3.5", "cli-progress": "^3.11.2", "colors": "^1.4.0", "csv-parse": "^5.6.0", @@ -121,6 +121,7 @@ "io-ts": "^2.2.21", "io-ts-types": "^0.5.16", "js-yaml": "^4.1.0", + "JSONStream": "^1.3.5", "jsonwebtoken": "^9.0.2", "lodash-es": "^4.17.21", "monocle-ts": "^2.3.13", @@ -135,15 +136,14 @@ "@ianvs/prettier-plugin-sort-imports": "^4.5.1", "@tsconfig/node22": "^22.0.2", "@tsconfig/strictest": "^2.0.5", - "@types/JSONStream": "npm:@types/jsonstream@^0.8.33", "@types/cli-progress": "^3.11.0", - "@types/eslint": "^9.6.1", "@types/fuzzysearch": "^1.0.0", "@types/global-agent": "^2.1.1", "@types/inquirer": "^7.3.1", - "@types/inquirer-autocomplete-prompt": "^3.0.0", + "@types/inquirer-autocomplete-prompt": "1.x", "@types/js-yaml": "^4.0.5", "@types/json-schema": "^7.0.15", + "@types/JSONStream": "npm:@types/jsonstream@^0.8.33", "@types/jsonwebtoken": "^9", "@types/lodash-es": "^4.17.12", "@types/node": "^22.x", @@ -154,6 +154,7 @@ "eslint": "^9.31.0", "eslint-plugin-unicorn": "^59.0.1", "fdir": "^6.4.6", + "knip": "^5.61.3", "prettier": "^3.6.2", "publint": "^0.3.12", "tsup": "^8.5.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4678d557..577a1f72 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -126,9 +126,6 @@ importers: '@types/cli-progress': specifier: ^3.11.0 version: 3.11.6 - '@types/eslint': - specifier: ^9.6.1 - version: 9.6.1 '@types/fuzzysearch': specifier: ^1.0.0 version: 1.0.2 @@ -139,8 +136,8 @@ importers: specifier: ^7.3.1 version: 7.3.3 '@types/inquirer-autocomplete-prompt': - specifier: ^3.0.0 - version: 3.0.3 + specifier: 1.x + version: 1.3.5 '@types/js-yaml': specifier: ^4.0.5 version: 4.0.9 @@ -170,13 +167,16 @@ importers: version: 2.2.1 eslint: specifier: ^9.31.0 - version: 9.31.0 + version: 9.31.0(jiti@2.4.2) eslint-plugin-unicorn: specifier: ^59.0.1 - version: 59.0.1(eslint@9.31.0) + version: 59.0.1(eslint@9.31.0(jiti@2.4.2)) fdir: specifier: ^6.4.6 version: 6.4.6(picomatch@4.0.2) + knip: + specifier: ^5.61.3 + version: 5.61.3(@types/node@22.16.4)(typescript@5.8.3) prettier: specifier: ^3.6.2 version: 3.6.2 @@ -185,7 +185,7 @@ importers: version: 0.3.12 tsup: specifier: ^8.5.0 - version: 8.5.0(postcss@8.5.6)(tsx@4.20.3)(typescript@5.8.3) + version: 8.5.0(jiti@2.4.2)(postcss@8.5.6)(tsx@4.20.3)(typescript@5.8.3) tsx: specifier: ^4.20.3 version: 4.20.3 @@ -194,13 +194,13 @@ importers: version: 5.8.3 typescript-eslint: specifier: ^8.37.0 - version: 8.37.0(eslint@9.31.0)(typescript@5.8.3) + version: 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) vite-tsconfig-paths: specifier: ^5.1.4 - version: 5.1.4(typescript@5.8.3)(vite@7.0.2(@types/node@22.16.4)(tsx@4.20.3)) + version: 5.1.4(typescript@5.8.3)(vite@7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3)) vitest: specifier: ^3.2.4 - version: 3.2.4(@types/node@22.16.4)(tsx@4.20.3) + version: 3.2.4(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3) packages: @@ -241,6 +241,15 @@ packages: resolution: {integrity: sha512-jYnje+JyZG5YThjHiF28oT4SIZLnYOcSBb6+SDaFIyzDVSkXQmQQYclJ2R+YxcdmK0AX6x1E5OQNtuh3jHDrUg==} engines: {node: '>=6.9.0'} + '@emnapi/core@1.4.4': + resolution: {integrity: sha512-A9CnAbC6ARNMKcIcrQwq6HeHCjpcBZ5wSx4U01WXCqEKlrzB9F9315WDNHkrs2xbx7YjjSxbUYxuN6EQzpcY2g==} + + '@emnapi/runtime@1.4.4': + resolution: {integrity: sha512-hHyapA4A3gPaDCNfiqyZUStTMqIkKRshqPIuDOXv1hcBnD4U3l8cP0T1HMCfGRxQ6V64TGCcoswChANyOAwbQg==} + + '@emnapi/wasi-threads@1.0.3': + resolution: {integrity: sha512-8K5IFFsQqF9wQNJptGbS6FNKgUTsSRYnTqNCG1vPP8jFdjSv18n2mQfJpkt2Oibo9iBEzcDnDxNwKTzC7svlJw==} + '@esbuild/aix-ppc64@0.25.6': resolution: {integrity: sha512-ShbM/3XxwuxjFiuVBHA+d3j5dyac0aEVVq1oluIDf71hUw0aRF59dV/efUsIwFnR6m8JNM2FjZOzmaZ8yG61kw==} engines: {node: '>=18'} @@ -507,6 +516,9 @@ packages: '@jridgewell/trace-mapping@0.3.29': resolution: {integrity: sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==} + '@napi-rs/wasm-runtime@0.2.12': + resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==} + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -519,6 +531,101 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@oxc-resolver/binding-android-arm-eabi@11.5.2': + resolution: {integrity: sha512-g3Dh0uN8E1fJAi+m5LxDU1frUz5q4ox/arqXGpEmt+u7wRXBpXnGsxDV/GFB59AmVWbQAiyhVCiM2GymkaxwwQ==} + cpu: [arm] + os: [android] + + '@oxc-resolver/binding-android-arm64@11.5.2': + resolution: {integrity: sha512-bij8HIMXYGsxdxuvycpkgvTfBpj6tv5jKaZ4tcPKPJjewH5WYIaSAT4PJYlAidP/0m8jyPu5GGkslF7/qPUhAg==} + cpu: [arm64] + os: [android] + + '@oxc-resolver/binding-darwin-arm64@11.5.2': + resolution: {integrity: sha512-C2hjujTOPgyi4sgc4UL+JHlEiClTNncLUdwiilMnwjiEcxSe7ubBmeZRENUd9bx8P9DbS1ApaBjwv13ZngrZRw==} + cpu: [arm64] + os: [darwin] + + '@oxc-resolver/binding-darwin-x64@11.5.2': + resolution: {integrity: sha512-Llf2qMBzs4PdbnrA7s3tVjW7MXnjUXepfqQkEXM2klxIggcbtbIESe3KupYHoo0Q0p6hLHwWoadyM32Ho2hLzA==} + cpu: [x64] + os: [darwin] + + '@oxc-resolver/binding-freebsd-x64@11.5.2': + resolution: {integrity: sha512-dKCHhqgKW3eqnJBlgLC03qoDSVeZSZJVcSVpyomu0XrrNha3wVyv6aJjN7A8HnjUCqJDibbZfTtD3/gnsm30eQ==} + cpu: [x64] + os: [freebsd] + + '@oxc-resolver/binding-linux-arm-gnueabihf@11.5.2': + resolution: {integrity: sha512-AMV4MbHdUvwA6oBLk90/gPo3gPMZl9+DHeas8BxRdq/uX1BFQ05s+mhy9ATGElGQsRVVOPya9qczOdb8eAlM6w==} + cpu: [arm] + os: [linux] + + '@oxc-resolver/binding-linux-arm-musleabihf@11.5.2': + resolution: {integrity: sha512-hTCkii4HwQushiD3L86cefvojTY6OSDzcrQZHVaUmrtkL0OQnRT9qUff83lJIQhb94rjaEfQsgUdVl1bvuUK/Q==} + cpu: [arm] + os: [linux] + + '@oxc-resolver/binding-linux-arm64-gnu@11.5.2': + resolution: {integrity: sha512-EXkMvem90Pdw0bw0TlOhAHFAGLopb1LaVwsxF+iSc/zQtuR62kl2jGMQRvsW4NHaF+nUN29H8IYQDzox4gxsRw==} + cpu: [arm64] + os: [linux] + + '@oxc-resolver/binding-linux-arm64-musl@11.5.2': + resolution: {integrity: sha512-UvA2QZB73XPXmFweDRyXyUchN1YnEx+cca7a/ojdhT+stDe0WKMK32y27oabWokJJsZZOd+W40dD7sxjzx7K/g==} + cpu: [arm64] + os: [linux] + + '@oxc-resolver/binding-linux-ppc64-gnu@11.5.2': + resolution: {integrity: sha512-0rllGQIAmeb+vAtmco0PnTzqlMs0DQs+QvHu/8AQAmgrlKBZDJJmRvLqMv6EXgTrLlWxoM0o9oNf7mZ0tEenUQ==} + cpu: [ppc64] + os: [linux] + + '@oxc-resolver/binding-linux-riscv64-gnu@11.5.2': + resolution: {integrity: sha512-kfE5ALnGsxEyz/e6lZbNUyPjZwTIuExTVJLVzjT/RjvaltSZ6J0u/6/CKsVFD3t686yqse1fnXuydUsgAFmuXg==} + cpu: [riscv64] + os: [linux] + + '@oxc-resolver/binding-linux-riscv64-musl@11.5.2': + resolution: {integrity: sha512-O6lbEl+heEd3QS2GOwm+iDGMqEWA18X/b9JNodzEHe2TJeOJAV/5xJ7jQTGA2seoy6/REhW744O35DyPFxZ2aQ==} + cpu: [riscv64] + os: [linux] + + '@oxc-resolver/binding-linux-s390x-gnu@11.5.2': + resolution: {integrity: sha512-6ZASmeqVq+xEQZz/EH+U4j1hPeqVQ8Eo58oYrt9FGJhseowAh6TAOHXe80HAJH6HQTcws1fhS/A7I4hm6NOgZA==} + cpu: [s390x] + os: [linux] + + '@oxc-resolver/binding-linux-x64-gnu@11.5.2': + resolution: {integrity: sha512-MYTtU3sKGZfvOYVpUfFHFcxLGOI8WN5BIQeWgNnNDEBHasthEDnyeNYpj6QbLd3XMz84gGA1G+mKMm/lVUF6hA==} + cpu: [x64] + os: [linux] + + '@oxc-resolver/binding-linux-x64-musl@11.5.2': + resolution: {integrity: sha512-7u1ANU1jkDUbC5ZxGXWDs0OLuUvV3DzqHUI+g41wHdz0iLoVSJ7rR+hl/crHIm4PpFkYbpU+joRslM5OLxeKlw==} + cpu: [x64] + os: [linux] + + '@oxc-resolver/binding-wasm32-wasi@11.5.2': + resolution: {integrity: sha512-2tOsCVH+THg9b9h6MiTymTrveSUWAOaQGj2CPQ4XJncxECsZY6MfxKLul+XsW4KLpstE89KBemRIQi6Il0Twew==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@oxc-resolver/binding-win32-arm64-msvc@11.5.2': + resolution: {integrity: sha512-NmpFIoT86wD2cNAweoEMLKZ4aaGzbYzmeMcYK65Ml9PbH53YXe5XZOXdzVExLKGJ3Rorf055n/67pRRvpIm/sQ==} + cpu: [arm64] + os: [win32] + + '@oxc-resolver/binding-win32-ia32-msvc@11.5.2': + resolution: {integrity: sha512-1EwjnPP5sEKdQl4+3edw+8xMZ79qk7iPXOJRUtdE0jLEdlFmzpnLBfsz54G7mOiQvnc6uR8YePBQb1iCRnysNA==} + cpu: [ia32] + os: [win32] + + '@oxc-resolver/binding-win32-x64-msvc@11.5.2': + resolution: {integrity: sha512-eB8eV8SdO+OpbJJ3dvTgSPOsDsW7SJp+ih5WIBWt7pWMlVbQyjBwDgTI8gGTqg2iwdEEUVqlfivEEs22hKnxRw==} + cpu: [x64] + os: [win32] + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -675,6 +782,9 @@ packages: '@tsconfig/strictest@2.0.5': resolution: {integrity: sha512-ec4tjL2Rr0pkZ5hww65c+EEPYwxOi4Ryv+0MtjeaSQRJyq322Q27eOQiFbuNgw2hpL4hB1/W/HBGk3VKS43osg==} + '@tybys/wasm-util@0.10.0': + resolution: {integrity: sha512-VyyPYFlOMNylG45GoAe0xDoLwWuowvf92F9kySqzYh8vmYm7D2u4iUJKa1tOUpS70Ku13ASrOkS4ScXFsTaCNQ==} + '@types/cacheable-request@6.0.3': resolution: {integrity: sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==} @@ -687,9 +797,6 @@ packages: '@types/deep-eql@4.0.2': resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} - '@types/eslint@9.6.1': - resolution: {integrity: sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==} - '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} @@ -702,12 +809,15 @@ packages: '@types/http-cache-semantics@4.0.4': resolution: {integrity: sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==} - '@types/inquirer-autocomplete-prompt@3.0.3': - resolution: {integrity: sha512-OQCW09mEECgvhcppbQRgZSmWskWv58l+WwyUvWB1oxTu3CZj8keYSDZR9U8owUzJ5Zeux5kacN9iVPJLXcoLXg==} + '@types/inquirer-autocomplete-prompt@1.3.5': + resolution: {integrity: sha512-xsydZ63gZt/2vqlqdSJQgxhbZd2NpRO6TawrDu1/IR6VbL3HfS709y6Yu7LwrkCcy4Lr05PeBInPizErcXSokw==} '@types/inquirer@7.3.3': resolution: {integrity: sha512-HhxyLejTHMfohAuhRun4csWigAMjXTmRyiJTU1Y/I1xmggikFMkOUoMQRlFm+zQcPEGHSs3io/0FAmNZf8EymQ==} + '@types/inquirer@8.2.11': + resolution: {integrity: sha512-15UboTvxb9SOaPG7CcXZ9dkv8lNqfiAwuh/5WxJDLjmElBt9tbx1/FDsEnJddUBKvN4mlPKvr8FyO1rAmBanzg==} + '@types/js-yaml@4.0.9': resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==} @@ -1414,6 +1524,9 @@ packages: fault@1.0.4: resolution: {integrity: sha512-CJ0HCB5tL5fYTEA7ToAq5+kTwd++Borf1/bifxd9iT70QcXr4MRrO3Llf8Ifs70q+SJcGHFtnIE/Nw6giCtECA==} + fd-package-json@2.0.0: + resolution: {integrity: sha512-jKmm9YtsNXN789RS/0mSzOC1NUq9mkVd65vbSSVsKdjGvYXBuE4oWe2QOEoFeRmJg+lPuZxpmrfFclNhoRMneQ==} + fdir@6.4.6: resolution: {integrity: sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==} peerDependencies: @@ -1476,6 +1589,11 @@ packages: resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} engines: {node: '>=0.4.x'} + formatly@0.2.4: + resolution: {integrity: sha512-lIN7GpcvX/l/i24r/L9bnJ0I8Qn01qijWpQpDDvTLL29nKqSaJJu4h20+7VJ6m2CAhQ2/En/GbxDiHCzq/0MyA==} + engines: {node: '>=18.3.0'} + hasBin: true + fp-ts@2.16.10: resolution: {integrity: sha512-vuROzbNVfCmUkZSUbnWSltR1sbheyQbTzug7LB/46fEa1c0EucLeBaCEUE0gF3ZGUGBt9lVUiziGOhhj6K1ORA==} @@ -1826,6 +1944,10 @@ packages: jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + jiti@2.4.2: + resolution: {integrity: sha512-rg9zJN+G4n2nfJl5MW3BMygZX56zKPNVEYYqq7adpmMh4Jn2QNEwhvQlFy6jPVdcod7txZtKHWnyZiA3a0zP7A==} + hasBin: true + joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} @@ -1891,6 +2013,14 @@ packages: keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + knip@5.61.3: + resolution: {integrity: sha512-8iSz8i8ufIjuUwUKzEwye7ROAW0RzCze7T770bUiz0PKL+SSwbs4RS32fjMztLwcOzSsNPlXdUAeqmkdzXxJ1Q==} + engines: {node: '>=18.18.0'} + hasBin: true + peerDependencies: + '@types/node': '>=18' + typescript: '>=5.0.4' + levn@0.4.1: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} @@ -2140,6 +2270,11 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + napi-postinstall@0.3.0: + resolution: {integrity: sha512-M7NqKyhODKV1gRLdkwE7pDsZP2/SC2a2vHkOYh9MCpKMbWVfyVfUw5MaH83Fv6XMjxr5jryUp3IDDL9rlxsTeA==} + engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} + hasBin: true + natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -2206,6 +2341,9 @@ packages: resolution: {integrity: sha512-qFOyK5PjiWZd+QQIh+1jhdb9LpxTF0qs7Pm8o5QHYZ0M3vKqSqzsZaEB6oWlxZ+q2sJBMI/Ktgd2N5ZwQoRHfg==} engines: {node: '>= 0.4'} + oxc-resolver@11.5.2: + resolution: {integrity: sha512-mYkOsrgvlm4OLPCgSR2XCMkJ203PwSOASxzHYzW7Kz3GXONVbe2VTpgwL/yBo0igSUwlZWTUKEbRJLscJ6N5QQ==} + p-cancelable@2.1.1: resolution: {integrity: sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==} engines: {node: '>=8'} @@ -2458,6 +2596,9 @@ packages: resolution: {integrity: sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ==} engines: {npm: '>=2.0.0'} + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + sade@1.8.1: resolution: {integrity: sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==} engines: {node: '>=6'} @@ -2541,6 +2682,10 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + smol-toml@1.4.1: + resolution: {integrity: sha512-CxdwHXyYTONGHThDbq5XdwbFsuY4wlClRGejfE2NtwUtiHYsP1QtNsHb/hnj31jKYSchztJsaA8pSQoVzkfCFg==} + engines: {node: '>= 18'} + snake-case@3.0.4: resolution: {integrity: sha512-LAOh4z89bGQvl9pFfNF8V146i7o7/CqFPbqzYgP+yYzDIDeS9HaNFtXABamRW+AQzEVODcvE79ljJ+8a9YSdMg==} @@ -2616,6 +2761,10 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} + strip-json-comments@5.0.2: + resolution: {integrity: sha512-4X2FR3UwhNUE9G49aIsJW5hRRR3GXGTBTZRMfv568O60ojM8HcWjV/VxAxCDW3SUND33O6ZY66ZuRcdkj73q2g==} + engines: {node: '>=14.16'} + strip-literal@3.0.0: resolution: {integrity: sha512-TcccoMhJOM3OebGhSBEmp3UZ2SfDMZUEBdRA/9ynfLi8yYajyWX3JiXArcJt4Umh4vISpspkQIY8ZZoCqjbviA==} @@ -2921,6 +3070,10 @@ packages: jsdom: optional: true + walk-up-path@4.0.0: + resolution: {integrity: sha512-3hu+tD8YzSLGuFYtPRb48vdhKMi0KQV5sn+uWr8+7dMEq/2G/dtLrdDinkLjqq5TIbIBjYJ4Ax/n3YiaW7QM8A==} + engines: {node: 20 || >=22} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -3005,6 +3158,15 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + zod-validation-error@3.5.3: + resolution: {integrity: sha512-OT5Y8lbUadqVZCsnyFaTQ4/O2mys4tj7PqhdbBCp7McPwvIEKfPtdA6QfPeFQK2/Rz5LgwmAXRJTugBNBi0btw==} + engines: {node: '>=18.0.0'} + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + zwitch@1.0.5: resolution: {integrity: sha512-V50KMwwzqJV0NpZIZFwfOD5/lyny3WlSzRiXgA0G7VUnRlqttta1L6UQIHzd6EuBY/cHGfwTIck7w1yH6Q5zUw==} @@ -3057,6 +3219,22 @@ snapshots: '@babel/helper-string-parser': 7.27.1 '@babel/helper-validator-identifier': 7.27.1 + '@emnapi/core@1.4.4': + dependencies: + '@emnapi/wasi-threads': 1.0.3 + tslib: 2.8.1 + optional: true + + '@emnapi/runtime@1.4.4': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/wasi-threads@1.0.3': + dependencies: + tslib: 2.8.1 + optional: true + '@esbuild/aix-ppc64@0.25.6': optional: true @@ -3135,9 +3313,9 @@ snapshots: '@esbuild/win32-x64@0.25.6': optional: true - '@eslint-community/eslint-utils@4.7.0(eslint@9.31.0)': + '@eslint-community/eslint-utils@4.7.0(eslint@9.31.0(jiti@2.4.2))': dependencies: - eslint: 9.31.0 + eslint: 9.31.0(jiti@2.4.2) eslint-visitor-keys: 3.4.3 '@eslint-community/regexpp@4.12.1': {} @@ -3262,6 +3440,13 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.4 + '@napi-rs/wasm-runtime@0.2.12': + dependencies: + '@emnapi/core': 1.4.4 + '@emnapi/runtime': 1.4.4 + '@tybys/wasm-util': 0.10.0 + optional: true + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -3274,6 +3459,65 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.19.1 + '@oxc-resolver/binding-android-arm-eabi@11.5.2': + optional: true + + '@oxc-resolver/binding-android-arm64@11.5.2': + optional: true + + '@oxc-resolver/binding-darwin-arm64@11.5.2': + optional: true + + '@oxc-resolver/binding-darwin-x64@11.5.2': + optional: true + + '@oxc-resolver/binding-freebsd-x64@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-arm-gnueabihf@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-arm-musleabihf@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-arm64-gnu@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-arm64-musl@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-ppc64-gnu@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-riscv64-gnu@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-riscv64-musl@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-s390x-gnu@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-x64-gnu@11.5.2': + optional: true + + '@oxc-resolver/binding-linux-x64-musl@11.5.2': + optional: true + + '@oxc-resolver/binding-wasm32-wasi@11.5.2': + dependencies: + '@napi-rs/wasm-runtime': 0.2.12 + optional: true + + '@oxc-resolver/binding-win32-arm64-msvc@11.5.2': + optional: true + + '@oxc-resolver/binding-win32-ia32-msvc@11.5.2': + optional: true + + '@oxc-resolver/binding-win32-x64-msvc@11.5.2': + optional: true + '@pkgjs/parseargs@0.11.0': optional: true @@ -3411,6 +3655,11 @@ snapshots: '@tsconfig/strictest@2.0.5': {} + '@tybys/wasm-util@0.10.0': + dependencies: + tslib: 2.8.1 + optional: true + '@types/cacheable-request@6.0.3': dependencies: '@types/http-cache-semantics': 4.0.4 @@ -3428,11 +3677,6 @@ snapshots: '@types/deep-eql@4.0.2': {} - '@types/eslint@9.6.1': - dependencies: - '@types/estree': 1.0.8 - '@types/json-schema': 7.0.15 - '@types/estree@1.0.8': {} '@types/fuzzysearch@1.0.2': {} @@ -3441,15 +3685,20 @@ snapshots: '@types/http-cache-semantics@4.0.4': {} - '@types/inquirer-autocomplete-prompt@3.0.3': + '@types/inquirer-autocomplete-prompt@1.3.5': dependencies: - '@types/inquirer': 7.3.3 + '@types/inquirer': 8.2.11 '@types/inquirer@7.3.3': dependencies: '@types/through': 0.0.33 rxjs: 6.6.7 + '@types/inquirer@8.2.11': + dependencies: + '@types/through': 0.0.33 + rxjs: 7.8.2 + '@types/js-yaml@4.0.9': {} '@types/json-schema@7.0.15': {} @@ -3503,15 +3752,15 @@ snapshots: '@types/yargs-parser@21.0.3': {} - '@typescript-eslint/eslint-plugin@8.37.0(@typescript-eslint/parser@8.37.0(eslint@9.31.0)(typescript@5.8.3))(eslint@9.31.0)(typescript@5.8.3)': + '@typescript-eslint/eslint-plugin@8.37.0(@typescript-eslint/parser@8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3))(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3)': dependencies: '@eslint-community/regexpp': 4.12.1 - '@typescript-eslint/parser': 8.37.0(eslint@9.31.0)(typescript@5.8.3) + '@typescript-eslint/parser': 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) '@typescript-eslint/scope-manager': 8.37.0 - '@typescript-eslint/type-utils': 8.37.0(eslint@9.31.0)(typescript@5.8.3) - '@typescript-eslint/utils': 8.37.0(eslint@9.31.0)(typescript@5.8.3) + '@typescript-eslint/type-utils': 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) + '@typescript-eslint/utils': 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) '@typescript-eslint/visitor-keys': 8.37.0 - eslint: 9.31.0 + eslint: 9.31.0(jiti@2.4.2) graphemer: 1.4.0 ignore: 7.0.5 natural-compare: 1.4.0 @@ -3520,14 +3769,14 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.37.0(eslint@9.31.0)(typescript@5.8.3)': + '@typescript-eslint/parser@8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3)': dependencies: '@typescript-eslint/scope-manager': 8.37.0 '@typescript-eslint/types': 8.37.0 '@typescript-eslint/typescript-estree': 8.37.0(typescript@5.8.3) '@typescript-eslint/visitor-keys': 8.37.0 debug: 4.4.1 - eslint: 9.31.0 + eslint: 9.31.0(jiti@2.4.2) typescript: 5.8.3 transitivePeerDependencies: - supports-color @@ -3550,13 +3799,13 @@ snapshots: dependencies: typescript: 5.8.3 - '@typescript-eslint/type-utils@8.37.0(eslint@9.31.0)(typescript@5.8.3)': + '@typescript-eslint/type-utils@8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3)': dependencies: '@typescript-eslint/types': 8.37.0 '@typescript-eslint/typescript-estree': 8.37.0(typescript@5.8.3) - '@typescript-eslint/utils': 8.37.0(eslint@9.31.0)(typescript@5.8.3) + '@typescript-eslint/utils': 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) debug: 4.4.1 - eslint: 9.31.0 + eslint: 9.31.0(jiti@2.4.2) ts-api-utils: 2.1.0(typescript@5.8.3) typescript: 5.8.3 transitivePeerDependencies: @@ -3580,13 +3829,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.37.0(eslint@9.31.0)(typescript@5.8.3)': + '@typescript-eslint/utils@8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3)': dependencies: - '@eslint-community/eslint-utils': 4.7.0(eslint@9.31.0) + '@eslint-community/eslint-utils': 4.7.0(eslint@9.31.0(jiti@2.4.2)) '@typescript-eslint/scope-manager': 8.37.0 '@typescript-eslint/types': 8.37.0 '@typescript-eslint/typescript-estree': 8.37.0(typescript@5.8.3) - eslint: 9.31.0 + eslint: 9.31.0(jiti@2.4.2) typescript: 5.8.3 transitivePeerDependencies: - supports-color @@ -3604,13 +3853,13 @@ snapshots: chai: 5.2.0 tinyrainbow: 2.0.0 - '@vitest/mocker@3.2.4(vite@7.0.2(@types/node@22.16.4)(tsx@4.20.3))': + '@vitest/mocker@3.2.4(vite@7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3))': dependencies: '@vitest/spy': 3.2.4 estree-walker: 3.0.3 magic-string: 0.30.17 optionalDependencies: - vite: 7.0.2(@types/node@22.16.4)(tsx@4.20.3) + vite: 7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3) '@vitest/pretty-format@3.2.4': dependencies: @@ -4219,15 +4468,15 @@ snapshots: escape-string-regexp@4.0.0: {} - eslint-plugin-unicorn@59.0.1(eslint@9.31.0): + eslint-plugin-unicorn@59.0.1(eslint@9.31.0(jiti@2.4.2)): dependencies: '@babel/helper-validator-identifier': 7.27.1 - '@eslint-community/eslint-utils': 4.7.0(eslint@9.31.0) + '@eslint-community/eslint-utils': 4.7.0(eslint@9.31.0(jiti@2.4.2)) '@eslint/plugin-kit': 0.2.8 ci-info: 4.3.0 clean-regexp: 1.0.0 core-js-compat: 3.44.0 - eslint: 9.31.0 + eslint: 9.31.0(jiti@2.4.2) esquery: 1.6.0 find-up-simple: 1.0.1 globals: 16.3.0 @@ -4249,9 +4498,9 @@ snapshots: eslint-visitor-keys@4.2.1: {} - eslint@9.31.0: + eslint@9.31.0(jiti@2.4.2): dependencies: - '@eslint-community/eslint-utils': 4.7.0(eslint@9.31.0) + '@eslint-community/eslint-utils': 4.7.0(eslint@9.31.0(jiti@2.4.2)) '@eslint-community/regexpp': 4.12.1 '@eslint/config-array': 0.21.0 '@eslint/config-helpers': 0.3.0 @@ -4286,6 +4535,8 @@ snapshots: minimatch: 3.1.2 natural-compare: 1.4.0 optionator: 0.9.4 + optionalDependencies: + jiti: 2.4.2 transitivePeerDependencies: - supports-color @@ -4358,6 +4609,10 @@ snapshots: dependencies: format: 0.2.2 + fd-package-json@2.0.0: + dependencies: + walk-up-path: 4.0.0 + fdir@6.4.6(picomatch@4.0.2): optionalDependencies: picomatch: 4.0.2 @@ -4421,6 +4676,10 @@ snapshots: format@0.2.2: {} + formatly@0.2.4: + dependencies: + fd-package-json: 2.0.0 + fp-ts@2.16.10: {} fsevents@2.3.3: @@ -4813,6 +5072,8 @@ snapshots: optionalDependencies: '@pkgjs/parseargs': 0.11.0 + jiti@2.4.2: {} + joycon@3.1.1: {} js-tokens@4.0.0: {} @@ -4874,6 +5135,24 @@ snapshots: dependencies: json-buffer: 3.0.1 + knip@5.61.3(@types/node@22.16.4)(typescript@5.8.3): + dependencies: + '@nodelib/fs.walk': 1.2.8 + '@types/node': 22.16.4 + fast-glob: 3.3.3 + formatly: 0.2.4 + jiti: 2.4.2 + js-yaml: 4.1.0 + minimist: 1.2.8 + oxc-resolver: 11.5.2 + picocolors: 1.1.1 + picomatch: 4.0.2 + smol-toml: 1.4.1 + strip-json-comments: 5.0.2 + typescript: 5.8.3 + zod: 3.25.76 + zod-validation-error: 3.5.3(zod@3.25.76) + levn@0.4.1: dependencies: prelude-ls: 1.2.1 @@ -5146,6 +5425,8 @@ snapshots: nanoid@3.3.11: {} + napi-postinstall@0.3.0: {} + natural-compare@1.4.0: {} neo-async@2.6.2: {} @@ -5208,6 +5489,30 @@ snapshots: object-keys: 1.1.1 safe-push-apply: 1.0.0 + oxc-resolver@11.5.2: + dependencies: + napi-postinstall: 0.3.0 + optionalDependencies: + '@oxc-resolver/binding-android-arm-eabi': 11.5.2 + '@oxc-resolver/binding-android-arm64': 11.5.2 + '@oxc-resolver/binding-darwin-arm64': 11.5.2 + '@oxc-resolver/binding-darwin-x64': 11.5.2 + '@oxc-resolver/binding-freebsd-x64': 11.5.2 + '@oxc-resolver/binding-linux-arm-gnueabihf': 11.5.2 + '@oxc-resolver/binding-linux-arm-musleabihf': 11.5.2 + '@oxc-resolver/binding-linux-arm64-gnu': 11.5.2 + '@oxc-resolver/binding-linux-arm64-musl': 11.5.2 + '@oxc-resolver/binding-linux-ppc64-gnu': 11.5.2 + '@oxc-resolver/binding-linux-riscv64-gnu': 11.5.2 + '@oxc-resolver/binding-linux-riscv64-musl': 11.5.2 + '@oxc-resolver/binding-linux-s390x-gnu': 11.5.2 + '@oxc-resolver/binding-linux-x64-gnu': 11.5.2 + '@oxc-resolver/binding-linux-x64-musl': 11.5.2 + '@oxc-resolver/binding-wasm32-wasi': 11.5.2 + '@oxc-resolver/binding-win32-arm64-msvc': 11.5.2 + '@oxc-resolver/binding-win32-ia32-msvc': 11.5.2 + '@oxc-resolver/binding-win32-x64-msvc': 11.5.2 + p-cancelable@2.1.1: {} p-limit@3.1.0: @@ -5298,10 +5603,11 @@ snapshots: possible-typed-array-names@1.1.0: {} - postcss-load-config@6.0.1(postcss@8.5.6)(tsx@4.20.3): + postcss-load-config@6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.20.3): dependencies: lilconfig: 3.1.3 optionalDependencies: + jiti: 2.4.2 postcss: 8.5.6 tsx: 4.20.3 @@ -5478,6 +5784,10 @@ snapshots: dependencies: tslib: 1.14.1 + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + sade@1.8.1: dependencies: mri: 1.2.0 @@ -5581,6 +5891,8 @@ snapshots: signal-exit@4.1.0: {} + smol-toml@1.4.1: {} + snake-case@3.0.4: dependencies: dot-case: 3.0.4 @@ -5660,6 +5972,8 @@ snapshots: strip-json-comments@3.1.1: {} + strip-json-comments@5.0.2: {} + strip-literal@3.0.0: dependencies: js-tokens: 9.0.1 @@ -5743,7 +6057,7 @@ snapshots: tslib@2.8.1: {} - tsup@8.5.0(postcss@8.5.6)(tsx@4.20.3)(typescript@5.8.3): + tsup@8.5.0(jiti@2.4.2)(postcss@8.5.6)(tsx@4.20.3)(typescript@5.8.3): dependencies: bundle-require: 5.1.0(esbuild@0.25.6) cac: 6.7.14 @@ -5754,7 +6068,7 @@ snapshots: fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(postcss@8.5.6)(tsx@4.20.3) + postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.20.3) resolve-from: 5.0.0 rollup: 4.44.2 source-map: 0.8.0-beta.0 @@ -5830,13 +6144,13 @@ snapshots: typed-array-buffer: 1.0.3 typed-array-byte-offset: 1.0.4 - typescript-eslint@8.37.0(eslint@9.31.0)(typescript@5.8.3): + typescript-eslint@8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3): dependencies: - '@typescript-eslint/eslint-plugin': 8.37.0(@typescript-eslint/parser@8.37.0(eslint@9.31.0)(typescript@5.8.3))(eslint@9.31.0)(typescript@5.8.3) - '@typescript-eslint/parser': 8.37.0(eslint@9.31.0)(typescript@5.8.3) + '@typescript-eslint/eslint-plugin': 8.37.0(@typescript-eslint/parser@8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3))(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) + '@typescript-eslint/parser': 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) '@typescript-eslint/typescript-estree': 8.37.0(typescript@5.8.3) - '@typescript-eslint/utils': 8.37.0(eslint@9.31.0)(typescript@5.8.3) - eslint: 9.31.0 + '@typescript-eslint/utils': 8.37.0(eslint@9.31.0(jiti@2.4.2))(typescript@5.8.3) + eslint: 9.31.0(jiti@2.4.2) typescript: 5.8.3 transitivePeerDependencies: - supports-color @@ -5916,13 +6230,13 @@ snapshots: unist-util-stringify-position: 2.0.3 vfile-message: 2.0.4 - vite-node@3.2.4(@types/node@22.16.4)(tsx@4.20.3): + vite-node@3.2.4(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3): dependencies: cac: 6.7.14 debug: 4.4.1 es-module-lexer: 1.7.0 pathe: 2.0.3 - vite: 7.0.2(@types/node@22.16.4)(tsx@4.20.3) + vite: 7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3) transitivePeerDependencies: - '@types/node' - jiti @@ -5937,18 +6251,18 @@ snapshots: - tsx - yaml - vite-tsconfig-paths@5.1.4(typescript@5.8.3)(vite@7.0.2(@types/node@22.16.4)(tsx@4.20.3)): + vite-tsconfig-paths@5.1.4(typescript@5.8.3)(vite@7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3)): dependencies: debug: 4.4.1 globrex: 0.1.2 tsconfck: 3.1.6(typescript@5.8.3) optionalDependencies: - vite: 7.0.2(@types/node@22.16.4)(tsx@4.20.3) + vite: 7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3) transitivePeerDependencies: - supports-color - typescript - vite@7.0.2(@types/node@22.16.4)(tsx@4.20.3): + vite@7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3): dependencies: esbuild: 0.25.6 fdir: 6.4.6(picomatch@4.0.2) @@ -5959,13 +6273,14 @@ snapshots: optionalDependencies: '@types/node': 22.16.4 fsevents: 2.3.3 + jiti: 2.4.2 tsx: 4.20.3 - vitest@3.2.4(@types/node@22.16.4)(tsx@4.20.3): + vitest@3.2.4(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3): dependencies: '@types/chai': 5.2.2 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@7.0.2(@types/node@22.16.4)(tsx@4.20.3)) + '@vitest/mocker': 3.2.4(vite@7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 @@ -5983,8 +6298,8 @@ snapshots: tinyglobby: 0.2.14 tinypool: 1.1.1 tinyrainbow: 2.0.0 - vite: 7.0.2(@types/node@22.16.4)(tsx@4.20.3) - vite-node: 3.2.4(@types/node@22.16.4)(tsx@4.20.3) + vite: 7.0.2(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3) + vite-node: 3.2.4(@types/node@22.16.4)(jiti@2.4.2)(tsx@4.20.3) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 22.16.4 @@ -6002,6 +6317,8 @@ snapshots: - tsx - yaml + walk-up-path@4.0.0: {} + webidl-conversions@3.0.1: {} webidl-conversions@4.0.2: {} @@ -6109,4 +6426,10 @@ snapshots: yocto-queue@0.1.0: {} + zod-validation-error@3.5.3(zod@3.25.76): + dependencies: + zod: 3.25.76 + + zod@3.25.76: {} + zwitch@1.0.5: {} diff --git a/scripts/buildPathfinderJsonSchema.ts b/scripts/buildPathfinderJsonSchema.ts index ba22da89..2bf96766 100644 --- a/scripts/buildPathfinderJsonSchema.ts +++ b/scripts/buildPathfinderJsonSchema.ts @@ -13,7 +13,7 @@ */ import { writeFileSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import { toJsonSchema } from '@transcend-io/type-utils'; import { PathfinderPolicy } from '../src/codecs'; @@ -30,6 +30,9 @@ const jsonSchema = { ...toJsonSchema(PathfinderPolicy, true), }; -const schemaFilePath = join(process.cwd(), 'pathfinder-policy-yml-schema.json'); +const schemaFilePath = path.join( + process.cwd(), + 'pathfinder-policy-yml-schema.json', +); -writeFileSync(schemaFilePath, `${JSON.stringify(jsonSchema, null, 2)}\n`); +writeFileSync(schemaFilePath, `${JSON.stringify(jsonSchema, undefined, 2)}\n`); diff --git a/scripts/buildReadmeDocs.ts b/scripts/buildReadme.ts similarity index 65% rename from scripts/buildReadmeDocs.ts rename to scripts/buildReadme.ts index c02b9cb9..0dcc1fe7 100644 --- a/scripts/buildReadmeDocs.ts +++ b/scripts/buildReadme.ts @@ -14,17 +14,20 @@ const documentFiles = new fdir() .crawl('./src/commands') .sync(); -// For each src/commands/**/readme.ts file, create a key-value pair of the command and the exported Markdown documentation -const additionalDocumentation: Record = Object.fromEntries( - await Promise.all( - documentFiles.map(async (file) => { - const command = `transcend ${file.split('/').slice(0, -1).join(' ')}`; - const readme = (await import(`../src/commands/${file}`)).default; - return [command, readme]; - }), - ), +const entriesOfAdditionalDocumentation: [string, string][] = await Promise.all( + documentFiles.map(async (file) => { + const command = `transcend ${file.split('/').slice(0, -1).join(' ')}`; + const { default: readme } = (await import(`../src/commands/${file}`)) as { + default: string; + }; + return [command, readme]; + }), ); +// For each src/commands/**/readme.ts file, create a key-value pair of the command and the exported Markdown documentation +const additionalDocumentation: Record = + Object.fromEntries(entriesOfAdditionalDocumentation); + const helpTextForAllCommands = generateHelpTextForAllCommands( app as Application, ); @@ -48,7 +51,10 @@ const newReadme = readme.replaceAll( fs.writeFileSync('README.md', newReadme); -execSync('doctoc README.md --title "\n## Table of Contents" --maxlevel 5', { - stdio: 'inherit', -}); -execSync('prettier --write README.md', { stdio: 'inherit' }); +execSync( + 'pnpm exec doctoc README.md --title "\n## Table of Contents" --maxlevel 5', + { + stdio: 'inherit', + }, +); +execSync('pnpm exec prettier --write README.md', { stdio: 'inherit' }); diff --git a/scripts/buildTranscendJsonSchema.ts b/scripts/buildTranscendJsonSchema.ts index 9c00ec6c..da9361af 100644 --- a/scripts/buildTranscendJsonSchema.ts +++ b/scripts/buildTranscendJsonSchema.ts @@ -13,7 +13,7 @@ */ import { writeFileSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import { toJsonSchema } from '@transcend-io/type-utils'; import * as packageJson from '../package.json'; import { TranscendInput } from '../src/codecs'; @@ -35,7 +35,10 @@ for (const key of [`v${majorVersion}`, 'latest']) { // Build the JSON schema from io-ts codec const jsonSchema = { ...schemaDefaults, ...toJsonSchema(TranscendInput) }; - const schemaFilePath = join(process.cwd(), fileName); + const schemaFilePath = path.join(process.cwd(), fileName); - writeFileSync(schemaFilePath, `${JSON.stringify(jsonSchema, null, 2)}\n`); + writeFileSync( + schemaFilePath, + `${JSON.stringify(jsonSchema, undefined, 2)}\n`, + ); } diff --git a/src/commands/consent/pull-consent-metrics/impl.ts b/src/commands/consent/pull-consent-metrics/impl.ts index 0b1c58b2..4b6c3501 100644 --- a/src/commands/consent/pull-consent-metrics/impl.ts +++ b/src/commands/consent/pull-consent-metrics/impl.ts @@ -1,5 +1,5 @@ import fs, { existsSync, mkdirSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import colors from 'colors'; import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; import type { LocalContext } from '../../../context'; @@ -116,7 +116,7 @@ export async function pullConsentMetrics( // Write to file for (const [metricName, metrics] of Object.entries(configuration)) { for (const { points, name } of metrics) { - const file = join(folder, `${metricName}_${name}.csv`); + const file = path.join(folder, `${metricName}_${name}.csv`); logger.info( colors.magenta(`Writing configuration to file "${file}"...`), ); @@ -165,7 +165,7 @@ export async function pullConsentMetrics( }); // ensure folder exists for that organization - const subFolder = join(folder, apiKey.organizationName); + const subFolder = path.join(folder, apiKey.organizationName); if (!existsSync(subFolder)) { mkdirSync(subFolder); } @@ -173,7 +173,7 @@ export async function pullConsentMetrics( // Write to file for (const [metricName, metrics] of Object.entries(configuration)) { for (const { points, name } of metrics) { - const file = join(subFolder, `${metricName}_${name}.csv`); + const file = path.join(subFolder, `${metricName}_${name}.csv`); logger.info( colors.magenta(`Writing configuration to file "${file}"...`), ); diff --git a/src/commands/consent/upload-preferences/impl.ts b/src/commands/consent/upload-preferences/impl.ts index d8c64d03..e0e903cf 100644 --- a/src/commands/consent/upload-preferences/impl.ts +++ b/src/commands/consent/upload-preferences/impl.ts @@ -1,5 +1,5 @@ import { readdirSync } from 'node:fs'; -import { basename, join } from 'node:path'; +import path from 'node:path'; import colors from 'colors'; import type { LocalContext } from '../../../context'; import { map } from '../../../lib/bluebird-replace'; @@ -79,7 +79,7 @@ export async function uploadPreferences( } // Add full paths for each CSV file - files.push(...csvFiles.map((file) => join(directory, file))); + files.push(...csvFiles.map((file) => path.join(directory, file))); } catch (error) { logger.error(colors.red(`Failed to read directory: ${directory}`)); logger.error(colors.red((error as Error).message)); @@ -118,9 +118,9 @@ export async function uploadPreferences( await map( files, async (filePath) => { - const fileName = basename(filePath).replace('.csv', ''); + const fileName = path.basename(filePath).replace('.csv', ''); await uploadPreferenceManagementPreferencesInteractive({ - receiptFilepath: join(receiptFileDir, `${fileName}-receipts.json`), + receiptFilepath: path.join(receiptFileDir, `${fileName}-receipts.json`), auth, sombraAuth, file: filePath, diff --git a/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts b/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts index 28ab8b64..d846f548 100644 --- a/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts +++ b/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts @@ -33,7 +33,7 @@ export function consentManagerServiceJsonToYml( // Read in each consent manager configuration const services = decodeCodec( t.array(ConsentManagerServiceMetadata), - readFileSync(file, 'utf-8'), + readFileSync(file, 'utf8'), ); // Create data flows and cookie configurations diff --git a/src/commands/inventory/consent-managers-to-business-entities/impl.ts b/src/commands/inventory/consent-managers-to-business-entities/impl.ts index e379557a..1913f227 100644 --- a/src/commands/inventory/consent-managers-to-business-entities/impl.ts +++ b/src/commands/inventory/consent-managers-to-business-entities/impl.ts @@ -1,5 +1,5 @@ import { existsSync, lstatSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import colors from 'colors'; import type { LocalContext } from '../../../context'; import { listFiles } from '../../../lib/api-keys'; @@ -36,7 +36,7 @@ export function consentManagersToBusinessEntities( // Read in each consent manager configuration const inputs = listFiles(consentManagerYmlFolder).map((directory) => { const { 'consent-manager': consentManager } = readTranscendYaml( - join(consentManagerYmlFolder, directory), + path.join(consentManagerYmlFolder, directory), ); return { name: directory, input: consentManager }; }); diff --git a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts index 9b29f0ba..2a61680b 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts @@ -1,8 +1,6 @@ import { existsSync, lstatSync } from 'node:fs'; -import { join } from 'node:path'; -import colors from 'colors'; +import path from 'node:path'; import { difference } from 'lodash-es'; -import { DataFlowInput } from '../../../codecs'; import type { LocalContext } from '../../../context'; import { listFiles } from '../../../lib/api-keys'; import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; @@ -36,12 +34,9 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( ): Promise { // Ensure folder is passed to dataFlowsYmlFolder if (!dataFlowsYmlFolder) { - logger.error( - colors.red( - 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', - ), + throw new Error( + 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', ); - process.exit(1); } // Ensure folder is passed @@ -49,8 +44,7 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( !existsSync(dataFlowsYmlFolder) || !lstatSync(dataFlowsYmlFolder).isDirectory() ) { - logger.error(colors.red(`Folder does not exist: "${dataFlowsYmlFolder}"`)); - process.exit(1); + throw new Error(`Folder does not exist: "${dataFlowsYmlFolder}"`); } // Ignore the data flows in these yml files @@ -60,7 +54,7 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( const dataSiloInputs = listFiles(dataFlowsYmlFolder).map((directory) => { // read in the data flows for a specific instance const { 'data-flows': dataFlows = [] } = readTranscendYaml( - join(dataFlowsYmlFolder, directory), + path.join(dataFlowsYmlFolder, directory), ); // map the data flows to data silos @@ -88,11 +82,9 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( } of dataSiloInputs) { const allDataSilos = [...adTechDataSilos, ...siteTechDataSilos]; for (const dataSilo of allDataSilos) { - const service = dataSilo['outer-type'] || dataSilo.integrationName; + const service = dataSilo['outer-type'] ?? dataSilo.integrationName; // create mapping to instance - if (!serviceToInstance[service]) { - serviceToInstance[service] = []; - } + serviceToInstance[service] ??= []; serviceToInstance[service].push(organizationName); serviceToInstance[service] = [...new Set(serviceToInstance[service])]; } @@ -103,7 +95,7 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( ...new Set( dataSiloInputs.flatMap(({ adTechDataSilos }) => adTechDataSilos.map( - (silo) => silo['outer-type'] || silo.integrationName, + (silo) => silo['outer-type'] ?? silo.integrationName, ), ), ), @@ -115,7 +107,7 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( ...new Set( dataSiloInputs.flatMap(({ siteTechDataSilos }) => siteTechDataSilos.map( - (silo) => silo['outer-type'] || silo.integrationName, + (silo) => silo['outer-type'] ?? silo.integrationName, ), ), ), @@ -128,15 +120,13 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( for (const { adTechDataSilos, siteTechDataSilos } of dataSiloInputs) { const allDataSilos = [...adTechDataSilos, ...siteTechDataSilos]; for (const dataSilo of allDataSilos) { - const service = dataSilo['outer-type'] || dataSilo.integrationName; + const service = dataSilo['outer-type'] ?? dataSilo.integrationName; const foundOnDomain = dataSilo.attributes?.find( (attribute) => attribute.key === 'Found On Domain', ); // create mapping to instance - if (!serviceToFoundOnDomain[service]) { - serviceToFoundOnDomain[service] = []; - } - serviceToFoundOnDomain[service].push(...(foundOnDomain?.values || [])); + serviceToFoundOnDomain[service] ??= []; + serviceToFoundOnDomain[service].push(...(foundOnDomain?.values ?? [])); serviceToFoundOnDomain[service] = [ ...new Set(serviceToFoundOnDomain[service]), ]; @@ -163,22 +153,27 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( { key: 'Business Units', values: difference( - serviceToInstance[service] || [], + service in serviceToInstance ? serviceToInstance[service] : [], instancesToIgnore, ), }, { key: 'Found On Domain', - values: serviceToFoundOnDomain[service] || [], + values: + service in serviceToFoundOnDomain + ? serviceToFoundOnDomain[service] + : [], }, ], }), ); // Log output - logger.log(`Total Services: ${dataSilos.length}`); - logger.log(`Ad Tech Services: ${adTechIntegrations.length}`); - logger.log(`Site Tech Services: ${siteTechIntegrations.length}`); + logger.log(`Total Services: ${dataSilos.length.toLocaleString()}`); + logger.log(`Ad Tech Services: ${adTechIntegrations.length.toLocaleString()}`); + logger.log( + `Site Tech Services: ${siteTechIntegrations.length.toLocaleString()}`, + ); // Write to yaml writeTranscendYaml(output, { diff --git a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts index 27bb7b51..0c41c7a6 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts @@ -1,7 +1,5 @@ import { existsSync, lstatSync } from 'node:fs'; -import { join } from 'node:path'; -import colors from 'colors'; -import { DataFlowInput } from '../../../codecs'; +import path from 'node:path'; import type { LocalContext } from '../../../context'; import { listFiles } from '../../../lib/api-keys'; import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; @@ -35,12 +33,9 @@ export async function deriveDataSilosFromDataFlows( ): Promise { // Ensure folder is passed to dataFlowsYmlFolder if (!dataFlowsYmlFolder) { - logger.error( - colors.red( - 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', - ), + throw new Error( + 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', ); - process.exit(1); } // Ensure folder is passed @@ -48,18 +43,14 @@ export async function deriveDataSilosFromDataFlows( !existsSync(dataFlowsYmlFolder) || !lstatSync(dataFlowsYmlFolder).isDirectory() ) { - logger.error(colors.red(`Folder does not exist: "${dataFlowsYmlFolder}"`)); - process.exit(1); + throw new Error(`Folder does not exist: "${dataFlowsYmlFolder}"`); } // Ensure folder is passed to dataSilosYmlFolder if (!dataSilosYmlFolder) { - logger.error( - colors.red( - 'Missing required arg: --dataSilosYmlFolder=./working/data-silos/', - ), + throw new Error( + 'Missing required arg: --dataSilosYmlFolder=./working/data-silos/', ); - process.exit(1); } // Ensure folder is passed @@ -67,8 +58,7 @@ export async function deriveDataSilosFromDataFlows( !existsSync(dataSilosYmlFolder) || !lstatSync(dataSilosYmlFolder).isDirectory() ) { - logger.error(colors.red(`Folder does not exist: "${dataSilosYmlFolder}"`)); - process.exit(1); + throw new Error(`Folder does not exist: "${dataSilosYmlFolder}"`); } // Fetch all integrations in the catalog @@ -80,7 +70,7 @@ export async function deriveDataSilosFromDataFlows( for (const directory of listFiles(dataFlowsYmlFolder)) { // read in the data flows for a specific instance const { 'data-flows': dataFlows = [] } = readTranscendYaml( - join(dataFlowsYmlFolder, directory), + path.join(dataFlowsYmlFolder, directory), ); // map the data flows to data silos @@ -94,10 +84,12 @@ export async function deriveDataSilosFromDataFlows( // combine and write to yml file const dataSilos = [...adTechDataSilos, ...siteTechDataSilos]; - logger.log(`Total Services: ${dataSilos.length}`); - logger.log(`Ad Tech Services: ${adTechDataSilos.length}`); - logger.log(`Site Tech Services: ${siteTechDataSilos.length}`); - writeTranscendYaml(join(dataSilosYmlFolder, directory), { + logger.log(`Total Services: ${dataSilos.length.toLocaleString()}`); + logger.log(`Ad Tech Services: ${adTechDataSilos.length.toLocaleString()}`); + logger.log( + `Site Tech Services: ${siteTechDataSilos.length.toLocaleString()}`, + ); + writeTranscendYaml(path.join(dataSilosYmlFolder, directory), { 'data-silos': ignoreYmls.includes(directory) ? [] : dataSilos, }); } diff --git a/src/commands/inventory/pull/impl.ts b/src/commands/inventory/pull/impl.ts index a6802e91..322edf4d 100644 --- a/src/commands/inventory/pull/impl.ts +++ b/src/commands/inventory/pull/impl.ts @@ -1,5 +1,5 @@ import fs from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; import colors from 'colors'; import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; @@ -128,7 +128,7 @@ export async function pull( trackerStatuses, }); - const filePath = join(file, `${apiKey.organizationName}.yml`); + const filePath = path.join(file, `${apiKey.organizationName}.yml`); logger.info( colors.magenta(`Writing configuration to file "${filePath}"...`), ); diff --git a/src/commands/inventory/push/impl.ts b/src/commands/inventory/push/impl.ts index 3ba81b28..bd490ab3 100644 --- a/src/commands/inventory/push/impl.ts +++ b/src/commands/inventory/push/impl.ts @@ -1,5 +1,5 @@ import { existsSync, lstatSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; import colors from 'colors'; import { TranscendInput } from '../../../codecs'; import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; @@ -104,7 +104,7 @@ export async function push( let fileList: string[]; fileList = Array.isArray(apiKeyOrList) && lstatSync(file).isDirectory() - ? listFiles(file).map((filePath) => join(file, filePath)) + ? listFiles(file).map((filePath) => path.join(file, filePath)) : file.split(','); // Ensure at least one file is parsed diff --git a/src/commands/inventory/scan-packages/impl.ts b/src/commands/inventory/scan-packages/impl.ts index 6b6abe12..d036821c 100644 --- a/src/commands/inventory/scan-packages/impl.ts +++ b/src/commands/inventory/scan-packages/impl.ts @@ -40,7 +40,7 @@ export async function scanPackages( `cd ${scanPath} && git config --get remote.origin.url`, ); // Trim and parse the URL - const url = name.toString('utf-8').trim(); + const url = name.toString('utf8').trim(); [gitRepositoryName] = url.includes('https:') ? url.split('/').slice(3).join('/').split('.') : (url.split(':').pop() || '').split('.'); diff --git a/src/lib/@types/fuzzysearch.d.ts b/src/lib/@types/fuzzysearch.d.ts deleted file mode 100644 index 77b581e8..00000000 --- a/src/lib/@types/fuzzysearch.d.ts +++ /dev/null @@ -1,13 +0,0 @@ -declare module 'fuzzysearch' { - /** - * Fuzzy search - * - * @param needle - Needle to search - * @param haystack - Hay to search through - * @returns True if matching - */ - export default function fuzzysearch( - needle: string, - haystack: string, - ): boolean; -} diff --git a/src/lib/@types/inquirer-autocomplete-prompt.d.ts b/src/lib/@types/inquirer-autocomplete-prompt.d.ts deleted file mode 100644 index 0c0db0d3..00000000 --- a/src/lib/@types/inquirer-autocomplete-prompt.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -declare module 'inquirer-autocomplete-prompt' { - export default import('inquirer').PromptConstructor; -} diff --git a/src/lib/ai/TranscendPromptManager.ts b/src/lib/ai/TranscendPromptManager.ts index 86571d25..f1d4db36 100644 --- a/src/lib/ai/TranscendPromptManager.ts +++ b/src/lib/ai/TranscendPromptManager.ts @@ -277,7 +277,9 @@ export class TranscendPromptManager< */ async fetchPromptsAndMetadata(): Promise { // Determine what to fetch - const promptDefinitions = getValues(this.prompts); + const promptDefinitions = getValues>( + this.prompts, + ); const promptIds = promptDefinitions .map(({ id }) => id) .filter((x): x is string => !!x); @@ -285,7 +287,7 @@ export class TranscendPromptManager< .map(({ title }) => title) .filter((x): x is string => !!x); const agentNames = uniq( - promptDefinitions.flatMap(({ agentNames }) => agentNames || []), + promptDefinitions.flatMap(({ agentNames }) => agentNames ?? []), ); // Fetch prompts and data @@ -624,22 +626,28 @@ export class TranscendPromptManager< `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}`, ); } - if ( - options.promptRunMessages.at(-1).role !== ChatCompletionRole.Assistant - ) { + + const lastMessage = options.promptRunMessages.at(-1); + if (!lastMessage) { + throw new Error('promptRunMessages is expected to have length > 0'); + } + if (lastMessage.role !== ChatCompletionRole.Assistant) { throw new Error( - `promptRunMessages[${ + `promptRunMessages[${( options.promptRunMessages.length - 1 - }].role is expected to be = ${ChatCompletionRole.Assistant}`, + ).toString()}].role is expected to be = ${ChatCompletionRole.Assistant}`, ); } - const response = options.promptRunMessages.at(-1).content; + const response = lastMessage.content; let parsed: t.TypeOf; try { // Parse the response parsed = this.parseAiResponse(promptName, response); } catch (error) { + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } await reportPromptRun(this.graphQLClient, { productArea: PromptRunProductArea.PromptManager, ...options, diff --git a/src/lib/ai/getGitFilesThatChanged.ts b/src/lib/ai/getGitFilesThatChanged.ts index 7ccd2034..bd2a4f5d 100644 --- a/src/lib/ai/getGitFilesThatChanged.ts +++ b/src/lib/ai/getGitFilesThatChanged.ts @@ -42,12 +42,12 @@ export function getGitFilesThatChanged({ // Latest commit on base branch. If we are on the base branch, we take the prior commit const latestBasedCommit = execSync( `git ls-remote ${githubRepo} "refs/heads/${baseBranch}" | cut -f 1`, - { encoding: 'utf-8' }, + { encoding: 'utf8' }, ).split('\n')[0]; // This commit const latestThisCommit = execSync('git rev-parse HEAD', { - encoding: 'utf-8', + encoding: 'utf8', }).split('\n')[0]; // Ensure commits are present @@ -60,7 +60,7 @@ export function getGitFilesThatChanged({ `git fetch && git diff --name-only "${ baseBranch || latestBasedCommit }...${latestThisCommit}" -- ${rootDirectory}`, - { encoding: 'utf-8' }, + { encoding: 'utf8' }, ); // Filter out block list @@ -79,7 +79,7 @@ export function getGitFilesThatChanged({ const fileDiffs: Record = {}; for (const file of filteredChanges) { const contents = execSync(`git show ${latestThisCommit}:${file}`, { - encoding: 'utf-8', + encoding: 'utf8', }); fileDiffs[file] = contents; } diff --git a/src/lib/api-keys/listDirectories.ts b/src/lib/api-keys/listDirectories.ts index fb9a29e0..d2ab5e23 100644 --- a/src/lib/api-keys/listDirectories.ts +++ b/src/lib/api-keys/listDirectories.ts @@ -1,5 +1,5 @@ import { readdirSync, statSync } from 'node:fs'; -import { join } from 'node:path'; +import path from 'node:path'; /** * List the folders in a directory @@ -9,6 +9,6 @@ import { join } from 'node:path'; */ export function listDirectories(startDir: string): string[] { return readdirSync(startDir).filter((entryName) => - statSync(join(startDir, entryName)).isDirectory(), + statSync(path.join(startDir, entryName)).isDirectory(), ); } diff --git a/src/lib/api-keys/validateTranscendAuth.ts b/src/lib/api-keys/validateTranscendAuth.ts index 7d69ba4a..9e7a240b 100644 --- a/src/lib/api-keys/validateTranscendAuth.ts +++ b/src/lib/api-keys/validateTranscendAuth.ts @@ -26,7 +26,7 @@ export function validateTranscendAuth(auth: string): string | StoredApiKey[] { // Read from disk if (existsSync(auth)) { // validate that file is a list of API keys - return decodeCodec(t.array(StoredApiKey), readFileSync(auth, 'utf-8')); + return decodeCodec(t.array(StoredApiKey), readFileSync(auth, 'utf8')); } // Return as single API key diff --git a/src/lib/code-scanning/integrations/cocoaPods.ts b/src/lib/code-scanning/integrations/cocoaPods.ts index 30dfebc5..0b6a9109 100644 --- a/src/lib/code-scanning/integrations/cocoaPods.ts +++ b/src/lib/code-scanning/integrations/cocoaPods.ts @@ -11,7 +11,7 @@ export const cocoaPods: CodeScanningConfig = { supportedFiles: ['Podfile'], ignoreDirs: ['Pods'], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, 'utf8'); const targets = findAllWithRegex( { diff --git a/src/lib/code-scanning/integrations/composerJson.ts b/src/lib/code-scanning/integrations/composerJson.ts index 3cf697c5..ff8ede44 100644 --- a/src/lib/code-scanning/integrations/composerJson.ts +++ b/src/lib/code-scanning/integrations/composerJson.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import path from 'node:path'; import { CodePackageSdk } from '../../../codecs'; import { CodeScanningConfig } from '../types'; @@ -7,8 +7,8 @@ export const composerJson: CodeScanningConfig = { supportedFiles: ['composer.json'], ignoreDirs: ['vendor', 'node_modules', 'cache', 'build', 'dist'], scanFunction: (filePath) => { - const file = readFileSync(filePath, 'utf-8'); - const directory = dirname(filePath); + const file = readFileSync(filePath, 'utf8'); + const directory = path.dirname(filePath); const asJson = JSON.parse(file); const { name, diff --git a/src/lib/code-scanning/integrations/gemfile.ts b/src/lib/code-scanning/integrations/gemfile.ts index 28b631df..fd171ae5 100644 --- a/src/lib/code-scanning/integrations/gemfile.ts +++ b/src/lib/code-scanning/integrations/gemfile.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import path from 'node:path'; import { CodePackageType } from '@transcend-io/privacy-types'; import { findAllWithRegex } from '@transcend-io/type-utils'; import { listFiles } from '../../api-keys'; @@ -15,15 +15,13 @@ export const gemfile: CodeScanningConfig = { supportedFiles: ['Gemfile'], ignoreDirs: ['bin'], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); - const directory = dirname(filePath); + const fileContents = readFileSync(filePath, 'utf8'); + const directory = path.dirname(filePath); const filesInFolder = listFiles(directory); // parse gemspec file for name const gemspec = filesInFolder.find((file) => file === '.gemspec'); - const gemspecContents = gemspec - ? readFileSync(gemspec, 'utf-8') - : undefined; + const gemspecContents = gemspec ? readFileSync(gemspec, 'utf8') : undefined; const gemfileName = gemspecContents ? (GEMFILE_PACKAGE_NAME_REGEX.exec(gemspecContents) || [])[2] : undefined; diff --git a/src/lib/code-scanning/integrations/gradle.ts b/src/lib/code-scanning/integrations/gradle.ts index 70983adf..341dde17 100644 --- a/src/lib/code-scanning/integrations/gradle.ts +++ b/src/lib/code-scanning/integrations/gradle.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import path from 'node:path'; import { findAllWithRegex } from '@transcend-io/type-utils'; import { CodeScanningConfig } from '../types'; @@ -28,8 +28,8 @@ export const gradle: CodeScanningConfig = { 'gradle-wrapper.properties', ], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); - const directory = dirname(filePath); + const fileContents = readFileSync(filePath, 'utf8'); + const directory = path.dirname(filePath); const targets = findAllWithRegex( { diff --git a/src/lib/code-scanning/integrations/javascriptPackageJson.ts b/src/lib/code-scanning/integrations/javascriptPackageJson.ts index fe437c32..e01b2a8a 100644 --- a/src/lib/code-scanning/integrations/javascriptPackageJson.ts +++ b/src/lib/code-scanning/integrations/javascriptPackageJson.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import path from 'node:path'; import { CodePackageSdk } from '../../../codecs'; import { CodeScanningConfig } from '../types'; @@ -7,8 +7,8 @@ export const javascriptPackageJson: CodeScanningConfig = { supportedFiles: ['package.json'], ignoreDirs: ['node_modules', 'serverless-build', 'lambda-build'], scanFunction: (filePath) => { - const file = readFileSync(filePath, 'utf-8'); - const directory = dirname(filePath); + const file = readFileSync(filePath, 'utf8'); + const directory = path.dirname(filePath); const asJson = JSON.parse(file); const { name, diff --git a/src/lib/code-scanning/integrations/pubspec.ts b/src/lib/code-scanning/integrations/pubspec.ts index 0f8e1b72..ef4eaf89 100644 --- a/src/lib/code-scanning/integrations/pubspec.ts +++ b/src/lib/code-scanning/integrations/pubspec.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import path from 'node:path'; import { CodePackageType } from '@transcend-io/privacy-types'; import yaml from 'js-yaml'; import { CodeScanningConfig } from '../types'; @@ -33,8 +33,8 @@ export const pubspec: CodeScanningConfig = { supportedFiles: ['pubspec.yml'], ignoreDirs: ['build'], scanFunction: (filePath) => { - const directory = dirname(filePath); - const fileContents = readFileSync(filePath, 'utf-8'); + const directory = path.dirname(filePath); + const fileContents = readFileSync(filePath, 'utf8'); const { name, description, diff --git a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts index fa119fe9..8e01c59a 100644 --- a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts +++ b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; +import path from 'node:path'; import { CodePackageType } from '@transcend-io/privacy-types'; import { findAllWithRegex } from '@transcend-io/type-utils'; import { listFiles } from '../../api-keys'; @@ -13,20 +13,20 @@ export const pythonRequirementsTxt: CodeScanningConfig = { supportedFiles: ['requirements.txt'], ignoreDirs: ['build', 'lib', 'lib64'], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); - const directory = dirname(filePath); + const fileContents = readFileSync(filePath, 'utf8'); + const directory = path.dirname(filePath); const filesInFolder = listFiles(directory); // parse setup file for name const setupFile = filesInFolder.find((file) => file === 'setup.py'); const setupFileContents = setupFile - ? readFileSync(join(directory, setupFile), 'utf-8') + ? readFileSync(path.join(directory, setupFile), 'utf8') : undefined; const packageName = setupFileContents - ? (PACKAGE_NAME.exec(setupFileContents) || [])[2] + ? (PACKAGE_NAME.exec(setupFileContents) ?? [])[2] : undefined; const packageDescription = setupFileContents - ? (PACKAGE_DESCRIPTION.exec(setupFileContents) || [])[2] + ? (PACKAGE_DESCRIPTION.exec(setupFileContents) ?? [])[2] : undefined; const targets = findAllWithRegex( @@ -39,7 +39,9 @@ export const pythonRequirementsTxt: CodeScanningConfig = { return [ { - name: packageName || directory.split('/').pop()!, + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + name: packageName || path.basename(directory), + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing description: packageDescription || undefined, type: CodePackageType.RequirementsTxt, softwareDevelopmentKits: targets.map((package_) => ({ diff --git a/src/lib/code-scanning/integrations/swift.ts b/src/lib/code-scanning/integrations/swift.ts index df1f1804..9379cd30 100644 --- a/src/lib/code-scanning/integrations/swift.ts +++ b/src/lib/code-scanning/integrations/swift.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { dirname } from 'node:path'; +import path from 'node:path'; import { CodePackageType } from '@transcend-io/privacy-types'; import { decodeCodec } from '@transcend-io/type-utils'; import * as t from 'io-ts'; @@ -24,13 +24,13 @@ export const swift: CodeScanningConfig = { supportedFiles: ['Package.resolved'], ignoreDirs: [], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, 'utf8'); const parsed = decodeCodec(SwiftPackage, fileContents); return [ { - name: dirname(filePath).split('/').pop() || '', // FIXME pull from Package.swift ->> name if possible + name: path.dirname(filePath).split('/').pop() || '', // FIXME pull from Package.swift ->> name if possible type: CodePackageType.CocoaPods, // FIXME should be swift softwareDevelopmentKits: parsed.pins.map((target) => ({ name: target.identity, diff --git a/src/lib/consent-manager/types.ts b/src/lib/consent-manager/types.ts index f538b5f9..265d33c1 100644 --- a/src/lib/consent-manager/types.ts +++ b/src/lib/consent-manager/types.ts @@ -1,6 +1,6 @@ import * as t from 'io-ts'; -export const ConsentPreferenceBase = t.intersection([ +const ConsentPreferenceBase = t.intersection([ t.type({ /** User ID */ userId: t.string, @@ -28,7 +28,7 @@ export const ConsentPreferenceBase = t.intersection([ ]); /** Type override */ -export type ConsentPreferenceBase = t.TypeOf; +type ConsentPreferenceBase = t.TypeOf; export const ConsentPreferenceUpload = t.intersection([ ConsentPreferenceBase, diff --git a/src/lib/consent-manager/uploadConsents.ts b/src/lib/consent-manager/uploadConsents.ts index c682baca..4845103c 100644 --- a/src/lib/consent-manager/uploadConsents.ts +++ b/src/lib/consent-manager/uploadConsents.ts @@ -6,7 +6,10 @@ import * as t from 'io-ts'; import { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants'; import { logger } from '../../logger'; import { map } from '../bluebird-replace'; -import { createTranscendConsentGotInstance } from '../graphql'; +import { + createTranscendConsentGotInstance, + isTranscendConsentError, +} from '../graphql'; import { createConsentToken } from './createConsentToken'; import type { ConsentPreferenceUpload } from './types'; @@ -100,7 +103,7 @@ export async function uploadConsents({ logger.info( colors.magenta( - `Uploading ${preferences.length} user preferences to partition ${partition}`, + `Uploading ${preferences.length.toLocaleString()} user preferences to partition ${partition}`, ), ); @@ -133,7 +136,7 @@ export async function uploadConsents({ // parse usp string const [, saleStatus] = consent.usp - ? USP_STRING_REGEX.exec(consent.usp) || [] + ? (USP_STRING_REGEX.exec(consent.usp) ?? []) : []; const input = { @@ -160,17 +163,27 @@ export async function uploadConsents({ }) .json(); } catch (error) { - try { - const parsed = JSON.parse(error?.response?.body || '{}'); - if (parsed.error) { - logger.error(colors.red(`Error: ${parsed.error}`)); + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + + if (isTranscendConsentError(error)) { + try { + const parsed = JSON.parse(error.response.body) as Record< + string, + unknown + >; + if ('error' in parsed && typeof parsed.error === 'string') { + logger.error(colors.red(`Error: ${parsed.error}`)); + } + } catch { + // continue } - } catch { - // continue } + throw new Error( `Received an error from server: ${ - error?.response?.body || error?.message + isTranscendConsentError(error) ? error.response.body : error.message }`, ); } @@ -187,11 +200,11 @@ export async function uploadConsents({ logger.info( colors.green( - `Successfully uploaded ${ - preferences.length - } user preferences to partition ${partition} in "${ + `Successfully uploaded ${preferences.length.toLocaleString()} user preferences to partition ${partition} in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); } diff --git a/src/lib/cron/markCronIdentifierCompleted.ts b/src/lib/cron/markCronIdentifierCompleted.ts index 47b70c9b..6da7c4a6 100644 --- a/src/lib/cron/markCronIdentifierCompleted.ts +++ b/src/lib/cron/markCronIdentifierCompleted.ts @@ -1,5 +1,6 @@ -import type { Got } from 'got'; +import { RequestError, type Got } from 'got'; import * as t from 'io-ts'; +import { isSombraError } from '../graphql'; /** * Minimal set required to mark as completed @@ -41,12 +42,17 @@ export async function markCronIdentifierCompleted( return true; } catch (error) { // handle gracefully - if (error.response?.statusCode === 409) { + if (error instanceof RequestError && error.response?.statusCode === 409) { return false; } + + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + throw new Error( `Received an error from server: ${ - error?.response?.body || error?.message + isSombraError(error) ? error.response.body : error.message }`, ); } diff --git a/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts b/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts deleted file mode 100644 index 607d9134..00000000 --- a/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { RequestAction } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { - buildTranscendGraphQLClient, - createSombraGotInstance, - fetchRequestDataSiloActiveCount, -} from '../graphql'; -import { - CronIdentifier, - pullCronPageOfIdentifiers, -} from './pullCronPageOfIdentifiers'; - -/** - * A CSV formatted identifier - */ -export type CsvFormattedIdentifier = Record< - string, - string | null | boolean | number ->; - -export interface CronIdentifierWithAction extends CronIdentifier { - /** The request action that the identifier relates to */ - action: RequestAction; -} - -/** - * Pull the set of identifiers outstanding for a cron or AVC integration - * - * This function is designed to be used in a loop, and will call the onSave callback - * with a chunk of identifiers when the savePageSize is reached. - * - * @param options - Options - * @returns The identifiers and identifiers formatted for CSV - */ -export async function pullChunkedCustomSiloOutstandingIdentifiers({ - dataSiloId, - auth, - sombraAuth, - actions, - apiPageSize = 100, - savePageSize = 1000, - onSave, - transcendUrl = DEFAULT_TRANSCEND_API, - skipRequestCount = false, -}: { - /** Transcend API key authentication */ - auth: string; - /** Data Silo ID to pull down jobs for */ - dataSiloId: string; - /** The request actions to fetch */ - actions: RequestAction[]; - /** How many identifiers to pull in a single call to the backend */ - apiPageSize: number; - /** How many identifiers to save at a time (usually to a CSV file, should be a multiple of apiPageSize) */ - savePageSize: number; - /** Callback function called when a chunk of identifiers is ready to be saved */ - onSave: (chunk: CsvFormattedIdentifier[]) => Promise; - /** API URL for Transcend backend */ - transcendUrl?: string; - /** Sombra API key authentication */ - sombraAuth?: string; - /** Skip request count */ - skipRequestCount?: boolean; -}): Promise<{ - /** Raw Identifiers */ - identifiers: CronIdentifierWithAction[]; -}> { - // Validate savePageSize - if (savePageSize % apiPageSize !== 0) { - throw new Error( - `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`, - ); - } - - // Create sombra instance to communicate with - const sombra = await createSombraGotInstance(transcendUrl, auth, sombraAuth); - - // Create GraphQL client to connect to Transcend backend - const client = buildTranscendGraphQLClient(transcendUrl, auth); - - let totalRequestCount = 0; - if (!skipRequestCount) { - totalRequestCount = await fetchRequestDataSiloActiveCount(client, { - dataSiloId, - }); - } - - logger.info( - colors.magenta( - `Pulling ${ - skipRequestCount ? 'all' : totalRequestCount - } outstanding request identifiers ` + - `for data silo: "${dataSiloId}" for requests of types "${actions.join( - '", "', - )}"`, - ), - ); - - // Time duration - const t0 = Date.now(); - // create a new progress bar instance and use shades_classic theme - const progressBar = new cliProgress.SingleBar( - {}, - cliProgress.Presets.shades_classic, - ); - const foundRequestIds = new Set(); - - // identifiers found in total - const identifiers: CronIdentifierWithAction[] = []; - // current chunk of identifiers to be saved - let currentChunk: CsvFormattedIdentifier[] = []; - - // map over each action - if (!skipRequestCount) { - progressBar.start(totalRequestCount, 0); - } - await mapSeries(actions, async (action) => { - let offset = 0; - let shouldContinue = true; - - // Fetch a page of identifiers - while (shouldContinue) { - const pageIdentifiers = await pullCronPageOfIdentifiers(sombra, { - dataSiloId, - limit: apiPageSize, - offset, - requestType: action, - }); - - const identifiersWithAction: CronIdentifierWithAction[] = - pageIdentifiers.map((identifier) => { - foundRequestIds.add(identifier.requestId); - return { - ...identifier, - action, - }; - }); - - const csvFormattedIdentifiers = identifiersWithAction.map( - ({ attributes, ...identifier }) => ({ - ...identifier, - ...Object.fromEntries( - attributes.map((value) => [value.key, value.values.join(',')]), - ), - }), - ); - - identifiers.push(...identifiersWithAction); - currentChunk.push(...csvFormattedIdentifiers); - - // Check if we've reached the savePageSize and call the onSave callback - if (currentChunk.length >= savePageSize) { - await onSave(currentChunk); - currentChunk = []; - } - - shouldContinue = pageIdentifiers.length === apiPageSize; - offset += apiPageSize; - if (skipRequestCount) { - logger.info( - colors.magenta( - `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`, - ), - ); - } else { - progressBar.update(foundRequestIds.size); - } - } - }); - - // Save any remaining identifiers in the current chunk - if (currentChunk.length > 0) { - await onSave(currentChunk); - } - - if (!skipRequestCount) { - progressBar.stop(); - } - const t1 = Date.now(); - const totalTime = t1 - t0; - - logger.info( - colors.green( - `Successfully pulled ${identifiers.length} outstanding identifiers from ${ - foundRequestIds.size - } requests in "${totalTime / 1000}" seconds!`, - ), - ); - - return { identifiers }; -} diff --git a/src/lib/graphql/createSombraGotInstance.ts b/src/lib/graphql/createSombraGotInstance.ts index 9bd392f0..38d7e36e 100644 --- a/src/lib/graphql/createSombraGotInstance.ts +++ b/src/lib/graphql/createSombraGotInstance.ts @@ -1,8 +1,18 @@ -import got, { Got } from 'got'; +import got, { Got, RequestError, Response } from 'got'; import { buildTranscendGraphQLClient } from './buildTranscendGraphQLClient'; import { ORGANIZATION } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; +interface SombraError extends Omit { + response: Response; +} + +export function isSombraError(error: unknown): error is SombraError { + return ( + error instanceof RequestError && typeof error.response?.body === 'string' + ); +} + /** * Instantiate an instance of got that is capable of making requests * to a sombra gateway. diff --git a/src/lib/graphql/createTranscendConsentGotInstance.ts b/src/lib/graphql/createTranscendConsentGotInstance.ts index 8fde4f07..8f27c262 100644 --- a/src/lib/graphql/createTranscendConsentGotInstance.ts +++ b/src/lib/graphql/createTranscendConsentGotInstance.ts @@ -1,4 +1,16 @@ -import got, { Got } from 'got'; +import got, { Got, RequestError, type Response } from 'got'; + +interface TranscendConsentError extends Omit { + response: Response; +} + +export function isTranscendConsentError( + error: unknown, +): error is TranscendConsentError { + return ( + error instanceof RequestError && typeof error.response?.body === 'string' + ); +} /** * Instantiate an instance of got that is capable of making requests diff --git a/src/lib/graphql/fetchCatalogs.ts b/src/lib/graphql/fetchCatalogs.ts index 75502a95..b5ca8298 100644 --- a/src/lib/graphql/fetchCatalogs.ts +++ b/src/lib/graphql/fetchCatalogs.ts @@ -73,7 +73,7 @@ export async function fetchAndIndexCatalogs(client: GraphQLClient): Promise< // Create mapping from service name to service title const serviceToTitle = Object.fromEntries( - catalogs.map>((catalog) => [ + catalogs.map<[string, string]>((catalog) => [ catalog.integrationName, catalog.title, ]), @@ -81,7 +81,7 @@ export async function fetchAndIndexCatalogs(client: GraphQLClient): Promise< // Create mapping from service name to boolean indicate if service has API integration support const serviceToSupportedIntegration = Object.fromEntries( - catalogs.map>((catalog) => [ + catalogs.map<[string, boolean]>((catalog) => [ catalog.integrationName, catalog.hasApiFunctionality, ]), diff --git a/src/lib/graphql/formatAttributeValues.ts b/src/lib/graphql/formatAttributeValues.ts index 87af79a8..642aa64c 100644 --- a/src/lib/graphql/formatAttributeValues.ts +++ b/src/lib/graphql/formatAttributeValues.ts @@ -1,6 +1,6 @@ import type { DataSiloAttributeValue } from './syncDataSilos'; -export interface FormattedAttribute { +interface FormattedAttribute { /** Attribute key */ key: string; /** Attribute values */ diff --git a/src/lib/graphql/makeGraphQLRequest.ts b/src/lib/graphql/makeGraphQLRequest.ts index ea13b0f8..3edf1a0b 100644 --- a/src/lib/graphql/makeGraphQLRequest.ts +++ b/src/lib/graphql/makeGraphQLRequest.ts @@ -1,8 +1,10 @@ +/* eslint-disable unicorn/filename-case */ import colors from 'colors'; -import type { - GraphQLClient, - RequestDocument, - Variables, +import { + ClientError, + type GraphQLClient, + type RequestDocument, + type Variables, } from 'graphql-request'; import { logger } from '../../logger'; @@ -40,29 +42,31 @@ const KNOWN_ERRORS = [ * @param maxRequests - Max number of requests * @returns Response */ -export async function makeGraphQLRequest( +export async function makeGraphQLRequest( client: GraphQLClient, document: RequestDocument, - variables?: V, + variables?: Variables, requestHeaders?: Record | string[][] | Headers, maxRequests = MAX_RETRIES, ): Promise { let retryCount = 0; + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition while (true) { try { const result = await client.request(document, variables, requestHeaders); return result as T; } catch (error) { + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + if (error.message.includes('API key is invalid')) { - logger.error( - colors.red( - 'API key is invalid. ' + - 'Please ensure that the key provided to `transcendAuth` has the proper scope and is not expired, ' + - 'and that `transcendUrl` corresponds to the correct backend for your organization.', - ), + throw new Error( + 'API key is invalid. ' + + 'Please ensure that the key provided to `transcendAuth` has the proper scope and is not expired, ' + + 'and that `transcendUrl` corresponds to the correct backend for your organization.', ); - process.exit(1); } if (KNOWN_ERRORS.some((message) => error.message.includes(message))) { @@ -70,15 +74,18 @@ export async function makeGraphQLRequest( } // wait for rate limit to resolve - if (error.message.startsWith('Client error: Too many requests')) { - const rateLimitResetAt = - error.response.headers?.get('x-ratelimit-reset'); + if ( + error instanceof ClientError && + error.message.startsWith('Client error: Too many requests') + ) { + const headers = error.response.headers as Headers | undefined; + const rateLimitResetAt = headers?.get('x-ratelimit-reset'); const sleepTime = rateLimitResetAt ? new Date(rateLimitResetAt).getTime() - Date.now() + 100 : 1000 * 10; logger.log( colors.yellow( - `DETECTED RATE LIMIT: ${error.message}. Sleeping for ${sleepTime}ms`, + `DETECTED RATE LIMIT: ${error.message}. Sleeping for ${sleepTime.toLocaleString()}ms`, ), ); @@ -91,7 +98,7 @@ export async function makeGraphQLRequest( retryCount += 1; logger.log( colors.yellow( - `REQUEST FAILED: ${error.message}. Retrying ${retryCount}/${maxRequests}...`, + `REQUEST FAILED: ${error.message}. Retrying ${retryCount.toLocaleString()}/${maxRequests.toLocaleString()}...`, ), ); } diff --git a/src/lib/graphql/syncPromptGroups.ts b/src/lib/graphql/syncPromptGroups.ts index 178ba8d1..c1456c5a 100644 --- a/src/lib/graphql/syncPromptGroups.ts +++ b/src/lib/graphql/syncPromptGroups.ts @@ -9,7 +9,7 @@ import { fetchAllPrompts } from './fetchPrompts'; import { CREATE_PROMPT_GROUP, UPDATE_PROMPT_GROUPS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -export interface EditPromptGroupInput { +interface EditPromptGroupInput { /** Title of prompt group */ title: string; /** Prompt group description */ @@ -25,7 +25,7 @@ export interface EditPromptGroupInput { * @param input - Prompt input * @returns Prompt group ID */ -export async function createPromptGroup( +async function createPromptGroup( client: GraphQLClient, input: EditPromptGroupInput, ): Promise { @@ -55,7 +55,7 @@ export async function createPromptGroup( * @param client - GraphQL client * @param input - Prompt input */ -export async function updatePromptGroups( +async function updatePromptGroups( client: GraphQLClient, input: [EditPromptGroupInput, string][], ): Promise { diff --git a/src/lib/graphql/syncPromptPartials.ts b/src/lib/graphql/syncPromptPartials.ts index dcdbe2af..bfc13644 100644 --- a/src/lib/graphql/syncPromptPartials.ts +++ b/src/lib/graphql/syncPromptPartials.ts @@ -15,7 +15,7 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; * @param input - Prompt input * @returns Prompt partial ID */ -export async function createPromptPartial( +async function createPromptPartial( client: GraphQLClient, input: { /** Title of prompt partial */ @@ -50,7 +50,7 @@ export async function createPromptPartial( * @param client - GraphQL client * @param input - Prompt input */ -export async function updatePromptPartials( +async function updatePromptPartials( client: GraphQLClient, input: [PromptPartialInput, string][], ): Promise { diff --git a/src/lib/graphql/syncTeams.ts b/src/lib/graphql/syncTeams.ts index 7681bee0..98c126ad 100644 --- a/src/lib/graphql/syncTeams.ts +++ b/src/lib/graphql/syncTeams.ts @@ -15,7 +15,7 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; * @param team - Input * @returns Created team */ -export async function createTeam( +async function createTeam( client: GraphQLClient, team: TeamInput, ): Promise> { @@ -49,7 +49,7 @@ export async function createTeam( * @param teamId - ID of team * @returns Updated team */ -export async function updateTeam( +async function updateTeam( client: GraphQLClient, input: TeamInput, teamId: string, diff --git a/src/lib/manual-enrichment/enrichPrivacyRequest.ts b/src/lib/manual-enrichment/enrichPrivacyRequest.ts index 91b1475a..82baa54a 100644 --- a/src/lib/manual-enrichment/enrichPrivacyRequest.ts +++ b/src/lib/manual-enrichment/enrichPrivacyRequest.ts @@ -1,8 +1,9 @@ import colors from 'colors'; -import type { Got } from 'got'; +import { type Got } from 'got'; import * as t from 'io-ts'; import { uniq } from 'lodash-es'; import { logger } from '../../logger'; +import { isSombraError } from '../graphql'; import { splitCsvToList } from '../requests/splitCsvToList'; const ADMIN_URL = @@ -33,7 +34,7 @@ export async function enrichPrivacyRequest( if (!rawId) { // error const message = `Request ID must be provided to enricher request.${ - index ? ` Found error in row: ${index}` : '' + index ? ` Found error in row: ${index.toLocaleString()}` : '' }`; logger.error(colors.red(message)); throw new Error(message); @@ -42,6 +43,7 @@ export async function enrichPrivacyRequest( const id = rawId.toLowerCase(); // Pull out the identifiers + // eslint-disable-next-line unicorn/no-array-reduce const enrichedIdentifiers = Object.entries(rest).reduce< Record >((accumulator, [key, value]) => { @@ -76,7 +78,7 @@ export async function enrichPrivacyRequest( } catch (error) { // skip if already enriched if ( - typeof error.response.body === 'string' && + isSombraError(error) && error.response.body.includes('Cannot update a resolved RequestEnricher') ) { logger.warn( @@ -87,12 +89,15 @@ export async function enrichPrivacyRequest( return false; } - // error - logger.error( - colors.red( - `Failed to enricher identifiers for request with id: ${ADMIN_URL}${id} - ${error.message} - ${error.response.body}`, - ), - ); + // Error message + let message = `Failed to enricher identifiers for request with id: ${ADMIN_URL}${id}`; + if (error instanceof Error) { + message += ` - ${error.message}`; + } + if (isSombraError(error)) { + message += ` - ${error.response.body}`; + } + logger.error(colors.red(message)); throw error; } } diff --git a/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts b/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts index 58e5f10c..d97395e8 100644 --- a/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts +++ b/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts @@ -54,7 +54,9 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ // Notify Transcend logger.info( - colors.magenta(`Enriching "${activeResults.length}" privacy requests.`), + colors.magenta( + `Enriching "${activeResults.length.toLocaleString()}" privacy requests.`, + ), ); let successCount = 0; @@ -99,17 +101,21 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ logger.info( colors.green( - `Successfully notified Transcend! \n Success count: ${successCount}.`, + `Successfully notified Transcend! \n Success count: ${successCount.toLocaleString()}.`, ), ); if (skippedCount > 0) { - logger.info(colors.magenta(`Skipped count: ${skippedCount}.`)); + logger.info( + colors.magenta(`Skipped count: ${skippedCount.toLocaleString()}.`), + ); } if (errorCount > 0) { - logger.info(colors.red(`Error Count: ${errorCount}.`)); - throw new Error(`Failed to enrich: ${errorCount} requests.`); + logger.info(colors.red(`Error Count: ${errorCount.toLocaleString()}.`)); + throw new Error( + `Failed to enrich: ${errorCount.toLocaleString()} requests.`, + ); } return activeResults.length; diff --git a/src/lib/oneTrust/helpers/convertToEmptyStrings.ts b/src/lib/oneTrust/helpers/convertToEmptyStrings.ts deleted file mode 100644 index 067e325a..00000000 --- a/src/lib/oneTrust/helpers/convertToEmptyStrings.ts +++ /dev/null @@ -1,59 +0,0 @@ -/* eslint-disable eslint-comments/disable-enable-pair */ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -/** - * Converts all primitive values in an object or array to empty strings while maintaining structure - * - * @param input - A primitive value, object, or array to be processed - * @returns The input structure with all primitive values converted to empty strings - * @example - * // Simple primitive - * convertToEmptyStrings(42) // returns '' - * - * // Array - * convertToEmptyStrings([1, 'hello', true]) // returns ['', '', ''] - * - * // Complex object - * convertToEmptyStrings({ - * id: 123, - * name: 'test', - * nested: { - * active: true, - * count: 0 - * }, - * items: [1, 2, 3] - * }) - * // returns { - * // id: '', - * // name: '', - * // nested: { - * // active: '', - * // count: '' - * // }, - * // items: ['', '', ''] - * // } - */ -export function convertToEmptyStrings(input: T): any { - // Handle null/undefined - if (input === null || input === undefined) { - return ''; - } - - // Handle arrays - if (Array.isArray(input)) { - return input.map((item) => convertToEmptyStrings(item)); - } - - // Handle objects - if (typeof input === 'object') { - return Object.fromEntries( - Object.entries(input).map>(([key, value]) => [ - key, - convertToEmptyStrings(value), - ]), - ); - } - - // Handle primitives - return ''; -} diff --git a/src/lib/oneTrust/helpers/index.ts b/src/lib/oneTrust/helpers/index.ts index bdf24b7f..38d7c8aa 100644 --- a/src/lib/oneTrust/helpers/index.ts +++ b/src/lib/oneTrust/helpers/index.ts @@ -1,4 +1,3 @@ -export * from './parseCliSyncOtArguments'; export * from './syncOneTrustAssessmentToDisk'; export * from './syncOneTrustAssessmentsFromOneTrust'; export * from './syncOneTrustAssessmentsFromFile'; diff --git a/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts b/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts deleted file mode 100644 index 5dde3ee2..00000000 --- a/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts +++ /dev/null @@ -1,185 +0,0 @@ -import colors from 'colors'; -import yargs from 'yargs-parser'; -import { - OneTrustFileFormat, - OneTrustPullResource, - OneTrustPullSource, -} from '../../../enums'; -import { logger } from '../../../logger'; - -const VALID_RESOURCES = Object.values(OneTrustPullResource); - -interface OneTrustCliArguments { - /** The name of the file to write the resources to */ - file: string; - /** The OneTrust hostname to send the requests to */ - hostname?: string; - /** The OAuth Bearer token used to authenticate the requests to OneTrust */ - oneTrustAuth?: string; - /** The Transcend API key to authenticate the requests to Transcend */ - transcendAuth: string; - /** The Transcend URL where to forward requests */ - transcendUrl: string; - /** The resource to pull from OneTrust */ - resource: OneTrustPullResource; - /** Whether to enable debugging while reporting errors */ - debug: boolean; - /** Whether to export the resource into a file rather than push to transcend */ - dryRun: boolean; - /** Where to read the OneTrust resource from */ - source: OneTrustPullSource; -} - -/** - * Parse the command line arguments - * - * @returns the parsed arguments - */ -export const parseCliSyncOtArguments = (): OneTrustCliArguments => { - const { - file, - hostname, - oneTrustAuth, - resource, - debug, - dryRun, - transcendAuth, - transcendUrl, - source, - } = yargs(process.argv.slice(2), { - string: [ - 'file', - 'hostname', - 'oneTrustAuth', - 'resource', - 'dryRun', - 'transcendAuth', - 'transcendUrl', - 'source', - ], - boolean: ['debug', 'dryRun'], - default: { - resource: OneTrustPullResource.Assessments, - debug: false, - dryRun: false, - transcendUrl: 'https://api.transcend.io', - source: OneTrustPullSource.OneTrust, - }, - }); - - // Must be able to authenticate to transcend to sync resources to it - if (!dryRun && !transcendAuth) { - logger.error( - colors.red( - 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}', - ), - ); - return process.exit(1); - } - if (!dryRun && !transcendUrl) { - logger.error( - colors.red( - 'Must specify a "transcendUrl" parameter to sync resources to Transcend. e.g. --transcendUrl=https://api.transcend.io', - ), - ); - return process.exit(1); - } - - // If trying to sync to disk, must specify a file path - if (dryRun && !file) { - logger.error( - colors.red( - 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json', - ), - ); - return process.exit(1); - } - - if (file) { - const splitFile = file.split('.'); - if (splitFile.length < 2) { - logger.error( - colors.red( - 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.', - ), - ); - return process.exit(1); - } - if (splitFile.at(-1) !== OneTrustFileFormat.Json) { - logger.error( - colors.red( - `Expected the format of the "file" parameters '${file}' to be '${ - OneTrustFileFormat.Json - }', but got '${splitFile.at(-1)}'.`, - ), - ); - return process.exit(1); - } - } - - // if reading assessments from a OneTrust - if (source === OneTrustPullSource.OneTrust) { - // must specify the OneTrust hostname - if (!hostname) { - logger.error( - colors.red( - 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com', - ), - ); - return process.exit(1); - } - // must specify the OneTrust auth - if (!oneTrustAuth) { - logger.error( - colors.red( - 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN', - ), - ); - return process.exit(1); - } - } else { - // if reading the assessments from a file, must specify a file to read from - if (!file) { - logger.error( - colors.red( - 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json', - ), - ); - return process.exit(1); - } - - // Cannot try reading from file and save assessments to a file simultaneously - if (dryRun) { - logger.error( - colors.red( - 'Cannot read and write to a file simultaneously.' + - ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.`, - ), - ); - return process.exit(1); - } - } - - if (!VALID_RESOURCES.includes(resource)) { - logger.error( - colors.red( - `Received invalid resource value: "${resource}". Allowed: ${VALID_RESOURCES.join( - ',', - )}`, - ), - ); - return process.exit(1); - } - - return { - file, - ...(hostname && { hostname }), - ...(oneTrustAuth && { oneTrustAuth }), - resource, - debug, - dryRun, - transcendAuth, - transcendUrl, - source, - }; -}; diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts index e18c1d26..38f08f0e 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts @@ -9,7 +9,7 @@ import { } from '../../graphql'; import { oneTrustAssessmentToJson } from './oneTrustAssessmentToJson'; -export interface AssessmentForm { +interface AssessmentForm { /** ID of Assessment Form */ id: string; /** Title of Assessment Form */ diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts index e2d3ae23..1b649fdb 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts @@ -26,7 +26,7 @@ export const syncOneTrustAssessmentsFromFile = ({ return new Promise((resolve, reject) => { // Create a readable stream from the file const fileStream = createReadStream(file, { - encoding: 'utf-8', + encoding: 'utf8', highWaterMark: 64 * 1024, // 64KB chunks }); diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts index c06eb645..0ed1ee93 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts @@ -21,13 +21,6 @@ import { enrichOneTrustAssessment } from './enrichOneTrustAssessment'; import { syncOneTrustAssessmentToDisk } from './syncOneTrustAssessmentToDisk'; import { syncOneTrustAssessmentToTranscend } from './syncOneTrustAssessmentToTranscend'; -export interface AssessmentForm { - /** ID of Assessment Form */ - id: string; - /** Title of Assessment Form */ - name: string; -} - /** * Reads all the assessments from a OneTrust instance and syncs them to Transcend or to Disk. * diff --git a/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts b/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts deleted file mode 100644 index a5de0dd5..00000000 --- a/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { convertToEmptyStrings } from '../convertToEmptyStrings'; - -describe('buildDefaultCodecWrapper', () => { - it('should correctly build a default codec for null', () => { - const result = convertToEmptyStrings(null); - expect(result).to.equal(''); - }); - - it('should correctly build a default codec for number', () => { - const result = convertToEmptyStrings(0); - expect(result).to.equal(''); - }); - - it('should correctly build a default codec for boolean', () => { - const result = convertToEmptyStrings(false); - expect(result).to.equal(''); - }); - - it('should correctly build a default codec for undefined', () => { - const result = convertToEmptyStrings(); - expect(result).to.equal(''); - }); - - it('should correctly build a default codec for string', () => { - const result = convertToEmptyStrings('1'); - expect(result).to.equal(''); - }); - - it('should correctly build a default codec for an object', () => { - const result = convertToEmptyStrings({ name: 'joe' }); - expect(result).to.deep.equal({ name: '' }); - }); - - it('should correctly build a default codec for an array of primitive types', () => { - const result = convertToEmptyStrings(['name', 0, false]); - expect(result).to.deep.equal(['', '', '']); - }); - - it('should correctly build a default codec for an array of object', () => { - const result = convertToEmptyStrings([ - { name: 'john', age: 52 }, - { name: 'jane', age: 15, isAdult: true }, - ]); - // should default to the array with object if the union contains an array of objects - expect(result).to.deep.equal([ - { name: '', age: '' }, - { name: '', age: '', isAdult: '' }, - ]); - }); -}); diff --git a/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts b/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts index c0845f58..eb9088a1 100644 --- a/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts @@ -6,7 +6,7 @@ import { logger } from '../../logger'; import { mapSeries } from '../bluebird-replace'; import { PreferenceTopic } from '../graphql'; import { splitCsvToList } from '../requests'; -import { FileMetadataState } from './codecs'; +import { FileMetadataState, type PurposeRowMapping } from './codecs'; /** * Parse out the purpose.enabled and preference values from a CSV file @@ -59,7 +59,9 @@ export async function parsePreferenceAndPurposeValuesFromCsv( const uniqueValues = uniq(preferences.map((x) => x[col])); // Map the column to a purpose - let purposeMapping = currentState.columnToPurposeName[col]; + let purposeMapping = currentState.columnToPurposeName[col] as + | PurposeRowMapping + | undefined; if (purposeMapping) { logger.info( colors.magenta( @@ -164,7 +166,9 @@ export async function parsePreferenceAndPurposeValuesFromCsv( return; } - if (preferenceTopic.type === PreferenceTopicType.MultiSelect) { + if ( + (preferenceTopic.type as string) === PreferenceTopicType.MultiSelect + ) { const parsedValues = splitCsvToList(value); // need to do this serially await mapSeries(parsedValues, async (parsedValue) => { diff --git a/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts b/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts index df9357ce..581ec484 100644 --- a/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts @@ -47,7 +47,7 @@ export async function parsePreferenceIdentifiersFromCsv( default: remainingColumnsForIdentifier.find((col) => col.toLowerCase().includes('email'), - ) || remainingColumnsForIdentifier[0], + ) ?? remainingColumnsForIdentifier[0], choices: remainingColumnsForIdentifier, }, ]); @@ -59,9 +59,18 @@ export async function parsePreferenceIdentifiersFromCsv( ), ); + if (!currentState.identifierColumn) { + throw new TypeError('No identifier column found'); + } + if (!currentState.timestampColum) { + throw new TypeError('No timestamp column found'); + } + + const { identifierColumn, timestampColum } = currentState; + // Validate that the identifier column is present for all rows and unique const identifierColumnsMissing = preferences - .map((pref, ind) => (pref[currentState.identifierColumn!] ? null : [ind])) + .map((pref, ind) => (pref[identifierColumn] ? null : [ind])) .filter((x): x is number[] => !!x) .flat(); if (identifierColumnsMissing.length > 0) { @@ -82,32 +91,30 @@ export async function parsePreferenceIdentifiersFromCsv( // Filter out rows missing an identifier const previous = preferences.length; - preferences = preferences.filter( - (pref) => pref[currentState.identifierColumn!], - ); + preferences = preferences.filter((pref) => pref[identifierColumn]); logger.info( colors.yellow( - `Skipped ${previous - preferences.length} rows missing an identifier`, + `Skipped ${(previous - preferences.length).toLocaleString()} rows missing an identifier`, ), ); } logger.info( colors.magenta( - `The identifier column "${currentState.identifierColumn}" is present for all rows`, + `The identifier column "${identifierColumn}" is present for all rows`, ), ); // Validate that all identifiers are unique - const rowsByUserId = groupBy(preferences, currentState.identifierColumn); + const rowsByUserId = groupBy(preferences, identifierColumn); const duplicateIdentifiers = Object.entries(rowsByUserId).filter( ([, rows]) => rows.length > 1, ); if (duplicateIdentifiers.length > 0) { const message = `The identifier column "${ - currentState.identifierColumn + identifierColumn }" has duplicate values for the following rows: ${duplicateIdentifiers .slice(0, 10) - .map(([userId, rows]) => `${userId} (${rows.length})`) + .map(([userId, rows]) => `${userId} (${rows.length.toLocaleString()})`) .join('\n')}`; logger.warn(colors.yellow(message)); @@ -123,8 +130,8 @@ export async function parsePreferenceIdentifiersFromCsv( .map(([, rows]) => { const sorted = rows.sort( (a, b) => - new Date(b[currentState.timestampColum!]).getTime() - - new Date(a[currentState.timestampColum!]).getTime(), + new Date(b[timestampColum]).getTime() - + new Date(a[timestampColum]).getTime(), ); return sorted[0]; }) diff --git a/src/lib/preference-management/parsePreferenceManagementCsv.ts b/src/lib/preference-management/parsePreferenceManagementCsv.ts index c0d629f5..e41e8b1f 100644 --- a/src/lib/preference-management/parsePreferenceManagementCsv.ts +++ b/src/lib/preference-management/parsePreferenceManagementCsv.ts @@ -1,4 +1,5 @@ import { PersistedState } from '@transcend-io/persisted-state'; +import type { PreferenceQueryResponseItem } from '@transcend-io/privacy-types'; import colors from 'colors'; import type { Got } from 'got'; import * as t from 'io-ts'; @@ -67,7 +68,7 @@ export async function parsePreferenceManagementCsvWithCache( pendingConflictUpdates: {}, skippedUpdates: {}, // Load in the last fetched time - ...((fileMetadata[file] || {}) as Partial), + ...((fileMetadata[file] ?? {}) as Partial), lastFetchedAt: new Date().toISOString(), }; @@ -103,10 +104,14 @@ export async function parsePreferenceManagementCsvWithCache( fileMetadata[file] = currentState; await cache.setValue(fileMetadata, 'fileMetadata'); + if (!currentState.identifierColumn) { + throw new TypeError('No identifier column found'); + } + + const { identifierColumn } = currentState; + // Grab existing preference store records - const identifiers = preferences.map( - (pref) => pref[currentState.identifierColumn!], - ); + const identifiers = preferences.map((pref) => pref[identifierColumn]); const existingConsentRecords = skipExistingRecordCheck ? [] : await getPreferencesForIdentifiers(sombra, { @@ -123,7 +128,7 @@ export async function parsePreferenceManagementCsvWithCache( // Process each row for (const pref of preferences) { // Grab unique Id for the user - const userId = pref[currentState.identifierColumn!]; + const userId = pref[identifierColumn]; // determine updates for user const pendingUpdates = getPreferenceUpdatesFromRow({ @@ -134,7 +139,9 @@ export async function parsePreferenceManagementCsvWithCache( }); // Grab current state of the update - const currentConsentRecord = consentRecordByIdentifier[userId]; + const currentConsentRecord = consentRecordByIdentifier[userId] as + | PreferenceQueryResponseItem + | undefined; if (forceTriggerWorkflows && !currentConsentRecord) { throw new Error( `No existing consent record found for user with id: ${userId}. @@ -183,7 +190,12 @@ export async function parsePreferenceManagementCsvWithCache( const t1 = Date.now(); logger.info( colors.green( - `Successfully pre-processed file: "${file}" in ${(t1 - t0) / 1000}s`, + `Successfully pre-processed file: "${file}" in ${( + (t1 - t0) / + 1000 + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}s`, ), ); } diff --git a/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts b/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts index 09b2fb48..6014dc75 100644 --- a/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts @@ -45,10 +45,10 @@ export async function parsePreferenceTimestampsFromCsv( default: remainingColumnsForTimestamp.find((col) => col.toLowerCase().includes('date'), - ) || + ) ?? remainingColumnsForTimestamp.find((col) => col.toLowerCase().includes('time'), - ) || + ) ?? remainingColumnsForTimestamp[0], choices: [...remainingColumnsForTimestamp, NONE_PREFERENCE_MAP], }, @@ -60,9 +60,13 @@ export async function parsePreferenceTimestampsFromCsv( ); // Validate that all rows have valid timestamp - if (currentState.timestampColum !== NONE_PREFERENCE_MAP) { + if ( + currentState.timestampColum !== NONE_PREFERENCE_MAP && + currentState.timestampColum + ) { + const { timestampColum } = currentState; const timestampColumnsMissing = preferences - .map((pref, ind) => (pref[currentState.timestampColum!] ? null : [ind])) + .map((pref, ind) => (pref[timestampColum] ? null : [ind])) .filter((x): x is number[] => !!x) .flat(); if (timestampColumnsMissing.length > 0) { diff --git a/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts b/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts index 1c4a3c90..534fc029 100644 --- a/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts +++ b/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts @@ -526,7 +526,7 @@ describe('getPreferenceUpdatesFromRow', () => { }); expect.fail('Should have thrown'); } catch (error) { - expect(error.message).to.include('No mapping provided'); + expect((error as Error).message).to.include('No mapping provided'); } }); @@ -590,7 +590,7 @@ describe('getPreferenceUpdatesFromRow', () => { }); expect.fail('Should have thrown'); } catch (error) { - expect(error.message).to.equal( + expect((error as Error).message).to.equal( 'Invalid purpose slug: InvalidPurpose, expected: Marketing, Advertising', ); } @@ -652,7 +652,7 @@ describe('getPreferenceUpdatesFromRow', () => { }); expect.fail('Should have thrown'); } catch (error) { - expect(error.message).to.equal( + expect((error as Error).message).to.equal( 'Invalid value for select preference: SingleSelectPreference, expected string or null, got: true', ); } @@ -714,7 +714,7 @@ describe('getPreferenceUpdatesFromRow', () => { }); expect.fail('Should have thrown'); } catch (error) { - expect(error.message).to.equal( + expect((error as Error).message).to.equal( 'Invalid value for multi select preference: MultiSelectPreference, expected one of: Value1, Value2, got: true', ); } diff --git a/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts b/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts index 0f592e5e..b7270194 100644 --- a/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts +++ b/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts @@ -12,6 +12,7 @@ import { createSombraGotInstance, fetchAllPreferenceTopics, fetchAllPurposes, + isSombraError, PreferenceTopic, Purpose, } from '../graphql'; @@ -87,12 +88,12 @@ export async function uploadPreferenceManagementPreferencesInteractive({ logger.info( colors.magenta( 'Restored cache, there are: \n' + - `${ - Object.values(failingRequests).length - } failing requests to be retried\n` + - `${ - Object.values(pendingRequests).length - } pending requests to be processed\n` + + `${Object.values( + failingRequests, + ).length.toLocaleString()} failing requests to be retried\n` + + `${Object.values( + pendingRequests, + ).length.toLocaleString()} pending requests to be processed\n` + `The following files are stored in cache and will be used:\n${Object.keys( fileMetadata, ) @@ -138,23 +139,23 @@ export async function uploadPreferenceManagementPreferencesInteractive({ logger.info( colors.magenta( - `Found ${ - Object.entries(metadata.pendingSafeUpdates).length - } safe updates in ${file}`, + `Found ${Object.entries( + metadata.pendingSafeUpdates, + ).length.toLocaleString()} safe updates in ${file}`, ), ); logger.info( colors.magenta( - `Found ${ - Object.entries(metadata.pendingConflictUpdates).length - } conflict updates in ${file}`, + `Found ${Object.entries( + metadata.pendingConflictUpdates, + ).length.toLocaleString()} conflict updates in ${file}`, ), ); logger.info( colors.magenta( - `Found ${ - Object.entries(metadata.skippedUpdates).length - } skipped updates in ${file}`, + `Found ${Object.entries( + metadata.skippedUpdates, + ).length.toLocaleString()} skipped updates in ${file}`, ), ); @@ -167,9 +168,10 @@ export async function uploadPreferenceManagementPreferencesInteractive({ })) { // Determine timestamp const timestamp = - metadata.timestampColum === NONE_PREFERENCE_MAP + metadata.timestampColum === NONE_PREFERENCE_MAP || + metadata.timestampColum === undefined ? new Date() - : new Date(update[metadata.timestampColum!]); + : new Date(update[metadata.timestampColum]); // Determine updates const updates = getPreferenceUpdatesFromRow({ @@ -200,9 +202,9 @@ export async function uploadPreferenceManagementPreferencesInteractive({ if (dryRun) { logger.info( colors.green( - `Dry run complete, exiting. ${ - Object.values(pendingUpdates).length - } pending updates. Check file: ${receiptFilepath}`, + `Dry run complete, exiting. ${Object.values( + pendingUpdates, + ).length.toLocaleString()} pending updates. Check file: ${receiptFilepath}`, ), ); return; @@ -210,9 +212,9 @@ export async function uploadPreferenceManagementPreferencesInteractive({ logger.info( colors.magenta( - `Uploading ${ - Object.values(pendingUpdates).length - } preferences to partition: ${partition}`, + `Uploading ${Object.values( + pendingUpdates, + ).length.toLocaleString()} preferences to partition: ${partition}`, ), ); @@ -245,20 +247,27 @@ export async function uploadPreferenceManagementPreferencesInteractive({ }) .json(); } catch (error) { - try { - const parsed = JSON.parse(error?.response?.body || '{}'); - if (parsed.error) { - logger.error(colors.red(`Error: ${parsed.error}`)); + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + + if (isSombraError(error)) { + try { + const parsed = JSON.parse(error.response.body) as Record< + string, + unknown + >; + if ('error' in parsed && typeof parsed.error === 'string') { + logger.error(colors.red(`Error: ${parsed.error}`)); + } + } catch { + // continue } - } catch { - // continue } logger.error( colors.red( - `Failed to upload ${ - currentChunk.length - } user preferences to partition ${partition}: ${ - error?.response?.body || error?.message + `Failed to upload ${currentChunk.length.toLocaleString()} user preferences to partition ${partition}: ${ + isSombraError(error) ? error.response.body : error.message }`, ), ); @@ -267,7 +276,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ failingUpdates[userId] = { uploadedAt: new Date().toISOString(), update, - error: error?.response?.body || error?.message || 'Unknown error', + error: isSombraError(error) ? error.response.body : error.message, }; } await preferenceState.setValue(failingUpdates, 'failingUpdates'); @@ -286,11 +295,11 @@ export async function uploadPreferenceManagementPreferencesInteractive({ const totalTime = t1 - t0; logger.info( colors.green( - `Successfully uploaded ${ - updatesToRun.length - } user preferences to partition ${partition} in "${ + `Successfully uploaded ${updatesToRun.length.toLocaleString()} user preferences to partition ${partition} in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); } diff --git a/src/lib/readTranscendYaml.ts b/src/lib/readTranscendYaml.ts index ea8e645e..d4141e24 100644 --- a/src/lib/readTranscendYaml.ts +++ b/src/lib/readTranscendYaml.ts @@ -1,5 +1,5 @@ import { readFileSync, writeFileSync } from 'node:fs'; -import { decodeCodec, ObjByString } from '@transcend-io/type-utils'; +import { decodeCodec } from '@transcend-io/type-utils'; import yaml from 'js-yaml'; import { TranscendInput } from '../codecs'; @@ -17,7 +17,7 @@ export const VARIABLE_PARAMETERS_NAME = 'parameters'; */ export function replaceVariablesInYaml( input: string, - variables: ObjByString, + variables: Record, extraErrorMessage = '', ): string { let contents = input; @@ -30,7 +30,7 @@ export function replaceVariablesInYaml( // Throw error if unfilled variables if (VARIABLE_PARAMETERS_REGEXP.test(contents)) { - const [, name] = VARIABLE_PARAMETERS_REGEXP.exec(contents) || []; + const [, name] = VARIABLE_PARAMETERS_REGEXP.exec(contents) ?? []; throw new Error( `Found variable that was not set: ${name}. Make sure you are passing all parameters through the --${VARIABLE_PARAMETERS_NAME}=${name}:value-for-param flag. @@ -51,10 +51,10 @@ ${extraErrorMessage}`, */ export function readTranscendYaml( filePath: string, - variables: ObjByString = {}, + variables: Record = {}, ): TranscendInput { // Read in contents - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, 'utf8'); // Replace variables const replacedVariables = replaceVariablesInYaml( diff --git a/src/lib/requests/approvePrivacyRequests.ts b/src/lib/requests/approvePrivacyRequests.ts index dd331f2e..3bc0904e 100644 --- a/src/lib/requests/approvePrivacyRequests.ts +++ b/src/lib/requests/approvePrivacyRequests.ts @@ -70,7 +70,11 @@ export async function approvePrivacyRequests({ }); // Notify Transcend - logger.info(colors.magenta(`Approving "${allRequests.length}" requests.`)); + logger.info( + colors.magenta( + `Approving "${allRequests.length.toLocaleString()}" requests.`, + ), + ); let total = 0; let skipped = 0; @@ -98,7 +102,10 @@ export async function approvePrivacyRequests({ input: { requestId: requestToApprove.id }, }); } catch (error) { - if (error.message.includes('Request must be in an approving state,')) { + if ( + error instanceof Error && + error.message.includes('Request must be in an approving state,') + ) { skipped += 1; } } @@ -113,13 +120,17 @@ export async function approvePrivacyRequests({ const t1 = Date.now(); const totalTime = t1 - t0; if (skipped > 0) { - logger.info(colors.yellow(`${skipped} requests were skipped.`)); + logger.info( + colors.yellow(`${skipped.toLocaleString()} requests were skipped.`), + ); } logger.info( colors.green( - `Successfully approved ${total} requests in "${ + `Successfully approved ${total.toLocaleString()} requests in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); return allRequests.length; diff --git a/src/lib/requests/bulkRestartRequests.ts b/src/lib/requests/bulkRestartRequests.ts index 07a04e91..fd2ac075 100644 --- a/src/lib/requests/bulkRestartRequests.ts +++ b/src/lib/requests/bulkRestartRequests.ts @@ -1,4 +1,4 @@ -import { join } from 'node:path'; +import path from 'node:path'; import { PersistedState } from '@transcend-io/persisted-state'; import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; import cliProgress from 'cli-progress'; @@ -14,6 +14,7 @@ import { fetchAllRequestIdentifiers, fetchAllRequests, } from '../graphql'; +import { isSombraError } from '../graphql/createSombraGotInstance'; import { SuccessfulRequest } from './constants'; import { extractClientError } from './extractClientError'; import { restartPrivacyRequest } from './restartPrivacyRequest'; @@ -100,7 +101,7 @@ export async function bulkRestartRequests({ ); // Create a new state file to store the requests from this run - const cacheFile = join( + const cacheFile = path.join( requestReceiptFolder, `tr-request-restart-${new Date().toISOString()}`, ); @@ -124,7 +125,7 @@ export async function bulkRestartRequests({ const requests = allRequests.filter( (request) => new Date(request.createdAt) < createdAt, ); - logger.info(`Found ${requests.length} requests to process`); + logger.info(`Found ${requests.length.toLocaleString()} requests to process`); if (copyIdentifiers) { logger.info('copyIdentifiers detected - All Identifiers will be copied.'); @@ -143,14 +144,11 @@ export async function bulkRestartRequests({ requests.map(({ id }) => id), ); if (missingRequests.length > 0) { - logger.error( - colors.red( - `Failed to find the following requests by ID: ${missingRequests.join( - ',', - )}.`, - ), + throw new Error( + `Failed to find the following requests by ID: ${missingRequests.join( + ',', + )}.`, ); - process.exit(1); } } @@ -199,11 +197,15 @@ export async function bulkRestartRequests({ }); await state.setValue(restartedRequests, 'restartedRequests'); } catch (error) { - const message = `${error.message} - ${JSON.stringify( - error.response?.body, - null, - 2, - )}`; + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + + const message = `${error.message} - ${ + isSombraError(error) + ? JSON.stringify(error.response.body, null, 2) + : '' + }`; const clientError = extractClientError(message); const failingRequests = state.getValue('failingRequests'); @@ -213,7 +215,7 @@ export async function bulkRestartRequests({ rowIndex: ind, coreIdentifier: request.coreIdentifier, attemptedAt: new Date().toISOString(), - error: clientError || message, + error: clientError ?? message, }); await state.setValue(failingRequests, 'failingRequests'); } @@ -230,18 +232,20 @@ export async function bulkRestartRequests({ // Log completion time logger.info( colors.green( - `Completed restarting of requests in "${totalTime / 1000}" seconds.`, + `Completed restarting of requests in "${(totalTime / 1000).toLocaleString( + undefined, + { + maximumFractionDigits: 2, + }, + )}" seconds.`, ), ); // Log errors if (state.getValue('failingRequests').length > 0) { - logger.error( - colors.red( - `Encountered "${state.getValue('failingRequests').length}" errors. ` + - `See "${cacheFile}" to review the error messages and inputs.`, - ), + throw new Error( + `Encountered "${state.getValue('failingRequests').length.toLocaleString()}" errors. ` + + `See "${cacheFile}" to review the error messages and inputs.`, ); - process.exit(1); } } diff --git a/src/lib/requests/bulkRetryEnrichers.ts b/src/lib/requests/bulkRetryEnrichers.ts index d854fb35..92bc34ba 100644 --- a/src/lib/requests/bulkRetryEnrichers.ts +++ b/src/lib/requests/bulkRetryEnrichers.ts @@ -81,14 +81,11 @@ export async function bulkRetryEnrichers({ requests.map(({ id }) => id), ); if (missingRequests.length > 0) { - logger.error( - colors.red( - `Failed to find the following requests by ID: ${missingRequests.join( - ',', - )}.`, - ), + throw new Error( + `Failed to find the following requests by ID: ${missingRequests.join( + ',', + )}.`, ); - process.exit(1); } } @@ -126,11 +123,11 @@ export async function bulkRetryEnrichers({ // Log completion time logger.info( colors.green( - `Completed restarting of ${ - requests.length - } requests and ${totalRestarted} enrichers in "${ + `Completed restarting of ${requests.length.toLocaleString()} requests and ${totalRestarted.toLocaleString()} enrichers in "${( totalTime / 1000 - }" seconds.`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds.`, ), ); } diff --git a/src/lib/requests/cancelPrivacyRequests.ts b/src/lib/requests/cancelPrivacyRequests.ts index 815ff5d5..3525196e 100644 --- a/src/lib/requests/cancelPrivacyRequests.ts +++ b/src/lib/requests/cancelPrivacyRequests.ts @@ -103,7 +103,7 @@ export async function cancelPrivacyRequests({ // Notify Transcend logger.info( colors.magenta( - `Canceling "${allRequests.length}" requests${ + `Canceling "${allRequests.length.toLocaleString()}" requests${ cancelationTemplate ? ` Using template: ${cancelationTemplate.title}` : '' @@ -155,9 +155,11 @@ export async function cancelPrivacyRequests({ logger.info( colors.green( - `Successfully canceled ${total} requests in "${ + `Successfully canceled ${total.toLocaleString()} requests in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); return allRequests.length; diff --git a/src/lib/requests/downloadPrivacyRequestFiles.ts b/src/lib/requests/downloadPrivacyRequestFiles.ts index 83fc4ee1..f9d4df9b 100644 --- a/src/lib/requests/downloadPrivacyRequestFiles.ts +++ b/src/lib/requests/downloadPrivacyRequestFiles.ts @@ -1,5 +1,5 @@ import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; +import path from 'node:path'; import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; import cliProgress from 'cli-progress'; import colors from 'colors'; @@ -99,7 +99,7 @@ export async function downloadPrivacyRequestFiles({ requestFileMetadata, async ([request, metadata]) => { // Create a new folder to store request files - const requestFolder = join(folderPath, request.id); + const requestFolder = path.join(folderPath, request.id); if (!existsSync(requestFolder)) { mkdirSync(requestFolder); } @@ -111,8 +111,8 @@ export async function downloadPrivacyRequestFiles({ onFileDownloaded: (fil, stream) => { // Ensure a folder exists for the file // filename looks like Health/heartbeat.csv - const filePath = join(requestFolder, fil.fileName); - const folder = dirname(filePath); + const filePath = path.join(requestFolder, fil.fileName); + const folder = path.dirname(filePath); if (!existsSync(folder)) { mkdirSync(folder, { recursive: true }); } @@ -143,14 +143,18 @@ export async function downloadPrivacyRequestFiles({ logger.info( colors.green( - `Successfully downloaded ${total} requests in "${ + `Successfully downloaded ${total.toLocaleString()} requests in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); if (totalApproved > 0) { logger.info( - colors.green(`Approved ${totalApproved} requests in Transcend.`), + colors.green( + `Approved ${totalApproved.toLocaleString()} requests in Transcend.`, + ), ); } return allRequests.length; diff --git a/src/lib/requests/extractClientError.ts b/src/lib/requests/extractClientError.ts index 41eb2e58..c1500730 100644 --- a/src/lib/requests/extractClientError.ts +++ b/src/lib/requests/extractClientError.ts @@ -7,5 +7,7 @@ const CLIENT_ERROR = /{\\"message\\":\\"(.+?)\\",/; * @returns Client error or null */ export function extractClientError(error: string): string | null { - return CLIENT_ERROR.test(error) ? CLIENT_ERROR.exec(error)![1] : null; + return CLIENT_ERROR.test(error) + ? (CLIENT_ERROR.exec(error)?.[1] ?? null) + : null; } diff --git a/src/lib/requests/filterRows.ts b/src/lib/requests/filterRows.ts index ac289466..184be160 100644 --- a/src/lib/requests/filterRows.ts +++ b/src/lib/requests/filterRows.ts @@ -1,4 +1,3 @@ -import { ObjByString } from '@transcend-io/type-utils'; import colors from 'colors'; import inquirer from 'inquirer'; import { uniq } from 'lodash-es'; @@ -13,7 +12,9 @@ import { getUniqueValuesForColumn } from './getUniqueValuesForColumn'; * @param rows - Rows to filter * @returns Filtered rows */ -export async function filterRows(rows: ObjByString[]): Promise { +export async function filterRows( + rows: Record[], +): Promise[]> { // Determine set of column names const columnNames = uniq(rows.flatMap((x) => Object.keys(x))); @@ -32,7 +33,7 @@ export async function filterRows(rows: ObjByString[]): Promise { { name: 'filterColumnName', - message: `If you need to filter the list of requests to import, choose the column to filter on. Currently ${filteredRows.length} rows.`, + message: `If you need to filter the list of requests to import, choose the column to filter on. Currently ${filteredRows.length.toLocaleString()} rows.`, type: 'list', default: columnNames, choices: [NONE, ...columnNames], @@ -63,6 +64,10 @@ export async function filterRows(rows: ObjByString[]): Promise { } } - logger.info(colors.magenta(`Importing ${filteredRows.length} requests`)); + logger.info( + colors.magenta( + `Importing ${filteredRows.length.toLocaleString()} requests`, + ), + ); return filteredRows; } diff --git a/src/lib/requests/getFileMetadataForPrivacyRequests.ts b/src/lib/requests/getFileMetadataForPrivacyRequests.ts index 0c856642..0fab1960 100644 --- a/src/lib/requests/getFileMetadataForPrivacyRequests.ts +++ b/src/lib/requests/getFileMetadataForPrivacyRequests.ts @@ -6,7 +6,7 @@ import type { Got } from 'got'; import * as t from 'io-ts'; import { logger } from '../../logger'; import { map } from '../bluebird-replace'; -import { PrivacyRequest } from '../graphql'; +import { isSombraError, PrivacyRequest } from '../graphql'; export const IntlMessage = t.type({ /** The message key */ @@ -108,7 +108,9 @@ export async function getFileMetadataForPrivacyRequests( }, ): Promise<[Pick, RequestFileMetadata[]][]> { logger.info( - colors.magenta(`Pulling file metadata for ${requests.length} requests`), + colors.magenta( + `Pulling file metadata for ${requests.length.toLocaleString()} requests`, + ), ); // Time duration @@ -160,9 +162,13 @@ export async function getFileMetadataForPrivacyRequests( shouldContinue = !!response._links.next && response.nodes.length === limit; } catch (error) { + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + throw new Error( `Received an error from server: ${ - error?.response?.body || error?.message + isSombraError(error) ? error.response.body : error.message }`, ); } @@ -181,9 +187,11 @@ export async function getFileMetadataForPrivacyRequests( logger.info( colors.green( - `Successfully downloaded file metadata ${requests.length} requests in "${ + `Successfully downloaded file metadata ${requests.length.toLocaleString()} requests in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); diff --git a/src/lib/requests/getUniqueValuesForColumn.ts b/src/lib/requests/getUniqueValuesForColumn.ts index 99527b67..55f197dc 100644 --- a/src/lib/requests/getUniqueValuesForColumn.ts +++ b/src/lib/requests/getUniqueValuesForColumn.ts @@ -1,4 +1,3 @@ -import { ObjByString } from '@transcend-io/type-utils'; import { uniq } from 'lodash-es'; /** @@ -9,8 +8,8 @@ import { uniq } from 'lodash-es'; * @returns Unique set of values in that column */ export function getUniqueValuesForColumn( - rows: ObjByString[], + rows: Record[], columnName: string, ): string[] { - return uniq(rows.flatMap((row) => row[columnName] || '')); + return uniq(rows.flatMap((row) => row[columnName] ?? '')); } diff --git a/src/lib/requests/mapCsvRowsToRequestInputs.ts b/src/lib/requests/mapCsvRowsToRequestInputs.ts index da8ebefa..a9837dc6 100644 --- a/src/lib/requests/mapCsvRowsToRequestInputs.ts +++ b/src/lib/requests/mapCsvRowsToRequestInputs.ts @@ -8,7 +8,7 @@ import { NORMALIZE_PHONE_NUMBER, RequestAction, } from '@transcend-io/privacy-types'; -import { ObjByString, valuesOf } from '@transcend-io/type-utils'; +import { valuesOf } from '@transcend-io/type-utils'; import * as t from 'io-ts'; import { DateFromISOString } from 'io-ts-types'; import { AttributeKey } from '../graphql'; @@ -132,7 +132,7 @@ export function normalizeIdentifierValue( * @returns [raw input, request input] list */ export function mapCsvRowsToRequestInputs( - requestInputs: ObjByString[], + requestInputs: Record[], state: PersistedState, { columnNameMap, @@ -154,8 +154,16 @@ export function mapCsvRowsToRequestInputs( }, ): [Record, PrivacyRequestInput][] { // map the CSV to request input - const getMappedName = (attribute: ColumnName): string => - state.getValue('columnNames', attribute) || columnNameMap[attribute]!; + // Get mapped value + const getMappedName = (attribute: ColumnName): string => { + const value = + state.getValue('columnNames', attribute) ?? columnNameMap[attribute]; + if (value === undefined) { + throw new Error(`Column name ${attribute} is not mapped`); + } + return value; + }; + return requestInputs.map( (input): [Record, PrivacyRequestInput] => { // The extra identifiers to upload for this request @@ -166,9 +174,9 @@ export function mapCsvRowsToRequestInputs( // filter out skipped identifiers .filter(([, columnName]) => columnName !== NONE)) { // Determine the identifier type being specified - const identifierType = Object.values(IdentifierType).includes( - identifierName as any, // eslint-disable-line @typescript-eslint/no-explicit-any - ) + const identifierType = ( + Object.values(IdentifierType) as string[] + ).includes(identifierName) ? (identifierName as IdentifierType) : IdentifierType.Custom; @@ -182,9 +190,7 @@ export function mapCsvRowsToRequestInputs( ); if (normalized) { // Initialize - if (!attestedExtraIdentifiers[identifierType]) { - attestedExtraIdentifiers[identifierType] = []; - } + attestedExtraIdentifiers[identifierType] ??= []; // Add the identifier attestedExtraIdentifiers[identifierType].push({ @@ -211,7 +217,7 @@ export function mapCsvRowsToRequestInputs( attributes.push({ values: isMulti ? splitCsvToList(attributeValueString) - : attributeValueString, + : [attributeValueString], key: attributeName, }); } diff --git a/src/lib/requests/mapRequestEnumValues.ts b/src/lib/requests/mapRequestEnumValues.ts index baf2ee61..21c3800e 100644 --- a/src/lib/requests/mapRequestEnumValues.ts +++ b/src/lib/requests/mapRequestEnumValues.ts @@ -6,7 +6,6 @@ import { IsoCountrySubdivisionCode, RequestAction, } from '@transcend-io/privacy-types'; -import { ObjByString } from '@transcend-io/type-utils'; import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; import { logger } from '../../logger'; @@ -25,7 +24,7 @@ import { mapEnumValues } from './mapEnumValues'; */ export async function mapRequestEnumValues( client: GraphQLClient, - requests: ObjByString[], + requests: Record[], { state, columnNameMap, @@ -37,8 +36,14 @@ export async function mapRequestEnumValues( }, ): Promise { // Get mapped value - const getMappedName = (attribute: ColumnName): string => - state.getValue('columnNames', attribute) || columnNameMap[attribute]!; + const getMappedName = (attribute: ColumnName): string => { + const value = + state.getValue('columnNames', attribute) ?? columnNameMap[attribute]; + if (value === undefined) { + throw new Error(`Column name ${attribute} is not mapped`); + } + return value; + }; // Fetch all data subjects in the organization const { internalSubjects } = await makeGraphQLRequest<{ diff --git a/src/lib/requests/markSilentPrivacyRequests.ts b/src/lib/requests/markSilentPrivacyRequests.ts index eb6dc7e7..ccbaa4c2 100644 --- a/src/lib/requests/markSilentPrivacyRequests.ts +++ b/src/lib/requests/markSilentPrivacyRequests.ts @@ -76,7 +76,9 @@ export async function markSilentPrivacyRequests({ // Notify Transcend logger.info( - colors.magenta(`Marking "${allRequests.length}" as silent mode.`), + colors.magenta( + `Marking "${allRequests.length.toLocaleString()}" as silent mode.`, + ), ); let total = 0; @@ -103,9 +105,11 @@ export async function markSilentPrivacyRequests({ logger.info( colors.green( - `Successfully marked ${total} requests as silent mode in "${ + `Successfully marked ${total.toLocaleString()} requests as silent mode in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); return allRequests.length; diff --git a/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts b/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts index 9da01de9..835ff784 100644 --- a/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts +++ b/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts @@ -95,7 +95,7 @@ export async function notifyPrivacyRequestsAdditionalTime({ // Notify Transcend logger.info( colors.magenta( - `Notifying "${allRequests.length}" that more time is needed.`, + `Notifying "${allRequests.length.toLocaleString()}" that more time is needed.`, ), ); @@ -125,9 +125,11 @@ export async function notifyPrivacyRequestsAdditionalTime({ logger.info( colors.green( - `Successfully marked ${total} requests as silent mode in "${ + `Successfully marked ${total.toLocaleString()} requests as silent mode in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); return allRequests.length; diff --git a/src/lib/requests/pullPrivacyRequests.ts b/src/lib/requests/pullPrivacyRequests.ts index 5ba70bbb..54b36e0e 100644 --- a/src/lib/requests/pullPrivacyRequests.ts +++ b/src/lib/requests/pullPrivacyRequests.ts @@ -120,7 +120,9 @@ export async function pullPrivacyRequests({ ); logger.info( - colors.magenta(`Pulled ${requestsWithRequestIdentifiers.length} requests`), + colors.magenta( + `Pulled ${requestsWithRequestIdentifiers.length.toLocaleString()} requests`, + ), ); // Write out to CSV diff --git a/src/lib/requests/readCsv.ts b/src/lib/requests/readCsv.ts index e949fe64..be0478f2 100644 --- a/src/lib/requests/readCsv.ts +++ b/src/lib/requests/readCsv.ts @@ -15,16 +15,17 @@ import * as t from 'io-ts'; export function readCsv( pathToFile: string, codec: T, - options: Options = { columns: true }, + options: Options = { columns: true } as Options, ): t.TypeOf[] { // read file contents and parse - const fileContent = parse(readFileSync(pathToFile, 'utf-8'), options); + const fileContent: unknown = parse(readFileSync(pathToFile, 'utf8'), options); // validate codec const data = decodeCodec(t.array(codec), fileContent); // remove any special characters from object keys const parsed = data.map((datum) => + // eslint-disable-next-line unicorn/no-array-reduce Object.entries(datum).reduce( (accumulator, [key, value]) => Object.assign(accumulator, { diff --git a/src/lib/requests/removeUnverifiedRequestIdentifiers.ts b/src/lib/requests/removeUnverifiedRequestIdentifiers.ts index 0d4effc4..40c20908 100644 --- a/src/lib/requests/removeUnverifiedRequestIdentifiers.ts +++ b/src/lib/requests/removeUnverifiedRequestIdentifiers.ts @@ -98,9 +98,11 @@ export async function removeUnverifiedRequestIdentifiers({ logger.info( colors.green( - `Successfully cleared out unverified identifiers "${ + `Successfully cleared out unverified identifiers "${( totalTime / 1000 - }" seconds for ${total} requests, ${processed} identifiers were cleared out!`, + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds for ${total.toLocaleString()} requests, ${processed.toLocaleString()} identifiers were cleared out!`, ), ); return allRequests.length; diff --git a/src/lib/requests/restartPrivacyRequest.ts b/src/lib/requests/restartPrivacyRequest.ts index 1e41f41c..8fd49ff8 100644 --- a/src/lib/requests/restartPrivacyRequest.ts +++ b/src/lib/requests/restartPrivacyRequest.ts @@ -57,9 +57,9 @@ export async function restartPrivacyRequest( ) .map((ri) => ({ ...ri, - type: Object.values(IdentifierType).includes( - ri.name as any, // eslint-disable-line @typescript-eslint/no-explicit-any - ) + type: ( + Object.values(IdentifierType) as string[] + ).includes(ri.name) ? ri.name : IdentifierType.Custom, })), diff --git a/src/lib/requests/retryRequestDataSilos.ts b/src/lib/requests/retryRequestDataSilos.ts index bdcb9525..2a92a5a7 100644 --- a/src/lib/requests/retryRequestDataSilos.ts +++ b/src/lib/requests/retryRequestDataSilos.ts @@ -56,7 +56,7 @@ export async function retryRequestDataSilos({ // Notify Transcend logger.info( colors.magenta( - `Retrying requests for Data Silo: "${dataSiloId}", restarting "${allRequests.length}" requests.`, + `Retrying requests for Data Silo: "${dataSiloId}", restarting "${allRequests.length.toLocaleString()}" requests.`, ), ); @@ -79,6 +79,10 @@ export async function retryRequestDataSilos({ requestDataSiloId: requestDataSilo.id, }); } catch (error) { + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + // some requests may not have this data silo connected if (!error.message.includes('Failed to find RequestDataSilo')) { throw error; @@ -98,9 +102,12 @@ export async function retryRequestDataSilos({ logger.info( colors.green( - `Successfully notified Transcend in "${ - totalTime / 1000 - }" seconds for ${total} requests, ${skipped} requests were skipped because data silo was not attached to the request!`, + `Successfully notified Transcend in "${(totalTime / 1000).toLocaleString( + undefined, + { + maximumFractionDigits: 2, + }, + )}" seconds for ${total.toLocaleString()} requests, ${skipped.toLocaleString()} requests were skipped because data silo was not attached to the request!`, ), ); return allRequests.length; diff --git a/src/lib/requests/skipPreflightJobs.ts b/src/lib/requests/skipPreflightJobs.ts index 893f6259..c9555b99 100644 --- a/src/lib/requests/skipPreflightJobs.ts +++ b/src/lib/requests/skipPreflightJobs.ts @@ -52,9 +52,7 @@ export async function skipPreflightJobs({ // Notify Transcend logger.info( colors.magenta( - `Processing enricher: "${enricherIds.join(',')}" fetched "${ - requests.length - }" in enriching status.`, + `Processing enricher: "${enricherIds.join(',')}" fetched "${requests.length.toLocaleString()}" in enriching status.`, ), ); @@ -77,11 +75,12 @@ export async function skipPreflightJobs({ const requestEnrichersFiltered = requestEnrichers.filter( (enricher) => enricherIds.includes(enricher.enricher.id) && - ![ - RequestEnricherStatus.Resolved, - RequestEnricherStatus.Skipped, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ].includes(enricher.status as any), + !( + [ + RequestEnricherStatus.Resolved, + RequestEnricherStatus.Skipped, + ] as RequestEnricherStatus[] + ).includes(enricher.status), ); // FIXME @@ -96,6 +95,10 @@ export async function skipPreflightJobs({ }); totalSkipped += 1; } catch (error) { + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + if ( !error.message.includes( 'Client error: Cannot skip Request enricher because it has already completed', @@ -118,9 +121,11 @@ export async function skipPreflightJobs({ logger.info( colors.green( - `Successfully skipped "${totalSkipped}" for "${ - requests.length - }" requests in "${totalTime / 1000}" seconds!`, + `Successfully skipped "${totalSkipped.toLocaleString()}" for "${requests.length.toLocaleString()}" requests in "${( + totalTime / 1000 + ).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds!`, ), ); return requests.length; diff --git a/src/lib/requests/skipRequestDataSilos.ts b/src/lib/requests/skipRequestDataSilos.ts index 23000495..db643a21 100644 --- a/src/lib/requests/skipRequestDataSilos.ts +++ b/src/lib/requests/skipRequestDataSilos.ts @@ -53,7 +53,7 @@ export async function skipRequestDataSilos({ // Notify Transcend logger.info( colors.magenta( - `Processing data silo: "${dataSiloId}" marking "${requestDataSilos.length}" requests as skipped.`, + `Processing data silo: "${dataSiloId}" marking "${requestDataSilos.length.toLocaleString()}" requests as skipped.`, ), ); @@ -77,7 +77,10 @@ export async function skipRequestDataSilos({ status, }); } catch (error) { - if (!error.message.includes('Client error: Request must be active:')) { + if ( + !(error instanceof Error) || + !error.message.includes('Client error: Request must be active:') + ) { throw error; } } @@ -94,9 +97,9 @@ export async function skipRequestDataSilos({ logger.info( colors.green( - `Successfully skipped "${requestDataSilos.length}" requests in "${ + `Successfully skipped "${requestDataSilos.length.toLocaleString()}" requests in "${( totalTime / 1000 - }" seconds!`, + ).toLocaleString(undefined, { maximumFractionDigits: 2 })}" seconds!`, ), ); return requestDataSilos.length; diff --git a/src/lib/requests/streamPrivacyRequestFiles.ts b/src/lib/requests/streamPrivacyRequestFiles.ts index 35d20a85..8659164a 100644 --- a/src/lib/requests/streamPrivacyRequestFiles.ts +++ b/src/lib/requests/streamPrivacyRequestFiles.ts @@ -1,7 +1,8 @@ import colors from 'colors'; -import type { Got } from 'got'; +import { type Got } from 'got'; import { logger } from '../../logger'; import { map } from '../bluebird-replace'; +import { isSombraError } from '../graphql'; import { RequestFileMetadata } from './getFileMetadataForPrivacyRequests'; /** @@ -47,7 +48,10 @@ export async function streamPrivacyRequestFiles( onFileDownloaded(metadata, fileResponse); }); } catch (error) { - if (error?.response?.body?.includes('fileMetadata#verify')) { + if ( + isSombraError(error) && + error.response.body.includes('fileMetadata#verify') + ) { logger.error( colors.red( `Failed to pull file for: ${metadata.fileName} (request:${requestId}) - JWT expired. ` + @@ -58,9 +62,14 @@ export async function streamPrivacyRequestFiles( ); return; } + + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + throw new Error( `Received an error from server: ${ - error?.response?.body || error?.message + isSombraError(error) ? error.response.body : error.message }`, ); } diff --git a/src/lib/requests/submitPrivacyRequest.ts b/src/lib/requests/submitPrivacyRequest.ts index ffb7cba7..54704633 100644 --- a/src/lib/requests/submitPrivacyRequest.ts +++ b/src/lib/requests/submitPrivacyRequest.ts @@ -8,6 +8,7 @@ import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; import type { Got } from 'got'; import * as t from 'io-ts'; import { uniq } from 'lodash-es'; +import { isSombraError } from '../graphql'; import { PrivacyRequestInput } from './mapCsvRowsToRequestInputs'; import { ParsedAttributeInput } from './parseAttributesFromString'; @@ -70,7 +71,7 @@ export async function submitPrivacyRequest( // Merge the per-request attributes with the // global attributes const mergedAttributes = [...additionalAttributes]; - for (const attribute of input.attributes || []) { + for (const attribute of input.attributes ?? []) { const existing = mergedAttributes.find( (attribute_) => attribute_.key === attribute.key, ); @@ -125,9 +126,13 @@ export async function submitPrivacyRequest( }) .json(); } catch (error) { + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + throw new Error( `Received an error from server: ${ - error?.response?.body || error?.message + isSombraError(error) ? error.response.body : error.message }`, ); } diff --git a/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts b/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts index 1c1523f9..b101e35c 100644 --- a/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts +++ b/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts @@ -1,4 +1,6 @@ import { describe } from 'vitest'; // TODO: https://transcend.height.app/T-10772 - mapCsvRowsToRequestInputs test -describe.skip('mapCsvRowsToRequestInputs', () => {}); +describe.skip('mapCsvRowsToRequestInputs', () => { + return; +}); diff --git a/src/lib/requests/tests/readCsv.test.ts b/src/lib/requests/tests/readCsv.test.ts index b3c3ae84..587833a8 100644 --- a/src/lib/requests/tests/readCsv.test.ts +++ b/src/lib/requests/tests/readCsv.test.ts @@ -1,4 +1,4 @@ -import { join } from 'node:path'; +import path from 'node:path'; import * as t from 'io-ts'; import { describe, expect, it } from 'vitest'; import { readCsv } from '../index'; @@ -6,7 +6,10 @@ import { readCsv } from '../index'; describe('readCsv', () => { it('should successfully parse a csv', () => { expect( - readCsv(join(__dirname, 'sample.csv'), t.record(t.string, t.string)), + readCsv( + path.join(import.meta.dirname, 'sample.csv'), + t.record(t.string, t.string), + ), ).to.deep.equal([ { CASL_STATUS: 'Undefined', @@ -36,19 +39,28 @@ describe('readCsv', () => { it('throw an error for invalid file', () => { expect(() => - readCsv(join(__dirname, 'sample.csv'), t.type({ notValid: t.string })), + readCsv( + path.join(import.meta.dirname, 'sample.csv'), + t.type({ notValid: t.string }), + ), ).to.throw('Failed to decode codec'); }); it('throw an error for invalid codec', () => { expect(() => - readCsv(join(__dirname, 'sample.csvs'), t.record(t.string, t.string)), + readCsv( + path.join(import.meta.dirname, 'sample.csvs'), + t.record(t.string, t.string), + ), ).to.throw('ENOENT: no such file or directory, open'); }); it('throw an error for invalid format', () => { expect(() => - readCsv(join(__dirname, 'readCsv.test.ts'), t.record(t.string, t.string)), + readCsv( + path.join(import.meta.dirname, 'readCsv.test.ts'), + t.record(t.string, t.string), + ), ).to.throw(); }); }); diff --git a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts index 4fc243c5..a79094f8 100644 --- a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts +++ b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts @@ -1,4 +1,4 @@ -import { join } from 'node:path'; +import path from 'node:path'; import { PersistedState } from '@transcend-io/persisted-state'; import cliProgress from 'cli-progress'; import colors from 'colors'; @@ -11,6 +11,7 @@ import { buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequestAttributeKeys, + isSombraError, } from '../graphql'; import { CachedFileState, CachedRequestState } from './constants'; import { extractClientError } from './extractClientError'; @@ -106,11 +107,10 @@ export async function uploadPrivacyRequestsFromCsv({ }); // Create a new state file to store the requests from this run - const requestCacheFile = join( + const filenameWithoutExtension = path.basename(file, path.extname(file)); + const requestCacheFile = path.join( requestReceiptFolder, - `tr-request-upload-${new Date().toISOString()}-${file - .split('/') - .pop()}`.replace('.csv', '.json'), + `tr-request-upload-${new Date().toISOString()}-${filenameWithoutExtension}.json`, ); const requestState = new PersistedState( requestCacheFile, @@ -190,12 +190,12 @@ export async function uploadPrivacyRequestsFromCsv({ // The identifier to log, only include personal data if debug mode is on const requestLogId = debug ? `email:${requestInput.email} | coreIdentifier:${requestInput.coreIdentifier}` - : `row:${ind.toString()}`; + : `row:${ind.toLocaleString()}`; if (debug) { logger.info( colors.magenta( - `[${ind + 1}/${requestInputs.length}] Importing: ${JSON.stringify( + `[${(ind + 1).toLocaleString()}/${requestInputs.length.toLocaleString()}] Importing: ${JSON.stringify( requestInput, null, 2, @@ -235,14 +235,12 @@ export async function uploadPrivacyRequestsFromCsv({ if (debug) { logger.info( colors.green( - `[${ind + 1}/${ - requestInputs.length - }] Successfully submitted the test data subject request: "${requestLogId}"`, + `[${(ind + 1).toLocaleString()}/${requestInputs.length.toLocaleString()}] Successfully submitted the test data subject request: "${requestLogId}"`, ), ); logger.info( colors.green( - `[${ind + 1}/${requestInputs.length}] View it at: "${ + `[${(ind + 1).toLocaleString()}/${requestInputs.length.toLocaleString()}] View it at: "${ requestResponse.link }"`, ), @@ -260,11 +258,14 @@ export async function uploadPrivacyRequestsFromCsv({ }); await requestState.setValue(successfulRequests, 'successfulRequests'); } catch (error) { - const message = `${error.message} - ${JSON.stringify( - error.response?.body, - null, - 2, - )}`; + if (!(error instanceof Error)) { + throw new TypeError('Unknown CLI Error', { cause: error }); + } + const message = `${error.message} - ${ + isSombraError(error) + ? JSON.stringify(error.response.body, null, 2) + : '' + }`; const clientError = extractClientError(message); if ( @@ -273,9 +274,7 @@ export async function uploadPrivacyRequestsFromCsv({ if (debug) { logger.info( colors.yellow( - `[${ind + 1}/${ - requestInputs.length - }] Skipping request as it is a duplicate`, + `[${(ind + 1).toLocaleString()}/${requestInputs.length.toLocaleString()}] Skipping request as it is a duplicate`, ), ); } @@ -291,17 +290,17 @@ export async function uploadPrivacyRequestsFromCsv({ failingRequests.push({ ...requestInput, rowIndex: ind, + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing error: clientError || message, attemptedAt: new Date().toISOString(), }); await requestState.setValue(failingRequests, 'failingRequests'); if (debug) { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing logger.error(colors.red(clientError || message)); logger.error( colors.red( - `[${ind + 1}/${ - requestInputs.length - }] Failed to submit request for: "${requestLogId}"`, + `[${(ind + 1).toLocaleString()}/${requestInputs.length.toLocaleString()}] Failed to submit request for: "${requestLogId}"`, ), ); } @@ -324,16 +323,20 @@ export async function uploadPrivacyRequestsFromCsv({ // Log completion time logger.info( - colors.green(`Completed upload in "${totalTime / 1000}" seconds.`), + colors.green( + `Completed upload in "${(totalTime / 1000).toLocaleString(undefined, { + maximumFractionDigits: 2, + })}" seconds.`, + ), ); // Log duplicates if (requestState.getValue('duplicateRequests').length > 0) { logger.info( colors.yellow( - `Encountered "${ - requestState.getValue('duplicateRequests').length - }" duplicate requests. ` + + `Encountered "${requestState + .getValue('duplicateRequests') + .length.toLocaleString()}" duplicate requests. ` + `See "${requestCacheFile}" to review the core identifiers for these requests.`, ), ); @@ -341,14 +344,11 @@ export async function uploadPrivacyRequestsFromCsv({ // Log errors if (requestState.getValue('failingRequests').length > 0) { - logger.error( - colors.red( - `Encountered "${ - requestState.getValue('failingRequests').length - }" errors. ` + - `See "${requestCacheFile}" to review the error messages and inputs.`, - ), + throw new Error( + `Encountered "${requestState + .getValue('failingRequests') + .length.toLocaleString()}" errors. ` + + `See "${requestCacheFile}" to review the error messages and inputs.`, ); - process.exit(1); } } diff --git a/src/lib/tests/codebase.test.ts b/src/lib/tests/codebase.test.ts index 78c2e332..85363c3d 100644 --- a/src/lib/tests/codebase.test.ts +++ b/src/lib/tests/codebase.test.ts @@ -75,7 +75,10 @@ async function checkExport( .relative(process.cwd(), filePath) .replaceAll('\\', '/'); - const module = await import(`../../../${relativePath}`); + const module = (await import(`../../../${relativePath}`)) as Record< + string, + unknown + >; return exportName in module && module[exportName] !== undefined; } catch { diff --git a/src/lib/tests/findCodePackagesInFolder.test.ts b/src/lib/tests/findCodePackagesInFolder.test.ts index 078f92e3..4cd6c622 100644 --- a/src/lib/tests/findCodePackagesInFolder.test.ts +++ b/src/lib/tests/findCodePackagesInFolder.test.ts @@ -1,4 +1,4 @@ -import { join } from 'node:path'; +import path from 'node:path'; import { describe, expect, it } from 'vitest'; import type { CodePackageInput } from '../../codecs'; import { findCodePackagesInFolder } from '../code-scanning/findCodePackagesInFolder'; @@ -723,7 +723,10 @@ describe('findCodePackagesInFolder', () => { it('should remove links', async () => { const result = await findCodePackagesInFolder({ repositoryName: 'transcend-io/cli', - scanPath: join(__dirname, '../../../examples/code-scanning'), + scanPath: path.join( + import.meta.dirname, + '../../../examples/code-scanning', + ), }); expect(sortCodePackages(result)).to.deep.equal(sortCodePackages(expected)); }); diff --git a/src/lib/tests/getGitFilesThatChanged.test.ts b/src/lib/tests/getGitFilesThatChanged.test.ts index 837126de..138aaf11 100644 --- a/src/lib/tests/getGitFilesThatChanged.test.ts +++ b/src/lib/tests/getGitFilesThatChanged.test.ts @@ -1,4 +1,4 @@ -import { join } from 'node:path'; +import path from 'node:path'; import { describe, expect, it } from 'vitest'; import { getGitFilesThatChanged } from '../ai/getGitFilesThatChanged'; @@ -9,7 +9,7 @@ describe.skip('getGitFilesThatChanged', () => { getGitFilesThatChanged({ baseBranch: 'main', githubRepo: 'https://github.com/transcend-io/cli.git', - rootDirectory: join(__dirname, '../../'), + rootDirectory: path.join(import.meta.dirname, '../../'), }), ).to.deep.equal({ changedFiles: [ diff --git a/src/lib/tests/readTranscendYaml.test.ts b/src/lib/tests/readTranscendYaml.test.ts index 050ae708..0b2d9a48 100644 --- a/src/lib/tests/readTranscendYaml.test.ts +++ b/src/lib/tests/readTranscendYaml.test.ts @@ -1,18 +1,24 @@ -import { join } from 'node:path'; +import path from 'node:path'; import { describe, expect, it } from 'vitest'; import { readTranscendYaml } from '../../index'; -const EXAMPLE_DIR = join(__dirname, '..', '..', '..', 'examples'); +const EXAMPLE_DIR = path.join( + import.meta.dirname, + '..', + '..', + '..', + 'examples', +); describe('readTranscendYaml', () => { it('simple.yml should pass the codec validation for TranscendInput', () => { expect(() => - readTranscendYaml(join(EXAMPLE_DIR, 'simple.yml')), + readTranscendYaml(path.join(EXAMPLE_DIR, 'simple.yml')), ).to.not.throw(); }); it('invalid.yml should fail the codec validation for TranscendInput', () => { - expect(() => readTranscendYaml(join(EXAMPLE_DIR, 'invalid.yml'))).to + expect(() => readTranscendYaml(path.join(EXAMPLE_DIR, 'invalid.yml'))).to .throw(` ".enrichers.0.0.title expected type 'string'", ".enrichers.1.0.output-identifiers expected type 'Array'", ".data-silos.0.0.title expected type 'string'", @@ -24,23 +30,26 @@ describe('readTranscendYaml', () => { it('multi-instance.yml should fail when no variables are provided', () => { expect(() => - readTranscendYaml(join(EXAMPLE_DIR, 'multi-instance.yml')), + readTranscendYaml(path.join(EXAMPLE_DIR, 'multi-instance.yml')), ).to.throw('Found variable that was not set: domain'); }); it('multi-instance.yml should be successful when variables are provided', () => { - const result = readTranscendYaml(join(EXAMPLE_DIR, 'multi-instance.yml'), { - domain: 'acme.com', - stage: 'Staging', - }); + const result = readTranscendYaml( + path.join(EXAMPLE_DIR, 'multi-instance.yml'), + { + domain: 'acme.com', + stage: 'Staging', + }, + ); - expect(result.enrichers![0].url).to.equal( + expect(result.enrichers?.[0].url).to.equal( 'https://example.acme.com/transcend-enrichment-webhook', ); - expect(result.enrichers![1].url).to.equal( + expect(result.enrichers?.[1].url).to.equal( 'https://example.acme.com/transcend-fraud-check', ); - expect(result['data-silos']![0].description).to.equal( + expect(result['data-silos']?.[0].description).to.equal( 'The mega-warehouse that contains a copy over all SQL backed databases - Staging', ); }); diff --git a/src/logger.ts b/src/logger.ts index 4d25a1c6..5ffebdb1 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -7,7 +7,7 @@ export const logger = console; // When the proxy env var of flag is specified, initiate the proxy const { httpProxy = process.env.http_proxy } = yargs(process.argv.slice(2)); -if (httpProxy) { +if (httpProxy && typeof httpProxy === 'string') { logger.info(colors.green(`Initializing proxy: ${httpProxy}`)); // Use global-agent, which overrides `request` based requests diff --git a/tsconfig.json b/tsconfig.json index d241dbb0..929c7cca 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -24,7 +24,6 @@ "noFallthroughCasesInSwitch": true, "useUnknownInCatchVariables": false, - "typeRoots": ["src/lib/@types", "node_modules/@types"], "baseUrl": ".", "checkJs": true }, diff --git a/vitest.config.ts b/vitest.config.ts index 9ecfec02..497f8153 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,4 +3,7 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ plugins: [tsconfigPaths()], + test: { + include: ['src/**/*.test.ts'], + }, });